This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c7ffbda3877 [fix](iceberg) Preserve nullability in schema metadata
(#68640)
c7ffbda3877 is described below
commit c7ffbda38772c84c41139bab930a2e704866f50e
Author: daidai <[email protected]>
AuthorDate: Tue Oct 20 10:57:02 2026 +0800
[fix](iceberg) Preserve nullability in schema metadata (#68640)
### What problem does this PR solve?
Related PR: #67856
Problem Summary:
Iceberg columns declared `NOT NULL` are shown as nullable by `DESC`,
`SHOW CREATE TABLE`, `SHOW [FULL] COLUMNS` and
`information_schema.columns`. The connector reads every top-level column
as nullable on purpose, since old data files can miss a column, and the
display paths reuse that read schema.
`ConnectorColumn` now also carries the nullability the source declares.
The Iceberg connector sets it from the field's required flag, and these
metadata commands use it. The columns used for reads and writes stay
nullable.
### Release note
Show the declared nullability of Iceberg columns in `DESC`, `SHOW CREATE
TABLE`, `SHOW COLUMNS` and `information_schema.columns`.
### Check List (For Author)
- Test
- [x] Regression test
- [x] Unit Test
- Behavior changed:
- [x] Yes. Required Iceberg columns are now shown as `NOT NULL`.
- Does this need documentation?
- [x] No.
---
.../iceberg/IcebergConnectorMetadata.java | 4 +-
.../iceberg/IcebergConnectorMetadataTest.java | 19 ++++++
.../doris/connector/spi/ConnectorColumn.java | 53 +++++++++++++---
.../connector/spi/ConnectorPluginSurfaceTest.java | 5 +-
.../test/resources/connector-plugin-surface.txt | 2 +
fe/fe-connector/pom.xml | 2 +-
.../main/java/org/apache/doris/catalog/Env.java | 2 +-
.../java/org/apache/doris/catalog/TableIf.java | 25 ++++++++
.../apache/doris/common/proc/IndexInfoProcDir.java | 2 +-
.../plugin/PluginDrivenExternalTable.java | 69 ++++++++++++++++++++-
.../plugin/PluginDrivenSchemaCacheValue.java | 32 ++++++++++
.../doris/httpv2/rest/TableSchemaAction.java | 14 ++++-
.../doris/httpv2/restv2/MetaInfoActionV2.java | 7 ++-
.../org/apache/doris/mysql/MysqlSerializer.java | 18 +++++-
.../doris/mysql/protocol/MysqlResultSender.java | 4 +-
.../trees/plans/commands/ShowColumnsCommand.java | 2 +-
.../org/apache/doris/qe/MysqlConnectProcessor.java | 4 +-
.../apache/doris/service/FrontendServiceImpl.java | 2 +-
.../catalog/EnvShowCreatePluginTableTest.java | 4 ++
.../plugin/PluginDrivenExternalTableTest.java | 64 +++++++++++++++++++
.../rest/TableSchemaActionColumnInfoTest.java | 22 +++++++
.../mysql/protocol/MysqlResultSenderTest.java | 55 +++++++++++++++++
.../iceberg/iceberg_schema_change_ddl.out | 72 +++++++++++-----------
.../iceberg/test_iceberg_initial_defaults.out | 8 +--
.../iceberg/test_iceberg_show_nullable.out | 29 +++++++++
.../iceberg/test_iceberg_show_nullable.groovy | 71 +++++++++++++++++++++
26 files changed, 527 insertions(+), 64 deletions(-)
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java
index 1e3aa164e26..43901064454 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadata.java
@@ -2487,6 +2487,8 @@ public class IcebergConnectorMetadata implements
ConnectorMetadata {
// isKey is always true (external-table semantics: DESC shows
Key=true),
// and isAllowNull is always true regardless of the Iceberg
required/optional flag (rows can
// still read NULL under schema-evolution default-fill; do NOT
propagate the NOT NULL constraint).
+ // The required/optional flag travels separately as the SOURCE
nullability, which is what DESC /
+ // SHOW CREATE TABLE / SHOW COLUMNS / information_schema display
instead of the read nullability.
// Iceberg write defaults are statement-scoped writer metadata,
not catalog display metadata.
// IcebergWritePlanProvider pins them before write analysis;
keeping the cached default null prevents
// DESCRIBE/SHOW CREATE from exposing them and avoids using a
stale default after schema evolution.
@@ -2497,7 +2499,7 @@ public class IcebergConnectorMetadata implements
ConnectorMetadata {
field.doc() != null ? field.doc() : "",
true,
null,
- true);
+ true).withSourceNullable(field.isOptional());
// Carry the stable iceberg field-id as the column's uniqueId
(legacy
// IcebergUtils.updateIcebergColumnUniqueId set the top-level
Column.uniqueId = field.fieldId()).
// fe-core's ConnectorColumnConverter re-applies it (>= 0); the BE
field-id scan path keys the
diff --git
a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataTest.java
b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataTest.java
index a8975d561ad..4b28492c4ff 100644
---
a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataTest.java
+++
b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataTest.java
@@ -888,6 +888,25 @@ public class IcebergConnectorMetadataTest {
"a doc-less Iceberg field must yield an empty (not null)
comment");
}
+ @Test
+ public void getTableSchemaDeclaresRequiredFieldsNotNull() {
+ RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps();
+ ops.table = new FakeIcebergTable(
+ "t1", idNameSchema(), PartitionSpec.unpartitioned(),
+ "s3://bucket/db1/t1", Collections.emptyMap());
+
+ ConnectorTableSchema schema =
+ metadataWith(ops).getTableSchema(null, new
IcebergTableHandle("db1", "t1"));
+
+ // Required fields are read as nullable but displayed as NOT NULL.
+ ConnectorColumn id = schema.getColumns().get(0);
+ Assertions.assertTrue(id.isNullable());
+ Assertions.assertFalse(id.isSourceNullable());
+ ConnectorColumn name = schema.getColumns().get(1);
+ Assertions.assertTrue(name.isNullable());
+ Assertions.assertTrue(name.isSourceNullable());
+ }
+
@Test
public void getTableSchemaCopiesTablePropertiesAndLocation() {
RecordingIcebergCatalogOps ops = new RecordingIcebergCatalogOps();
diff --git
a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorColumn.java
b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorColumn.java
index 007f2d12fc9..8b678e02535 100644
---
a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorColumn.java
+++
b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorColumn.java
@@ -74,6 +74,22 @@ public final class ConnectorColumn {
private final String defaultValueSql;
// Optional semantic type for textual write input, separate from the
exposed binary column type.
private final ConnectorType stringWriteType;
+ // The nullability the SOURCE schema declares (its required/optional
flag), which is what the engine's
+ // metadata statements show: DESC, SHOW CREATE TABLE, SHOW [FULL] COLUMNS,
information_schema.columns and
+ // the field list a MySQL client asks for. It is deliberately separate
from the read nullability above:
+ // a connector may read a source-required column as nullable
(iceberg/paimon do, because a row written
+ // before the column existed reads back NULL under schema-evolution
default-fill) while the source
+ // contract still declares it NOT NULL. Execution — scans, the optimizer,
write validation, the BE
+ // descriptor — always follows `nullable`, never this field.
+ //
+ // Defaults to `nullable`, so every connector that does not opt in keeps
display == read and this field
+ // changes nothing. Set via withSourceNullable().
+ //
+ // Excluded from equals()/hashCode() like the "specified" markers above:
it is a presentation fact that
+ // never round-trips (a column converted back from a Doris Column carries
no source contract), so it must
+ // not become part of column identity — otherwise a MODIFY COLUMN would
look like a change just because
+ // the request-side column lacks the display flag.
+ private final boolean sourceNullable;
public ConnectorColumn(String name, ConnectorType type, String comment,
boolean nullable, String defaultValue) {
@@ -94,14 +110,14 @@ public final class ConnectorColumn {
boolean nullable, String defaultValue, boolean isKey, boolean
isAutoInc,
boolean isAggregated) {
this(name, type, comment, nullable, defaultValue, isKey, isAutoInc,
isAggregated, false, true,
- UNSET_UNIQUE_ID, false, false, false, null, null);
+ UNSET_UNIQUE_ID, false, false, false, null, null, nullable);
}
private ConnectorColumn(String name, ConnectorType type, String comment,
boolean nullable, String defaultValue, boolean isKey, boolean
isAutoInc,
boolean isAggregated, boolean withTimeZone, boolean visible, int
uniqueId,
boolean reservedPassthrough, boolean nullableSpecified, boolean
commentSpecified,
- String defaultValueSql, ConnectorType stringWriteType) {
+ String defaultValueSql, ConnectorType stringWriteType, boolean
sourceNullable) {
this.name = Objects.requireNonNull(name, "name");
this.type = Objects.requireNonNull(type, "type");
this.comment = comment;
@@ -118,6 +134,7 @@ public final class ConnectorColumn {
this.commentSpecified = commentSpecified;
this.defaultValueSql = defaultValueSql;
this.stringWriteType = stringWriteType;
+ this.sourceNullable = sourceNullable;
}
/**
@@ -128,7 +145,7 @@ public final class ConnectorColumn {
public ConnectorColumn withTimeZone() {
return new ConnectorColumn(name, type, comment, nullable, defaultValue,
isKey, isAutoInc, isAggregated, true, visible, uniqueId,
reservedPassthrough,
- nullableSpecified, commentSpecified, defaultValueSql,
stringWriteType);
+ nullableSpecified, commentSpecified, defaultValueSql,
stringWriteType, sourceNullable);
}
/**
@@ -138,7 +155,7 @@ public final class ConnectorColumn {
public ConnectorColumn invisible() {
return new ConnectorColumn(name, type, comment, nullable, defaultValue,
isKey, isAutoInc, isAggregated, withTimeZone, false, uniqueId,
reservedPassthrough,
- nullableSpecified, commentSpecified, defaultValueSql,
stringWriteType);
+ nullableSpecified, commentSpecified, defaultValueSql,
stringWriteType, sourceNullable);
}
/**
@@ -150,7 +167,7 @@ public final class ConnectorColumn {
public ConnectorColumn reservedPassthrough() {
return new ConnectorColumn(name, type, comment, nullable, defaultValue,
isKey, isAutoInc, isAggregated, withTimeZone, visible,
uniqueId, true,
- nullableSpecified, commentSpecified, defaultValueSql,
stringWriteType);
+ nullableSpecified, commentSpecified, defaultValueSql,
stringWriteType, sourceNullable);
}
/**
@@ -161,7 +178,7 @@ public final class ConnectorColumn {
public ConnectorColumn withSpecified(boolean nullableSpecified, boolean
commentSpecified) {
return new ConnectorColumn(name, type, comment, nullable, defaultValue,
isKey, isAutoInc, isAggregated, withTimeZone, visible,
uniqueId, reservedPassthrough,
- nullableSpecified, commentSpecified, defaultValueSql,
stringWriteType);
+ nullableSpecified, commentSpecified, defaultValueSql,
stringWriteType, sourceNullable);
}
/**
@@ -173,7 +190,7 @@ public final class ConnectorColumn {
public ConnectorColumn withUniqueId(int uniqueId) {
return new ConnectorColumn(name, type, comment, nullable, defaultValue,
isKey, isAutoInc, isAggregated, withTimeZone, visible,
uniqueId, reservedPassthrough,
- nullableSpecified, commentSpecified, defaultValueSql,
stringWriteType);
+ nullableSpecified, commentSpecified, defaultValueSql,
stringWriteType, sourceNullable);
}
/**
@@ -182,7 +199,7 @@ public final class ConnectorColumn {
public ConnectorColumn withDefaultValueSql(String defaultValueSql) {
return new ConnectorColumn(name, type, comment, nullable, defaultValue,
isKey, isAutoInc, isAggregated, withTimeZone, visible,
uniqueId, reservedPassthrough,
- nullableSpecified, commentSpecified, defaultValueSql,
stringWriteType);
+ nullableSpecified, commentSpecified, defaultValueSql,
stringWriteType, sourceNullable);
}
/**
@@ -193,13 +210,23 @@ public final class ConnectorColumn {
public ConnectorColumn withStringWriteType(ConnectorType stringWriteType) {
return new ConnectorColumn(name, type, comment, nullable, defaultValue,
isKey, isAutoInc, isAggregated, withTimeZone, visible,
uniqueId, reservedPassthrough,
- nullableSpecified, commentSpecified, defaultValueSql,
stringWriteType);
+ nullableSpecified, commentSpecified, defaultValueSql,
stringWriteType, sourceNullable);
}
public ConnectorType getStringWriteType() {
return stringWriteType;
}
+ /**
+ * Returns a copy carrying the nullability the source schema declares, for
metadata statements only.
+ * See {@link #isSourceNullable()}; execution keeps following {@link
#isNullable()}.
+ */
+ public ConnectorColumn withSourceNullable(boolean sourceNullable) {
+ return new ConnectorColumn(name, type, comment, nullable, defaultValue,
+ isKey, isAutoInc, isAggregated, withTimeZone, visible,
uniqueId, reservedPassthrough,
+ nullableSpecified, commentSpecified, defaultValueSql,
stringWriteType, sourceNullable);
+ }
+
public String getName() {
return name;
}
@@ -262,6 +289,14 @@ public final class ConnectorColumn {
return defaultValueSql;
}
+ /**
+ * The nullability the source schema declares, used by metadata statements
only; see the field comment.
+ * Defaults to {@link #isNullable()}. Execution always follows {@link
#isNullable()}.
+ */
+ public boolean isSourceNullable() {
+ return sourceNullable;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git
a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java
b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java
index 7a28ab6aa69..cce4173053f 100644
---
a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java
+++
b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java
@@ -88,8 +88,9 @@ public class ConnectorPluginSurfaceTest {
Assertions.assertNotNull(in, "missing connector plugin API version
resource");
version.load(in);
}
- // Major 18 adds DiagnosticException to the major 17 scoped streaming
surface.
- Assertions.assertEquals("18.0", version.getProperty("api.version"));
+ // Major 19 adds
ConnectorColumn#isSourceNullable()/withSourceNullable(boolean) - the
nullability the
+ // source declares, kept separate from the nullability the engine
reads the column with.
+ Assertions.assertEquals("19.0", version.getProperty("api.version"));
}
/** Root entry points plus provider/handle types returned to connector
plugins. */
diff --git
a/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt
b/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt
index aa67d2b9468..81918e6daa1 100644
---
a/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt
+++
b/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt
@@ -52,11 +52,13 @@
org.apache.doris.connector.spi.ConnectorColumn#isKey():boolean
org.apache.doris.connector.spi.ConnectorColumn#isNullable():boolean
org.apache.doris.connector.spi.ConnectorColumn#isNullableSpecified():boolean
org.apache.doris.connector.spi.ConnectorColumn#isReservedPassthrough():boolean
+org.apache.doris.connector.spi.ConnectorColumn#isSourceNullable():boolean
org.apache.doris.connector.spi.ConnectorColumn#isVisible():boolean
org.apache.doris.connector.spi.ConnectorColumn#isWithTimeZone():boolean
org.apache.doris.connector.spi.ConnectorColumn#reservedPassthrough():org.apache.doris.connector.spi.ConnectorColumn
org.apache.doris.connector.spi.ConnectorColumn#toString():java.lang.String
org.apache.doris.connector.spi.ConnectorColumn#withDefaultValueSql(java.lang.String):org.apache.doris.connector.spi.ConnectorColumn
+org.apache.doris.connector.spi.ConnectorColumn#withSourceNullable(boolean):org.apache.doris.connector.spi.ConnectorColumn
org.apache.doris.connector.spi.ConnectorColumn#withSpecified(boolean,boolean):org.apache.doris.connector.spi.ConnectorColumn
org.apache.doris.connector.spi.ConnectorColumn#withStringWriteType(org.apache.doris.connector.spi.ConnectorType):org.apache.doris.connector.spi.ConnectorColumn
org.apache.doris.connector.spi.ConnectorColumn#withTimeZone():org.apache.doris.connector.spi.ConnectorColumn
diff --git a/fe/fe-connector/pom.xml b/fe/fe-connector/pom.xml
index eab1885c677..50adfde9967 100644
--- a/fe/fe-connector/pom.xml
+++ b/fe/fe-connector/pom.xml
@@ -55,7 +55,7 @@ under the License.
of the latter two means bumping this property as well (and
fe-extension-spi means bumping
all five families).
-->
- <connector.plugin.api.version>18.0</connector.plugin.api.version>
+ <connector.plugin.api.version>19.0</connector.plugin.api.version>
</properties>
<modules>
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 46a11792b6f..2c041999427 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -4735,7 +4735,7 @@ public class Env {
sb.append(" (\n");
int idx = 0;
- List<Column> columns = table.getBaseSchema(false);
+ List<Column> columns = table.getBaseSchemaForDisplay(false);
for (Column column : columns) {
if (idx++ != 0) {
sb.append(",\n");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
index 154fa21cc03..3bcebc6e9bf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
@@ -173,8 +173,33 @@ public interface TableIf {
}
}
+ /** {@link #getBaseSchemaOrEmpty()} over the metadata view; see {@link
#getBaseSchemaForDisplay()}. */
+ default List<Column> getBaseSchemaForDisplayOrEmpty() {
+ try {
+ return getBaseSchemaForDisplay();
+ } catch (Exception e) {
+ LOG.warn("failed to get base schema for table {}", getName(), e);
+ return Lists.newArrayList();
+ }
+ }
+
List<Column> getBaseSchema(boolean full);
+ /**
+ * The base schema shown by metadata statements (DESC, SHOW CREATE TABLE,
SHOW COLUMNS and
+ * information_schema.columns). It differs from {@link #getBaseSchema()}
only for a table whose source
+ * declares columns NOT NULL that are still read as nullable. Execution
(scans, the optimizer, write
+ * validation, the BE descriptor) must keep using {@link #getBaseSchema()}.
+ */
+ default List<Column> getBaseSchemaForDisplay() {
+ return getBaseSchema();
+ }
+
+ /** Same as {@link #getBaseSchemaForDisplay()}, with the hidden-column
choice of {@link #getBaseSchema(boolean)}. */
+ default List<Column> getBaseSchemaForDisplay(boolean full) {
+ return getBaseSchema(full);
+ }
+
void setNewFullSchema(List<Column> newSchema);
Column getColumn(String name);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java
index 8d6d57bdeb4..71912a1916b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java
@@ -144,7 +144,7 @@ public class IndexInfoProcDir implements ProcDirInterface {
return new RemoteIndexSchemaProcDir(table, schema,
bfColumns);
}
} else {
- schema = table.getBaseSchema();
+ schema = table.getBaseSchemaForDisplay();
}
return new IndexSchemaProcNode(schema, bfColumns);
} finally {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java
index cc95d843239..6cbea40af75 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java
@@ -719,6 +719,10 @@ public class PluginDrivenExternalTable extends
ExternalTable {
}
List<Column> columns =
ConnectorColumnConverter.convertColumns(mappedColumns);
+ // The display view is projected HERE, from the same mapped columns,
so the read and display schemas
+ // always belong to one schema generation (a connector may
deliberately read a column as nullable
+ // while its source contract declares it NOT NULL; only metadata
statements show the latter).
+ List<Column> displayColumns = toDisplayColumns(columns, mappedColumns);
// Identify partition columns from the connector's reserved
PARTITION_COLUMNS_KEY property (a CSV of
// RAW remote column names; producer:
hive/hudi/iceberg/paimon/maxcompute). We keep two aligned
@@ -749,7 +753,40 @@ public class PluginDrivenExternalTable extends
ExternalTable {
}
return new PluginDrivenSchemaCacheValue(columns, partitionColumns,
partitionColumnRemoteNames,
tableSchema.getProperties(),
tableSchema.getTableCapabilities(),
- tableSchema.getWriteMetadataIdentity());
+ tableSchema.getWriteMetadataIdentity(), displayColumns);
+ }
+
+ /**
+ * Projects {@code readColumns} onto the nullability the connector's
source declares
+ * ({@link ConnectorColumn#isSourceNullable()}), index-aligned with and
the same size as {@code readColumns}
+ * (the converter maps connector columns 1:1 and in order). A column whose
declared nullability equals its
+ * read nullability is shared, not copied, so a connector that declares no
divergence — and every
+ * never-opted-in connector — keeps a display view that is literally the
read schema. A copied column
+ * re-applies the one display marker {@link Column#Column(Column)} does
not carry (the WITH_TIMEZONE
+ * "Extra" marker the connector declared), so a required TIMESTAMP WITH
TIME ZONE column keeps showing it.
+ */
+ private static List<Column> toDisplayColumns(List<Column> readColumns,
List<ConnectorColumn> connectorColumns) {
+ List<Column> displayColumns = new ArrayList<>(readColumns.size());
+ for (int i = 0; i < readColumns.size(); i++) {
+ Column readColumn = readColumns.get(i);
+ ConnectorColumn connectorColumn = connectorColumns.get(i);
+ boolean sourceNullable = connectorColumn.isSourceNullable();
+ if (sourceNullable == readColumn.isAllowNull()) {
+ displayColumns.add(readColumn);
+ continue;
+ }
+ Column displayColumn = new Column(readColumn);
+ displayColumn.setIsAllowNull(sourceNullable);
+ // Column's copy constructor does not carry `extraInfo`, so the
WITH_TIMEZONE marker
+ // (ConnectorColumn.withTimeZone(), re-applied by
ConnectorColumnConverter on the read column)
+ // has to be re-applied here as well — otherwise DESC would show
"Extra: NONE" for exactly the
+ // required TZ columns this projection touches.
+ if (connectorColumn.isWithTimeZone()) {
+ displayColumn.setWithTZExtraInfo();
+ }
+ displayColumns.add(displayColumn);
+ }
+ return displayColumns;
}
@Override
@@ -885,6 +922,36 @@ public class PluginDrivenExternalTable extends
ExternalTable {
return appendSyntheticWriteColumns(super.getFullSchema());
}
+ @Override
+ public List<Column> getBaseSchemaForDisplay() {
+ boolean showHidden = Util.showHiddenColumns();
+ if (!showHidden && needInternalHiddenColumns()) {
+ showHidden = true;
+ }
+ return getBaseSchemaForDisplay(showHidden);
+ }
+
+ /**
+ * The metadata view of this table's base schema: the same columns and
order, with the nullability the
+ * connector's source declares. Mirrors {@link
ExternalTable#getBaseSchema(boolean)} exactly — {@code full}
+ * only decides whether invisible columns are kept, and request-scoped
synthetic write columns join the
+ * hidden view — but serves one schema-cache generation and never the read
nullability.
+ */
+ @Override
+ public List<Column> getBaseSchemaForDisplay(boolean full) {
+ // ONE cache read: the columns and the declared-nullability projection
were produced together at
+ // schema load, so this can never mix generations or hit the connector
twice (which it would under
+ // a bypassed schema cache, e.g. a per-user delegated credential).
Mirrors getWriteSchemaSnapshot().
+ List<Column> displayColumns = getSchemaCacheValue()
+ .map(value -> ((PluginDrivenSchemaCacheValue)
value).getDisplayColumns())
+ .orElse(null);
+ if (displayColumns == null) {
+ return null;
+ }
+ List<Column> schema = appendSyntheticWriteColumns(displayColumns);
+ return full ? schema :
schema.stream().filter(Column::isVisible).collect(Collectors.toList());
+ }
+
/**
* Same as {@link #getFullSchema()}, but the BASE schema is resolved AS OF
{@code snapshot} (this
* reference's own pin). The synthetic write columns are request-scoped,
not version-scoped, so they are
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenSchemaCacheValue.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenSchemaCacheValue.java
index f89662035e2..4e7650aa2a8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenSchemaCacheValue.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenSchemaCacheValue.java
@@ -46,6 +46,12 @@ import java.util.Set;
* the raw-keyed partition-value maps returned by the connector SPI
* ({@code ConnectorPartitionInfo.getPartitionValues()}).</li>
* </ul>
+ *
+ * <p>It also caches the DISPLAY view of the schema next to the read schema
+ * ({@link #getDisplayColumns()}): the same columns with the nullability the
source
+ * declares rather than the nullability the engine reads them with. Metadata
statements
+ * use it; everything that executes uses {@link #getSchema()}. See the field
comment for
+ * why both views are projected together, once per generation.</p>
*/
public class PluginDrivenSchemaCacheValue extends SchemaCacheValue {
@@ -61,6 +67,15 @@ public class PluginDrivenSchemaCacheValue extends
SchemaCacheValue {
// carrier. Empty for every connector that does not refine per table.
private final Set<ConnectorCapability> tableCapabilities;
private final String writeMetadataIdentity;
+ // The same columns with the nullability the SOURCE declares
(ConnectorColumn.isSourceNullable()) instead
+ // of the read nullability, index-aligned with the full schema above.
Metadata statements (DESC, SHOW
+ // CREATE TABLE, SHOW [FULL] COLUMNS, information_schema.columns,
COM_FIELD_LIST) serve THIS view;
+ // scans, the optimizer, write binding and the BE descriptor keep serving
getSchema(), which stays as
+ // nullable as the connector declared it to be read. Projected once per
schema generation from the same
+ // ConnectorTableSchema, so the two views can never straddle a refresh;
entries are shared with the read
+ // schema (same Column instance) whenever the connector declares no
divergence, and are full copies
+ // otherwise. Identical to the read schema for every connector that does
not opt in.
+ private final List<Column> displayColumns;
public PluginDrivenSchemaCacheValue(List<Column> schema, List<Column>
partitionColumns,
List<String> partitionColumnRemoteNames) {
@@ -82,12 +97,21 @@ public class PluginDrivenSchemaCacheValue extends
SchemaCacheValue {
public PluginDrivenSchemaCacheValue(List<Column> schema, List<Column>
partitionColumns,
List<String> partitionColumnRemoteNames, Map<String, String>
tableProperties,
Set<ConnectorCapability> tableCapabilities, String
writeMetadataIdentity) {
+ this(schema, partitionColumns, partitionColumnRemoteNames,
tableProperties, tableCapabilities,
+ writeMetadataIdentity, schema);
+ }
+
+ public PluginDrivenSchemaCacheValue(List<Column> schema, List<Column>
partitionColumns,
+ List<String> partitionColumnRemoteNames, Map<String, String>
tableProperties,
+ Set<ConnectorCapability> tableCapabilities, String
writeMetadataIdentity,
+ List<Column> displayColumns) {
super(schema);
this.partitionColumns = partitionColumns;
this.partitionColumnRemoteNames = partitionColumnRemoteNames;
this.tableProperties = tableProperties == null ?
Collections.emptyMap() : tableProperties;
this.tableCapabilities = tableCapabilities == null ?
Collections.emptySet() : tableCapabilities;
this.writeMetadataIdentity = writeMetadataIdentity;
+ this.displayColumns = displayColumns == null ? schema : displayColumns;
}
public List<Column> getPartitionColumns() {
@@ -109,4 +133,12 @@ public class PluginDrivenSchemaCacheValue extends
SchemaCacheValue {
public String getWriteMetadataIdentity() {
return writeMetadataIdentity;
}
+
+ /**
+ * The columns as the source declares them, index-aligned with {@link
#getSchema()}. Only metadata
+ * statements read this; see the field comment for why the two views must
come from one generation.
+ */
+ public List<Column> getDisplayColumns() {
+ return displayColumns;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableSchemaAction.java
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableSchemaAction.java
index 6761e7fdc6f..eda09df97a3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableSchemaAction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableSchemaAction.java
@@ -67,6 +67,15 @@ import java.util.stream.Collectors;
@RestController
public class TableSchemaAction extends RestBaseController {
+ /**
+ * The columns {@code /_schema} reports: the display view, so its {@code
is_nullable} agrees with DESC /
+ * SHOW COLUMNS. Split out so the choice of view is pinned by a unit test.
+ */
+ @VisibleForTesting
+ static List<Column> schemaColumns(TableIf table) {
+ return table.getBaseSchemaForDisplay();
+ }
+
/**
* Build column information map for a given column
* @param column the column to build info for
@@ -137,7 +146,10 @@ public class TableSchemaAction extends RestBaseController {
table.readLock();
try {
try {
- List<Column> columns = table.getBaseSchema();
+ // The public schema REST response is a metadata view:
report the nullability the source
+ // declares (same as DESC / SHOW COLUMNS), not the
nullability the engine reads with.
+ // Internal tables keep read == display (TableIf's default
implementation).
+ List<Column> columns = schemaColumns(table);
List<Map<String, Object>> propList = new
ArrayList<>(columns.size());
for (Column column : columns) {
Map<String, Object> baseInfo = buildColumnInfo(column);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/MetaInfoActionV2.java
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/MetaInfoActionV2.java
index 9427f4e2f72..8e2f48970df 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/MetaInfoActionV2.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/restv2/MetaInfoActionV2.java
@@ -295,7 +295,9 @@ public class MetaInfoActionV2 extends RestBaseController {
} else {
TableSchema tableSchema = new TableSchema();
tableSchema.setBaseIndex(false);
- List<Schema> schema = generateSchame(tbl.getBaseSchema());
+ // The meta REST API reports table metadata, so it shows the
nullability the source declares
+ // (same view as DESC / SHOW COLUMNS), not the read nullability
execution uses.
+ List<Schema> schema =
generateSchame(tbl.getBaseSchemaForDisplay());
tableSchema.setSchema(schema);
schemaMap.put(tbl.getName(), tableSchema);
schemaInfo.setSchemaMap(schemaMap);
@@ -332,7 +334,8 @@ public class MetaInfoActionV2 extends RestBaseController {
if (tbl.getType() == Table.TableType.OLAP) {
propMap.put("keyType", ((OlapTable) tbl).getKeysType());
}
- propMap.put("schema", generateSchema(tbl.getBaseSchema()));
+ // Metadata endpoint: the display view, like the schema map above.
+ propMap.put("schema", generateSchema(tbl.getBaseSchemaForDisplay()));
}
List<Map<String, String>> generateSchema(List<Column> columns) throws
UserException {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java
b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java
index e67f843acef..24d3e6a21d1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java
@@ -33,6 +33,8 @@ import java.nio.ByteBuffer;
// used for serialize memory data to byte stream of MySQL protocol
public class MysqlSerializer {
private static final Logger LOG =
LogManager.getLogger(MysqlSerializer.class);
+ // ColumnDefinition41 flags: the column cannot be NULL (mysql_com.h
NOT_NULL_FLAG).
+ private static final int NOT_NULL_FLAG = 0x0001;
private ByteArrayOutputStream out;
private MysqlCapability capability;
@@ -198,6 +200,16 @@ public class MysqlSerializer {
}
public void writeField(String db, String table, Column column, boolean
sendDefault) {
+ writeField(db, table, column, sendDefault, false);
+ }
+
+ /**
+ * @param encodeNotNull whether ColumnDefinition41's NOT_NULL flag follows
{@code column.isAllowNull()}.
+ * COM_FIELD_LIST passes true (the column is the display schema
column this packet describes);
+ * prepared-statement metadata passes false because its packet
describes an OUTPUT slot whose
+ * nullability can differ from the original column (e.g. the
nullable side of an outer join).
+ */
+ public void writeField(String db, String table, Column column, boolean
sendDefault, boolean encodeNotNull) {
// Catalog Name: length encoded string
writeLenEncodedString("def");
// Schema: length encoded string
@@ -219,7 +231,11 @@ public class MysqlSerializer {
// Column type: one byte integer
writeInt1(getMysqlResultType(column.getType()).getCode());
// Flags: two byte integer
- writeInt2(getMysqlFlags(column.getType()));
+ int flags = getMysqlFlags(column.getType());
+ if (encodeNotNull && !column.isAllowNull()) {
+ flags |= NOT_NULL_FLAG;
+ }
+ writeInt2(flags);
// Decimals: one byte integer
writeInt1(getMysqlDecimals(column.getType()));
// filler: two byte integer
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mysql/protocol/MysqlResultSender.java
b/fe/fe-core/src/main/java/org/apache/doris/mysql/protocol/MysqlResultSender.java
index 7ff8cfd257f..d4327d117a9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/mysql/protocol/MysqlResultSender.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/mysql/protocol/MysqlResultSender.java
@@ -234,7 +234,9 @@ public class MysqlResultSender implements ResultSender {
MysqlChannel channel = channel();
for (Column column : columns) {
serializer.reset();
- serializer.writeField(dbName, tableName, column, true);
+ // COM_FIELD_LIST is a metadata statement: carry the display
column's declared nullability in the
+ // ColumnDefinition41 NOT_NULL flag, like SHOW COLUMNS does.
+ serializer.writeField(dbName, tableName, column, true, true);
channel.sendOnePacket(serializer.toByteBuffer());
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowColumnsCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowColumnsCommand.java
index bd921e533e3..8e91c0d1c89 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowColumnsCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowColumnsCommand.java
@@ -193,7 +193,7 @@ public class ShowColumnsCommand extends ShowCommand {
boolean mysqlCompatible =
ctx.getSessionVariable().enableMysqlCompatibleIndexMetadata();
Map<String, String> columnKeys = mysqlCompatible
? TableKeyMeta.buildColumnKeys(table) :
Collections.emptyMap();
- List<Column> columns = table.getBaseSchema();
+ List<Column> columns = table.getBaseSchemaForDisplay();
for (Column col : columns) {
if (matcher != null && !matcher.match(col.getName())) {
continue;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java
index 44495a1bf0e..af8c804259c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java
@@ -343,7 +343,9 @@ public class MysqlConnectProcessor extends ConnectProcessor
{
table.readLock();
try {
MysqlProtocolAdapter.of(ctx).resultSender(ctx)
- .sendFieldList(db.getFullName(), table.getName(),
table.getBaseSchema());
+ // COM_FIELD_LIST is a metadata statement: it must agree
with SHOW COLUMNS, so it serves
+ // the display schema (the nullability the source
declares), not the read schema.
+ .sendFieldList(db.getFullName(), table.getName(),
table.getBaseSchemaForDisplay());
} catch (Throwable throwable) {
handleQueryException(throwable, "", null, null);
} finally {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index ee1e5fa9f19..4825fd767ce 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -1002,7 +1002,7 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
// values such as AGG and DUP that no MySQL client
knows what to do with.
Map<String, String> columnKeys =
params.isMysqlCompatibleIndexMetadata()
? TableKeyMeta.buildColumnKeys(table) :
Collections.emptyMap();
- List<Column> baseSchema = table.getBaseSchemaOrEmpty();
+ List<Column> baseSchema =
table.getBaseSchemaForDisplayOrEmpty();
for (Column column : baseSchema) {
final TColumnDesc desc = getColumnDesc(column);
final TColumnDef colDef = new TColumnDef(desc);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/EnvShowCreatePluginTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/EnvShowCreatePluginTableTest.java
index aa96b0baea4..ae4a81093bc 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/catalog/EnvShowCreatePluginTableTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/catalog/EnvShowCreatePluginTableTest.java
@@ -53,6 +53,10 @@ public class EnvShowCreatePluginTableTest {
Mockito.doReturn("t").when(table).getName();
Mockito.doReturn("").when(table).getComment();
Mockito.doReturn(COLUMNS).when(table).getBaseSchema(false);
+ // SHOW CREATE TABLE reads the DISPLAY schema (see
TableIf.getBaseSchemaForDisplay). Stub it with the
+ // same columns: this test pins the Env render arm, and resolving the
display view for real would need
+ // a connector-backed schema cache (the real accessor is unit-tested
in PluginDrivenExternalTableTest).
+ Mockito.doReturn(COLUMNS).when(table).getBaseSchemaForDisplay(false);
// The engine name a connector declares (here: an iceberg catalog's).
Stubbed because this test is about
// the Env render arm, not about where the name comes from — that is
pinned by
// PluginDrivenExternalTableEngineTest.
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableTest.java
index e54812e27eb..d57a6dd73df 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableTest.java
@@ -1025,6 +1025,70 @@ public class PluginDrivenExternalTableTest {
.getViewDefinition(Mockito.any(), Mockito.anyString(),
Mockito.anyString());
}
+ @Test
+ public void displaySchemaUsesSourceNullability() {
+ ConnectorMetadata metadata = Mockito.mock(ConnectorMetadata.class);
+ ConnectorTableHandle handle = Mockito.mock(ConnectorTableHandle.class);
+ Mockito.when(metadata.getTableHandle(Mockito.any(), Mockito.eq("db1"),
Mockito.eq("t1")))
+ .thenReturn(Optional.of(handle));
+ Mockito.when(metadata.getTableSchema(Mockito.any(),
Mockito.eq(handle)))
+ .thenReturn(new ConnectorTableSchema("t1", ImmutableList.of(
+ new ConnectorColumn("id", ConnectorType.of("INT"), "",
true, null, true)
+ .withSourceNullable(false),
+ new ConnectorColumn("name",
ConnectorType.of("STRING"), "", true, null, true),
+ // Required + WITH_TIMEZONE: the display copy must
keep the marker Column's copy
+ // constructor drops, otherwise DESC loses "Extra:
WITH_TIMEZONE" for exactly the
+ // required TZ columns this projection touches.
+ new ConnectorColumn("ts",
ConnectorType.of("TIMESTAMPTZ"), "", true, null, true)
+ .withTimeZone().withSourceNullable(false)),
+ "ICEBERG", Collections.emptyMap()));
+ Mockito.when(metadata.fromRemoteColumnName(Mockito.any(),
Mockito.any(), Mockito.any(),
+ Mockito.anyString())).thenAnswer(inv -> inv.getArgument(3));
+ ConnectorSession session = Mockito.mock(ConnectorSession.class);
+
Mockito.when(session.getStatementScope()).thenReturn(ConnectorStatementScope.NONE);
+ Connector connector = Mockito.mock(Connector.class);
+
Mockito.when(connector.getMetadata(Mockito.any())).thenReturn(metadata);
+ ExternalDatabase db = Mockito.mock(ExternalDatabase.class);
+ Mockito.when(db.getRemoteName()).thenReturn("db1");
+ PluginDrivenExternalCatalog catalog =
Mockito.mock(PluginDrivenExternalCatalog.class);
+ Mockito.when(catalog.getConnector()).thenReturn(connector);
+ Mockito.when(catalog.buildConnectorSession()).thenReturn(session);
+ Mockito.when(catalog.buildCrossStatementSession()).thenReturn(session);
+ PluginDrivenExternalTable table =
+ Mockito.mock(PluginDrivenExternalTable.class,
Mockito.CALLS_REAL_METHODS);
+ Deencapsulation.setField(table, "catalog", catalog);
+ Deencapsulation.setField(table, "db", db);
+ Deencapsulation.setField(table, "remoteName", "t1");
+ Mockito.doReturn(false).when(table).isView();
+ SchemaCacheValue cached = table.initSchema().get();
+
Mockito.doReturn(Optional.of(cached)).when(table).getSchemaCacheValue();
+ Mockito.doReturn(cached.getSchema()).when(table).getFullSchema();
+ // Count the schema-cache reads of the display call alone, not of the
setup above.
+ Mockito.clearInvocations(table);
+
+ List<Column> display = table.getBaseSchemaForDisplay(false);
+
+ // Index-aligned with the read schema: same size, same names, same
order.
+ List<Column> read = table.getBaseSchema(false);
+ Assertions.assertEquals(read.size(), display.size());
+ for (int i = 0; i < read.size(); i++) {
+ Assertions.assertEquals(read.get(i).getName(),
display.get(i).getName());
+ }
+ // The source-required column is shown NOT NULL; the optional one
keeps its nullable view.
+ Assertions.assertFalse(display.get(0).isAllowNull());
+ Assertions.assertTrue(display.get(1).isAllowNull());
+ Assertions.assertFalse(display.get(2).isAllowNull());
+
Assertions.assertTrue(read.get(2).getExtraInfo().contains("WITH_TIMEZONE"));
+
Assertions.assertTrue(display.get(2).getExtraInfo().contains("WITH_TIMEZONE"));
+ // The cached columns keep the nullable read semantics used by scans.
+ Assertions.assertTrue(read.stream().allMatch(Column::isAllowNull));
+ // A column the source does not diverge on is shared, not copied: no
per-call projection is left.
+ Assertions.assertSame(read.get(1), display.get(1));
+ // One schema-cache read per display call (it must not re-read for the
declared nullability, which
+ // would hit the connector twice under a bypassed cache such as a
delegated credential).
+ Mockito.verify(table, Mockito.times(1)).getSchemaCacheValue();
+ }
+
@Test
public void systemTableOverridesResolveIsViewToFalse() {
// A system/metadata table ($snapshots etc.) overrides resolveIsView
to a constant false so the base
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/TableSchemaActionColumnInfoTest.java
b/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/TableSchemaActionColumnInfoTest.java
index 20e2c5e3be7..377c5f91f97 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/TableSchemaActionColumnInfoTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/TableSchemaActionColumnInfoTest.java
@@ -24,6 +24,7 @@ import org.apache.doris.catalog.MapType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.StructField;
import org.apache.doris.catalog.StructType;
+import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.Type;
import org.apache.doris.catalog.VariantField;
import org.apache.doris.catalog.VariantType;
@@ -34,11 +35,32 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+import java.util.List;
import java.util.Map;
public class TableSchemaActionColumnInfoTest {
+ @Test
+ public void schemaRouteReportsDeclaredNullability() {
+ Column declaredNotNull = new Column("id", ScalarType.INT);
+ declaredNotNull.setIsAllowNull(false);
+ Column readNullable = new Column("id", ScalarType.INT);
+ readNullable.setIsAllowNull(true);
+
+ TableIf table = Mockito.mock(TableIf.class);
+
Mockito.when(table.getBaseSchemaForDisplay()).thenReturn(Lists.newArrayList(declaredNotNull));
+ // The read schema deliberately differs: switching /_schema back to it
must fail this test.
+
Mockito.when(table.getBaseSchema()).thenReturn(Lists.newArrayList(readNullable));
+
+ List<Column> columns = TableSchemaAction.schemaColumns(table);
+
+ Assertions.assertSame(declaredNotNull, columns.get(0));
+ Assertions.assertEquals("No",
TableSchemaAction.buildColumnInfo(columns.get(0)).get("is_nullable"));
+ Assertions.assertEquals("Yes",
TableSchemaAction.buildColumnInfo(readNullable).get("is_nullable"));
+ }
+
@Test
public void testArrayDecimalColumnInfo() {
Column column = new Column("array_decimal",
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/mysql/protocol/MysqlResultSenderTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mysql/protocol/MysqlResultSenderTest.java
index 8fdf3d0709a..2a91ae1feb8 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/mysql/protocol/MysqlResultSenderTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/mysql/protocol/MysqlResultSenderTest.java
@@ -20,10 +20,13 @@ package org.apache.doris.mysql.protocol;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.FeConstants;
import org.apache.doris.mysql.MysqlChannel;
import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.trees.expressions.ExprId;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.qe.CommonResultSet;
import org.apache.doris.qe.CommonResultSet.CommonResultSetMetaData;
import org.apache.doris.qe.ConnectContext;
@@ -218,6 +221,58 @@ public class MysqlResultSenderTest {
Assertions.assertEquals(2, channel.getOutbound().size());
}
+ @Test
+ public void testFieldListReportsDeclaredNullability() throws IOException {
+ RecordingMysqlChannel channel = newChannel(true);
+ ConnectContext ctx = newContext(channel);
+ Column required = new Column("id", PrimitiveType.INT);
+ required.setIsAllowNull(false);
+ Column optional = new Column("name", PrimitiveType.VARCHAR);
+ optional.setIsAllowNull(true);
+
+ // COM_FIELD_LIST is fed the display schema, so its ColumnDefinition41
packets must carry the
+ // NOT_NULL flag for a source-required column (mysql_list_fields /
MySQL clients read it).
+ MysqlProtocolAdapter.of(ctx).resultSender(ctx)
+ .sendFieldList("db1", "tbl1", Lists.newArrayList(required,
optional));
+
+ List<byte[]> packets = payloads(channel);
+ Assertions.assertEquals(2, packets.size());
+ Assertions.assertEquals(1, fieldFlags(packets.get(0)) & 1);
+ Assertions.assertEquals(0, fieldFlags(packets.get(1)) & 1);
+ }
+
+ @Test
+ public void testPrepareMetadataKeepsOuterJoinOutputNullable() throws
IOException {
+ RecordingMysqlChannel channel = newChannel(true);
+ ConnectContext ctx = newContext(channel);
+ Column required = new Column("id", PrimitiveType.INT);
+ required.setIsAllowNull(false);
+ TableIf table = Mockito.mock(TableIf.class);
+ Mockito.when(table.getName()).thenReturn("r");
+ // The nullable side of an outer join: the output slot is nullable
while the original column (which
+ // the prepare metadata packet describes) is NOT NULL.
+ SlotReference output = SlotReference.fromColumn(new ExprId(1), table,
required, Lists.newArrayList("r"))
+ .withNullable(true);
+
+ MysqlProtocolAdapter.of(ctx).resultSender(ctx)
+ .sendStmtPrepareOK(1, Lists.newArrayList(),
Lists.newArrayList(output));
+
+ List<byte[]> packets = payloads(channel);
+ Assertions.assertEquals(2, packets.size()); // prepare OK + one column
definition
+ Assertions.assertEquals(0, fieldFlags(packets.get(1)) & 1);
+ }
+
+ /** The 2-byte flags of a ColumnDefinition41 packet; all fixtures here use
one-byte length-encoded names. */
+ private static int fieldFlags(byte[] payload) {
+ int off = 0;
+ for (int i = 0; i < 6; i++) { // catalog, schema, table, org_table,
name, org_name
+ off += 1 + (payload[off] & 0xFF);
+ }
+ Assertions.assertEquals(0x0c, payload[off++] & 0xFF);
+ off += 2 + 4 + 1; // charset, column length, column type
+ return (payload[off] & 0xFF) | ((payload[off + 1] & 0xFF) << 8);
+ }
+
@Test
public void testResetClearsTheChannel() {
MysqlChannel channel = Mockito.mock(MysqlChannel.class);
diff --git
a/regression-test/data/external_table_p0/iceberg/iceberg_schema_change_ddl.out
b/regression-test/data/external_table_p0/iceberg/iceberg_schema_change_ddl.out
index 4565871aa8a..a67ef7f8a16 100644
---
a/regression-test/data/external_table_p0/iceberg/iceberg_schema_change_ddl.out
+++
b/regression-test/data/external_table_p0/iceberg/iceberg_schema_change_ddl.out
@@ -1,9 +1,9 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !init_1 --
-id int Yes true \N
-name text Yes true \N
-age int Yes true \N
-score double Yes true \N
+id int No true \N
+name text No true \N
+age int No true \N
+score double No true \N
-- !init_2 --
1 Alice 25 95.5
@@ -11,11 +11,11 @@ score double Yes true \N
3 Charlie 22 92.8
-- !add_1 --
-id int Yes true \N
-name text Yes true \N
-age int Yes true \N
+id int No true \N
+name text No true \N
+age int No true \N
phone text Yes true \N User phone number
-score double Yes true \N
+score double No true \N
email text Yes true \N
-- !add_2 --
@@ -33,11 +33,11 @@ email text Yes true \N
3 \N
-- !add_multi_1 --
-id int Yes true \N
-name text Yes true \N
-age int Yes true \N
+id int No true \N
+name text No true \N
+age int No true \N
phone text Yes true \N User phone number
-score double Yes true \N
+score double No true \N
email text Yes true \N
address struct<city:text,country:text> Yes true \N
@@ -52,11 +52,11 @@ address struct<city:text,country:text> Yes true
\N
5 {"city":"New York", "country":"USA"}
-- !rename_1 --
-id int Yes true \N
-name text Yes true \N
-age int Yes true \N
+id int No true \N
+name text No true \N
+age int No true \N
phone text Yes true \N User phone number
-grade double Yes true \N
+grade double No true \N
email text Yes true \N
address struct<city:text,country:text> Yes true \N
@@ -73,10 +73,10 @@ address struct<city:text,country:text> Yes true
\N
5 Eve 26 223-345-132 91.3 [email protected]
{"city":"New York", "country":"USA"}
-- !drop_1 --
-id int Yes true \N
-age int Yes true \N
+id int No true \N
+age int No true \N
phone text Yes true \N User phone number
-grade double Yes true \N
+grade double No true \N
email text Yes true \N
address struct<city:text,country:text> Yes true \N
@@ -93,10 +93,10 @@ address struct<city:text,country:text> Yes true
\N
5 26
-- !add_columns_1 --
-id int Yes true \N
-age int Yes true \N
+id int No true \N
+age int No true \N
phone text Yes true \N User phone number
-grade double Yes true \N
+grade double No true \N
email text Yes true \N
address struct<city:text,country:text> Yes true \N
col1 float Yes true \N User defined column1
@@ -111,9 +111,9 @@ col2 text Yes true \N User
defined column2
-- !modify_1 --
age bigint Yes true \N
-id int Yes true \N
+id int No true \N
phone text Yes true \N User phone number
-grade double Yes true \N
+grade double No true \N
email text Yes true \N
address struct<city:text,country:text> Yes true \N
col1 double Yes true \N Updated column1 type
@@ -134,9 +134,9 @@ col2 text Yes true \N User
defined column2
-- !before_no_comment --
age bigint Yes true \N
-id int Yes true \N
+id int No true \N
phone text Yes true \N User phone number
-grade double Yes true \N
+grade double No true \N
email text Yes true \N
address struct<city:text,country:text> Yes true \N
col1 double Yes true \N Updated column1 type
@@ -144,9 +144,9 @@ col2 text Yes true \N User
defined column2
-- !after_no_comment --
age bigint Yes true \N
-id int Yes true \N
+id int No true \N
phone text Yes true \N User phone number
-grade double Yes true \N
+grade double No true \N
email text Yes true \N
address struct<city:text,country:text> Yes true \N
col1 double Yes true \N Updated column1 type
@@ -154,9 +154,9 @@ col2 text Yes true \N User
defined column2
-- !after_no_comment --
age bigint Yes true \N
-id int Yes true \N
+id int No true \N
phone text Yes true \N User phone number
-grade double Yes true \N
+grade double No true \N
email text Yes true \N
address struct<city:text,country:text> Yes true \N
col1 double Yes true \N Updated column1 type
@@ -164,9 +164,9 @@ col2 text Yes true \N User
defined column2
-- !modify_positive_1 --
age bigint Yes true \N
-id int Yes true \N
+id int No true \N
phone text Yes true \N User phone number
-grade double Yes true \N
+grade double No true \N
email text Yes true \N
address struct<city:text,country:text> Yes true \N
col1 double Yes true \N Updated column1 type
@@ -178,11 +178,11 @@ test_decimal decimal(10,2) Yes true \N
7 3.140000104904175 123.45
-- !reorder_1 --
-id int Yes true \N
+id int No true \N
age bigint Yes true \N
col1 double Yes true \N Updated column1 type
col2 text Yes true \N User defined column2
-grade double Yes true \N
+grade double No true \N
phone text Yes true \N User phone number
email text Yes true \N
address struct<city:text,country:text> Yes true \N
@@ -209,11 +209,11 @@ iceberg_ddl_test
iceberg_ddl_test_renamed
-- !rename_table_1 --
-id int Yes true \N
+id int No true \N
age bigint Yes true \N
col1 double Yes true \N Updated column1 type
col2 text Yes true \N User defined column2
-grade double Yes true \N
+grade double No true \N
phone text Yes true \N User phone number
email text Yes true \N
address struct<city:text,country:text> Yes true \N
diff --git
a/regression-test/data/external_table_p0/iceberg/test_iceberg_initial_defaults.out
b/regression-test/data/external_table_p0/iceberg/test_iceberg_initial_defaults.out
index a465b704587..bae87ce9b5f 100644
---
a/regression-test/data/external_table_p0/iceberg/test_iceberg_initial_defaults.out
+++
b/regression-test/data/external_table_p0/iceberg/test_iceberg_initial_defaults.out
@@ -82,7 +82,7 @@ default_decimal decimal(20,4) Yes true \N
default_date date Yes true \N
default_timestamp datetime(6) Yes true \N
default_timestamptz timestamptz(6) Yes true \N WITH_TIMEZONE
-default_string text Yes true \N
+default_string text No true \N
default_uuid uuid Yes true \N
default_fixed varbinary(4) Yes true \N
default_binary varbinary(2147483647) Yes true \N
@@ -179,7 +179,7 @@ default_decimal decimal(20,4) Yes true \N
default_date date Yes true \N
default_timestamp datetime(6) Yes true \N
default_timestamptz timestamptz(6) Yes true \N WITH_TIMEZONE
-default_string text Yes true \N
+default_string text No true \N
default_uuid uuid Yes true \N
default_fixed varbinary(4) Yes true \N
default_binary varbinary(2147483647) Yes true \N
@@ -276,7 +276,7 @@ default_decimal decimal(20,4) Yes true \N
default_date date Yes true \N
default_timestamp datetime(6) Yes true \N
default_timestamptz timestamptz(6) Yes true \N WITH_TIMEZONE
-default_string text Yes true \N
+default_string text No true \N
default_uuid uuid Yes true \N
default_fixed varbinary(4) Yes true \N
default_binary varbinary(2147483647) Yes true \N
@@ -373,7 +373,7 @@ default_decimal decimal(20,4) Yes true \N
default_date date Yes true \N
default_timestamp datetime(6) Yes true \N
default_timestamptz timestamptz(6) Yes true \N WITH_TIMEZONE
-default_string text Yes true \N
+default_string text No true \N
default_uuid uuid Yes true \N
default_fixed varbinary(4) Yes true \N
default_binary varbinary(2147483647) Yes true \N
diff --git
a/regression-test/data/external_table_p0/iceberg/test_iceberg_show_nullable.out
b/regression-test/data/external_table_p0/iceberg/test_iceberg_show_nullable.out
new file mode 100644
index 00000000000..b14cf5aacb7
--- /dev/null
+++
b/regression-test/data/external_table_p0/iceberg/test_iceberg_show_nullable.out
@@ -0,0 +1,29 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !desc --
+id bigint No true \N
+value text No true \N
+event_time datetime(6) Yes true \N
+
+-- !show_columns --
+id bigint NO YES \N
+value text NO YES \N
+event_time datetime(6) YES YES \N
+
+-- !show_columns_where --
+id bigint NO \N
+value string NO \N
+
+-- !information_schema --
+event_time YES
+id NO
+value NO
+
+-- !select --
+1 a 2026-01-01T00:00
+2 b \N
+
+-- !desc_after_alter --
+id bigint No true \N
+value text Yes true \N
+event_time datetime(6) Yes true \N
+
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_nullable.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_nullable.groovy
new file mode 100644
index 00000000000..59b5cade746
--- /dev/null
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_nullable.groovy
@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_iceberg_show_nullable", "p0,external") {
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable iceberg test.")
+ return
+ }
+
+ String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port")
+ String minio_port = context.config.otherConfigs.get("iceberg_minio_port")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String catalog_name = "test_iceberg_show_nullable"
+
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """
+ CREATE CATALOG ${catalog_name} PROPERTIES (
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='rest',
+ 'uri' = 'http://${externalEnvIp}:${rest_port}',
+ "s3.access_key" = "admin",
+ "s3.secret_key" = "password",
+ "s3.endpoint" = "http://${externalEnvIp}:${minio_port}";,
+ "s3.region" = "us-east-1"
+ );"""
+
+ sql """switch ${catalog_name}"""
+ sql """drop database if exists test_iceberg_show_nullable_db force"""
+ sql """create database test_iceberg_show_nullable_db"""
+ sql """use test_iceberg_show_nullable_db"""
+
+ sql """
+ create table iceberg_show_nullable (
+ id bigint not null,
+ value string not null,
+ event_time datetime
+ )"""
+ sql """insert into iceberg_show_nullable values (1, 'a', '2026-01-01
00:00:00'), (2, 'b', null)"""
+
+ qt_desc """desc iceberg_show_nullable"""
+ qt_show_columns """show columns from iceberg_show_nullable"""
+ // SHOW COLUMNS ... WHERE is rewritten to information_schema.columns; no
ORDER BY, so the rows are
+ // sorted by the framework (order_qt_) instead of being compared in an
unspecified scan order.
+ order_qt_show_columns_where """show columns from iceberg_show_nullable
where `Null` = 'NO'"""
+ order_qt_information_schema """select COLUMN_NAME, IS_NULLABLE from
information_schema.columns
+ where TABLE_SCHEMA = 'test_iceberg_show_nullable_db' and TABLE_NAME =
'iceberg_show_nullable'"""
+ String ddl = sql("""show create table iceberg_show_nullable""")[0][1]
+ assertTrue(ddl.contains("`id` bigint NOT NULL"), ddl)
+ assertTrue(ddl.contains("`value` text NOT NULL"), ddl)
+ assertTrue(ddl.contains("`event_time` datetimev2(6) NULL"), ddl)
+ order_qt_select """select * from iceberg_show_nullable"""
+
+ // Relaxing the constraint is reflected without a new data snapshot.
+ sql """alter table iceberg_show_nullable modify column value string null"""
+ qt_desc_after_alter """desc iceberg_show_nullable"""
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
