This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 4714241b993 branch-4.1: [fix](iceberg) Fix planning failure for
Iceberg Variant subpaths (#68121)
4714241b993 is described below
commit 4714241b9936c434d22ceb5cfa84288f28b05124
Author: daidai <[email protected]>
AuthorDate: Thu Sep 17 21:20:50 2026 +0800
branch-4.1: [fix](iceberg) Fix planning failure for Iceberg Variant
subpaths (#68121)
### What problem does this PR solve?
Issue Number: None
Related PR: None
Problem Summary:
Querying a subpath of an Iceberg `VARIANT` column fails during planning
when `enable_prune_nested_column` is enabled (the default):
```sql
SELECT CAST(message['mainDomain'] AS STRING) FROM iceberg_tbl;
-- ERROR 1105 (HY000): errCode = 2, detailMessage = Iceberg access path
continues below primitive column message
```
`IcebergScanNode` walks projected access paths when checking backend
compatibility and treats every path component as an Iceberg schema
child. Variant object keys are data, so the walk fails on them. This PR
stops the walk at Variant columns after checking the Variant field
itself.
Master is not affected.
### Release note
None
### Check List (For Author)
- Test
- [x] Regression test
- [x] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason
- Behavior changed:
- [x] No.
- [ ] Yes.
- Does this need documentation?
- [x] No.
- [ ] Yes.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../datasource/iceberg/source/IcebergScanNode.java | 5 +
.../iceberg/source/IcebergScanNodeTest.java | 48 ++++++++++
.../iceberg/test_iceberg_variant_access_path.out | 29 ++++++
.../test_iceberg_variant_access_path.groovy | 104 +++++++++++++++++++++
4 files changed, 186 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
index ebae13b7d4c..3188e7a73e6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
@@ -1144,6 +1144,11 @@ public class IcebergScanNode extends FileQueryScanNode {
if (requiresIcebergField(column, fieldById, isTopLevel, requirement)) {
return true;
}
+ if (column.getType().isVariantType()) {
+ // VARIANT has no Iceberg schema children. Any remaining
components are object keys or
+ // array indexes inside the encoded value, not Iceberg field IDs
or access tokens.
+ return false;
+ }
if (pathIndex == path.size()) {
return requiresProjectedIcebergField(column, fieldById,
requirement);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java
index 8bb684ce1b8..50279f24fab 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java
@@ -2462,6 +2462,54 @@ public class IcebergScanNodeTest {
"20", AccessPathInfo.ACCESS_NULL);
}
+ @Test
+ public void testVariantAccessPathTerminatesIcebergFieldTraversal() {
+ Types.NestedField nestedDefault = Types.NestedField.optional("added")
+ .withId(4)
+ .ofType(Types.IntegerType.get())
+ .withInitialDefault(7)
+ .build();
+ Schema historicalSchema = new Schema(
+ Types.NestedField.optional(1, "message",
Types.VariantType.get()));
+ Schema schema = new Schema(
+ Types.NestedField.optional(1, "message",
Types.VariantType.get()),
+ Types.NestedField.optional(2, "info", Types.StructType.of(
+ Types.NestedField.optional(3, "payload",
Types.VariantType.get()),
+ nestedDefault)),
+ Types.NestedField.optional(5, "attrs",
Types.MapType.ofOptional(
+ 6, 7, Types.StringType.get(),
Types.VariantType.get())),
+ Types.NestedField.required(8, "required_variant",
Types.VariantType.get()));
+ List<Column> columns = IcebergUtils.parseSchema(schema, false, false);
+ SlotDescriptor messageSlot = slotDescriptor(1);
+ messageSlot.setColumn(columns.get(0));
+ SlotDescriptor infoSlot = slotDescriptor(2);
+ infoSlot.setColumn(columns.get(1));
+ SlotDescriptor attrsSlot = slotDescriptor(5);
+ attrsSlot.setColumn(columns.get(2));
+ SlotDescriptor requiredVariantSlot = slotDescriptor(8);
+ requiredVariantSlot.setColumn(columns.get(3));
+
+ assertRequiresRecursiveInitialDefault(schema, messageSlot, false, "1",
"mainDomain");
+ // Variant keys are data selectors, even when they spell an Iceberg
field ID or access token.
+ assertRequiresRecursiveInitialDefault(schema, messageSlot, false, "1",
"4");
+ assertRequiresRecursiveInitialDefault(schema, messageSlot, false,
+ "1", AccessPathInfo.ACCESS_ALL);
+ assertRequiresRecursiveInitialDefault(schema, infoSlot, false, "2",
"3", "kind");
+ assertRequiresRecursiveInitialDefault(schema, infoSlot, true, "2",
"4");
+ assertRequiresRecursiveInitialDefault(schema, attrsSlot, false,
+ "5", AccessPathInfo.ACCESS_ALL, "object", "kind");
+
+ requiredVariantSlot.setAllAccessPaths(Collections.singletonList(
+ dataAccessPath(ImmutableList.of("8", "mainDomain"))));
+
Assert.assertTrue(IcebergScanNode.requiresMissingRequiredFieldRejection(
+ schema, Collections.singletonList(requiredVariantSlot),
+ ImmutableList.of(historicalSchema)));
+ messageSlot.setAllAccessPaths(Collections.singletonList(
+ dataAccessPath(ImmutableList.of("1", "mainDomain"))));
+
Assert.assertFalse(IcebergScanNode.requiresMissingRequiredFieldRejection(
+ schema, Collections.singletonList(messageSlot),
ImmutableList.of(historicalSchema)));
+ }
+
@Test
public void testPotentiallyMissingRequiredFieldsFollowProjection() {
Types.NestedField existing = Types.NestedField.optional(
diff --git
a/regression-test/data/external_table_p0/iceberg/test_iceberg_variant_access_path.out
b/regression-test/data/external_table_p0/iceberg/test_iceberg_variant_access_path.out
new file mode 100644
index 00000000000..2032fdd908b
--- /dev/null
+++
b/regression-test/data/external_table_p0/iceberg/test_iceberg_variant_access_path.out
@@ -0,0 +1,29 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !variant_leaf --
+1 a.com dns
+2 b.com http
+3 a.com dns
+
+-- !variant_group_by --
+a.com dns 2
+b.com http 1
+
+-- !variant_filter --
+1
+3
+
+-- !struct_variant --
+1 x
+2 y
+3 \N
+
+-- !map_variant --
+1 m1
+2 m2
+3 \N
+
+-- !array_variant --
+1 e1
+2 e2
+3 \N
+
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_variant_access_path.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_variant_access_path.groovy
new file mode 100644
index 00000000000..bc1c4da3f4b
--- /dev/null
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_variant_access_path.groovy
@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_iceberg_variant_access_path",
+ "p0,external,iceberg,external_docker,external_docker_iceberg") {
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable iceberg test")
+ return
+ }
+
+ String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port")
+ String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String catalogName = "test_iceberg_variant_access_path"
+ String dbName = "iceberg_variant_access_path_db"
+ String tableName = "variant_access_path"
+
+ spark_iceberg_multi """
+ CREATE NAMESPACE IF NOT EXISTS demo.${dbName};
+ DROP TABLE IF EXISTS demo.${dbName}.${tableName};
+ CREATE TABLE demo.${dbName}.${tableName} (
+ id INT,
+ message VARIANT,
+ info STRUCT<label: STRING, payload: VARIANT>,
+ attrs MAP<STRING, VARIANT>,
+ events ARRAY<VARIANT>
+ ) USING iceberg
+ TBLPROPERTIES ('format-version'='3', 'write.format.default'='parquet');
+ INSERT INTO demo.${dbName}.${tableName} VALUES
+ (1,
+ parse_json('{"mainDomain":"a.com","anomalyType":"dns"}'),
+ named_struct('label', 'one', 'payload',
parse_json('{"kind":"x"}')),
+ map('object', parse_json('{"kind":"m1"}')),
+ array(parse_json('{"kind":"e1"}'))),
+ (2,
+ parse_json('{"mainDomain":"b.com","anomalyType":"http"}'),
+ named_struct('label', 'two', 'payload',
parse_json('{"kind":"y"}')),
+ map('object', parse_json('{"kind":"m2"}')),
+ array(parse_json('{"kind":"e2"}'))),
+ (3,
+ parse_json('{"mainDomain":"a.com","anomalyType":"dns"}'),
+ NULL, NULL, NULL)
+ """
+
+ sql """DROP CATALOG IF EXISTS ${catalogName}"""
+ sql """
+ CREATE CATALOG ${catalogName} PROPERTIES (
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='rest',
+ 'uri'='http://${externalEnvIp}:${restPort}',
+ 's3.access_key'='admin',
+ 's3.secret_key'='password',
+ 's3.endpoint'='http://${externalEnvIp}:${minioPort}',
+ 's3.region'='us-east-1'
+ )
+ """
+ sql """SWITCH ${catalogName}"""
+ sql """USE ${dbName}"""
+ sql """SET enable_file_scanner_v2=true"""
+ sql """SET enable_prune_nested_column=true"""
+
+ explain {
+ sql """SELECT CAST(message['mainDomain'] AS STRING) FROM
${tableName}"""
+ contains "all access paths: [message(2).mainDomain]"
+ }
+
+ order_qt_variant_leaf """
+ SELECT id, CAST(message['mainDomain'] AS STRING),
CAST(message['anomalyType'] AS STRING)
+ FROM ${tableName}
+ """
+ order_qt_variant_group_by """
+ SELECT CAST(message['mainDomain'] AS STRING),
CAST(message['anomalyType'] AS STRING),
+ COUNT(*)
+ FROM ${tableName}
+ GROUP BY CAST(message['mainDomain'] AS STRING),
CAST(message['anomalyType'] AS STRING)
+ """
+ order_qt_variant_filter """
+ SELECT id FROM ${tableName} WHERE CAST(message['anomalyType'] AS
STRING) = 'dns'
+ """
+ order_qt_struct_variant """
+ SELECT id, CAST(info.payload['kind'] AS STRING) FROM ${tableName}
+ """
+ order_qt_map_variant """
+ SELECT id, CAST(attrs['object']['kind'] AS STRING) FROM ${tableName}
+ """
+ order_qt_array_variant """
+ SELECT id, CAST(events[1]['kind'] AS STRING) FROM ${tableName}
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
