This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 688e3ce5e41 [opt](paimon) support prune complex type in paimon jni
reader (#66573)
688e3ce5e41 is described below
commit 688e3ce5e41b191faa310040f2aeaf3f83bf5594
Author: zhangstar333 <[email protected]>
AuthorDate: Sat Aug 8 01:42:33 2026 +0800
[opt](paimon) support prune complex type in paimon jni reader (#66573)
### What problem does this PR solve?
Problem Summary:
before disable prune complex struct/array/map type in paimon. now
support those in paimon jni reader, so could enable it.
```
| 0:VPAIMON_SCAN_NODE(116)
|
| table:
test_paimon_variant.test_paimon_spark.jni_complex_column_pruning
|
| predicates: (element_at(profile[#1], 'zip') >= 200000),
(element_at(element_at(events[#2], 1), 'score') >= 90),
(element_at(element_at(attributes[#3], 'primary'), 'code') = 20) |
| inputSplitNum=1, totalFileSize=0, scanRanges=1
|
| partition=1/0
|
| cardinality=2, numNodes=1
|
| nested columns:
|
| profile:
|
| origin type: struct<city:text,zip:int,street:text>
|
| pruned type: struct<city:text,zip:int>
|
| all access paths: [profile.city, profile.zip]
|
| predicate access paths: [profile.zip]
```
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [x] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../create_preinstalled_scripts/paimon/run14.sql | 35 ++++++
.../org/apache/doris/paimon/PaimonJniScanner.java | 19 ++--
.../doris/paimon/PaimonReadTypeProjection.java | 126 +++++++++++++++++++++
.../doris/paimon/PaimonReadTypeProjectionTest.java | 93 +++++++++++++++
.../nereids/rules/rewrite/SlotTypeReplacer.java | 8 +-
.../trees/plans/logical/LogicalFileScan.java | 12 --
.../plans/logical/SupportPruneNestedColumn.java | 7 --
.../trees/plans/logical/LogicalFileScanTest.java | 14 +--
.../test_paimon_jni_complex_column_pruning.out | 16 +++
.../test_paimon_jni_complex_column_pruning.groovy | 106 +++++++++++++++++
10 files changed, 391 insertions(+), 45 deletions(-)
diff --git
a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run14.sql
b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run14.sql
new file mode 100644
index 00000000000..392b0c99382
--- /dev/null
+++
b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run14.sql
@@ -0,0 +1,35 @@
+use paimon;
+create database if not exists test_paimon_spark;
+use test_paimon_spark;
+
+-- Dedicated append-only table for Doris Paimon JNI nested-column pruning
regressions.
+drop table if exists jni_complex_column_pruning;
+create table jni_complex_column_pruning (
+ id BIGINT,
+ profile STRUCT<city: STRING, zip: INT, street: STRING>,
+ events ARRAY<STRUCT<name: STRING, score: INT, note: STRING>>,
+ attributes MAP<STRING, STRUCT<code: INT, label: STRING, note: STRING>>
+) using paimon
+tblproperties (
+ 'file.format' = 'parquet'
+);
+
+insert into jni_complex_column_pruning values
+ (
+ 1,
+ struct('beijing', 100000, 'road-a'),
+ array(struct('login', 90, 'web'), struct('purchase', 70, 'app')),
+ map(
+ 'primary', struct(10, 'alpha', 'keep-primary'),
+ 'backup', struct(11, 'alpha-backup', 'keep-backup')
+ )
+ ),
+ (
+ 2,
+ struct('shanghai', 200000, 'road-b'),
+ array(struct('purchase', 95, 'app'), struct('logout', 60, 'web')),
+ map(
+ 'primary', struct(20, 'beta', 'keep-primary'),
+ 'backup', struct(21, 'beta-backup', 'keep-backup')
+ )
+ );
diff --git
a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
index 03a12bc8245..2ef37f7645a 100644
---
a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
+++
b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
@@ -203,7 +203,7 @@ public class PaimonJniScanner extends JniScanner {
int[] projected = getProjected();
List<DataField> readFields = new ArrayList<>(projected.length);
variantProjections = new ArrayList<>(projected.length);
- boolean hasVariantProjection = false;
+ boolean hasReadTypeProjection = false;
for (int outputIndex = 0; outputIndex < projected.length;
outputIndex++) {
DataField tableField =
table.rowType().getFields().get(projected[outputIndex]);
PaimonVariantProjection projection = tableField.type() instanceof
VariantType
@@ -212,17 +212,22 @@ public class PaimonJniScanner extends JniScanner {
: null;
variantProjections.add(projection);
if (projection == null) {
- readFields.add(tableField);
+ DataType projectedType = PaimonReadTypeProjection.project(
+ tableField.type(), types[outputIndex]);
+ if (!projectedType.equals(tableField.type())) {
+ hasReadTypeProjection = true;
+ }
+ readFields.add(tableField.newType(projectedType));
} else {
- hasVariantProjection = true;
+ hasReadTypeProjection = true;
readFields.add(tableField.newType(
projection.readType().copy(tableField.type().isNullable())));
}
}
- if (hasVariantProjection) {
- // Paimon recognizes a metadata-marked RowType as a list of
Variant extraction fields.
- // It prunes matching shredded Parquet fields per file and reads
the raw Variant value
- // as a correctness fallback for unshredded or non-matching files.
+ if (hasReadTypeProjection) {
+ // For static complex types, the recursively pruned read type
tells Paimon which nested
+ // ROW/ARRAY/MAP fields to read. For Variant, Paimon recognizes a
metadata-marked RowType
+ // as a list of extraction fields and falls back to the raw value
when needed.
RowType requestedReadType = new RowType(readFields);
readBuilder.withReadType(requestedReadType);
} else {
diff --git
a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonReadTypeProjection.java
b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonReadTypeProjection.java
new file mode 100644
index 00000000000..6304bfdabbe
--- /dev/null
+++
b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonReadTypeProjection.java
@@ -0,0 +1,126 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.paimon;
+
+import org.apache.doris.common.jni.vec.ColumnType;
+
+import org.apache.paimon.types.ArrayType;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.MapType;
+import org.apache.paimon.types.RowType;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Builds the Paimon read type that matches the nested column shape requested
by Doris.
+ *
+ * <p>The BE applies access paths to the JNI {@link ColumnType} before
creating this scanner. This
+ * class mirrors that shape with Paimon's types so {@code withReadType} can
push the same projection
+ * through ROW, ARRAY, and MAP readers while retaining Paimon's field IDs and
nullability.
+ */
+final class PaimonReadTypeProjection {
+ private PaimonReadTypeProjection() {
+ }
+
+ static DataType project(DataType tableType, ColumnType requiredType) {
+ if (requiredType.isStruct()) {
+ return projectRow(tableType, requiredType);
+ }
+ if (requiredType.isArray()) {
+ return projectArray(tableType, requiredType);
+ }
+ if (requiredType.isMap()) {
+ return projectMap(tableType, requiredType);
+ }
+ // Doris' scalar type can differ from Paimon's logical type (for
example timestamp
+ // precision). Keep the table type and only use ColumnType to describe
nested shape.
+ return tableType;
+ }
+
+ private static DataType projectRow(DataType tableType, ColumnType
requiredType) {
+ if (!(tableType instanceof RowType)) {
+ throw incompatibleType(requiredType, tableType);
+ }
+ RowType rowType = (RowType) tableType;
+ List<String> childNames = requiredType.getChildNames();
+ List<ColumnType> childTypes = requiredType.getChildTypes();
+ if (childNames == null || childTypes == null || childNames.size() !=
childTypes.size()) {
+ throw new IllegalArgumentException(
+ "Invalid Doris STRUCT projection for column " +
requiredType.getName());
+ }
+
+ List<DataField> projectedFields = new ArrayList<>(childNames.size());
+ for (int i = 0; i < childNames.size(); i++) {
+ DataField tableField = findField(rowType, childNames.get(i));
+ if (tableField == null) {
+ throw new IllegalArgumentException(String.format(
+ "Doris requested nested field '%s' which does not
exist in Paimon type %s",
+ childNames.get(i), rowType.asSQLString()));
+ }
+ DataType projectedType = project(tableField.type(),
childTypes.get(i));
+ projectedFields.add(tableField.newType(projectedType));
+ }
+ return rowType.copy(projectedFields);
+ }
+
+ private static DataType projectArray(DataType tableType, ColumnType
requiredType) {
+ if (!(tableType instanceof ArrayType)) {
+ throw incompatibleType(requiredType, tableType);
+ }
+ List<ColumnType> childTypes = requiredType.getChildTypes();
+ if (childTypes == null || childTypes.size() != 1) {
+ throw new IllegalArgumentException(
+ "Invalid Doris ARRAY projection for column " +
requiredType.getName());
+ }
+ ArrayType arrayType = (ArrayType) tableType;
+ return arrayType.newElementType(project(arrayType.getElementType(),
childTypes.get(0)));
+ }
+
+ private static DataType projectMap(DataType tableType, ColumnType
requiredType) {
+ if (!(tableType instanceof MapType)) {
+ throw incompatibleType(requiredType, tableType);
+ }
+ List<ColumnType> childTypes = requiredType.getChildTypes();
+ if (childTypes == null || childTypes.size() != 2) {
+ throw new IllegalArgumentException(
+ "Invalid Doris MAP projection for column " +
requiredType.getName());
+ }
+ MapType mapType = (MapType) tableType;
+ return mapType.newKeyValueType(
+ project(mapType.getKeyType(), childTypes.get(0)),
+ project(mapType.getValueType(), childTypes.get(1)));
+ }
+
+ private static DataField findField(RowType rowType, String requiredName) {
+ for (DataField field : rowType.getFields()) {
+ if (field.name().equalsIgnoreCase(requiredName)) {
+ return field;
+ }
+ }
+ return null;
+ }
+
+ private static IllegalArgumentException incompatibleType(
+ ColumnType requiredType, DataType tableType) {
+ return new IllegalArgumentException(String.format(
+ "Doris requested %s for column '%s', but the Paimon type is
%s",
+ requiredType.getType(), requiredType.getName(),
tableType.asSQLString()));
+ }
+}
diff --git
a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonReadTypeProjectionTest.java
b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonReadTypeProjectionTest.java
new file mode 100644
index 00000000000..bc4acf2e1bc
--- /dev/null
+++
b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonReadTypeProjectionTest.java
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.paimon;
+
+import org.apache.doris.common.jni.vec.ColumnType;
+
+import org.apache.paimon.types.ArrayType;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.MapType;
+import org.apache.paimon.types.RowType;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+public class PaimonReadTypeProjectionTest {
+ @Test
+ public void testProjectNestedRowArrayAndMap() {
+ RowType profileType = new RowType(false, Arrays.asList(
+ new DataField(2, "city", DataTypes.STRING(), "city
description"),
+ new DataField(3, "zip", DataTypes.INT())));
+ RowType eventType = new RowType(false, Arrays.asList(
+ new DataField(5, "score", DataTypes.INT()),
+ new DataField(6, "detail", DataTypes.STRING())));
+ RowType attributeType = new RowType(false, Arrays.asList(
+ new DataField(8, "code", DataTypes.BIGINT()),
+ new DataField(9, "detail", DataTypes.STRING())));
+ RowType tableType = new RowType(false, Arrays.asList(
+ new DataField(1, "profile", profileType, "profile
description"),
+ new DataField(4, "events", new ArrayType(false, eventType)),
+ new DataField(7, "attributes",
+ new MapType(false, DataTypes.STRING(),
attributeType))));
+ String requestedType = "struct<PROFILE:struct<city:string>,"
+ + "events:array<struct<score:int>>,"
+ + "attributes:map<string,struct<code:bigint>>>";
+ ColumnType requiredType = ColumnType.parseType("root", requestedType);
+
+ RowType projected = (RowType)
PaimonReadTypeProjection.project(tableType, requiredType);
+
+ Assert.assertFalse(projected.isNullable());
+ Assert.assertEquals(Arrays.asList("profile", "events", "attributes"),
+ projected.getFieldNames());
+
+ DataField profile = projected.getFields().get(0);
+ Assert.assertEquals(1, profile.id());
+ Assert.assertEquals("profile description", profile.description());
+ RowType projectedProfile = (RowType) profile.type();
+ Assert.assertFalse(projectedProfile.isNullable());
+ Assert.assertEquals(Arrays.asList("city"),
projectedProfile.getFieldNames());
+ Assert.assertEquals(2, projectedProfile.getFields().get(0).id());
+ Assert.assertEquals("city description",
+ projectedProfile.getFields().get(0).description());
+
+ ArrayType projectedEvents = (ArrayType) projected.getTypeAt(1);
+ Assert.assertFalse(projectedEvents.isNullable());
+ Assert.assertEquals(Arrays.asList("score"),
+ ((RowType) projectedEvents.getElementType()).getFieldNames());
+
+ MapType projectedAttributes = (MapType) projected.getTypeAt(2);
+ Assert.assertFalse(projectedAttributes.isNullable());
+ Assert.assertEquals(DataTypes.STRING(),
projectedAttributes.getKeyType());
+ Assert.assertEquals(Arrays.asList("code"),
+ ((RowType)
projectedAttributes.getValueType()).getFieldNames());
+ }
+
+ @Test
+ public void testRejectMissingNestedField() {
+ RowType tableType = new RowType(Arrays.asList(
+ new DataField(1, "known", DataTypes.INT())));
+ ColumnType requiredType = ColumnType.parseType("root",
"struct<missing:int>");
+
+ IllegalArgumentException exception = Assert.assertThrows(
+ IllegalArgumentException.class,
+ () -> PaimonReadTypeProjection.project(tableType,
requiredType));
+ Assert.assertTrue(exception.getMessage().contains("missing"));
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
index de3de9adef6..c90d85d55e6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
@@ -723,18 +723,14 @@ public class SlotTypeReplacer extends
DefaultPlanRewriter<Void> {
}
private void tryRecordReplaceSlots(Plan plan, Object checkObj,
Set<Integer> shouldReplaceSlots) {
- if (checkObj instanceof SupportPruneNestedColumn) {
- SupportPruneNestedColumn supportPruneNestedColumn =
(SupportPruneNestedColumn) checkObj;
- if (!supportPruneNestedColumn.supportPruneNestedColumn()) {
- return;
- }
+ if (checkObj instanceof SupportPruneNestedColumn
+ && ((SupportPruneNestedColumn)
checkObj).supportPruneNestedColumn()) {
List<Slot> output = plan.getOutput();
boolean shouldPrune = false;
for (Slot slot : output) {
int slotId = slot.getExprId().asInt();
if ((slot.getDataType() instanceof NestedColumnPrunable
|| slot.getDataType().isVariantType())
- &&
supportPruneNestedColumn.supportPruneNestedColumn(slot.getDataType())
&& replacedDataTypes.containsKey(slotId)) {
shouldReplaceSlots.add(slotId);
shouldPrune = true;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
index c6727bf93c1..53480d193ba 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
@@ -43,7 +43,6 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
-import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;
@@ -358,17 +357,6 @@ public class LogicalFileScan extends
LogicalCatalogRelation implements SupportPr
return false;
}
- @Override
- public boolean supportPruneNestedColumn(DataType dataType) {
- ExternalTable table = getTable();
- if (table instanceof PaimonExternalTable || table instanceof
PaimonSysExternalTable) {
- // Paimon JNI currently supports nested projection only for
Variant. Its static complex
- // types still return the full value, which would misalign pruned
ROW/ARRAY/MAP slots.
- return dataType.isVariantType();
- }
- return supportPruneNestedColumn();
- }
-
private boolean hasSameSnapshot(Optional<TableSnapshot> left,
Optional<TableSnapshot> right) {
if (!left.isPresent() || !right.isPresent()) {
return left.isPresent() == right.isPresent();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/SupportPruneNestedColumn.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/SupportPruneNestedColumn.java
index 9776bb920a5..44f1b733dd1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/SupportPruneNestedColumn.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/SupportPruneNestedColumn.java
@@ -17,15 +17,8 @@
package org.apache.doris.nereids.trees.plans.logical;
-import org.apache.doris.nereids.types.DataType;
-
/** SupportPruneNestedColumn */
public interface SupportPruneNestedColumn {
// return false will not prune the nested column
boolean supportPruneNestedColumn();
-
- // Allows a scan implementation to restrict pruning to selected root types.
- default boolean supportPruneNestedColumn(DataType dataType) {
- return supportPruneNestedColumn();
- }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScanTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScanTest.java
index 5fc6d422e09..eaaa24ebaf0 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScanTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScanTest.java
@@ -34,12 +34,6 @@ import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
import org.apache.doris.nereids.trees.plans.RelationId;
import
org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions;
-import org.apache.doris.nereids.types.ArrayType;
-import org.apache.doris.nereids.types.IntegerType;
-import org.apache.doris.nereids.types.MapType;
-import org.apache.doris.nereids.types.StructField;
-import org.apache.doris.nereids.types.StructType;
-import org.apache.doris.nereids.types.VariantType;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -105,7 +99,7 @@ public class LogicalFileScanTest {
}
@Test
- public void testPaimonSupportsOnlyVariantNestedColumnPruning() {
+ public void testPaimonSupportsNestedColumnPruning() {
PaimonExternalTable table = Mockito.mock(PaimonExternalTable.class);
Mockito.when(table.getName()).thenReturn("paimon_tbl");
TableScanParams scanParams = new TableScanParams(
@@ -119,12 +113,6 @@ public class LogicalFileScanTest {
Optional.empty(), Optional.empty(), Optional.of(scanParams),
Optional.empty());
Assertions.assertTrue(scan.supportPruneNestedColumn());
-
Assertions.assertTrue(scan.supportPruneNestedColumn(VariantType.INSTANCE));
-
Assertions.assertFalse(scan.supportPruneNestedColumn(ArrayType.of(IntegerType.INSTANCE)));
- Assertions.assertFalse(scan.supportPruneNestedColumn(
- MapType.of(IntegerType.INSTANCE, IntegerType.INSTANCE)));
- Assertions.assertFalse(scan.supportPruneNestedColumn(new
StructType(Collections.singletonList(
- new StructField("field", IntegerType.INSTANCE, true, "")))));
}
@Test
diff --git
a/regression-test/data/external_table_p0/paimon/test_paimon_jni_complex_column_pruning.out
b/regression-test/data/external_table_p0/paimon/test_paimon_jni_complex_column_pruning.out
new file mode 100644
index 00000000000..cfddfee825c
--- /dev/null
+++
b/regression-test/data/external_table_p0/paimon/test_paimon_jni_complex_column_pruning.out
@@ -0,0 +1,16 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !struct_projection --
+1 beijing
+2 shanghai
+
+-- !array_projection --
+1 90
+2 95
+
+-- !map_projection --
+1 10
+2 20
+
+-- !combined_projection --
+2 shanghai purchase beta
+
diff --git
a/regression-test/suites/external_table_p0/paimon/test_paimon_jni_complex_column_pruning.groovy
b/regression-test/suites/external_table_p0/paimon/test_paimon_jni_complex_column_pruning.groovy
new file mode 100644
index 00000000000..ac407998525
--- /dev/null
+++
b/regression-test/suites/external_table_p0/paimon/test_paimon_jni_complex_column_pruning.groovy
@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_paimon_jni_complex_column_pruning",
+ "p0,external,doris,external_docker,external_docker_doris") {
+ String enabled = context.config.otherConfigs.get("enablePaimonTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable paimon test")
+ return
+ }
+
+ String minioPort = context.config.otherConfigs.get("iceberg_minio_port")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String catalogName = "test_paimon_jni_complex_column_pruning"
+
+ try {
+ sql "drop catalog if exists ${catalogName}"
+ sql """
+ create catalog ${catalogName} properties (
+ 'type' = 'paimon',
+ 'paimon.catalog.type' = 'filesystem',
+ 'warehouse' = 's3://warehouse/wh',
+ 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}',
+ 's3.access_key' = 'admin',
+ 's3.secret_key' = 'password',
+ 's3.region' = 'us-east-1',
+ 's3.path.style.access' = 'true'
+ )
+ """
+ sql "use `${catalogName}`.`test_paimon_spark`"
+ sql "set force_jni_scanner = true"
+
+ explain {
+ sql "select id, profile.city from jni_complex_column_pruning order
by id"
+ contains "paimonNativeReadSplits=0/1"
+ contains "pruned type:"
+ contains "all access paths: [profile.city]"
+ }
+
+ explain {
+ sql "select id, events[1].score from jni_complex_column_pruning
order by id"
+ contains "paimonNativeReadSplits=0/1"
+ contains "pruned type:"
+ contains "all access paths: [events.*.score]"
+ }
+
+ explain {
+ sql """
+ select id, element_at(attributes, 'primary').code
+ from jni_complex_column_pruning
+ order by id
+ """
+ contains "paimonNativeReadSplits=0/1"
+ contains "pruned type:"
+ contains "all access paths: [attributes.*.code]"
+ }
+
+ order_qt_struct_projection """
+ select id, profile.city
+ from jni_complex_column_pruning
+ order by id
+ """
+
+ order_qt_array_projection """
+ select id, events[1].score
+ from jni_complex_column_pruning
+ order by id
+ """
+
+ order_qt_map_projection """
+ select id, element_at(attributes, 'primary').code
+ from jni_complex_column_pruning
+ order by id
+ """
+
+ // Different projected and predicate children must be combined into
one requested read type.
+ order_qt_combined_projection """
+ select id,
+ profile.city,
+ events[1].name,
+ element_at(attributes, 'primary').label
+ from jni_complex_column_pruning
+ where profile.zip >= 200000
+ and events[1].score >= 90
+ and element_at(attributes, 'primary').code = 20
+ order by id
+ """
+ } finally {
+ sql "set force_jni_scanner = false"
+ sql "drop catalog if exists ${catalogName}"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
