This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new fc7c49e3d5b [opt](paimon) support mapping Paimon column type "Row" to
Doris type "Struct" (#33786)
fc7c49e3d5b is described below
commit fc7c49e3d5be395c72639a1a770db041379754dd
Author: 苏小刚 <suxiaogang...@icloud.com>
AuthorDate: Sun Apr 28 19:38:00 2024 +0800
[opt](paimon) support mapping Paimon column type "Row" to Doris type
"Struct" (#33786)
---
.../main/java/org/apache/doris/paimon/PaimonColumnValue.java | 6 +++++-
.../main/java/org/apache/doris/paimon/PaimonTypeUtils.java | 10 +++++++++-
.../apache/doris/datasource/paimon/PaimonExternalTable.java | 10 ++++++++++
.../data/external_table_p0/paimon/test_paimon_catalog.out | 10 ++++++++++
.../external_table_p0/paimon/test_paimon_catalog.groovy | 12 ++++++++----
5 files changed, 42 insertions(+), 6 deletions(-)
diff --git
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java
index b1b7440649e..d102ebd2fdf 100644
---
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java
+++
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java
@@ -166,6 +166,10 @@ public class PaimonColumnValue implements ColumnValue {
@Override
public void unpackStruct(List<Integer> structFieldIndex, List<ColumnValue>
values) {
-
+ // todo: support pruned struct fields
+ InternalRow row = record.getRow(idx, structFieldIndex.size());
+ for (int i : structFieldIndex) {
+ values.add(new PaimonColumnValue(row, i,
dorisType.getChildTypes().get(i)));
+ }
}
}
diff --git
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java
index c0bb94dd8e4..1b8046494d2 100644
---
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java
+++
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTypeUtils.java
@@ -25,6 +25,7 @@ import org.apache.paimon.types.BigIntType;
import org.apache.paimon.types.BinaryType;
import org.apache.paimon.types.BooleanType;
import org.apache.paimon.types.CharType;
+import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypeDefaultVisitor;
import org.apache.paimon.types.DateType;
@@ -48,6 +49,7 @@ import org.slf4j.LoggerFactory;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.stream.Collectors;
/**
* Convert paimon type to doris type.
@@ -190,7 +192,13 @@ public class PaimonTypeUtils {
@Override
public PaimonColumnType visit(RowType rowType) {
- return this.defaultMethod(rowType);
+ PaimonColumnType paimonColumnType = new
PaimonColumnType(Type.STRUCT);
+ List<DataField> fields = rowType.getFields();
+ List<ColumnType> childTypes = fields.stream()
+ .map(field -> fromPaimonType(field.name(), field.type()))
+ .collect(Collectors.toList());
+ paimonColumnType.setChildTypes(childTypes);
+ return paimonColumnType;
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
index af2da32374f..7d870f36059 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
@@ -39,9 +39,12 @@ import org.apache.paimon.types.ArrayType;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DecimalType;
import org.apache.paimon.types.MapType;
+import org.apache.paimon.types.RowType;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
+import java.util.stream.Collectors;
public class PaimonExternalTable extends ExternalTable {
@@ -131,6 +134,13 @@ public class PaimonExternalTable extends ExternalTable {
MapType mapType = (MapType) dataType;
return new org.apache.doris.catalog.MapType(
paimonTypeToDorisType(mapType.getKeyType()),
paimonTypeToDorisType(mapType.getValueType()));
+ case ROW:
+ RowType rowType = (RowType) dataType;
+ List<DataField> fields = rowType.getFields();
+ return new org.apache.doris.catalog.StructType(fields.stream()
+ .map(field -> new
org.apache.doris.catalog.StructField(field.name(),
+ paimonTypeToDorisType(field.type())))
+ .collect(Collectors.toCollection(ArrayList::new)));
case TIME_WITHOUT_TIME_ZONE:
return Type.UNSUPPORTED;
default:
diff --git
a/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
b/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
index 430292ceb86..92cfe08294c 100644
--- a/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
+++ b/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
@@ -548,3 +548,13 @@ bbb
2 [[0, 0, 0], [1, 0, 0], [0, 1, 1]] [["a", "5", "f"], ["7", "2",
"3"], ["1", "b", "f"]] [["11", "a8", "e3"], ["f4", "ee", "c3"], ["0f",
"c6", "05"]]
[["b723869515b24e9fbb54503f8a7584f083479998766213b784a9c530cbc0376bd5035054c657437251b85fa3dd41a0483776",
"ee780d81d3e9faa36aad06522a09cf9b18e00614c991d2c079243dcc7190f3dd6559e75e2c1992270272d9a9c01e950c7bd4",
"6e825a52cdae65786801caae53182956c80f88a48fa258a90d2c93302023c78b83f1dce758615a74731c9eef993a8c1dc4ad"],
["2ae97869372970f7ad7d0007a1 [...]
3 [[1, 0, 0], [1, 0, 1], [1, 0, 1]] [["c", "6", "4"], ["1", "e",
"7"], ["7", "1", "d"]] [["50", "a9", "b0"], ["02", "bf", "3a"], ["0a",
"1d", "9a"]]
[["2df05ba6ce8661f0fb9bc88386a1ba67188e3e99a4142a0703e1cd8bdf041fbc20131e50bea2a9891498c638ebac842d3d46",
"f6fecf3a4263ed8d4c5b63e4b3d9f4084b83835d9fba2046bf48d0a8068f2044c48271a1e9726741a09badea72c37cf18de3",
"d23af3266db4eda12673f5c451d36343ba1cea00fbfeeff2165de40e834778eb96a1199cb523dd394b4f08824f6af2a7d894"],
["dc00e9b27b9540e170caf93805 [...]
+-- !c102 --
+1 {"c_boolean": 1, "c_tinyint": 1, "c_smallint": 2, "c_int": 3,
"c_bigint": 4, "c_float": 5.1, "c_double": 6.2, "c_decimal": 7.30, "c_char":
"6", "c_varchar": "9", "c_binary": "a", "c_varbinary": "bbbb", "c_date":
"2020-01-01", "c_timestamp": "2020-01-01 00:00:00.000000", "c_array": [1, 2,
3], "c_map": {"a":1, "b":2}, "c_row": {"c_int": 3, "c_float": 4.1, "c_varchar":
"5"}}
+2 {"c_boolean": 1, "c_tinyint": 5, "c_smallint": 6, "c_int": 7,
"c_bigint": 8, "c_float": 10.2, "c_double": 11.4, "c_decimal": 12.60, "c_char":
"7", "c_varchar": "10", "c_binary": "c", "c_varbinary": "dddd", "c_date":
"2021-06-15", "c_timestamp": "2021-06-15 06:00:00.000000", "c_array": [4, 5,
6], "c_map": {"a":6, "b":7}, "c_row": {"c_int": 8, "c_float": 9.2, "c_varchar":
"10"}}
+3 {"c_boolean": 1, "c_tinyint": 9, "c_smallint": 10, "c_int": 11,
"c_bigint": 12, "c_float": 13.3, "c_double": 14.6, "c_decimal": 15.90,
"c_char": "8", "c_varchar": "11", "c_binary": "e", "c_varbinary": "ffff",
"c_date": "2022-12-31", "c_timestamp": "2022-12-31 12:59:59.000000", "c_array":
[7, 8, 9], "c_map": {"a":11, "b":12}, "c_row": {"c_int": 13, "c_float": 14.3,
"c_varchar": "15"}}
+
+-- !c103 --
+1 {"c_boolean": 1, "c_tinyint": 1, "c_smallint": 2, "c_int": 3,
"c_bigint": 4, "c_float": 5.1, "c_double": 6.2, "c_decimal": 7.30, "c_char":
"6", "c_varchar": "9", "c_binary": "a", "c_varbinary": "bbbb", "c_date":
"2020-01-01", "c_timestamp": "2020-01-01 00:00:00.000000", "c_array": [1, 2,
3], "c_map": {"a":1, "b":2}, "c_row": {"c_int": 3, "c_float": 4.1, "c_varchar":
"5"}}
+2 {"c_boolean": 1, "c_tinyint": 5, "c_smallint": 6, "c_int": 7,
"c_bigint": 8, "c_float": 10.2, "c_double": 11.4, "c_decimal": 12.60, "c_char":
"7", "c_varchar": "10", "c_binary": "c", "c_varbinary": "dddd", "c_date":
"2021-06-15", "c_timestamp": "2021-06-15 06:00:00.000000", "c_array": [4, 5,
6], "c_map": {"a":6, "b":7}, "c_row": {"c_int": 8, "c_float": 9.2, "c_varchar":
"10"}}
+3 {"c_boolean": 1, "c_tinyint": 9, "c_smallint": 10, "c_int": 11,
"c_bigint": 12, "c_float": 13.3, "c_double": 14.6, "c_decimal": 15.90,
"c_char": "8", "c_varchar": "11", "c_binary": "e", "c_varbinary": "ffff",
"c_date": "2022-12-31", "c_timestamp": "2022-12-31 12:59:59.000000", "c_array":
[7, 8, 9], "c_map": {"a":11, "b":12}, "c_row": {"c_int": 13, "c_float": 14.3,
"c_varchar": "15"}}
+
diff --git
a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
index 70ccb1c97d7..53ab596a82f 100644
--- a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
@@ -56,10 +56,7 @@ suite("test_paimon_catalog",
"p0,external,doris,external_docker,external_docker_
sql """drop catalog ${hms_ctl_name}""";
String enabled = context.config.otherConfigs.get("enablePaimonTest")
- if (enabled != null &&
enabled.equalsIgnoreCase("enable_deprecated_case")) {
- // The timestamp type of paimon has no logical or converted type,
- // and is conflict with column type change from bigint to
timestamp.
- // Deprecated currently.
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
def qt_all_type = { String table_name ->
qt_all """select * from ${table_name} order by c1"""
qt_predict_like_1 """select * from ${table_name} where c13
like '%3%' order by c1"""
@@ -174,6 +171,9 @@ suite("test_paimon_catalog",
"p0,external,doris,external_docker,external_docker_
def c100= """select * from array_nested order by c1;"""
+ def c102= """select * from row_native_test order by id;"""
+ def c103= """select * from row_jni_test order by id;"""
+
String hdfs_port = context.config.otherConfigs.get("hive2HdfsPort")
String catalog_name = "ctl_test_paimon_catalog"
String externalEnvIp =
context.config.otherConfigs.get("externalEnvIp")
@@ -271,6 +271,10 @@ suite("test_paimon_catalog",
"p0,external,doris,external_docker,external_docker_
qt_c98 c98
qt_c99 c99
qt_c100 c100
+ qt_c102 c102
+ sql """ set force_jni_scanner=true; """
+ qt_c103 c103
+ sql """ set force_jni_scanner=false; """
// test view from jion paimon
sql """ switch internal """
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
