This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 38fe4a6439e [fix](paimon)Modify the data type conversion for
varchar/char (#49623)
38fe4a6439e is described below
commit 38fe4a6439e7bf4e7aa9bbe388436e9177eae421
Author: wuwenchi <wuwen...@selectdb.com>
AuthorDate: Mon Mar 31 16:32:23 2025 +0800
[fix](paimon)Modify the data type conversion for varchar/char (#49623)
### What problem does this PR solve?
Problem Summary:
schema from paimon to doris:
before:
varchar(1) -> text
char(1) -> text
after:
varchar(1) -> varchar(1)
char(1) -> char(1)
---
.../create_preinstalled_scripts/paimon/run03.sql | 15 +++++++
.../apache/doris/datasource/paimon/PaimonUtil.java | 6 ++-
.../doris/datasource/paimon/PaimonUtilTest.java | 39 ++++++++++++++++++
.../paimon/test_paimon_char_varchar_type.out | Bin 0 -> 282 bytes
.../paimon/test_paimon_schema_change.out | Bin 4601 -> 4751 bytes
.../paimon/test_paimon_char_varchar_type.groovy | 44 +++++++++++++++++++++
6 files changed, 103 insertions(+), 1 deletion(-)
diff --git
a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run03.sql
b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run03.sql
new file mode 100644
index 00000000000..bd8d673636c
--- /dev/null
+++
b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run03.sql
@@ -0,0 +1,15 @@
+use paimon;
+
+create database if not exists test_paimon_spark;
+use test_paimon_spark;
+
+drop table if exists test_varchar_char_type;
+
+create table test_varchar_char_type (
+ c1 int,
+ c2 char(1),
+ c3 char(2147483647),
+ c4 varchar(1),
+ c6 varchar(2147483646),
+ c5 varchar(2147483647)
+);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
index 4119f978d24..0d65d53ba40 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
@@ -42,10 +42,12 @@ import org.apache.paimon.reader.RecordReader;
import org.apache.paimon.table.Table;
import org.apache.paimon.table.source.ReadBuilder;
import org.apache.paimon.types.ArrayType;
+import org.apache.paimon.types.CharType;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DecimalType;
import org.apache.paimon.types.MapType;
import org.apache.paimon.types.RowType;
+import org.apache.paimon.types.VarCharType;
import org.apache.paimon.utils.Pair;
import org.apache.paimon.utils.Projection;
@@ -165,8 +167,10 @@ public class PaimonUtil {
case TINYINT:
return Type.TINYINT;
case VARCHAR:
- case BINARY:
+ return ScalarType.createVarcharType(((VarCharType)
dataType).getLength());
case CHAR:
+ return ScalarType.createCharType(((CharType)
dataType).getLength());
+ case BINARY:
case VARBINARY:
return Type.STRING;
case DECIMAL:
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java
new file mode 100644
index 00000000000..449cf4c70dc
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.paimon;
+
+import org.apache.doris.catalog.Type;
+
+import org.apache.paimon.types.CharType;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.VarCharType;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class PaimonUtilTest {
+ @Test
+ public void testSchemaForVarcharAndChar() {
+ DataField c1 = new DataField(1, "c1", new VarCharType(32));
+ DataField c2 = new DataField(2, "c2", new CharType(14));
+ Type type1 = PaimonUtil.paimonTypeToDorisType(c1.type());
+ Type type2 = PaimonUtil.paimonTypeToDorisType(c2.type());
+ Assert.assertTrue(type1.isVarchar());
+ Assert.assertEquals(32, type1.getLength());
+ Assert.assertEquals(14, type2.getLength());
+ }
+}
diff --git
a/regression-test/data/external_table_p0/paimon/test_paimon_char_varchar_type.out
b/regression-test/data/external_table_p0/paimon/test_paimon_char_varchar_type.out
new file mode 100644
index 00000000000..80fca7ed402
Binary files /dev/null and
b/regression-test/data/external_table_p0/paimon/test_paimon_char_varchar_type.out
differ
diff --git
a/regression-test/data/external_table_p0/paimon/test_paimon_schema_change.out
b/regression-test/data/external_table_p0/paimon/test_paimon_schema_change.out
index 5d33ed9f7e6..fed740a8803 100644
Binary files
a/regression-test/data/external_table_p0/paimon/test_paimon_schema_change.out
and
b/regression-test/data/external_table_p0/paimon/test_paimon_schema_change.out
differ
diff --git
a/regression-test/suites/external_table_p0/paimon/test_paimon_char_varchar_type.groovy
b/regression-test/suites/external_table_p0/paimon/test_paimon_char_varchar_type.groovy
new file mode 100644
index 00000000000..5ebef0b85ec
--- /dev/null
+++
b/regression-test/suites/external_table_p0/paimon/test_paimon_char_varchar_type.groovy
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_paimon_char_varchar_type",
"p0,external,doris,external_docker,external_docker_doris") {
+ String enabled = context.config.otherConfigs.get("enablePaimonTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String minio_port =
context.config.otherConfigs.get("iceberg_minio_port")
+ String catalog_name = "test_paimon_char_varchar_type"
+ String externalEnvIp =
context.config.otherConfigs.get("externalEnvIp")
+
+ sql """drop catalog if exists ${catalog_name}"""
+
+ sql """
+ CREATE CATALOG ${catalog_name} PROPERTIES (
+ 'type' = 'paimon',
+ 'warehouse' = 's3://warehouse/wh',
+ 's3.endpoint' =
'http://${externalEnvIp}:${minio_port}',
+ 's3.access_key' = 'admin',
+ 's3.secret_key' = 'password',
+ 's3.path.style.access' = 'true'
+ );
+ """
+ sql """switch `${catalog_name}`"""
+ sql """show databases; """
+ sql """use `${catalog_name}`.`test_paimon_spark`"""
+ qt_q0 """ desc test_varchar_char_type """
+ }
+}
+
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
