This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5789b7e3801 [fix](jin) add datetimev2 precision (#29528)
5789b7e3801 is described below
commit 5789b7e380116431d7511d740016cd90caa76f48
Author: Ashin Gau <ashin...@users.noreply.github.com>
AuthorDate: Sat Jan 6 13:35:26 2024 +0800
[fix](jin) add datetimev2 precision (#29528)
---
be/src/vec/exec/jni_connector.cpp | 14 ++++++++------
.../org/apache/doris/common/jni/vec/ColumnType.java | 14 +++++++++++---
.../org/apache/doris/paimon/PaimonJniScanner.java | 19 +++++++++++++++++++
.../doris/catalog/external/PaimonExternalTable.java | 13 ++++++++++---
.../external_table_p0/paimon/test_paimon_catalog.out | 12 ++++++------
5 files changed, 54 insertions(+), 18 deletions(-)
diff --git a/be/src/vec/exec/jni_connector.cpp
b/be/src/vec/exec/jni_connector.cpp
index 59ad1015976..f0494f931a3 100644
--- a/be/src/vec/exec/jni_connector.cpp
+++ b/be/src/vec/exec/jni_connector.cpp
@@ -480,9 +480,10 @@ std::string JniConnector::get_jni_type(const DataTypePtr&
data_type) {
return "datetimev1";
case TYPE_DATETIMEV2:
[[fallthrough]];
- case TYPE_TIMEV2:
- // can ignore precision of timestamp in jni
- return "datetimev2";
+ case TYPE_TIMEV2: {
+ buffer << "datetimev2(" << type->get_scale() << ")";
+ return buffer.str();
+ }
case TYPE_BINARY:
return "binary";
case TYPE_DECIMALV2: {
@@ -563,9 +564,10 @@ std::string JniConnector::get_jni_type(const
TypeDescriptor& desc) {
return "datetimev1";
case TYPE_DATETIMEV2:
[[fallthrough]];
- case TYPE_TIMEV2:
- // can ignore precision of timestamp in jni
- return "datetimev2";
+ case TYPE_TIMEV2: {
+ buffer << "datetimev2(" << desc.scale << ")";
+ return buffer.str();
+ }
case TYPE_BINARY:
return "binary";
case TYPE_CHAR: {
diff --git
a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java
b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java
index e981c4d0418..1a919d84631 100644
---
a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java
+++
b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java
@@ -76,8 +76,8 @@ public class ColumnType {
// only used in char & varchar
private final int length;
// only used in decimal
- private final int precision;
- private final int scale;
+ private int precision;
+ private int scale;
public ColumnType(String name, Type type) {
this.name = name;
@@ -188,10 +188,18 @@ public class ColumnType {
return length;
}
+ public void setPrecision(int precision) {
+ this.precision = precision;
+ }
+
public int getPrecision() {
return precision;
}
+ public void setScale(int scale) {
+ this.scale = scale;
+ }
+
public int getScale() {
return scale;
}
@@ -226,7 +234,7 @@ public class ColumnType {
}
}
- private static final Pattern digitPattern = Pattern.compile("(\\d+)");
+ private static final Pattern digitPattern =
Pattern.compile("\\((\\d+)\\)");
private static int findNextNestedField(String commaSplitFields) {
int numLess = 0;
diff --git
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
index be1505c4ef1..ac497776004 100644
---
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
+++
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
@@ -30,6 +30,8 @@ import org.apache.paimon.reader.RecordReader;
import org.apache.paimon.table.Table;
import org.apache.paimon.table.source.ReadBuilder;
import org.apache.paimon.table.source.Split;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.TimestampType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -87,6 +89,7 @@ public class PaimonJniScanner extends JniScanner {
public void open() throws IOException {
initTable();
initReader();
+ resetDatetimeV2Precision();
}
private void initReader() throws IOException {
@@ -112,6 +115,22 @@ public class PaimonJniScanner extends JniScanner {
return split;
}
+ private void resetDatetimeV2Precision() {
+ for (int i = 0; i < types.length; i++) {
+ if (types[i].isDateTimeV2()) {
+ // paimon support precision > 6, but it has been reset as 6 in
FE
+ // try to get the right precision for datetimev2
+ int index = paimonAllFieldNames.indexOf(fields[i]);
+ if (index != -1) {
+ DataType dataType = table.rowType().getTypeAt(index);
+ if (dataType instanceof TimestampType) {
+ types[i].setPrecision(((TimestampType)
dataType).getPrecision());
+ }
+ }
+ }
+ }
+ }
+
@Override
public void close() throws IOException {
if (reader != null) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/PaimonExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/PaimonExternalTable.java
index 679a7842975..c8ea253671d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/PaimonExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/PaimonExternalTable.java
@@ -43,7 +43,6 @@ public class PaimonExternalTable extends ExternalTable {
private static final Logger LOG =
LogManager.getLogger(PaimonExternalTable.class);
- public static final int PAIMON_DATETIME_SCALE_MS = 3;
private Table originTable = null;
public PaimonExternalTable(long id, String name, String dbName,
PaimonExternalCatalog catalog) {
@@ -112,7 +111,14 @@ public class PaimonExternalTable extends ExternalTable {
return ScalarType.createDateV2Type();
case TIMESTAMP_WITHOUT_TIME_ZONE:
case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
- return
ScalarType.createDatetimeV2Type(PAIMON_DATETIME_SCALE_MS);
+ int scale = 3; // default
+ if (dataType instanceof org.apache.paimon.types.TimestampType)
{
+ scale = ((org.apache.paimon.types.TimestampType)
dataType).getPrecision();
+ if (scale > 6) {
+ scale = 6;
+ }
+ }
+ return ScalarType.createDatetimeV2Type(scale);
case ARRAY:
ArrayType arrayType = (ArrayType) dataType;
Type innerType =
paimonPrimitiveTypeToDorisType(arrayType.getElementType());
@@ -124,7 +130,8 @@ public class PaimonExternalTable extends ExternalTable {
case TIME_WITHOUT_TIME_ZONE:
return Type.UNSUPPORTED;
default:
- throw new IllegalArgumentException("Cannot transform unknown
type: " + dataType.getTypeRoot());
+ LOG.warn("Cannot transform unknown type: " +
dataType.getTypeRoot());
+ return Type.UNSUPPORTED;
}
}
diff --git
a/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
b/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
index 9661bf56cf8..dff28c84a9d 100644
--- a/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
+++ b/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
@@ -105,9 +105,9 @@ true \N \N \N
9999
-- !c27 --
-\N [1, null, null] [1, null, null, null] [4, null, null] [null, null]
[null, 12, 13] [null, 15] [null, null] [18, 19, 20] [21.11, 22, 22]
[null] [null] [null, null, "2023-10-24"] ["13str", "arr_string"]
["14varchar", "arr_varcha"] [null, null] [null, null] [null, null,
null] [null, null] {1:null, 0:null} {1:null, 0:null}
{1:null, 2:null} {1:6, 0:null} {1:7} {0:null, 1:20} {1:null}
{1:null} {0:21.11, 1:22.22} {1:23.333} {0:24.44, 1:null}
{1:null, 4:"2023-02-03"} {0:"byte_string"} {0:"aa", [...]
-1 [1, 0] [1, 2, 3] [4, 5, 6] [7, 8, 9] [10, 11, 12,
13] [14, 15] [16, 17] [18, 19, 20] [21.11, 22, 22]
[23.333] [24.44] ["2020-02-02", "2023-10-24"] ["13str", "arr_string"]
["14varchar", "arr_varcha"] ["a", "b"] [1, 0, 1] ["aaaa", "bbb"]
["2023-10-24 16:10:40.529", "2023-10-24 16:10:40.529"] {1:0, 0:1} {0:2,
1:3} {1:4, 0:5} {1:6, 0:9} {1:7} {0:19, 1:20} {1:21}
{1:22} {0:21.11, 1:22.22} {1:23.333} {0:24.44, 1:25.55}
{1:"2021-02-03", 4:"2023-02-03"} {0:"byte_string"} {0:"aa",
1:"bb"} {1:"o [...]
-2 [1, null, 0] [1, null, 2, 3] [4, 5, 6] [null, null] [null,
12, 13] [14, 15] [null, null] [18, 19, 20] [21.11, 22, 22]
[23.333] [null, 24.44] ["2020-02-02", null, "2023-10-24"]
["13str", "arr_string"] ["14varchar", "arr_varcha"] ["a", "b"] [1, 0,
1] ["aaaa", "bbb", null, null] ["2023-10-24 18:19:51.670", null,
"2023-10-24 18:19:51.670"] {1:null, 0:null} {1:null, 0:null}
{1:4, 2:null} {1:6, 0:9} {1:7} {0:19, 1:20} {1:21} {1:22}
{0:21.11, 1:22.22} {1:23.333} {0:24.44, 1:25.55} {1:null, 4:
[...]
+\N [1, null, null] [1, null, null, null] [4, null, null] [null, null]
[null, 12, 13] [null, 15] [null, null] [18, 19, 20] [21.11, 22, 22]
[null] [null] [null, null, "2023-10-24"] ["13str", "arr_string"]
["14varchar", "arr_varcha"] [null, null] [null, null] [null, null,
null] [null, null] {1:null, 0:null} {1:null, 0:null}
{1:null, 2:null} {1:6, 0:null} {1:7} {0:null, 1:20} {1:null}
{1:null} {0:21.11, 1:22.22} {1:23.333} {0:24.44, 1:null}
{1:null, 4:"2023-02-03"} {0:"byte_string"} {0:"aa", [...]
+1 [1, 0] [1, 2, 3] [4, 5, 6] [7, 8, 9] [10, 11, 12,
13] [14, 15] [16, 17] [18, 19, 20] [21.11, 22, 22]
[23.333] [24.44] ["2020-02-02", "2023-10-24"] ["13str", "arr_string"]
["14varchar", "arr_varcha"] ["a", "b"] [1, 0, 1] ["aaaa", "bbb"]
["2023-10-24 16:10:40.529000", "2023-10-24 16:10:40.529000"] {1:0, 0:1}
{0:2, 1:3} {1:4, 0:5} {1:6, 0:9} {1:7} {0:19, 1:20} {1:21}
{1:22} {0:21.11, 1:22.22} {1:23.333} {0:24.44, 1:25.55}
{1:"2021-02-03", 4:"2023-02-03"} {0:"byte_string"} {0:"aa",
1:"bb"} [...]
+2 [1, null, 0] [1, null, 2, 3] [4, 5, 6] [null, null] [null,
12, 13] [14, 15] [null, null] [18, 19, 20] [21.11, 22, 22]
[23.333] [null, 24.44] ["2020-02-02", null, "2023-10-24"]
["13str", "arr_string"] ["14varchar", "arr_varcha"] ["a", "b"] [1, 0,
1] ["aaaa", "bbb", null, null] ["2023-10-24 18:19:51.670000", null,
"2023-10-24 18:19:51.670000"] {1:null, 0:null} {1:null, 0:null}
{1:4, 2:null} {1:6, 0:9} {1:7} {0:19, 1:20} {1:21} {1:22}
{0:21.11, 1:22.22} {1:23.333} {0:24.44, 1:25.55} {1:nu [...]
-- !c28 --
0
@@ -470,9 +470,9 @@ bbb
\N
-- !c100 --
-1 [[0, 1, 1], [0, 1, 1], [1, 1, 1]] [["3", "7", "a"], ["0", "e",
"2"], ["b", "4", "5"]] [["77", "83", "1c"], ["af", "f0", "0b"], ["be",
"d2", "7b"]]
[["4a3aee3c345936f7a6eeb5307fc79d5f9ee3ae3a5e58edcfd4d40ec3d27353bf7b15a2eb6b6011e9ddf2c2cd1d6d50d6cc20",
"00cb283c1a2bb2f3f244f89eafb79effd8aea55d5e68ce9d707250a3b47b65c0ea2b591b7145a56c801b9e6bda853e2f0581",
"ae8fbe21e20f32a486757b5254faea09906bf451e096cc7d33b5a6fb56995c1601e7469a674f5e8475a2b86d2a69e02f9438"],
["6827f8e65869d476a9e2fdfa03 [...]
-2 [[0, 0, 0], [1, 0, 0], [0, 1, 1]] [["a", "5", "f"], ["7", "2",
"3"], ["1", "b", "f"]] [["11", "a8", "e3"], ["f4", "ee", "c3"], ["0f",
"c6", "05"]]
[["b723869515b24e9fbb54503f8a7584f083479998766213b784a9c530cbc0376bd5035054c657437251b85fa3dd41a0483776",
"ee780d81d3e9faa36aad06522a09cf9b18e00614c991d2c079243dcc7190f3dd6559e75e2c1992270272d9a9c01e950c7bd4",
"6e825a52cdae65786801caae53182956c80f88a48fa258a90d2c93302023c78b83f1dce758615a74731c9eef993a8c1dc4ad"],
["2ae97869372970f7ad7d0007a1 [...]
-3 [[1, 0, 0], [1, 0, 1], [1, 0, 1]] [["c", "6", "4"], ["1", "e",
"7"], ["7", "1", "d"]] [["50", "a9", "b0"], ["02", "bf", "3a"], ["0a",
"1d", "9a"]]
[["2df05ba6ce8661f0fb9bc88386a1ba67188e3e99a4142a0703e1cd8bdf041fbc20131e50bea2a9891498c638ebac842d3d46",
"f6fecf3a4263ed8d4c5b63e4b3d9f4084b83835d9fba2046bf48d0a8068f2044c48271a1e9726741a09badea72c37cf18de3",
"d23af3266db4eda12673f5c451d36343ba1cea00fbfeeff2165de40e834778eb96a1199cb523dd394b4f08824f6af2a7d894"],
["dc00e9b27b9540e170caf93805 [...]
+1 [[0, 1, 1], [0, 1, 1], [1, 1, 1]] [["3", "7", "a"], ["0", "e",
"2"], ["b", "4", "5"]] [["77", "83", "1c"], ["af", "f0", "0b"], ["be",
"d2", "7b"]]
[["4a3aee3c345936f7a6eeb5307fc79d5f9ee3ae3a5e58edcfd4d40ec3d27353bf7b15a2eb6b6011e9ddf2c2cd1d6d50d6cc20",
"00cb283c1a2bb2f3f244f89eafb79effd8aea55d5e68ce9d707250a3b47b65c0ea2b591b7145a56c801b9e6bda853e2f0581",
"ae8fbe21e20f32a486757b5254faea09906bf451e096cc7d33b5a6fb56995c1601e7469a674f5e8475a2b86d2a69e02f9438"],
["6827f8e65869d476a9e2fdfa03 [...]
+2 [[0, 0, 0], [1, 0, 0], [0, 1, 1]] [["a", "5", "f"], ["7", "2",
"3"], ["1", "b", "f"]] [["11", "a8", "e3"], ["f4", "ee", "c3"], ["0f",
"c6", "05"]]
[["b723869515b24e9fbb54503f8a7584f083479998766213b784a9c530cbc0376bd5035054c657437251b85fa3dd41a0483776",
"ee780d81d3e9faa36aad06522a09cf9b18e00614c991d2c079243dcc7190f3dd6559e75e2c1992270272d9a9c01e950c7bd4",
"6e825a52cdae65786801caae53182956c80f88a48fa258a90d2c93302023c78b83f1dce758615a74731c9eef993a8c1dc4ad"],
["2ae97869372970f7ad7d0007a1 [...]
+3 [[1, 0, 0], [1, 0, 1], [1, 0, 1]] [["c", "6", "4"], ["1", "e",
"7"], ["7", "1", "d"]] [["50", "a9", "b0"], ["02", "bf", "3a"], ["0a",
"1d", "9a"]]
[["2df05ba6ce8661f0fb9bc88386a1ba67188e3e99a4142a0703e1cd8bdf041fbc20131e50bea2a9891498c638ebac842d3d46",
"f6fecf3a4263ed8d4c5b63e4b3d9f4084b83835d9fba2046bf48d0a8068f2044c48271a1e9726741a09badea72c37cf18de3",
"d23af3266db4eda12673f5c451d36343ba1cea00fbfeeff2165de40e834778eb96a1199cb523dd394b4f08824f6af2a7d894"],
["dc00e9b27b9540e170caf93805 [...]
-- !c101 --
1 2 3 4 5 6 7 8 9.1 10.1
11.10 2020-02-02 13str 14varchar a true aaaa
2023-08-13T09:32:38.530
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
