This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 446b188927798dee991aa7911eda5d8b68f94f66
Author: zclllyybb <zhaochan...@selectdb.com>
AuthorDate: Fri Aug 25 09:59:38 2023 +0800
[fix](functions) fix function substitute for datetimeV1/V2 (#23344)
* fix
* function fe
---
be/src/vec/functions/function_cast.h | 4 +-
be/src/vec/io/io_helper.h | 44 ++++++++++++++++++----
.../main/java/org/apache/doris/catalog/Type.java | 1 +
.../main/java/org/apache/doris/analysis/Expr.java | 2 +
.../doris/analysis/TimestampArithmeticExpr.java | 10 +++++
.../datatype_p0/datetimev2/test_tz_streamload.out | 9 ++++-
.../datatype_p0/datetimev2/test_tz_streamload2.csv | 5 +++
.../datetime_functions/test_date_function.out | 3 ++
.../datetime_functions/test_date_function.out | 3 ++
.../datetimev2/test_tz_streamload.groovy | 40 ++++++++++++++++----
.../datetime_functions/test_date_function.groovy | 2 +
.../datetime_functions/test_date_function.groovy | 2 +
12 files changed, 108 insertions(+), 17 deletions(-)
diff --git a/be/src/vec/functions/function_cast.h
b/be/src/vec/functions/function_cast.h
index 528ba75a21..9133e1d724 100644
--- a/be/src/vec/functions/function_cast.h
+++ b/be/src/vec/functions/function_cast.h
@@ -844,11 +844,11 @@ bool try_parse_impl(typename DataType::FieldType& x,
ReadBuffer& rb,
const cctz::time_zone& local_time_zone, ZoneList&
time_zone_cache,
Additions additions [[maybe_unused]] = Additions()) {
if constexpr (IsDateTimeType<DataType>) {
- return try_read_datetime_text(x, rb);
+ return try_read_datetime_text(x, rb, local_time_zone, time_zone_cache);
}
if constexpr (IsDateType<DataType>) {
- return try_read_date_text(x, rb);
+ return try_read_date_text(x, rb, local_time_zone, time_zone_cache);
}
if constexpr (IsDateV2Type<DataType>) {
diff --git a/be/src/vec/io/io_helper.h b/be/src/vec/io/io_helper.h
index 5f07d8ced1..4d8b69d1fd 100644
--- a/be/src/vec/io/io_helper.h
+++ b/be/src/vec/io/io_helper.h
@@ -269,6 +269,33 @@ bool read_int_text_impl(T& x, ReadBuffer& buf) {
return true;
}
+template <typename T>
+bool read_date_text_impl(T& x, ReadBuffer& buf) {
+ static_assert(std::is_same_v<Int64, T>);
+ auto dv = binary_cast<Int64, VecDateTimeValue>(x);
+ auto ans = dv.from_date_str(buf.position(), buf.count());
+ dv.cast_to_date();
+
+ // only to match the is_all_read() check to prevent return null
+ buf.position() = buf.end();
+ x = binary_cast<VecDateTimeValue, Int64>(dv);
+ return ans;
+}
+
+template <typename T>
+bool read_date_text_impl(T& x, ReadBuffer& buf, const cctz::time_zone&
local_time_zone,
+ ZoneList& time_zone_cache) {
+ static_assert(std::is_same_v<Int64, T>);
+ auto dv = binary_cast<Int64, VecDateTimeValue>(x);
+ auto ans = dv.from_date_str(buf.position(), buf.count(), local_time_zone,
time_zone_cache);
+ dv.cast_to_date();
+
+ // only to match the is_all_read() check to prevent return null
+ buf.position() = buf.end();
+ x = binary_cast<VecDateTimeValue, Int64>(dv);
+ return ans;
+}
+
template <typename T>
bool read_datetime_text_impl(T& x, ReadBuffer& buf) {
static_assert(std::is_same_v<Int64, T>);
@@ -283,11 +310,12 @@ bool read_datetime_text_impl(T& x, ReadBuffer& buf) {
}
template <typename T>
-bool read_date_text_impl(T& x, ReadBuffer& buf) {
+bool read_datetime_text_impl(T& x, ReadBuffer& buf, const cctz::time_zone&
local_time_zone,
+ ZoneList& time_zone_cache) {
static_assert(std::is_same_v<Int64, T>);
auto dv = binary_cast<Int64, VecDateTimeValue>(x);
- auto ans = dv.from_date_str(buf.position(), buf.count());
- dv.cast_to_date();
+ auto ans = dv.from_date_str(buf.position(), buf.count(), local_time_zone,
time_zone_cache);
+ dv.to_datetime();
// only to match the is_all_read() check to prevent return null
buf.position() = buf.end();
@@ -422,13 +450,15 @@ bool try_read_decimal_text(T& x, ReadBuffer& in, UInt32
precision, UInt32 scale)
}
template <typename T>
-bool try_read_datetime_text(T& x, ReadBuffer& in) {
- return read_datetime_text_impl<T>(x, in);
+bool try_read_datetime_text(T& x, ReadBuffer& in, const cctz::time_zone&
local_time_zone,
+ ZoneList& time_zone_cache) {
+ return read_datetime_text_impl<T>(x, in, local_time_zone, time_zone_cache);
}
template <typename T>
-bool try_read_date_text(T& x, ReadBuffer& in) {
- return read_date_text_impl<T>(x, in);
+bool try_read_date_text(T& x, ReadBuffer& in, const cctz::time_zone&
local_time_zone,
+ ZoneList& time_zone_cache) {
+ return read_date_text_impl<T>(x, in, local_time_zone, time_zone_cache);
}
template <typename T>
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
index 436b5e951e..b9c17ce455 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
@@ -90,6 +90,7 @@ public abstract class Type {
public static final ScalarType DEFAULT_DECIMALV3 = DEFAULT_DECIMAL32;
public static final ScalarType DEFAULT_DATETIMEV2 =
ScalarType.createDatetimeV2Type(0);
public static final ScalarType DATETIMEV2 = DEFAULT_DATETIMEV2;
+ public static final ScalarType DATETIMEV2_WITH_MAX_SCALAR =
ScalarType.createDatetimeV2Type(6);
public static final ScalarType DEFAULT_TIMEV2 =
ScalarType.createTimeV2Type(0);
public static final ScalarType DECIMALV2 = DEFAULT_DECIMALV2;
public static final ScalarType DECIMAL32 = DEFAULT_DECIMAL32;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
index 24031bfb89..08f6d5b2f1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
@@ -2476,6 +2476,8 @@ public abstract class Expr extends TreeNode<Expr>
implements ParseNode, Cloneabl
return Type.DECIMAL128;
} else if (originType.getPrimitiveType() == PrimitiveType.DATETIMEV2) {
return Type.DATETIMEV2;
+ } else if (originType.getPrimitiveType() == PrimitiveType.DATEV2) {
+ return Type.DATEV2;
} else if (originType.getPrimitiveType() == PrimitiveType.VARCHAR) {
return Type.VARCHAR;
} else if (originType.getPrimitiveType() == PrimitiveType.CHAR) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/TimestampArithmeticExpr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/TimestampArithmeticExpr.java
index e344a674f3..e8960f7e89 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/TimestampArithmeticExpr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/TimestampArithmeticExpr.java
@@ -24,6 +24,7 @@ import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.thrift.TExprNode;
@@ -146,6 +147,15 @@ public class TimestampArithmeticExpr extends Expr {
return Type.DATEV2;
}
if (PrimitiveType.isImplicitCast(t1, PrimitiveType.DATETIME)) {
+ if (Config.enable_date_conversion) {
+ if (t1 == PrimitiveType.NULL_TYPE) {
+ getChild(0).type = Type.DATETIMEV2_WITH_MAX_SCALAR;
+ }
+ return Type.DATETIMEV2_WITH_MAX_SCALAR;
+ }
+ if (t1 == PrimitiveType.NULL_TYPE) {
+ getChild(0).type = Type.DATETIME;
+ }
return Type.DATETIME;
}
return Type.INVALID;
diff --git a/regression-test/data/datatype_p0/datetimev2/test_tz_streamload.out
b/regression-test/data/datatype_p0/datetimev2/test_tz_streamload.out
index 3fc4fb93b6..cc87e0fdfa 100644
--- a/regression-test/data/datatype_p0/datetimev2/test_tz_streamload.out
+++ b/regression-test/data/datatype_p0/datetimev2/test_tz_streamload.out
@@ -1,5 +1,5 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
--- !all --
+-- !table1 --
2022-01-01T01:02:55 2022-01-01
2022-02-01T03:02:55 2022-02-01
2022-02-28T19:02:55 2022-03-01
@@ -9,3 +9,10 @@
2022-06-30T20:02:55 2022-07-01
2022-07-31T21:00 2022-08-01
+-- !table2 --
+1 2023-08-17T17:41:18
+2 2023-08-17T17:41:18
+3 2023-08-17T17:41:18
+4 2023-08-17T14:41:18
+5 2023-08-17T09:41:18
+
diff --git
a/regression-test/data/datatype_p0/datetimev2/test_tz_streamload2.csv
b/regression-test/data/datatype_p0/datetimev2/test_tz_streamload2.csv
new file mode 100644
index 0000000000..9d94561af4
--- /dev/null
+++ b/regression-test/data/datatype_p0/datetimev2/test_tz_streamload2.csv
@@ -0,0 +1,5 @@
+1,2023-08-17T01:41:18Z
+2,2023-08-17T01:41:18uTc
+3,2023-08-17T01:41:18UTC
+4,2023-08-17T01:41:18UTC+3
+5,2023-08-17T01:41:18Asia/Shanghai
\ No newline at end of file
diff --git
a/regression-test/data/nereids_p0/sql_functions/datetime_functions/test_date_function.out
b/regression-test/data/nereids_p0/sql_functions/datetime_functions/test_date_function.out
index 617d55e6af..43eff1b302 100644
---
a/regression-test/data/nereids_p0/sql_functions/datetime_functions/test_date_function.out
+++
b/regression-test/data/nereids_p0/sql_functions/datetime_functions/test_date_function.out
@@ -609,3 +609,6 @@ true
2022-02-21 2022-02-21 2022-02-21 2022-02-21
2022-02-28 2022-02-28 2022-02-28 2022-02-28
+-- !sql --
+2023-08-17T17:41:18
+
diff --git
a/regression-test/data/query_p0/sql_functions/datetime_functions/test_date_function.out
b/regression-test/data/query_p0/sql_functions/datetime_functions/test_date_function.out
index 2f9cc35e4e..abfeacc037 100644
---
a/regression-test/data/query_p0/sql_functions/datetime_functions/test_date_function.out
+++
b/regression-test/data/query_p0/sql_functions/datetime_functions/test_date_function.out
@@ -662,3 +662,6 @@ true
-- !sql --
2022-01-20T00:00 2023-01-20T00:00
+-- !sql --
+2023-08-17T17:41:18
+
diff --git
a/regression-test/suites/datatype_p0/datetimev2/test_tz_streamload.groovy
b/regression-test/suites/datatype_p0/datetimev2/test_tz_streamload.groovy
index c4b0cdf051..359f432bd5 100644
--- a/regression-test/suites/datatype_p0/datetimev2/test_tz_streamload.groovy
+++ b/regression-test/suites/datatype_p0/datetimev2/test_tz_streamload.groovy
@@ -16,12 +16,12 @@
// under the License.
suite("test_tz_streamload") {
- def dbName = "tz_streamload"
- def tableName = "timezone"
+ def table1 = "timezone"
+ def table2 = "datetime"
- sql "drop table if exists ${tableName}"
+ sql "drop table if exists ${table1}"
sql """
- CREATE TABLE IF NOT EXISTS ${tableName} (
+ CREATE TABLE IF NOT EXISTS ${table1} (
`k1` datetimev2(3) NULL,
`k2` datev2 NULL
) ENGINE=OLAP
@@ -32,14 +32,40 @@ suite("test_tz_streamload") {
)
"""
+ sql "drop table if exists ${table2}"
+ sql """
+ CREATE TABLE ${table2} (
+ id int NULL,
+ createTime datetime NULL
+ )ENGINE=OLAP
+ UNIQUE KEY(`id`)
+ COMMENT "OLAP"
+ DISTRIBUTED BY HASH(`id`) BUCKETS 3
+ PROPERTIES (
+ "replication_num" = "1",
+ "colocate_with" = "lineitem_orders",
+ "enable_unique_key_merge_on_write" = "true"
+ );
+ """
+
streamLoad {
- table "${tableName}"
+ table "${table1}"
set 'column_separator', ','
set 'time_zone', '+02:00'
file "test_tz_streamload.csv"
- time 10000
+ time 20000
}
sql "sync"
+ qt_table1 "select * from ${table1} order by k1"
- qt_all "select * from ${tableName} order by k1"
+ streamLoad {
+ table "${table2}"
+ set 'column_separator', ','
+ set 'columns', 'id,createTime,createTime=date_add(createTime, INTERVAL
8 HOUR)'
+ // use default timezone for this
+ file "test_tz_streamload2.csv"
+ time 20000
+ }
+ sql "sync"
+ qt_table2 "select * from ${table2} order by id"
}
diff --git
a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy
b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy
index 8d74e47d98..073c5dc725 100644
---
a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy
+++
b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy
@@ -640,4 +640,6 @@ suite("test_date_function") {
sql "select cast('20230631' as date), cast('20230632' as date)"
result([[null, null]])
}
+
+ qt_sql """ select date_add("2023-08-17T01:41:18Z", interval 8 hour) """
}
diff --git
a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy
b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy
index ed8f4d032e..1ca3124e4c 100644
---
a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy
+++
b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy
@@ -715,4 +715,6 @@ suite("test_date_function") {
res = sql "explain select date_trunc('2022-04-24', 'day'),
date_trunc('1999-03-12 00:31:23', 'hour')"
assertFalse(res.contains("date_trunc"))
+
+ qt_sql """ select date_add("2023-08-17T01:41:18Z", interval 8 hour) """
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
