This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2d48f1a2292 [echcement](hive)support read hive table that change
timestamp column to bigint. (#52954)
2d48f1a2292 is described below
commit 2d48f1a229292b3e59358409637f2dd7a14aa75b
Author: daidai <[email protected]>
AuthorDate: Thu Jul 10 23:54:51 2025 +0800
[echcement](hive)support read hive table that change timestamp column to
bigint. (#52954)
### What problem does this PR solve?
Related PR: #47471
Problem Summary:
This pr is a supplement to #47471.
This pr is used to support reading hive tables that convert timestamp
columns to bigint columns and display them in `ms` precision.
(parquet/orc hive table.)
---
be/src/vec/exec/format/column_type_convert.cpp | 22 ++
be/src/vec/exec/format/column_type_convert.h | 57 ++++
.../exec/format/parquet/parquet_column_convert.cpp | 2 +-
be/test/vec/exec/column_type_convert_test.cpp | 304 +++++++++++++++------
.../hive/test_hive_schema_change_orc.out | Bin 10902 -> 11657 bytes
.../hive/test_hive_schema_change_parquet.out | Bin 11765 -> 12215 bytes
6 files changed, 293 insertions(+), 92 deletions(-)
diff --git a/be/src/vec/exec/format/column_type_convert.cpp
b/be/src/vec/exec/format/column_type_convert.cpp
index 492c3aadb17..d0cc2ba7342 100644
--- a/be/src/vec/exec/format/column_type_convert.cpp
+++ b/be/src/vec/exec/format/column_type_convert.cpp
@@ -278,6 +278,23 @@ static std::unique_ptr<ColumnTypeConverter>
_numeric_to_decimal_converter(
}
}
+static std::unique_ptr<ColumnTypeConverter> _datetime_to_numeric_converter(
+ const DataTypePtr& src_type, const DataTypePtr& dst_type) {
+ PrimitiveType dst_primitive_type = dst_type->get_primitive_type();
+
+ switch (dst_primitive_type) {
+#define DISPATCH(DST_TYPE) \
+ case DST_TYPE: { \
+ return std::make_unique<DateTimeToNumericConverter<DST_TYPE>>(); \
+ }
+ FOR_LOGICAL_INTEGER_TYPES(DISPATCH)
+#undef DISPATCH
+ default: {
+ return std::make_unique<UnsupportedConverter>(src_type, dst_type);
+ }
+ };
+}
+
static std::unique_ptr<ColumnTypeConverter> _decimal_to_numeric_converter(
const DataTypePtr& src_type, const DataTypePtr& dst_type) {
PrimitiveType src_primitive_type = src_type->get_primitive_type();
@@ -404,6 +421,11 @@ std::unique_ptr<ColumnTypeConverter>
ColumnTypeConverter::get_converter(const Da
return std::make_unique<TimeV2Converter<TYPE_DATETIMEV2,
TYPE_DATEV2>>();
}
+ // datetime to bigint (ms)
+ if (src_primitive_type == TYPE_DATETIMEV2 &&
_is_numeric_type(dst_primitive_type)) {
+ return _datetime_to_numeric_converter(src_type, dst_type);
+ }
+
// numeric to decimal
if (_is_numeric_type(src_primitive_type) &&
_is_decimal_type(dst_primitive_type)) {
return _numeric_to_decimal_converter(src_type, dst_type);
diff --git a/be/src/vec/exec/format/column_type_convert.h
b/be/src/vec/exec/format/column_type_convert.h
index 7381ab5ef4f..ddbaf2101e9 100644
--- a/be/src/vec/exec/format/column_type_convert.h
+++ b/be/src/vec/exec/format/column_type_convert.h
@@ -18,6 +18,7 @@
#pragma once
#include <absl/strings/numbers.h>
+#include <cctz/time_zone.h>
#include <cstdint>
#include <utility>
@@ -592,6 +593,62 @@ public:
}
};
+template <PrimitiveType DstPrimitiveType>
+class DateTimeToNumericConverter : public ColumnTypeConverter {
+public:
+ Status convert(ColumnPtr& src_col, MutableColumnPtr& dst_col) override {
+ using SrcColumnType = typename
PrimitiveTypeTraits<TYPE_DATETIMEV2>::ColumnType;
+ using DstColumnType = typename
PrimitiveTypeTraits<DstPrimitiveType>::ColumnType;
+ using SrcCppType = typename
PrimitiveTypeTraits<TYPE_DATETIMEV2>::CppType;
+ using DstCppType = typename
PrimitiveTypeTraits<DstPrimitiveType>::CppType;
+
+ ColumnPtr from_col = remove_nullable(src_col);
+ MutableColumnPtr to_col =
remove_nullable(dst_col->get_ptr())->assume_mutable();
+
+ NullMap* null_map = nullptr;
+ if (dst_col->is_nullable()) {
+ null_map =
&reinterpret_cast<vectorized::ColumnNullable*>(dst_col.get())
+ ->get_null_map_data();
+ }
+
+ size_t rows = from_col->size();
+ auto& src_data = static_cast<const
SrcColumnType*>(from_col.get())->get_data();
+ size_t start_idx = to_col->size();
+ to_col->resize(start_idx + rows);
+ auto& data = static_cast<DstColumnType&>(*to_col.get()).get_data();
+
+ for (int i = 0; i < rows; ++i) {
+ const SrcCppType& src_value = src_data[i];
+ auto& dst_value = reinterpret_cast<DstCppType&>(data[start_idx +
i]);
+
+ int64_t ts_s = 0;
+ if (!src_value.unix_timestamp(&ts_s, cctz::utc_time_zone())) {
+ if (null_map == nullptr) {
+ return Status::InternalError("Failed to cast value '{}' to
{} column",
+ src_data[i],
dst_col->get_name());
+ } else {
+ (*null_map)[start_idx + i] = 1;
+ }
+ }
+ auto micro = src_value.microsecond();
+ int64_t ts_ms = ts_s * 1000 + micro / 1000;
+ if constexpr (DstPrimitiveType != TYPE_LARGEINT &&
DstPrimitiveType != TYPE_BIGINT) {
+ if ((Int64)std::numeric_limits<DstCppType>::min() > ts_ms ||
+ ts_ms > (Int64)std::numeric_limits<DstCppType>::max()) {
+ if (null_map == nullptr) {
+ return Status::InternalError("Failed to cast value
'{}' to {} column",
+ src_data[i],
dst_col->get_name());
+ } else {
+ (*null_map)[start_idx + i] = 1;
+ }
+ }
+ }
+ dst_value = static_cast<DstCppType>(ts_ms);
+ }
+ return Status::OK();
+ }
+};
+
// only support date & datetime v2
template <PrimitiveType SrcPrimitiveType, PrimitiveType DstPrimitiveType>
class TimeV2Converter : public ColumnTypeConverter {
diff --git a/be/src/vec/exec/format/parquet/parquet_column_convert.cpp
b/be/src/vec/exec/format/parquet/parquet_column_convert.cpp
index 70350f5b83f..3f426cb338e 100644
--- a/be/src/vec/exec/format/parquet/parquet_column_convert.cpp
+++ b/be/src/vec/exec/format/parquet/parquet_column_convert.cpp
@@ -246,7 +246,7 @@ std::unique_ptr<PhysicalToLogicalConverter>
PhysicalToLogicalConverter::get_conv
convert_params->reset_time_scale_if_missing(9);
physical_converter = std::make_unique<Int96toTimestamp>();
} else if (src_physical_type == tparquet::Type::INT64) {
-
convert_params->reset_time_scale_if_missing(dst_logical_type->get_scale());
+
convert_params->reset_time_scale_if_missing(src_logical_type->get_scale());
physical_converter = std::make_unique<Int64ToTimestamp>();
} else {
physical_converter =
diff --git a/be/test/vec/exec/column_type_convert_test.cpp
b/be/test/vec/exec/column_type_convert_test.cpp
index ecfebadc263..9f2dd6fb9fa 100644
--- a/be/test/vec/exec/column_type_convert_test.cpp
+++ b/be/test/vec/exec/column_type_convert_test.cpp
@@ -1327,102 +1327,56 @@ TEST_F(ColumnTypeConverterTest,
TestUnsupportedConversions) {
{
std::vector<std::pair<PrimitiveType, PrimitiveType>>
unsupported_conversions = {
- {TYPE_BOOLEAN, TYPE_TINYINT},
- {TYPE_BOOLEAN, TYPE_SMALLINT},
- {TYPE_BOOLEAN, TYPE_INT},
- {TYPE_BOOLEAN, TYPE_BIGINT},
- {TYPE_BOOLEAN, TYPE_FLOAT},
- {TYPE_BOOLEAN, TYPE_DOUBLE},
- {TYPE_BOOLEAN, TYPE_DATE},
- {TYPE_BOOLEAN, TYPE_DATEV2},
- {TYPE_BOOLEAN, TYPE_TIMEV2},
- {TYPE_BOOLEAN, TYPE_DATETIME},
+ {TYPE_BOOLEAN, TYPE_TINYINT}, {TYPE_BOOLEAN, TYPE_SMALLINT},
+ {TYPE_BOOLEAN, TYPE_INT}, {TYPE_BOOLEAN, TYPE_BIGINT},
+ {TYPE_BOOLEAN, TYPE_FLOAT}, {TYPE_BOOLEAN, TYPE_DOUBLE},
+ {TYPE_BOOLEAN, TYPE_DATE}, {TYPE_BOOLEAN, TYPE_DATEV2},
+ {TYPE_BOOLEAN, TYPE_TIMEV2}, {TYPE_BOOLEAN, TYPE_DATETIME},
{TYPE_BOOLEAN, TYPE_DATETIMEV2},
- {TYPE_TINYINT, TYPE_BOOLEAN},
- {TYPE_SMALLINT, TYPE_BOOLEAN},
- {TYPE_INT, TYPE_BOOLEAN},
- {TYPE_BIGINT, TYPE_BOOLEAN},
-
- {TYPE_TINYINT, TYPE_DATE},
- {TYPE_SMALLINT, TYPE_DATE},
- {TYPE_INT, TYPE_DATE},
- {TYPE_BIGINT, TYPE_DATE},
- {TYPE_TINYINT, TYPE_DATEV2},
- {TYPE_SMALLINT, TYPE_DATEV2},
- {TYPE_INT, TYPE_DATEV2},
- {TYPE_BIGINT, TYPE_DATEV2},
- {TYPE_TINYINT, TYPE_DATETIME},
- {TYPE_SMALLINT, TYPE_DATETIME},
- {TYPE_INT, TYPE_DATETIME},
- {TYPE_BIGINT, TYPE_DATETIME},
- {TYPE_TINYINT, TYPE_DATETIMEV2},
- {TYPE_SMALLINT, TYPE_DATETIMEV2},
- {TYPE_INT, TYPE_DATETIMEV2},
- {TYPE_BIGINT, TYPE_DATETIMEV2},
- {TYPE_TINYINT, TYPE_TIMEV2},
- {TYPE_SMALLINT, TYPE_TIMEV2},
- {TYPE_INT, TYPE_TIMEV2},
- {TYPE_BIGINT, TYPE_TIMEV2},
-
- {TYPE_FLOAT, TYPE_BOOLEAN},
- {TYPE_FLOAT, TYPE_INT},
- {TYPE_FLOAT, TYPE_SMALLINT},
- {TYPE_FLOAT, TYPE_TINYINT},
- {TYPE_FLOAT, TYPE_BIGINT},
- {TYPE_FLOAT, TYPE_DATE},
- {TYPE_FLOAT, TYPE_DATEV2},
- {TYPE_FLOAT, TYPE_TIMEV2},
- {TYPE_FLOAT, TYPE_DATETIME},
- {TYPE_FLOAT, TYPE_DATETIMEV2},
-
- {TYPE_DOUBLE, TYPE_BOOLEAN},
- {TYPE_DOUBLE, TYPE_INT},
- {TYPE_DOUBLE, TYPE_SMALLINT},
- {TYPE_DOUBLE, TYPE_TINYINT},
- {TYPE_DOUBLE, TYPE_BIGINT},
- {TYPE_DOUBLE, TYPE_DATE},
- {TYPE_DOUBLE, TYPE_DATEV2},
- {TYPE_DOUBLE, TYPE_TIMEV2},
- {TYPE_DOUBLE, TYPE_DATETIME},
- {TYPE_DOUBLE, TYPE_DATETIMEV2},
+ {TYPE_TINYINT, TYPE_BOOLEAN}, {TYPE_SMALLINT, TYPE_BOOLEAN},
+ {TYPE_INT, TYPE_BOOLEAN}, {TYPE_BIGINT, TYPE_BOOLEAN},
+
+ {TYPE_TINYINT, TYPE_DATE}, {TYPE_SMALLINT, TYPE_DATE},
+ {TYPE_INT, TYPE_DATE}, {TYPE_BIGINT, TYPE_DATE},
+ {TYPE_TINYINT, TYPE_DATEV2}, {TYPE_SMALLINT, TYPE_DATEV2},
+ {TYPE_INT, TYPE_DATEV2}, {TYPE_BIGINT, TYPE_DATEV2},
+ {TYPE_TINYINT, TYPE_DATETIME}, {TYPE_SMALLINT,
TYPE_DATETIME},
+ {TYPE_INT, TYPE_DATETIME}, {TYPE_BIGINT, TYPE_DATETIME},
+ {TYPE_TINYINT, TYPE_DATETIMEV2}, {TYPE_SMALLINT,
TYPE_DATETIMEV2},
+ {TYPE_INT, TYPE_DATETIMEV2}, {TYPE_BIGINT,
TYPE_DATETIMEV2},
+ {TYPE_TINYINT, TYPE_TIMEV2}, {TYPE_SMALLINT, TYPE_TIMEV2},
+ {TYPE_INT, TYPE_TIMEV2}, {TYPE_BIGINT, TYPE_TIMEV2},
+
+ {TYPE_FLOAT, TYPE_BOOLEAN}, {TYPE_FLOAT, TYPE_INT},
+ {TYPE_FLOAT, TYPE_SMALLINT}, {TYPE_FLOAT, TYPE_TINYINT},
+ {TYPE_FLOAT, TYPE_BIGINT}, {TYPE_FLOAT, TYPE_DATE},
+ {TYPE_FLOAT, TYPE_DATEV2}, {TYPE_FLOAT, TYPE_TIMEV2},
+ {TYPE_FLOAT, TYPE_DATETIME}, {TYPE_FLOAT, TYPE_DATETIMEV2},
+
+ {TYPE_DOUBLE, TYPE_BOOLEAN}, {TYPE_DOUBLE, TYPE_INT},
+ {TYPE_DOUBLE, TYPE_SMALLINT}, {TYPE_DOUBLE, TYPE_TINYINT},
+ {TYPE_DOUBLE, TYPE_BIGINT}, {TYPE_DOUBLE, TYPE_DATE},
+ {TYPE_DOUBLE, TYPE_DATEV2}, {TYPE_DOUBLE, TYPE_TIMEV2},
+ {TYPE_DOUBLE, TYPE_DATETIME}, {TYPE_DOUBLE,
TYPE_DATETIMEV2},
{TYPE_DOUBLE, TYPE_FLOAT},
- {TYPE_DATE, TYPE_BOOLEAN},
- {TYPE_DATE, TYPE_TINYINT},
- {TYPE_DATE, TYPE_SMALLINT},
- {TYPE_DATE, TYPE_INT},
- {TYPE_DATE, TYPE_BIGINT},
- {TYPE_DATE, TYPE_FLOAT},
- {TYPE_DATE, TYPE_DOUBLE},
- {TYPE_DATEV2, TYPE_BOOLEAN},
- {TYPE_DATEV2, TYPE_TINYINT},
- {TYPE_DATEV2, TYPE_SMALLINT},
- {TYPE_DATEV2, TYPE_INT},
- {TYPE_DATEV2, TYPE_BIGINT},
- {TYPE_DATEV2, TYPE_FLOAT},
- {TYPE_DATEV2, TYPE_DOUBLE},
- {TYPE_TIMEV2, TYPE_BOOLEAN},
- {TYPE_TIMEV2, TYPE_TINYINT},
- {TYPE_TIMEV2, TYPE_SMALLINT},
- {TYPE_TIMEV2, TYPE_INT},
- {TYPE_TIMEV2, TYPE_BIGINT},
- {TYPE_TIMEV2, TYPE_FLOAT},
- {TYPE_TIMEV2, TYPE_DOUBLE},
- {TYPE_DATETIME, TYPE_BOOLEAN},
- {TYPE_DATETIME, TYPE_TINYINT},
- {TYPE_DATETIME, TYPE_SMALLINT},
- {TYPE_DATETIME, TYPE_INT},
- {TYPE_DATETIME, TYPE_BIGINT},
- {TYPE_DATETIME, TYPE_FLOAT},
- {TYPE_DATETIME, TYPE_DOUBLE},
- {TYPE_DATETIMEV2, TYPE_BOOLEAN},
- {TYPE_DATETIMEV2, TYPE_TINYINT},
- {TYPE_DATETIMEV2, TYPE_SMALLINT},
- {TYPE_DATETIMEV2, TYPE_INT},
- {TYPE_DATETIMEV2, TYPE_BIGINT},
- {TYPE_DATETIMEV2, TYPE_FLOAT},
+ {TYPE_DATE, TYPE_BOOLEAN}, {TYPE_DATE, TYPE_TINYINT},
+ {TYPE_DATE, TYPE_SMALLINT}, {TYPE_DATE, TYPE_INT},
+ {TYPE_DATE, TYPE_BIGINT}, {TYPE_DATE, TYPE_FLOAT},
+ {TYPE_DATE, TYPE_DOUBLE}, {TYPE_DATEV2, TYPE_BOOLEAN},
+ {TYPE_DATEV2, TYPE_TINYINT}, {TYPE_DATEV2, TYPE_SMALLINT},
+ {TYPE_DATEV2, TYPE_INT}, {TYPE_DATEV2, TYPE_BIGINT},
+ {TYPE_DATEV2, TYPE_FLOAT}, {TYPE_DATEV2, TYPE_DOUBLE},
+ {TYPE_TIMEV2, TYPE_BOOLEAN}, {TYPE_TIMEV2, TYPE_TINYINT},
+ {TYPE_TIMEV2, TYPE_SMALLINT}, {TYPE_TIMEV2, TYPE_INT},
+ {TYPE_TIMEV2, TYPE_BIGINT}, {TYPE_TIMEV2, TYPE_FLOAT},
+ {TYPE_TIMEV2, TYPE_DOUBLE}, {TYPE_DATETIME, TYPE_BOOLEAN},
+ {TYPE_DATETIME, TYPE_TINYINT}, {TYPE_DATETIME,
TYPE_SMALLINT},
+ {TYPE_DATETIME, TYPE_INT}, {TYPE_DATETIME, TYPE_BIGINT},
+ {TYPE_DATETIME, TYPE_FLOAT}, {TYPE_DATETIME, TYPE_DOUBLE},
+ {TYPE_DATETIMEV2, TYPE_BOOLEAN}, {TYPE_DATETIMEV2, TYPE_FLOAT},
{TYPE_DATETIMEV2, TYPE_DOUBLE},
};
@@ -1498,6 +1452,174 @@ TEST_F(ColumnTypeConverterTest,
TestUnsupportedConversions) {
}
}
+TEST_F(ColumnTypeConverterTest, TestDateTimeV2ToNumericConversions) {
+ using namespace doris::vectorized;
+
+ auto make_datetimev2_col =
+ [](const std::vector<std::tuple<int, int, int, int, int, int,
int>>& datetimes) {
+ auto col = ColumnDateTimeV2::create();
+ for (const auto& [y, m, d, h, min, s, micro] : datetimes) {
+ DateV2Value<DateTimeV2ValueType> v;
+ v.unchecked_set_time(y, m, d, h, min, s, micro);
+
col->get_data().push_back(*reinterpret_cast<vectorized::UInt64*>(&v));
+ }
+ return col;
+ };
+
+ auto parse_datetimev2_str = [](const std::string& datetime_str) {
+ UInt64 x = 0;
+ ReadBuffer buf((char*)datetime_str.data(), datetime_str.size());
+ bool ok = read_datetime_v2_text_impl(x, buf, 6);
+ CHECK(ok) << "parse_datetimev2_str failed for: " << datetime_str;
+ return x;
+ };
+
+ // 1. DATETIMEV2 -> BIGINT
+ {
+ auto src_type =
+
vectorized::DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2,
false);
+ auto dst_type = std::make_shared<DataTypeInt64>();
+ auto converter =
converter::ColumnTypeConverter::get_converter(src_type, dst_type,
+
converter::COMMON);
+
+ ASSERT_TRUE(converter->support());
+
+ // 2024-01-01 00:00:00.123456
+ auto src_col = make_datetimev2_col({{2024, 1, 1, 0, 0, 0, 123456}});
+ auto dst_col = dst_type->create_column();
+ auto mutable_dst = dst_col->assume_mutable();
+
+ Status st = converter->convert(reinterpret_cast<ColumnPtr&>(src_col),
mutable_dst);
+ ASSERT_TRUE(st.ok());
+
+ auto& dst_data = static_cast<ColumnInt64&>(*mutable_dst).get_data();
+ ASSERT_EQ(1, dst_data.size());
+ EXPECT_EQ(1704067200123, dst_data[0]);
+ }
+
+ // 2. DATETIMEV2 -> INT
+ {
+ auto src_type =
+
vectorized::DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2,
false);
+ auto dst_type = std::make_shared<DataTypeInt32>();
+ auto nullable_dst_type = std::make_shared<DataTypeNullable>(dst_type);
+ auto converter =
converter::ColumnTypeConverter::get_converter(src_type, nullable_dst_type,
+
converter::COMMON);
+
+ ASSERT_TRUE(converter->support());
+
+ // 1970-01-01 00:00:00.000000
+ // 3000-01-01 00:00:00.000000
+ auto src_col = make_datetimev2_col({{1970, 1, 1, 0, 0, 0, 0}, {3000,
1, 1, 0, 0, 0, 0}});
+ auto dst_col = nullable_dst_type->create_column();
+ auto mutable_dst = dst_col->assume_mutable();
+ auto& nullable_col = static_cast<ColumnNullable&>(*mutable_dst);
+ auto& null_map = nullable_col.get_null_map_data();
+ null_map.resize_fill(src_col->size(), 0);
+
+ Status st = converter->convert(reinterpret_cast<ColumnPtr&>(src_col),
mutable_dst);
+ ASSERT_TRUE(st.ok());
+ auto& nested_col =
static_cast<ColumnInt32&>(nullable_col.get_nested_column());
+ auto& dst_data = nested_col.get_data();
+
+ ASSERT_EQ(2, nested_col.size());
+ EXPECT_EQ(0, null_map[0]);
+ ASSERT_EQ(0, dst_data[0]);
+ EXPECT_EQ(1, null_map[1]);
+ }
+
+ // 3. DATETIMEV2 -> INT, non-nullable
+ {
+ auto src_type =
+
vectorized::DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2,
false);
+ auto dst_type = std::make_shared<DataTypeInt32>();
+ auto converter =
converter::ColumnTypeConverter::get_converter(src_type, dst_type,
+
converter::COMMON);
+
+ ASSERT_TRUE(converter->support());
+
+ // 3000-01-01 00:00:00.000000（会溢出int32）
+ auto src_col = make_datetimev2_col({{3000, 1, 1, 0, 0, 0, 0}});
+ auto dst_col = dst_type->create_column();
+ auto mutable_dst = dst_col->assume_mutable();
+
+ Status st = converter->convert(reinterpret_cast<ColumnPtr&>(src_col),
mutable_dst);
+ ASSERT_FALSE(st.ok());
+ }
+
+ {
+ auto src_type =
+
vectorized::DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2,
false);
+ auto dst_type = std::make_shared<DataTypeInt64>();
+ auto nullable_dst_type = std::make_shared<DataTypeNullable>(dst_type);
+ auto converter =
converter::ColumnTypeConverter::get_converter(src_type, nullable_dst_type,
+
converter::COMMON);
+
+ ASSERT_TRUE(converter->support());
+
+ auto src_col = ColumnDateTimeV2::create();
+ src_col->get_data().push_back(parse_datetimev2_str("2024-01-01
12:34:56.123456"));
+ src_col->get_data().push_back(parse_datetimev2_str("1970-01-01
00:00:00.000000"));
+ src_col->get_data().push_back(parse_datetimev2_str("3000-01-01
00:00:00.000000"));
+ src_col->get_data().push_back(parse_datetimev2_str("1900-01-01
00:00:00.000000"));
+ src_col->get_data().push_back(parse_datetimev2_str("1999-12-31
23:59:59.999999"));
+ src_col->get_data().push_back(parse_datetimev2_str("2000-01-01
00:00:00.000000"));
+ src_col->get_data().push_back(parse_datetimev2_str("2025-07-08
16:00:00.123456"));
+ src_col->get_data().push_back(parse_datetimev2_str("2100-01-01
00:00:00.000000"));
+ src_col->get_data().push_back(parse_datetimev2_str("9999-12-31
23:59:59.999999"));
+ src_col->get_data().push_back(parse_datetimev2_str("2022-05-01
12:00:00.000001"));
+ src_col->get_data().push_back(parse_datetimev2_str("2022-05-01
13:00:00.000002"));
+ src_col->get_data().push_back(parse_datetimev2_str("2022-05-01
14:00:00.000004"));
+ src_col->get_data().push_back(parse_datetimev2_str("2022-05-01
12:00:00"));
+ src_col->get_data().push_back(parse_datetimev2_str("2022-05-01
13:00:00"));
+ src_col->get_data().push_back(parse_datetimev2_str("2022-05-01
14:00:00"));
+
+ auto dst_col = nullable_dst_type->create_column();
+ auto mutable_dst = dst_col->assume_mutable();
+ auto& nullable_col = static_cast<ColumnNullable&>(*mutable_dst);
+ auto& null_map = nullable_col.get_null_map_data();
+ null_map.resize_fill(src_col->size(), 0);
+
+ Status st = converter->convert(reinterpret_cast<ColumnPtr&>(src_col),
mutable_dst);
+ ASSERT_TRUE(st.ok());
+
+ ASSERT_EQ(15, null_map.size());
+ EXPECT_EQ(0, null_map[0]);
+ EXPECT_EQ(0, null_map[1]);
+ EXPECT_EQ(0, null_map[2]);
+ EXPECT_EQ(0, null_map[3]);
+ EXPECT_EQ(0, null_map[4]);
+ EXPECT_EQ(0, null_map[5]);
+ EXPECT_EQ(0, null_map[6]);
+ EXPECT_EQ(0, null_map[7]);
+ EXPECT_EQ(0, null_map[8]);
+ EXPECT_EQ(0, null_map[9]);
+ EXPECT_EQ(0, null_map[10]);
+ EXPECT_EQ(0, null_map[11]);
+ EXPECT_EQ(0, null_map[12]);
+ EXPECT_EQ(0, null_map[13]);
+ EXPECT_EQ(0, null_map[14]);
+
+ auto& dst_data =
static_cast<ColumnInt64&>(nullable_col.get_nested_column()).get_data();
+ ASSERT_EQ(15, dst_data.size());
+ EXPECT_EQ(1704112496123L, dst_data[0]);
+ EXPECT_EQ(0L, dst_data[1]);
+ EXPECT_EQ(32503680000000L, dst_data[2]);
+ EXPECT_EQ(-2208988800000L, dst_data[3]);
+ EXPECT_EQ(946684799999L, dst_data[4]);
+ EXPECT_EQ(946684800000L, dst_data[5]);
+ EXPECT_EQ(1751990400123, dst_data[6]);
+ EXPECT_EQ(4102444800000L, dst_data[7]);
+ EXPECT_EQ(253402300799999, dst_data[8]);
+ EXPECT_EQ(1651406400000, dst_data[9]);
+ EXPECT_EQ(1651410000000, dst_data[10]);
+ EXPECT_EQ(1651413600000, dst_data[11]);
+ EXPECT_EQ(1651406400000, dst_data[12]);
+ EXPECT_EQ(1651410000000, dst_data[13]);
+ EXPECT_EQ(1651413600000, dst_data[14]);
+ }
+}
+
TEST_F(ColumnTypeConverterTest, TestEmptyColumnConversions) {
// Test empty column
{
diff --git
a/regression-test/data/external_table_p0/hive/test_hive_schema_change_orc.out
b/regression-test/data/external_table_p0/hive/test_hive_schema_change_orc.out
index 1028bdad184..9e819aded4f 100644
Binary files
a/regression-test/data/external_table_p0/hive/test_hive_schema_change_orc.out
and
b/regression-test/data/external_table_p0/hive/test_hive_schema_change_orc.out
differ
diff --git
a/regression-test/data/external_table_p0/hive/test_hive_schema_change_parquet.out
b/regression-test/data/external_table_p0/hive/test_hive_schema_change_parquet.out
index 8645792bfa8..719e9bbddbf 100644
Binary files
a/regression-test/data/external_table_p0/hive/test_hive_schema_change_parquet.out
and
b/regression-test/data/external_table_p0/hive/test_hive_schema_change_parquet.out
differ
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
