This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8d85f5bf7a4 [chore](SerDe) remove date_olap_format in FormatOptions
(#51260)
8d85f5bf7a4 is described below
commit 8d85f5bf7a4ec1cda566c8cd79b2277c9412374d
Author: Mryange <[email protected]>
AuthorDate: Tue Jun 10 09:01:21 2025 +0800
[chore](SerDe) remove date_olap_format in FormatOptions (#51260)
remove date_olap_format in FormatOptions
---
.../data_types/serde/data_type_date64_serde.cpp | 70 ++++------------------
.../serde/data_type_datetimev2_serde.cpp | 31 ++--------
.../data_types/serde/data_type_datev2_serde.cpp | 10 +---
be/src/vec/data_types/serde/data_type_serde.h | 5 --
.../serde/data_type_serde_agg_state_test.cpp | 6 --
.../serde/data_type_serde_bitmap_test.cpp | 2 -
.../data_types/serde/data_type_serde_csv_test.cpp | 53 +++++++++-------
.../data_type_serde_fixed_length_object_test.cpp | 3 -
.../data_types/serde/data_type_serde_hll_test.cpp | 2 -
.../serde/data_type_serde_quantile_state_test.cpp | 2 -
.../data_types/serde/data_type_serde_text_test.cpp | 53 +++++++++-------
11 files changed, 79 insertions(+), 158 deletions(-)
diff --git a/be/src/vec/data_types/serde/data_type_date64_serde.cpp
b/be/src/vec/data_types/serde/data_type_date64_serde.cpp
index 27834c656fa..9ac37d20755 100644
--- a/be/src/vec/data_types/serde/data_type_date64_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_date64_serde.cpp
@@ -43,23 +43,11 @@ Status DataTypeDate64SerDe<T>::serialize_one_cell_to_json(
bw.write('"');
}
Int64 int_val = assert_cast<const
ColumnVector<T>&>(*ptr).get_element(row_num);
- if (options.date_olap_format) {
- tm time_tm;
- memset(&time_tm, 0, sizeof(time_tm));
- time_tm.tm_mday = static_cast<int>(int_val & 31);
- time_tm.tm_mon = static_cast<int>(int_val >> 5 & 15) - 1;
- time_tm.tm_year = static_cast<int>(int_val >> 9) - 1900;
- char buf[20] = {'\0'};
- strftime(buf, sizeof(buf), "%Y-%m-%d", &time_tm);
- std::string s = std::string(buf);
- bw.write(s.c_str(), s.length());
- } else {
- doris::VecDateTimeValue value = binary_cast<Int64,
doris::VecDateTimeValue>(int_val);
+ doris::VecDateTimeValue value = binary_cast<Int64,
doris::VecDateTimeValue>(int_val);
- char buf[64];
- char* pos = value.to_string(buf);
- bw.write(buf, pos - buf - 1);
- }
+ char buf[64];
+ char* pos = value.to_string(buf);
+ bw.write(buf, pos - buf - 1);
if (DataTypeNumberSerDe<T>::_nesting_level > 1) {
bw.write('"');
}
@@ -83,16 +71,7 @@ Status
DataTypeDate64SerDe<T>::deserialize_one_cell_from_json(
slice.trim_quote();
}
Int64 val = 0;
- if (options.date_olap_format) {
- tm time_tm;
- char* res = strptime(slice.data, "%Y-%m-%d", &time_tm);
- if (nullptr != res) {
- val = (time_tm.tm_year + 1900) * 16 * 32 + (time_tm.tm_mon + 1) *
32 + time_tm.tm_mday;
- } else {
- // 1400 - 01 - 01
- val = 716833;
- }
- } else if (ReadBuffer rb(slice.data, slice.size);
!read_date_text_impl<Int64>(val, rb)) {
+ if (ReadBuffer rb(slice.data, slice.size);
!read_date_text_impl<Int64>(val, rb)) {
return Status::InvalidArgument("parse date fail, string: '{}'",
std::string(rb.position(),
rb.count()).c_str());
}
@@ -113,28 +92,11 @@ Status
DataTypeDateTimeSerDe::serialize_one_cell_to_json(const IColumn& column,
row_num = result.second;
Int64 int_val = assert_cast<const
ColumnDateTime&>(*ptr).get_element(row_num);
- if (options.date_olap_format) {
- tm time_tm;
- int64 part1 = (int_val / 1000000L);
- int64 part2 = (int_val - part1 * 1000000L);
- time_tm.tm_year = static_cast<int>((part1 / 10000L) % 10000) - 1900;
- time_tm.tm_mon = static_cast<int>((part1 / 100) % 100) - 1;
- time_tm.tm_mday = static_cast<int>(part1 % 100);
-
- time_tm.tm_hour = static_cast<int>((part2 / 10000L) % 10000);
- time_tm.tm_min = static_cast<int>((part2 / 100) % 100);
- time_tm.tm_sec = static_cast<int>(part2 % 100);
- char buf[20] = {'\0'};
- strftime(buf, 20, "%Y-%m-%d %H:%M:%S", &time_tm);
- std::string s = std::string(buf);
- bw.write(s.c_str(), s.length());
- } else {
- doris::VecDateTimeValue value = binary_cast<Int64,
doris::VecDateTimeValue>(int_val);
+ doris::VecDateTimeValue value = binary_cast<Int64,
doris::VecDateTimeValue>(int_val);
- char buf[64];
- char* pos = value.to_string(buf);
- bw.write(buf, pos - buf - 1);
- }
+ char buf[64];
+ char* pos = value.to_string(buf);
+ bw.write(buf, pos - buf - 1);
return Status::OK();
}
@@ -149,19 +111,7 @@ Status
DataTypeDateTimeSerDe::deserialize_one_cell_from_json(IColumn& column, Sl
const
FormatOptions& options) const {
auto& column_data = assert_cast<ColumnDateTime&>(column);
Int64 val = 0;
- if (options.date_olap_format) {
- tm time_tm;
- char* res = strptime(slice.data, "%Y-%m-%d %H:%M:%S", &time_tm);
- if (nullptr != res) {
- val = ((time_tm.tm_year + 1900) * 10000L + (time_tm.tm_mon + 1) *
100L +
- time_tm.tm_mday) *
- 1000000L +
- time_tm.tm_hour * 10000L + time_tm.tm_min * 100L +
time_tm.tm_sec;
- } else {
- // 1400 - 01 - 01
- val = 14000101000000L;
- }
- } else if (ReadBuffer rb(slice.data, slice.size);
!read_datetime_text_impl<Int64>(val, rb)) {
+ if (ReadBuffer rb(slice.data, slice.size);
!read_datetime_text_impl<Int64>(val, rb)) {
return Status::InvalidArgument("parse datetime fail, string: '{}'",
std::string(rb.position(),
rb.count()).c_str());
}
diff --git a/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp
b/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp
index fcf9dbc0651..06bd9635f95 100644
--- a/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp
@@ -57,18 +57,10 @@ Status
DataTypeDateTimeV2SerDe::serialize_one_cell_to_json(const IColumn& column
DateV2Value<DateTimeV2ValueType> val =
binary_cast<UInt64, DateV2Value<DateTimeV2ValueType>>(int_val);
- if (options.date_olap_format) {
- std::string format = "%Y-%m-%d %H:%i:%s.%f";
- char buf[30 + SAFE_FORMAT_STRING_MARGIN];
- val.to_format_string_conservative(format.c_str(),
cast_set<int>(format.size()), buf,
- 30 + SAFE_FORMAT_STRING_MARGIN);
- std::string s = std::string(buf);
- bw.write(s.c_str(), s.length());
- } else {
- char buf[64];
- char* pos = val.to_string(buf);
- bw.write(buf, pos - buf - 1);
- }
+ char buf[64];
+ char* pos = val.to_string(buf);
+ bw.write(buf, pos - buf - 1);
+
if (_nesting_level > 1) {
bw.write('"');
}
@@ -88,19 +80,8 @@ Status
DataTypeDateTimeV2SerDe::deserialize_one_cell_from_json(IColumn& column,
slice.trim_quote();
}
UInt64 val = 0;
- if (options.date_olap_format) {
- DateV2Value<DateTimeV2ValueType> datetimev2_value;
- std::string date_format = "%Y-%m-%d %H:%i:%s.%f";
- if (datetimev2_value.from_date_format_str(date_format.data(),
-
cast_set<int>(date_format.size()), slice.data,
- slice.size)) {
- val = datetimev2_value.to_date_int_val();
- } else {
- val = MIN_DATETIME_V2;
- }
-
- } else if (ReadBuffer rb(slice.data, slice.size);
- !read_datetime_v2_text_impl<UInt64>(val, rb, scale)) {
+ if (ReadBuffer rb(slice.data, slice.size);
+ !read_datetime_v2_text_impl<UInt64>(val, rb, scale)) {
return Status::InvalidArgument("parse date fail, string: '{}'",
std::string(rb.position(),
rb.count()).c_str());
}
diff --git a/be/src/vec/data_types/serde/data_type_datev2_serde.cpp
b/be/src/vec/data_types/serde/data_type_datev2_serde.cpp
index ac2e8943038..381c8c1764a 100644
--- a/be/src/vec/data_types/serde/data_type_datev2_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_datev2_serde.cpp
@@ -75,15 +75,7 @@ Status
DataTypeDateV2SerDe::deserialize_one_cell_from_json(IColumn& column, Slic
}
auto& column_data = assert_cast<ColumnDateV2&>(column);
UInt32 val = 0;
- if (options.date_olap_format) {
- tm time_tm;
- char* res = strptime(slice.data, "%Y-%m-%d", &time_tm);
- if (nullptr != res) {
- val = ((time_tm.tm_year + 1900) << 9) | ((time_tm.tm_mon + 1) <<
5) | time_tm.tm_mday;
- } else {
- val = MIN_DATE_V2;
- }
- } else if (ReadBuffer rb(slice.data, slice.size);
!read_date_v2_text_impl<UInt32>(val, rb)) {
+ if (ReadBuffer rb(slice.data, slice.size);
!read_date_v2_text_impl<UInt32>(val, rb)) {
return Status::InvalidArgument("parse date fail, string: '{}'",
std::string(rb.position(),
rb.count()).c_str());
}
diff --git a/be/src/vec/data_types/serde/data_type_serde.h
b/be/src/vec/data_types/serde/data_type_serde.h
index ae3c0426c70..75915b2dcc8 100644
--- a/be/src/vec/data_types/serde/data_type_serde.h
+++ b/be/src/vec/data_types/serde/data_type_serde.h
@@ -127,11 +127,6 @@ public:
// Text serialization/deserialization of data types depend on some
settings witch we define
// in formatOptions.
struct FormatOptions {
- /**
- * if true, we will use olap format which defined in src/olap/types.h,
but we do not suggest
- * use this format in olap, because it is more slower, keep this
option is for compatibility.
- */
- bool date_olap_format = false;
/**
* field delimiter is used to separate fields in one row
*/
diff --git a/be/test/vec/data_types/serde/data_type_serde_agg_state_test.cpp
b/be/test/vec/data_types/serde/data_type_serde_agg_state_test.cpp
index 981d94f42d6..6e9a03edf3a 100644
--- a/be/test/vec/data_types/serde/data_type_serde_agg_state_test.cpp
+++ b/be/test/vec/data_types/serde/data_type_serde_agg_state_test.cpp
@@ -209,7 +209,6 @@ TEST_F(AggStateSerdeTest, serializeOneCellToJson) {
*((int64_t*)&(column_fixed_length->get_data()[column_fixed_length->item_size()]))
= 22;
ASSERT_EQ(column_fixed_length->size(), 2);
DataTypeSerDe::FormatOptions formatOptions;
- formatOptions.date_olap_format = true;
auto ser_col = ColumnString::create();
VectorBufferWriter buffer_writer(*ser_col.get());
auto st = datatype_agg_state_serde_count->serialize_one_cell_to_json(
@@ -249,7 +248,6 @@ TEST_F(AggStateSerdeTest, serializeOneCellToJson2) {
column_string->insert_data(str2.c_str(), str2.size());
ASSERT_EQ(column_string->size(), 2);
DataTypeSerDe::FormatOptions formatOptions;
- formatOptions.date_olap_format = true;
auto ser_col = ColumnString::create();
VectorBufferWriter buffer_writer(*ser_col.get());
auto st = datatype_agg_state_serde_hll_union->serialize_one_cell_to_json(
@@ -288,7 +286,6 @@ TEST_F(AggStateSerdeTest, serializeColumnToJson) {
*((int64_t*)&(column_fixed_length->get_data()[column_fixed_length->item_size()]))
= 22;
ASSERT_EQ(column_fixed_length->size(), 2);
DataTypeSerDe::FormatOptions formatOptions;
- formatOptions.date_olap_format = true;
auto ser_col = ColumnString::create();
VectorBufferWriter buffer_writer(*ser_col.get());
auto st = datatype_agg_state_serde_count->serialize_column_to_json(
@@ -326,7 +323,6 @@ TEST_F(AggStateSerdeTest, serializeColumnToJson2) {
std::cout << "asd " << column_string->get_data_at(0).to_string() <<
std::endl;
ASSERT_EQ(column_string->size(), 2);
DataTypeSerDe::FormatOptions formatOptions;
- formatOptions.date_olap_format = true;
auto ser_col = ColumnString::create();
VectorBufferWriter buffer_writer(*ser_col.get());
auto st = datatype_agg_state_serde_hll_union->serialize_column_to_json(
@@ -365,7 +361,6 @@ TEST_F(AggStateSerdeTest, serializeOneCellToHiveText) {
*((int64_t*)&(column_fixed_length->get_data()[column_fixed_length->item_size()]))
= 22;
ASSERT_EQ(column_fixed_length->size(), 2);
DataTypeSerDe::FormatOptions formatOptions;
- formatOptions.date_olap_format = true;
auto ser_col = ColumnString::create();
VectorBufferWriter buffer_writer(*ser_col.get());
auto st = datatype_agg_state_serde_count->serialize_one_cell_to_hive_text(
@@ -406,7 +401,6 @@ TEST_F(AggStateSerdeTest, serializeOneCellToHiveText2) {
column_string->insert_data(str2.c_str(), str2.size());
ASSERT_EQ(column_string->size(), 2);
DataTypeSerDe::FormatOptions formatOptions;
- formatOptions.date_olap_format = true;
auto ser_col = ColumnString::create();
VectorBufferWriter buffer_writer(*ser_col.get());
auto st =
datatype_agg_state_serde_hll_union->serialize_one_cell_to_hive_text(
diff --git a/be/test/vec/data_types/serde/data_type_serde_bitmap_test.cpp
b/be/test/vec/data_types/serde/data_type_serde_bitmap_test.cpp
index 9d409bb2d80..ecfc34d1986 100644
--- a/be/test/vec/data_types/serde/data_type_serde_bitmap_test.cpp
+++ b/be/test/vec/data_types/serde/data_type_serde_bitmap_test.cpp
@@ -110,7 +110,6 @@ TEST(BitmapSerdeTest, serializeOneCellToJson) {
column_bitmap->insert_value(BitmapValue(123));
ASSERT_EQ(column_bitmap->size(), 2);
DataTypeSerDe::FormatOptions formatOptions;
- formatOptions.date_olap_format = true;
auto ser_col = ColumnString::create();
VectorBufferWriter buffer_writer(*ser_col.get());
auto st = bitmap_serde->serialize_one_cell_to_json(*column_bitmap, 0,
buffer_writer,
@@ -153,7 +152,6 @@ TEST(BitmapSerdeTest, serializeColumnToJson) {
column_bitmap->insert_value(BitmapValue(123));
ASSERT_EQ(column_bitmap->size(), 2);
DataTypeSerDe::FormatOptions formatOptions;
- formatOptions.date_olap_format = true;
auto ser_col = ColumnString::create();
VectorBufferWriter buffer_writer(*ser_col.get());
auto st = bitmap_serde->serialize_column_to_json(*column_bitmap, 0, 2,
buffer_writer,
diff --git a/be/test/vec/data_types/serde/data_type_serde_csv_test.cpp
b/be/test/vec/data_types/serde/data_type_serde_csv_test.cpp
index 08f41822dbb..6432f5866e9 100644
--- a/be/test/vec/data_types/serde/data_type_serde_csv_test.cpp
+++ b/be/test/vec/data_types/serde/data_type_serde_csv_test.cpp
@@ -181,32 +181,42 @@ TEST(CsvSerde, ScalaDataTypeSerdeCsvTest) {
// date and datetime type
{
- typedef std::pair<FieldType, string> FieldType_RandStr;
- std::vector<FieldType_RandStr> date_scala_field_types = {
- FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_DATE,
"2020-01-01"),
- FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_DATE,
"2020-01-01"),
- FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_DATEV2,
"2020-01-01"),
- FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_DATETIME,
"2020-01-01 12:00:00"),
- FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_DATETIMEV2,
- "2020-01-01 12:00:00.666666"),
+ struct DataTestField {
+ FieldType type;
+ string str;
+ string max_str;
+ string min_str;
+ };
+ std::vector<DataTestField> date_scala_field_types = {
+ DataTestField {.type = FieldType::OLAP_FIELD_TYPE_DATE,
+ .str = "2020-01-01",
+ .max_str = "9999-12-31",
+ .min_str = "0001-01-01"},
+ DataTestField {.type = FieldType::OLAP_FIELD_TYPE_DATE,
+ .str = "2020-01-01",
+ .max_str = "9999-12-31",
+ .min_str = "0001-01-01"},
+ DataTestField {.type = FieldType::OLAP_FIELD_TYPE_DATEV2,
+ .str = "2020-01-01",
+ .max_str = "9999-12-31",
+ .min_str = "0001-01-01"},
+ DataTestField {.type = FieldType::OLAP_FIELD_TYPE_DATETIME,
+ .str = "2020-01-01 12:00:00",
+ .max_str = "9999-12-31 23:59:59",
+ .min_str = "0001-01-01 00:00:00"},
+ DataTestField {.type = FieldType::OLAP_FIELD_TYPE_DATETIMEV2,
+ .str = "2020-01-01 12:00:00",
+ .max_str = "9999-12-31 23:59:59",
+ .min_str = "0001-01-01 00:00:00"},
};
for (auto pair : date_scala_field_types) {
- auto type = pair.first;
+ auto type = pair.type;
DataTypePtr data_type_ptr =
DataTypeFactory::instance().create_data_type(type, 0, 0);
std::cout << "========= This type is " <<
data_type_ptr->get_name() << ": "
<< fmt::format("{}", type) << std::endl;
-
- std::unique_ptr<WrapperField>
min_wf(WrapperField::create_by_type(type));
- std::unique_ptr<WrapperField>
max_wf(WrapperField::create_by_type(type));
- std::unique_ptr<WrapperField>
rand_wf(WrapperField::create_by_type(type));
-
- min_wf->set_to_min();
- max_wf->set_to_max();
- EXPECT_EQ(rand_wf->from_string(pair.second, 0, 0).ok(), true);
-
- string min_s = min_wf->to_string();
- string max_s = max_wf->to_string();
- string rand_date = rand_wf->to_string();
+ string min_s = pair.min_str;
+ string max_s = pair.max_str;
+ string rand_date = pair.str;
Slice min_rb(min_s.data(), min_s.size());
Slice max_rb(max_s.data(), max_s.size());
@@ -216,7 +226,6 @@ TEST(CsvSerde, ScalaDataTypeSerdeCsvTest) {
DataTypeSerDeSPtr serde = data_type_ptr->get_serde();
// make use c++ lib equals to wrapper field from_string behavior
DataTypeSerDe::FormatOptions formatOptions;
- formatOptions.date_olap_format = true;
Status st = serde->deserialize_one_cell_from_json(*col, min_rb,
formatOptions);
EXPECT_EQ(st.ok(), true);
diff --git
a/be/test/vec/data_types/serde/data_type_serde_fixed_length_object_test.cpp
b/be/test/vec/data_types/serde/data_type_serde_fixed_length_object_test.cpp
index 4836b8a12c3..d12310bb3e7 100644
--- a/be/test/vec/data_types/serde/data_type_serde_fixed_length_object_test.cpp
+++ b/be/test/vec/data_types/serde/data_type_serde_fixed_length_object_test.cpp
@@ -115,7 +115,6 @@ TEST(FixedLengthObjectSerdeTest, serializeOneCellToJson) {
*((int64_t*)&(column_fixed_length->get_data()[column_fixed_length->item_size()]))
= 22;
ASSERT_EQ(column_fixed_length->size(), 2);
DataTypeSerDe::FormatOptions formatOptions;
- formatOptions.date_olap_format = true;
auto ser_col = ColumnString::create();
VectorBufferWriter buffer_writer(*ser_col.get());
auto st =
fixed_length_serde->serialize_one_cell_to_json(*column_fixed_length, 0,
buffer_writer,
@@ -155,7 +154,6 @@ TEST(FixedLengthObjectSerdeTest, serializeColumnToJson) {
*((int64_t*)&(column_fixed_length->get_data()[column_fixed_length->item_size()]))
= 22;
ASSERT_EQ(column_fixed_length->size(), 2);
DataTypeSerDe::FormatOptions formatOptions;
- formatOptions.date_olap_format = true;
auto ser_col = ColumnString::create();
VectorBufferWriter buffer_writer(*ser_col.get());
auto st =
fixed_length_serde->serialize_column_to_json(*column_fixed_length, 0, 2,
@@ -192,7 +190,6 @@ TEST(FixedLengthObjectSerdeTest,
serializeOneCellToHiveText) {
*((int64_t*)&(column_fixed_length->get_data()[column_fixed_length->item_size()]))
= 22;
ASSERT_EQ(column_fixed_length->size(), 2);
DataTypeSerDe::FormatOptions formatOptions;
- formatOptions.date_olap_format = true;
auto ser_col = ColumnString::create();
VectorBufferWriter buffer_writer(*ser_col.get());
auto st =
fixed_length_serde->serialize_one_cell_to_hive_text(*column_fixed_length, 0,
diff --git a/be/test/vec/data_types/serde/data_type_serde_hll_test.cpp
b/be/test/vec/data_types/serde/data_type_serde_hll_test.cpp
index e10ccc3cf5d..8dce9df7272 100644
--- a/be/test/vec/data_types/serde/data_type_serde_hll_test.cpp
+++ b/be/test/vec/data_types/serde/data_type_serde_hll_test.cpp
@@ -118,7 +118,6 @@ TEST(HLLSerdeTest, serializeOneCellToJson) {
column_hll->insert_value(hll);
ASSERT_EQ(column_hll->size(), 2);
DataTypeSerDe::FormatOptions formatOptions;
- formatOptions.date_olap_format = true;
auto ser_col = ColumnString::create();
VectorBufferWriter buffer_writer(*ser_col.get());
auto st = hll_serde->serialize_one_cell_to_json(*column_hll, 0,
buffer_writer, formatOptions);
@@ -164,7 +163,6 @@ TEST(HLLSerdeTest, serializeColumnToJson) {
column_hll->insert_value(hll);
ASSERT_EQ(column_hll->size(), 2);
DataTypeSerDe::FormatOptions formatOptions;
- formatOptions.date_olap_format = true;
auto ser_col = ColumnString::create();
VectorBufferWriter buffer_writer(*ser_col.get());
auto st = hll_serde->serialize_column_to_json(*column_hll, 0, 2,
buffer_writer, formatOptions);
diff --git
a/be/test/vec/data_types/serde/data_type_serde_quantile_state_test.cpp
b/be/test/vec/data_types/serde/data_type_serde_quantile_state_test.cpp
index 7e7909da792..44681eee8bc 100644
--- a/be/test/vec/data_types/serde/data_type_serde_quantile_state_test.cpp
+++ b/be/test/vec/data_types/serde/data_type_serde_quantile_state_test.cpp
@@ -119,7 +119,6 @@ TEST(QuantileStateSerdeTest, serializeOneCellToJson) {
column_quantile_state->insert_value(quantile_state);
ASSERT_EQ(column_quantile_state->size(), 2);
DataTypeSerDe::FormatOptions formatOptions;
- formatOptions.date_olap_format = true;
auto ser_col = ColumnString::create();
VectorBufferWriter buffer_writer(*ser_col.get());
auto st =
quantile_state_serde->serialize_one_cell_to_json(*column_quantile_state, 0,
@@ -143,7 +142,6 @@ TEST(QuantileStateSerdeTest, serializeColumnToJson) {
column_quantile_state->insert_value(quantile_state);
ASSERT_EQ(column_quantile_state->size(), 2);
DataTypeSerDe::FormatOptions formatOptions;
- formatOptions.date_olap_format = true;
auto ser_col = ColumnString::create();
VectorBufferWriter buffer_writer(*ser_col.get());
auto st =
quantile_state_serde->serialize_column_to_json(*column_quantile_state, 0, 2,
diff --git a/be/test/vec/data_types/serde/data_type_serde_text_test.cpp
b/be/test/vec/data_types/serde/data_type_serde_text_test.cpp
index e2e4c61284e..91882e78dbb 100644
--- a/be/test/vec/data_types/serde/data_type_serde_text_test.cpp
+++ b/be/test/vec/data_types/serde/data_type_serde_text_test.cpp
@@ -178,32 +178,42 @@ TEST(TextSerde, ScalaDataTypeSerdeTextTest) {
// date and datetime type
{
- typedef std::pair<FieldType, string> FieldType_RandStr;
- std::vector<FieldType_RandStr> date_scala_field_types = {
- FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_DATE,
"2020-01-01"),
- FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_DATE,
"2020-01-01"),
- FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_DATEV2,
"2020-01-01"),
- FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_DATETIME,
"2020-01-01 12:00:00"),
- FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_DATETIMEV2,
- "2020-01-01 12:00:00.666666"),
+ struct DataTestField {
+ FieldType type;
+ string str;
+ string max_str;
+ string min_str;
+ };
+ std::vector<DataTestField> date_scala_field_types = {
+ DataTestField {.type = FieldType::OLAP_FIELD_TYPE_DATE,
+ .str = "2020-01-01",
+ .max_str = "9999-12-31",
+ .min_str = "0001-01-01"},
+ DataTestField {.type = FieldType::OLAP_FIELD_TYPE_DATE,
+ .str = "2020-01-01",
+ .max_str = "9999-12-31",
+ .min_str = "0001-01-01"},
+ DataTestField {.type = FieldType::OLAP_FIELD_TYPE_DATEV2,
+ .str = "2020-01-01",
+ .max_str = "9999-12-31",
+ .min_str = "0001-01-01"},
+ DataTestField {.type = FieldType::OLAP_FIELD_TYPE_DATETIME,
+ .str = "2020-01-01 12:00:00",
+ .max_str = "9999-12-31 23:59:59",
+ .min_str = "0001-01-01 00:00:00"},
+ DataTestField {.type = FieldType::OLAP_FIELD_TYPE_DATETIMEV2,
+ .str = "2020-01-01 12:00:00",
+ .max_str = "9999-12-31 23:59:59",
+ .min_str = "0001-01-01 00:00:00"},
};
for (auto pair : date_scala_field_types) {
- auto type = pair.first;
+ auto type = pair.type;
DataTypePtr data_type_ptr =
DataTypeFactory::instance().create_data_type(type, 0, 0);
std::cout << "========= This type is " <<
data_type_ptr->get_name() << ": "
<< fmt::format("{}", type) << std::endl;
-
- std::unique_ptr<WrapperField>
min_wf(WrapperField::create_by_type(type));
- std::unique_ptr<WrapperField>
max_wf(WrapperField::create_by_type(type));
- std::unique_ptr<WrapperField>
rand_wf(WrapperField::create_by_type(type));
-
- min_wf->set_to_min();
- max_wf->set_to_max();
- static_cast<void>(rand_wf->from_string(pair.second, 0, 0));
-
- string min_s = min_wf->to_string();
- string max_s = max_wf->to_string();
- string rand_date = rand_wf->to_string();
+ string min_s = pair.min_str;
+ string max_s = pair.max_str;
+ string rand_date = pair.str;
Slice min_rb(min_s.data(), min_s.size());
Slice max_rb(max_s.data(), max_s.size());
@@ -213,7 +223,6 @@ TEST(TextSerde, ScalaDataTypeSerdeTextTest) {
DataTypeSerDeSPtr serde = data_type_ptr->get_serde();
// make use c++ lib equals to wrapper field from_string behavior
DataTypeSerDe::FormatOptions formatOptions;
- formatOptions.date_olap_format = true;
Status st = serde->deserialize_one_cell_from_json(*col, min_rb,
formatOptions);
EXPECT_EQ(st.ok(), true);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
