This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2b5f2a17645 [fix](be) Accept year 0 when the V1 ORC reader decodes
TIMESTAMPTZ (#69243)
2b5f2a17645 is described below
commit 2b5f2a17645b5f042613660256c878662f30d2eb
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Mon Oct 19 07:32:13 2026 +0100
[fix](be) Accept year 0 when the V1 ORC reader decodes TIMESTAMPTZ (#69243)
### What problem does this PR solve?
Issue Number: None
Related PR: #68786
Problem Summary:
**In short.** The V1 ORC reader rejects TIMESTAMPTZ values in year 0,
which Doris stores and the other read paths return. Found while
comparing master with branch-4.x after #68786; the branch-4.x V1 reader
has no range check here.
**Background.** #68786 reads ORC `TIMESTAMP_INSTANT` as TIMESTAMPTZ and
added a UTC range check to the V1 ORC reader before it packs each value,
because ORC allows years that Doris cannot hold. Doris TIMESTAMPTZ holds
UTC years 0 through 9999: `TimestampTzValue` rejects only years below 0
or above 9999, and a TIMESTAMPTZ column stores `0000-01-01 00:00:00
+00:00`. In the review of #68786 the JNI `VectorColumn` check was
changed to reject only years below 0, and master's ORC serde uses the
same 0–9999 range.
**The problem, and what it cost.** The V1 check kept its lower bound at
0001-01-01T00:00:00Z (`MIN_SECONDS = -62135596800`). An ORC file with a
year-0 value fails the scan with `Decoded TIMESTAMPTZ column '...' is
outside the Doris 0001-9999 range`, while the V2 reader returns the
value.
**How this PR fixes it.** The lower bound becomes 0000-01-01T00:00:00Z
(`-62167219200`; year 0 is a leap year, 366 days before year 1). The
upper bound and the error for values outside the range are unchanged,
and the message now says 0000-9999.
**Results.** BE unit test on the V1 decode path:
| Value (UTC) | Before | After |
|---|---|---|
| 0000-01-01T00:00:00Z | DataQualityError | Decoded as year 0 |
| 0000-12-31T23:59:59Z | DataQualityError | Decoded as year 0 |
| 0001-01-01T00:00:00Z, 9999-12-31T23:59:59.999999Z | Decoded | Same |
| One second before year 0, 10000-01-01T00:00:00Z | DataQualityError |
Same |
---
be/src/format/orc/vorc_reader.h | 6 +++---
be/test/format/orc/orc_reader_fill_data_test.cpp | 24 ++++++++++++++++--------
2 files changed, 19 insertions(+), 11 deletions(-)
diff --git a/be/src/format/orc/vorc_reader.h b/be/src/format/orc/vorc_reader.h
index 2899f3abf8f..842ff89d429 100644
--- a/be/src/format/orc/vorc_reader.h
+++ b/be/src/format/orc/vorc_reader.h
@@ -657,12 +657,12 @@ private:
if (data->hasNulls && !data->notNull[i]) {
continue;
}
- // Check the UTC epoch before packing: ORC permits years outside
Doris's 0001-9999 range.
- static constexpr int64_t MIN_SECONDS = -62135596800LL;
+ // Check the UTC epoch before packing: ORC permits years outside
Doris's 0000-9999 range.
+ static constexpr int64_t MIN_SECONDS = -62167219200LL;
static constexpr int64_t MAX_SECONDS = 253402300799LL;
if (data->data[i] < MIN_SECONDS || data->data[i] > MAX_SECONDS) {
return Status::DataQualityError(
- "Decoded TIMESTAMPTZ column '{}' is outside the Doris
0001-9999 range: "
+ "Decoded TIMESTAMPTZ column '{}' is outside the Doris
0000-9999 range: "
"seconds={}",
col_name, data->data[i]);
}
diff --git a/be/test/format/orc/orc_reader_fill_data_test.cpp
b/be/test/format/orc/orc_reader_fill_data_test.cpp
index 811533038aa..0a672381feb 100644
--- a/be/test/format/orc/orc_reader_fill_data_test.cpp
+++ b/be/test/format/orc/orc_reader_fill_data_test.cpp
@@ -198,7 +198,9 @@ TEST_F(OrcReaderFillDataTest,
TimestampTzV1RejectsOutOfRangeUtcYears) {
TFileScanRangeParams params;
TFileRangeDesc range;
auto reader = OrcReader::create_unique(params, range, 4096, "UTC",
nullptr);
- for (int64_t seconds : {-62135596801LL, 253402300800LL}) {
+ // TIMESTAMPTZ holds UTC years 0 through 9999, so the bounds are
0000-01-01T00:00:00Z and
+ // 9999-12-31T23:59:59Z.
+ for (int64_t seconds : {-62167219201LL, 253402300800LL}) {
auto batch = create_timestamp_batch(1, {seconds}, {0});
auto column = ColumnTimeStampTz::create();
MutableColumnPtr mutable_column = column->assert_mutable();
@@ -206,17 +208,23 @@ TEST_F(OrcReaderFillDataTest,
TimestampTzV1RejectsOutOfRangeUtcYears) {
reader->_decode_timestamp_tz_column<false>("ts",
mutable_column, batch.get(), 1)
.ok());
}
- auto batch = create_timestamp_batch(3, {-62135596800LL, 253402300799LL,
253402300800LL},
- {0, 999999000, 0});
+ auto batch = create_timestamp_batch(
+ 4, {-62167219200LL, -62135596801LL, 253402300799LL,
253402300800LL},
+ {0, 0, 999999000, 0});
batch->hasNulls = true;
- batch->notNull[2] = false;
+ batch->notNull[3] = false;
auto column = ColumnTimeStampTz::create();
MutableColumnPtr mutable_column = column->assert_mutable();
ASSERT_TRUE(
- reader->_decode_timestamp_tz_column<false>("ts", mutable_column,
batch.get(), 3).ok());
- EXPECT_EQ(column->get_data()[0].year(), 1);
- EXPECT_EQ(column->get_data()[1].year(), 9999);
- EXPECT_EQ(column->get_data()[1].microsecond(), 999999);
+ reader->_decode_timestamp_tz_column<false>("ts", mutable_column,
batch.get(), 4).ok());
+ EXPECT_EQ(column->get_data()[0].year(), 0);
+ EXPECT_EQ(column->get_data()[0].month(), 1);
+ EXPECT_EQ(column->get_data()[0].day(), 1);
+ EXPECT_EQ(column->get_data()[1].year(), 0);
+ EXPECT_EQ(column->get_data()[1].month(), 12);
+ EXPECT_EQ(column->get_data()[1].day(), 31);
+ EXPECT_EQ(column->get_data()[2].year(), 9999);
+ EXPECT_EQ(column->get_data()[2].microsecond(), 999999);
}
TEST_F(OrcReaderFillDataTest, TimestampTzV1RejectsOutOfRangeArrayElements) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
