This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 7ed09f49eaf [fix](iceberg) Support fractional timestamps in time
travel (#67704)
7ed09f49eaf is described below
commit 7ed09f49eafb4f229cdafb8a3c86bbd882dd5f31
Author: Gabriel <[email protected]>
AuthorDate: Thu Sep 10 11:08:20 2026 +0800
[fix](iceberg) Support fractional timestamps in time travel (#67704)
### What problem does this PR solve?
Iceberg `FOR TIME AS OF` rejected timestamps containing fractional
seconds, even though the snapshots metadata exposes `committed_at` with
fractional precision. Truncating the value to whole seconds can also
select the wrong snapshot when multiple snapshots are committed within
the same second.
### What is changed and how does it work?
- Accept an optional 1-9 digit fractional-second component in the
Iceberg-specific time-travel parser while leaving other `TimeUtils`
consumers unchanged.
- Keep session time-zone interpretation unchanged.
- Add unit coverage that distinguishes two snapshots committed within
the same second.
- Update the Iceberg time-travel regression matrix to feed the
fractional `committed_at` value back into `FOR TIME AS OF` directly.
### Check List
- [x] Unit test: `IcebergUtilsTest` (38 tests)
- [x] FE Checkstyle
- [x] `git diff --check`
---
.../doris/datasource/iceberg/IcebergUtils.java | 21 ++++++++++++++++++++-
.../doris/datasource/iceberg/IcebergUtilsTest.java | 16 +++++++++++++---
.../test_iceberg_schema_time_travel_matrix.groovy | 4 ++--
3 files changed, 35 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
index c60f64c6f9c..a43ce9dc9d2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java
@@ -145,6 +145,8 @@ import java.time.Month;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.DateTimeParseException;
import java.time.temporal.ChronoField;
import java.time.temporal.TemporalAccessor;
import java.util.ArrayList;
@@ -256,6 +258,14 @@ public class IcebergUtils {
public static final String ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER_COL =
"_last_updated_sequence_number";
private static final Pattern SNAPSHOT_ID = Pattern.compile("\\d+");
+ // Iceberg's committed_at value may include fractional seconds, so the
time-travel parser must round-trip
+ // that value while preserving compatibility with existing whole-second
literals.
+ private static final DateTimeFormatter TIME_TRAVEL_DATETIME_FORMAT = new
DateTimeFormatterBuilder()
+ .appendPattern("yyyy-MM-dd HH:mm:ss")
+ .optionalStart()
+ .appendFraction(ChronoField.NANO_OF_SECOND, 1, 9, true)
+ .optionalEnd()
+ .toFormatter();
public static boolean hasIcebergCatalogFormatVersion(Map<String, String>
catalogProperties) {
return
catalogProperties.containsKey(CatalogProperties.TABLE_OVERRIDE_PREFIX +
TableProperties.FORMAT_VERSION)
@@ -1857,7 +1867,7 @@ public class IcebergUtils {
SnapshotUtil.schemaFor(table, value).schemaId()
);
} else {
- long timestamp = TimeUtils.timeStringToLong(value,
TimeUtils.getTimeZone());
+ long timestamp = timeTravelTimestampToLong(value);
if (timestamp < 0) {
throw new DateTimeException("can't parse time: " + value);
}
@@ -1870,6 +1880,15 @@ public class IcebergUtils {
}
}
+ private static long timeTravelTimestampToLong(String value) {
+ try {
+ return LocalDateTime.parse(value, TIME_TRAVEL_DATETIME_FORMAT)
+
.atZone(TimeUtils.getTimeZone().toZoneId()).toInstant().toEpochMilli();
+ } catch (DateTimeParseException e) {
+ return -1;
+ }
+ }
+
public static boolean isIcebergBranchOrTag(Optional<TableScanParams>
scanParams) {
if (scanParams == null || !scanParams.isPresent()) {
return false;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java
index 5ba783d1576..393d60cf6f5 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergUtilsTest.java
@@ -1007,6 +1007,10 @@ public class IcebergUtilsTest {
Mockito.when(table.snapshot(3)).thenReturn(s3);
Snapshot s4 = mockSnapshot(4, 1);
Mockito.when(table.snapshot(4)).thenReturn(s4);
+ Snapshot s5 = mockSnapshot(5, 2);
+ Mockito.when(table.snapshot(5)).thenReturn(s5);
+ Snapshot s6 = mockSnapshot(6, 2);
+ Mockito.when(table.snapshot(6)).thenReturn(s6);
// init history for snapshots
List<HistoryEntry> history = new ArrayList<>();
@@ -1014,6 +1018,8 @@ public class IcebergUtilsTest {
history.add(mockHistory(2, "2025-05-01 22:34:56"));
history.add(mockHistory(3, "2025-05-02 12:34:56"));
history.add(mockHistory(4, "2025-05-03 12:34:56"));
+ history.add(mockHistory(5, LocalDateTime.of(2025, 5, 4, 12, 34, 56,
125_000_000)));
+ history.add(mockHistory(6, LocalDateTime.of(2025, 5, 4, 12, 34, 56,
526_000_000)));
Mockito.when(table.history()).thenReturn(history);
// create some refs
@@ -1127,6 +1133,8 @@ public class IcebergUtilsTest {
assertQuerySpecSnapshotByTimeOf(table, "2025-05-02 11:34:56", 2, 0,
null);
assertQuerySpecSnapshotByTimeOf(table, "2025-05-02 12:34:56", 3, 1,
null);
assertQuerySpecSnapshotByTimeOf(table, "2025-05-03 12:34:56", 4, 1,
null);
+ assertQuerySpecSnapshotByTimeOf(table, "2025-05-04 12:34:56.125", 5,
2, null);
+ assertQuerySpecSnapshotByTimeOf(table, "2025-05-04 12:34:56.526000",
6, 2, null);
// query invalid time format
Assert.assertThrows(
@@ -1149,11 +1157,13 @@ public class IcebergUtilsTest {
}
private HistoryEntry mockHistory(long snapshotId, String time) {
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd
HH:mm:ss");
+ return mockHistory(snapshotId, LocalDateTime.parse(time, formatter));
+ }
+
+ private HistoryEntry mockHistory(long snapshotId, LocalDateTime dateTime) {
HistoryEntry historyEntry = Mockito.mock(HistoryEntry.class);
Mockito.when(historyEntry.snapshotId()).thenReturn(snapshotId);
-
- DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd
HH:mm:ss");
- LocalDateTime dateTime = LocalDateTime.parse(time, formatter);
long millis =
dateTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli();
Mockito.when(historyEntry.timestampMillis()).thenReturn(millis);
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_time_travel_matrix.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_time_travel_matrix.groovy
index f92f3b83c59..56f06cf5769 100644
---
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_time_travel_matrix.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_time_travel_matrix.groovy
@@ -458,7 +458,7 @@ suite("test_iceberg_schema_time_travel_matrix",
// Scenario TC02/T03/T04: complex-field time travel uses the
pre-change nested schema.
List<List<Object>> dorisNestedCp0Time = sql("""
- select date_format(date_add(committed_at, interval 1 second),
'%Y-%m-%d %H:%i:%s'),
+ select date_format(committed_at, '%Y-%m-%d %H:%i:%s.%f'),
cast(unix_timestamp(committed_at) * 1000 + 999 as bigint)
from ${dorisNestedTable}\$snapshots
where snapshot_id = ${dorisNestedCp0}
@@ -559,7 +559,7 @@ suite("test_iceberg_schema_time_travel_matrix",
// Scenario T03/T04: validate string time travel and reject
unsupported epoch millis.
List<List<Object>> cp0TimeRows = sql("""
- select date_format(date_add(committed_at, interval 1 second),
'%Y-%m-%d %H:%i:%s'),
+ select date_format(committed_at, '%Y-%m-%d %H:%i:%s.%f'),
cast(unix_timestamp(committed_at) * 1000 + 999 as bigint)
from ${topTable}\$snapshots
where snapshot_id = ${topCp0}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
