This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 0e974c8497 [iceberg] Refuse time precisions the mirror cannot publish
(#9467)
0e974c8497 is described below
commit 0e974c8497d0ead198ecf1bb49b785c44178e0c5
Author: Jiajia Li <[email protected]>
AuthorDate: Sun Aug 30 15:20:43 2026 +0800
[iceberg] Refuse time precisions the mirror cannot publish (#9467)
---
docs/docs/iceberg/index.md | 6 +++
.../org/apache/paimon/schema/SchemaValidation.java | 31 +++++++++++
.../apache/paimon/schema/SchemaManagerTest.java | 62 ++++++++++++++++++++++
3 files changed, 99 insertions(+)
diff --git a/docs/docs/iceberg/index.md b/docs/docs/iceberg/index.md
index a0bc183b73..1f94e90ef8 100644
--- a/docs/docs/iceberg/index.md
+++ b/docs/docs/iceberg/index.md
@@ -94,6 +94,8 @@ Paimon Iceberg compatibility currently supports the following
data types.
| `BINARY` | `binary` |
| `VARBINARY` | `binary` |
| `DATE` | `date` |
+| `TIME` (precision 0-3) | `time` |
+| `TIME` (other precisions) | not supported |
| `TIMESTAMP` (precision 3-6) | `timestamp` |
| `TIMESTAMP_LTZ` (precision 3-6) | `timestamptz` |
| `TIMESTAMP` (other precisions) | not supported |
@@ -112,6 +114,10 @@ Paimon Iceberg compatibility currently supports the
following data types.
as Parquet INT96, which Iceberg reads as a microsecond zoned timestamp
rather than the
nanoseconds the column declares. Use a precision from 3 to 6.
+**Note on Time Types:**
+`TIME` types with a precision above 3 are rejected while Iceberg metadata is
enabled: Iceberg
+compatibility publishes only millisecond time values.
+
**Note on Geospatial Types:**
- `GEOMETRY` and `GEOGRAPHY` values use OGC Well-Known Binary (WKB). The
default CRS is `OGC:CRS84`, and the default geography edge algorithm is
`spherical`.
- Geospatial columns require Parquet for data, per-level, and changelog files.
When Iceberg metadata is enabled, set `metadata.iceberg.format-version` to `3`.
diff --git
a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
index 136b5dc4bb..fe1239a208 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
@@ -53,6 +53,7 @@ import org.apache.paimon.types.LocalZonedTimestampType;
import org.apache.paimon.types.MapType;
import org.apache.paimon.types.MultisetType;
import org.apache.paimon.types.RowType;
+import org.apache.paimon.types.TimeType;
import org.apache.paimon.types.TimestampType;
import org.apache.paimon.types.VariantType;
import org.apache.paimon.types.VectorType;
@@ -124,6 +125,9 @@ import static
org.apache.paimon.utils.Preconditions.checkState;
/** Validation utilities for {@link TableSchema}. */
public class SchemaValidation {
+ /** The ceiling {@code IcebergDataField} converts. */
+ private static final int MAX_ICEBERG_TIME_PRECISION = 3;
+
/** The precisions {@code IcebergDataField} maps to the Iceberg timestamp
types. */
private static final int MIN_ICEBERG_TIMESTAMP_PRECISION = 3;
@@ -245,6 +249,7 @@ public class SchemaValidation {
RowType tableRowType = new RowType(schema.fields());
validateGeospatialTypes(schema, options, tableRowType);
validateIcebergTimestampPrecisions(tableRowType, options);
+ validateIcebergTimePrecisions(tableRowType, options);
validateBlobFields(tableRowType, options);
Set<String> blobDescriptorFields =
validateBlobDescriptorFields(tableRowType, options);
Set<String> blobViewFields =
@@ -581,6 +586,31 @@ public class SchemaValidation {
IcebergOptions.METADATA_ICEBERG_STORAGE.key());
}
+ /**
+ * Refuses the time precisions the mirror cannot publish: it writes whole
milliseconds into
+ * Iceberg's microsecond time values, and the conversion enforcing that
would only fail once the
+ * snapshot is durable.
+ */
+ public static void validateIcebergTimePrecisions(DataType dataType,
CoreOptions options) {
+ if
(options.toConfiguration().get(IcebergOptions.METADATA_ICEBERG_STORAGE)
+ == IcebergOptions.StorageType.DISABLED) {
+ return;
+ }
+ checkArgument(
+ !containsType(dataType, SchemaValidation::isUnpublishableTime),
+ "Time columns must have a precision of %s or less when Iceberg
metadata is "
+ + "enabled, the only precisions Iceberg compatibility
can publish. Use a "
+ + "precision of %s or less, or disable '%s'.",
+ MAX_ICEBERG_TIME_PRECISION,
+ MAX_ICEBERG_TIME_PRECISION,
+ IcebergOptions.METADATA_ICEBERG_STORAGE.key());
+ }
+
+ private static boolean isUnpublishableTime(DataType dataType) {
+ return dataType instanceof TimeType
+ && ((TimeType) dataType).getPrecision() >
MAX_ICEBERG_TIME_PRECISION;
+ }
+
private static boolean isUnpublishableTimestamp(DataType dataType) {
if (dataType instanceof TimestampType) {
return isUnpublishablePrecision(((TimestampType)
dataType).getPrecision());
@@ -607,6 +637,7 @@ public class SchemaValidation {
for (TableSchema schema : history.get()) {
validateIcebergGeospatialTypes(schema.logicalRowType(), options);
validateIcebergTimestampPrecisions(schema.logicalRowType(),
options);
+ validateIcebergTimePrecisions(schema.logicalRowType(), options);
}
}
diff --git
a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java
b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java
index b3094e7fc8..78396006a7 100644
--- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java
@@ -246,6 +246,68 @@ public class SchemaManagerTest {
.hasStackTraceContaining("precision from 3 to 6");
}
+ @ParameterizedTest
+ @ValueSource(ints = {4, 6, 9})
+ public void testIcebergMetadataRefusesUnsupportedTimePrecisions(int
precision)
+ throws Exception {
+ Map<String, String> options = new HashMap<>();
+ options.put(CoreOptions.BUCKET.key(), "-1");
+ options.put(IcebergOptions.METADATA_ICEBERG_STORAGE.key(),
"table-location");
+
+ assertThatThrownBy(
+ () ->
+ retryArtificialException(
+ () ->
manager.createTable(timeSchema(options, precision))))
+ .hasStackTraceContaining("precision of 3 or less");
+
+ assertThatCode(
+ () ->
+ retryArtificialException(
+ () ->
manager.createTable(timeSchema(options, 3))))
+ .doesNotThrowAnyException();
+ }
+
+ private Schema timeSchema(Map<String, String> options, int precision) {
+ return new Schema(
+ Arrays.asList(
+ new DataField(0, "id", DataTypes.INT()),
+ new DataField(1, "t", DataTypes.TIME(precision))),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ options,
+ "");
+ }
+
+ @Test
+ public void testEnableIcebergMetadataValidatesHistoricalTimePrecisions()
throws Exception {
+ Map<String, String> options = new HashMap<>();
+ options.put(CoreOptions.BUCKET.key(), "-1");
+ Schema micros =
+ new Schema(
+ Arrays.asList(
+ new DataField(0, "id", DataTypes.INT()),
+ new DataField(1, "t", DataTypes.TIME(6))),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ options,
+ "");
+
+ retryArtificialException(() -> manager.createTable(micros));
+ retryArtificialException(() ->
manager.commitChanges(SchemaChange.dropColumn("t")));
+
+ assertThatThrownBy(
+ () ->
+ retryArtificialException(
+ () ->
+ manager.commitChanges(
+ SchemaChange.setOption(
+ IcebergOptions
+
.METADATA_ICEBERG_STORAGE
+ .key(),
+
"table-location"))))
+ .hasStackTraceContaining("precision of 3 or less");
+ }
+
@ParameterizedTest
@ValueSource(ints = {2, 9})
public void
testEnableIcebergMetadataValidatesHistoricalTimestampPrecisions(int precision)
