This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new df3dd39c9c [iceberg] Pass table location when creating tables via the
REST committer (#8290)
df3dd39c9c is described below
commit df3dd39c9c6c50c468dd225f90b1c058221786ee
Author: Tejansh <[email protected]>
AuthorDate: Sat Jun 20 08:41:35 2026 +0100
[iceberg] Pass table location when creating tables via the REST committer
(#8290)
`IcebergRestMetadataCommitter#createTable` creates tables without a
location. Catalogs that auto-assign one tolerated this but AWS Glue's
Iceberg REST endpoint does not, so creation failed. This PR passes the
location Paimon already writes its metadata to normalised to s3:// (Glue
rejects the s3a:// scheme Paimon's warehouse uses).
---
.../iceberg/IcebergRestMetadataCommitter.java | 46 ++++++++++++++++++++--
.../iceberg/IcebergRestMetadataCommitterTest.java | 16 ++++++++
2 files changed, 58 insertions(+), 4 deletions(-)
diff --git
a/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java
b/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java
index 584966382e..f144ea96de 100644
---
a/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java
+++
b/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java
@@ -295,8 +295,7 @@ public class IcebergRestMetadataCommitter implements
IcebergMetadataCommitter {
LOG.info(
"Partition fieldId = 0. The Iceberg REST committer
will use partition evolution to support Iceberg compatibility with the Paimon
schema. If you want to avoid this, use a non-zero fieldId partition field");
}
- Schema emptySchema = new Schema();
- return restCatalog.createTable(icebergTableIdentifier,
emptySchema);
+ return createTable(new Schema(), null, newMetadata);
} else {
LOG.info(
"Partition fieldId > 0. In order to avoid partition
evlolution, dummy schema will be created first");
@@ -316,11 +315,50 @@ public class IcebergRestMetadataCommitter implements
IcebergMetadataCommitter {
columns[f.sourceId() - 1] =
newMetadata.schema().findField(f.sourceId());
}
- Schema dummySchema = new Schema(columns);
- return restCatalog.createTable(icebergTableIdentifier,
dummySchema, spec);
+ return createTable(new Schema(columns), spec, newMetadata);
}
}
+ private Table createTable(
+ Schema schema, @Nullable PartitionSpec spec, TableMetadata
newMetadata) {
+ try {
+ // Path-based catalogs (e.g. Hadoop) derive and assign the table
location themselves
+ // and reject a custom one, so first try letting the catalog
assign it.
+ return newTableBuilder(schema, spec).create();
+ } catch (RuntimeException e) {
+ // Some Iceberg REST catalogs (notably AWS Glue) do not
auto-assign a table location
+ // and reject creation without one. Retry with the location Paimon
writes its metadata
+ // to, normalised to the s3:// scheme such catalogs require.
+ try {
+ return newTableBuilder(schema, spec)
+ .withLocation(toRestLocation(newMetadata.location()))
+ .create();
+ } catch (RuntimeException retryError) {
+ e.addSuppressed(retryError);
+ throw e;
+ }
+ }
+ }
+
+ private Catalog.TableBuilder newTableBuilder(Schema schema, @Nullable
PartitionSpec spec) {
+ Catalog.TableBuilder builder =
restCatalog.buildTable(icebergTableIdentifier, schema);
+ return spec == null ? builder : builder.withPartitionSpec(spec);
+ }
+
+ /** Normalises a table location's URI scheme for the Iceberg REST catalog.
*/
+ static String toRestLocation(String location) {
+ if (location == null) {
+ return null;
+ }
+ if (location.startsWith("s3a://")) {
+ return "s3://" + location.substring("s3a://".length());
+ }
+ if (location.startsWith("s3n://")) {
+ return "s3://" + location.substring("s3n://".length());
+ }
+ return location;
+ }
+
private Table getTable() {
return restCatalog.loadTable(icebergTableIdentifier);
}
diff --git
a/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java
b/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java
index 7111726f36..de39ea6f6d 100644
---
a/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java
+++
b/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java
@@ -769,6 +769,22 @@ public class IcebergRestMetadataCommitterTest {
commit.close();
}
+ @Test
+ public void testToRestLocationNormalisesScheme() {
+ // s3a:// and legacy s3n:// are rewritten to s3:// (Glue's REST
endpoint only accepts
+ // s3://).
+
assertThat(IcebergRestMetadataCommitter.toRestLocation("s3a://bucket/db/t"))
+ .isEqualTo("s3://bucket/db/t");
+
assertThat(IcebergRestMetadataCommitter.toRestLocation("s3n://bucket/db/t"))
+ .isEqualTo("s3://bucket/db/t");
+ // s3:// and other schemes pass through unchanged; null is preserved.
+
assertThat(IcebergRestMetadataCommitter.toRestLocation("s3://bucket/db/t"))
+ .isEqualTo("s3://bucket/db/t");
+
assertThat(IcebergRestMetadataCommitter.toRestLocation("file:///tmp/db/t"))
+ .isEqualTo("file:///tmp/db/t");
+ assertThat(IcebergRestMetadataCommitter.toRestLocation(null)).isNull();
+ }
+
private static class TestRecord {
private final BinaryRow partition;
private final GenericRow record;
