This is an automated email from the ASF dual-hosted git repository.
jinsongzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new 957e52124 [AMORO-3084] Upgrade to iceberg 1.6.1 (#3071)
957e52124 is described below
commit 957e52124af7c7791af4e7196f421311fc55ceeb
Author: ConradJam <[email protected]>
AuthorDate: Thu Jan 16 17:00:19 2025 +0800
[AMORO-3084] Upgrade to iceberg 1.6.1 (#3071)
[Improvement]: Upgrade to iceberg 1.6.1 (#3084)
---
.../amoro/optimizing/IcebergRewriteExecutor.java | 13 ----------
.../org/apache/amoro/table/BasicUnkeyedTable.java | 6 +++++
.../org/apache/iceberg/TestIcebergFindFiles.java | 28 ++++++++++++++++++----
.../amoro-mixed-flink-common/pom.xml | 3 ++-
.../v1.15/amoro-mixed-flink-1.15/pom.xml | 1 +
.../v1.16/amoro-mixed-flink-1.16/pom.xml | 3 ++-
.../v1.17/amoro-mixed-flink-1.17/pom.xml | 3 ++-
.../amoro-mixed-spark-3-common/pom.xml | 3 ++-
.../v3.2/amoro-mixed-spark-3.2/pom.xml | 1 +
.../v3.3/amoro-mixed-spark-3.3/pom.xml | 1 +
pom.xml | 6 ++---
11 files changed, 44 insertions(+), 24 deletions(-)
diff --git
a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/IcebergRewriteExecutor.java
b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/IcebergRewriteExecutor.java
index fa355a8bf..d17222172 100644
---
a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/IcebergRewriteExecutor.java
+++
b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/IcebergRewriteExecutor.java
@@ -24,7 +24,6 @@ import
org.apache.amoro.io.writer.IcebergFanoutPosDeleteWriter;
import org.apache.amoro.table.MixedTable;
import org.apache.amoro.utils.map.StructLikeCollections;
import org.apache.iceberg.PartitionSpec;
-import org.apache.iceberg.StructLike;
import org.apache.iceberg.TableProperties;
import org.apache.iceberg.data.GenericAppenderFactory;
import org.apache.iceberg.data.IdentityPartitionConverters;
@@ -46,18 +45,6 @@ public class IcebergRewriteExecutor extends
AbstractRewriteFilesExecutor {
super(input, table, structLikeCollections);
}
- // TODO We can remove this override method after upgrading Iceberg version
to 1.5+.
- @Override
- protected StructLike partition() {
- StructLike partitionData = super.partition();
- if (partitionData != null && partitionData.size() == 0) {
- // Cast empty partition data to NULL to avoid creating empty partition
directory.
- return null;
- } else {
- return partitionData;
- }
- }
-
@Override
protected OptimizingDataReader dataReader() {
return new GenericCombinedIcebergDataReader(
diff --git
a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/BasicUnkeyedTable.java
b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/BasicUnkeyedTable.java
index d5f6f878d..6c5cceee6 100644
---
a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/BasicUnkeyedTable.java
+++
b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/BasicUnkeyedTable.java
@@ -65,6 +65,7 @@ import org.apache.iceberg.util.StructLikeMap;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
/** Basic implementation of {@link UnkeyedTable}, wrapping a {@link Table}. */
public class BasicUnkeyedTable implements UnkeyedTable, HasTableOperations {
@@ -318,4 +319,9 @@ public class BasicUnkeyedTable implements UnkeyedTable,
HasTableOperations {
public UpdatePartitionProperties updatePartitionProperties(Transaction
transaction) {
return new PartitionPropertiesUpdate(this, transaction);
}
+
+ @Override
+ public UUID uuid() {
+ return UUID.fromString(this.operations().current().uuid());
+ }
}
diff --git
a/amoro-format-iceberg/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java
b/amoro-format-iceberg/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java
index bd674c7d3..e7fa4a37e 100644
---
a/amoro-format-iceberg/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java
+++
b/amoro-format-iceberg/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java
@@ -25,24 +25,44 @@ import
org.apache.amoro.shade.guava32.com.google.common.collect.Sets;
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.types.Conversions;
import org.apache.iceberg.types.Types;
+import org.junit.After;
import org.junit.Assert;
import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import java.util.Arrays;
+import java.util.List;
import java.util.Set;
@RunWith(Parameterized.class)
-public class TestIcebergFindFiles extends TableTestBase {
+public class TestIcebergFindFiles extends TestBase {
+
@Parameterized.Parameters(name = "formatVersion = {0}")
- public static Object[] parameters() {
- return new Object[] {1, 2};
+ public static List<Object> parameters() {
+ return Arrays.asList(1, 2);
}
+ @Rule public TemporaryFolder tempDir = new TemporaryFolder();
+
public TestIcebergFindFiles(int formatVersion) {
- super(formatVersion);
+ this.formatVersion = formatVersion;
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ this.tableDir = tempDir.newFolder();
+ this.tableDir.delete();
+ super.setupTable();
+ }
+
+ @After
+ public void cleanUp() throws Exception {
+ super.cleanupTables();
}
@Test
diff --git
a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml
b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml
index 41a51b9d8..b173af469 100644
--- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml
@@ -34,10 +34,11 @@
<packaging>jar</packaging>
<properties>
+ <iceberg.version>1.4.3</iceberg.version>
<kafka.version>3.2.3</kafka.version>
<assertj.version>3.21.0</assertj.version>
<testcontainers.version>1.17.2</testcontainers.version>
- <flink.version>1.17.1</flink.version>
+ <flink.version>1.17.2</flink.version>
</properties>
<dependencies>
diff --git
a/amoro-format-mixed/amoro-mixed-flink/v1.15/amoro-mixed-flink-1.15/pom.xml
b/amoro-format-mixed/amoro-mixed-flink/v1.15/amoro-mixed-flink-1.15/pom.xml
index 0f9202e68..223f8e795 100644
--- a/amoro-format-mixed/amoro-mixed-flink/v1.15/amoro-mixed-flink-1.15/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-flink/v1.15/amoro-mixed-flink-1.15/pom.xml
@@ -34,6 +34,7 @@
<packaging>jar</packaging>
<properties>
+ <iceberg.version>1.4.3</iceberg.version>
<kafka.version>2.8.1</kafka.version>
<assertj.version>3.21.0</assertj.version>
<testcontainers.version>1.17.2</testcontainers.version>
diff --git
a/amoro-format-mixed/amoro-mixed-flink/v1.16/amoro-mixed-flink-1.16/pom.xml
b/amoro-format-mixed/amoro-mixed-flink/v1.16/amoro-mixed-flink-1.16/pom.xml
index cee742267..9ccfd0a6b 100644
--- a/amoro-format-mixed/amoro-mixed-flink/v1.16/amoro-mixed-flink-1.16/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-flink/v1.16/amoro-mixed-flink-1.16/pom.xml
@@ -34,10 +34,11 @@
<packaging>jar</packaging>
<properties>
+ <iceberg.version>1.4.3</iceberg.version>
<kafka.version>3.2.3</kafka.version>
<assertj.version>3.21.0</assertj.version>
<testcontainers.version>1.17.2</testcontainers.version>
- <flink.version>1.16.2</flink.version>
+ <flink.version>1.16.3</flink.version>
</properties>
<dependencies>
diff --git
a/amoro-format-mixed/amoro-mixed-flink/v1.17/amoro-mixed-flink-1.17/pom.xml
b/amoro-format-mixed/amoro-mixed-flink/v1.17/amoro-mixed-flink-1.17/pom.xml
index 5fd713cca..669b76b5b 100644
--- a/amoro-format-mixed/amoro-mixed-flink/v1.17/amoro-mixed-flink-1.17/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-flink/v1.17/amoro-mixed-flink-1.17/pom.xml
@@ -34,10 +34,11 @@
<packaging>jar</packaging>
<properties>
+ <iceberg.version>1.4.3</iceberg.version>
<kafka.version>3.2.3</kafka.version>
<assertj.version>3.21.0</assertj.version>
<testcontainers.version>1.17.2</testcontainers.version>
- <flink.version>1.17.1</flink.version>
+ <flink.version>1.17.2</flink.version>
</properties>
<dependencies>
diff --git
a/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/pom.xml
b/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/pom.xml
index dd4fa0995..e3ca63138 100644
--- a/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/pom.xml
@@ -33,6 +33,7 @@
<url>https://amoro.apache.org</url>
<properties>
+ <iceberg.version>1.4.3</iceberg.version>
<spark.version>3.2.4</spark.version>
<scala.version>2.12.15</scala.version>
</properties>
@@ -144,7 +145,7 @@
<dependency>
<groupId>org.apache.iceberg</groupId>
- <artifactId>iceberg-spark-3.2_2.12</artifactId>
+ <artifactId>iceberg-spark-3.3_2.12</artifactId>
<version>${iceberg.version}</version>
<scope>provided</scope>
<exclusions>
diff --git
a/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-3.2/pom.xml
b/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-3.2/pom.xml
index cc4e7a8d3..e5d5cf667 100644
--- a/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-3.2/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-3.2/pom.xml
@@ -34,6 +34,7 @@
<url>https://amoro.apache.org</url>
<properties>
+ <iceberg.version>1.4.3</iceberg.version>
<hive.version>2.3.9</hive.version>
<spark.version>3.2.4</spark.version>
<scala.version>2.12.15</scala.version>
diff --git
a/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml
b/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml
index 8d6cbff8b..2dd91e9ba 100644
--- a/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml
@@ -34,6 +34,7 @@
<url>https://amoro.apache.org</url>
<properties>
+ <iceberg.version>1.4.3</iceberg.version>
<hive.version>2.3.9</hive.version>
<spark.version>3.3.2</spark.version>
<scala.version>2.12.15</scala.version>
diff --git a/pom.xml b/pom.xml
index 97a882b18..458481f38 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,7 +101,7 @@
<maven-clean-plugin.version>3.3.2</maven-clean-plugin.version>
<maven-checkstyle-plugin.version>3.3.1</maven-checkstyle-plugin.version>
- <iceberg.version>1.4.3</iceberg.version>
+ <iceberg.version>1.6.1</iceberg.version>
<paimon.version>0.9.0</paimon.version>
<hive.version>3.1.1</hive.version>
<hadoop.version>3.4.0</hadoop.version>
@@ -1360,8 +1360,8 @@
<guava.version>14.0.1</guava.version>
<hive.version>2.3.9</hive.version>
<hadoop.version>2.10.2</hadoop.version>
- <terminal.spark.version>3.2.4</terminal.spark.version>
-
<terminal.spark.major.version>3.2</terminal.spark.major.version>
+ <terminal.spark.version>3.3.3</terminal.spark.version>
+
<terminal.spark.major.version>3.3</terminal.spark.major.version>
</properties>
</profile>
<profile>
