Re: [PR] Core: V3 Metadata Upgrade Validation and Testing [iceberg]

[via GitHub]

nastra commented on code in PR #10861:
URL: https://github.com/apache/iceberg/pull/10861#discussion_r1705039041


##########
core/src/test/java/org/apache/iceberg/TestFormatVersions.java:
##########
@@ -23,56 +23,100 @@
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 import org.junit.jupiter.api.TestTemplate;
 
 public class TestFormatVersions extends TestBase {
   @Parameters(name = "formatVersion = {0}")
   protected static List<Object> parameters() {
-    return Arrays.asList(1);
+    return Arrays.asList(1, 2);
   }
 
   @TestTemplate
-  public void testDefaultFormatVersion() {
-    assertThat(table.ops().current().formatVersion()).isEqualTo(1);
+  public void testFormatVersionUpgrade() {
+    TableOperations ops = table.ops();
+    TableMetadata base = ops.current();
+    int baseTableVersion = base.formatVersion();
+    int newTableVersion = baseTableVersion + 1;
+
+    TableMetadata newTableMetadata = 
base.upgradeToFormatVersion(newTableVersion);
+
+    assertThat(
+            newTableMetadata.changes().stream()
+                .filter(
+                    metadataUpdate -> metadataUpdate instanceof 
MetadataUpdate.UpgradeFormatVersion)
+                .map(
+                    metadataUpdate ->
+                        ((MetadataUpdate.UpgradeFormatVersion) 
metadataUpdate).formatVersion()))
+        .isEqualTo(List.of(newTableVersion));
+
+    ops.commit(base, newTableMetadata);
+
+    assertThat(ops.current().formatVersion()).isEqualTo(newTableVersion);
   }
 
   @TestTemplate
-  public void testFormatVersionUpgrade() {
+  public void testFormatVersionUpgradeToLatest() {
     TableOperations ops = table.ops();
     TableMetadata base = ops.current();
-    ops.commit(base, base.upgradeToFormatVersion(2));
+    int baseTableVersion = base.formatVersion();
+    int newTableVersion = TableMetadata.SUPPORTED_TABLE_FORMAT_VERSION;
+
+    TableMetadata newTableMetadata = 
base.upgradeToFormatVersion(newTableVersion);
+
+    // check that non-incremental updates are syntactic sugar for serial 
updates. E.g. upgrading
+    // from V1 to V3 will
+    // register changes in the table metadata for upgrading to V2 and V3 in 
order (V1->V2->V3)
+    assertThat(
+            newTableMetadata.changes().stream()
+                .filter(
+                    metadataUpdate -> metadataUpdate instanceof 
MetadataUpdate.UpgradeFormatVersion)
+                .map(
+                    metadataUpdate ->
+                        ((MetadataUpdate.UpgradeFormatVersion) 
metadataUpdate).formatVersion()))
+        .isEqualTo(
+            IntStream.rangeClosed(baseTableVersion + 1, newTableVersion)
+                .boxed()
+                .collect(Collectors.toList()));
+
+    ops.commit(base, newTableMetadata);
 
-    assertThat(ops.current().formatVersion()).isEqualTo(2);
+    assertThat(ops.current().formatVersion()).isEqualTo(newTableVersion);
   }
 
   @TestTemplate
   public void testFormatVersionDowngrade() {
     TableOperations ops = table.ops();
     TableMetadata base = ops.current();
-    ops.commit(base, base.upgradeToFormatVersion(2));
+    int baseTableVersion = base.formatVersion();
+    int newTableVersion = baseTableVersion + 1;
+    ops.commit(base, base.upgradeToFormatVersion(newTableVersion));
 
-    assertThat(ops.current().formatVersion()).isEqualTo(2);
+    assertThat(ops.current().formatVersion()).isEqualTo(newTableVersion);
 
-    assertThatThrownBy(() -> ops.current().upgradeToFormatVersion(1))
+    assertThatThrownBy(() -> 
ops.current().upgradeToFormatVersion(baseTableVersion))
         .isInstanceOf(IllegalArgumentException.class)
-        .hasMessage("Cannot downgrade v2 table to v1");
+        .hasMessage(
+            String.format("Cannot downgrade v%d table to v%d", 
newTableVersion, baseTableVersion));
 
-    assertThat(ops.current().formatVersion()).isEqualTo(2);
+    assertThat(ops.current().formatVersion()).isEqualTo(newTableVersion);
   }
 
   @TestTemplate
   public void testFormatVersionUpgradeNotSupported() {
     TableOperations ops = table.ops();
     TableMetadata base = ops.current();
+    int baseTableVersion = base.formatVersion();
+    int unsupportedTableVersion = TableMetadata.SUPPORTED_TABLE_FORMAT_VERSION 
+ 1;

Review Comment:
   ```suggestion
       int unsupportedFormatVersion = 
TableMetadata.SUPPORTED_TABLE_FORMAT_VERSION + 1;
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@iceberg.apache.org
For additional commands, e-mail: issues-h...@iceberg.apache.org