dramaticlly commented on code in PR #7581:
URL: https://github.com/apache/iceberg/pull/7581#discussion_r1240147358
##########
spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java:
##########
@@ -1283,6 +1296,158 @@ public void testUnpartitionedPartitionsTable() {
TestHelpers.assertEqualsSafe(expectedSchema, expectedRow, actual.get(0));
}
+ @Test
+ public void testPartitionsTableLastUpdatedSnapshot() {
+ TableIdentifier tableIdentifier = TableIdentifier.of("db",
"partitions_test");
+ Table table = createTable(tableIdentifier, SCHEMA, SPEC);
+ Table partitionsTable = loadTable(tableIdentifier, "partitions");
+ Dataset<Row> df1 =
+ spark.createDataFrame(
+ Lists.newArrayList(new SimpleRecord(1, "1"), new SimpleRecord(2,
"2")),
+ SimpleRecord.class);
+ Dataset<Row> df2 =
+ spark.createDataFrame(Lists.newArrayList(new SimpleRecord(2, "20")),
SimpleRecord.class);
+
+ df1.select("id", "data")
+ .write()
+ .format("iceberg")
+ .mode("append")
+ .save(loadLocation(tableIdentifier));
+
+ table.refresh();
+ long firstCommitId = table.currentSnapshot().snapshotId();
+
+ // add a second file
+ df2.select("id", "data")
+ .write()
+ .format("iceberg")
+ .mode("append")
+ .save(loadLocation(tableIdentifier));
+
+ table.refresh();
+ long secondCommitId = table.currentSnapshot().snapshotId();
+
+ // check if rewrite manifest does not override metadata about data file's
creating snapshot
+ RewriteManifests.Result rewriteManifestResult =
+ SparkActions.get().rewriteManifests(table).execute();
+ Assert.assertEquals(
+ "rewrite replaced 2 manifests",
+ 2,
+ Iterables.size(rewriteManifestResult.rewrittenManifests()));
+ Assert.assertEquals(
+ "rewrite added 1 manifests", 1,
Iterables.size(rewriteManifestResult.addedManifests()));
+
+ List<Row> actual =
+ spark
+ .read()
+ .format("iceberg")
+ .load(loadLocation(tableIdentifier, "partitions"))
+ .orderBy("partition.id")
+ .collectAsList();
+
+ GenericRecordBuilder builder =
+ new
GenericRecordBuilder(AvroSchemaUtil.convert(partitionsTable.schema(),
"partitions"));
+ GenericRecordBuilder partitionBuilder =
+ new GenericRecordBuilder(
+ AvroSchemaUtil.convert(
+ partitionsTable.schema().findType("partition").asStructType(),
"partition"));
+ List<GenericData.Record> expected = Lists.newArrayList();
+ expected.add(
+ builder
+ .set("partition", partitionBuilder.set("id", 1).build())
+ .set("record_count", 1L)
+ .set("file_count", 1)
+ .set("position_delete_record_count", 0L)
+ .set("position_delete_file_count", 0)
+ .set("equality_delete_record_count", 0L)
+ .set("equality_delete_file_count", 0)
+ .set("spec_id", 0)
+ .set("last_updated_ms",
table.snapshot(firstCommitId).timestampMillis() * 1000)
+ .set("last_updated_snapshot_id", firstCommitId)
+ .build());
+ expected.add(
+ builder
+ .set("partition", partitionBuilder.set("id", 2).build())
+ .set("record_count", 2L)
+ .set("file_count", 2)
+ .set("position_delete_record_count", 0L)
+ .set("position_delete_file_count", 0)
+ .set("equality_delete_record_count", 0L)
+ .set("equality_delete_file_count", 0)
+ .set("spec_id", 0)
+ .set("last_updated_ms",
table.snapshot(secondCommitId).timestampMillis() * 1000)
+ .set("last_updated_snapshot_id", secondCommitId)
+ .build());
+
+ Assert.assertEquals("Partitions table should have two rows", 2,
expected.size());
+ Assert.assertEquals("Actual results should have two rows", 2,
actual.size());
+ for (int i = 0; i < 2; i += 1) {
+ TestHelpers.assertEqualsSafe(
+ partitionsTable.schema().asStruct(), expected.get(i), actual.get(i));
+ }
+
+ // check time travel
+ List<Row> actualAfterFirstCommit =
+ spark
+ .read()
+ .format("iceberg")
+ .option(SparkReadOptions.SNAPSHOT_ID,
String.valueOf(secondCommitId))
+ .load(loadLocation(tableIdentifier, "partitions"))
+ .orderBy("partition.id")
+ .collectAsList();
+
+ Assert.assertEquals("Actual results should have one row", 2,
actualAfterFirstCommit.size());
Review Comment:
actually I think time travel is not really needed here as method is already
very long, let me just drop this section assert against `actualAfterFirstCommit`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
