rdblue commented on code in PR #19007:
URL: https://github.com/apache/iceberg/pull/19007#discussion_r4603798426
##########
core/src/test/java/org/apache/iceberg/TestV4ManifestWriter.java:
##########
@@ -546,6 +620,379 @@ public void rootDataFileFirstRowIdAssignment() throws
IOException {
}
}
+ @Test
+ public void rootManifestFileAppend() throws IOException {
+ OutputFile output = new InMemoryOutputFile("manifest.parquet");
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(output),
+ Map.of(),
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+ try (writer) {
+ writer.append(LEAF_MANIFEST);
+ }
+
+ ManifestInfo summary = LEAF_MANIFEST.manifestInfo();
+ long leafRowsAssigned =
+ summary.addedRowsCount() + summary.existingRowsCount() +
summary.modifiedRowsCount();
+ assertThat(writer.assignedRowIds()).isEqualTo(leafRowsAssigned);
+
+ InputFile in = output.toInputFile();
+ for (TrackedFile file : List.of(readOne(writer, in), rawReadOne(in))) {
+ assertThat(file)
+ .usingComparator(V4TestComparators.trackedFileWithoutStatus())
+ .isEqualTo(LEAF_MANIFEST);
+ assertThat(file.tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Tracking uses the commit metadata")
+ .isEqualTo(
+ new TrackingStruct(
+ EntryStatus.ADDED,
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_SEQUENCE_NUMBER,
+ null, // not modified
+ COMMIT_FIRST_ROW_ID,
+ null, // no change bitmaps
+ null));
+ }
+ }
+
+ @Test
+ public void rootManifestFileAppendAtSeq() throws IOException {
+ V4ManifestWriter root =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(new
InMemoryOutputFile("root.parquet")),
+ Map.of(),
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+
+ try (root) {
+ assertThatThrownBy(() -> root.appendAtSeq(LEAF_MANIFEST, 33L))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Cannot append DATA_MANIFEST at a data sequence number");
+ }
+ }
+
+ @Test
+ public void rootManifestFileWithMDVAppend() throws IOException {
+ OutputFile output = new InMemoryOutputFile("manifest.parquet");
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(output),
+ SPECS,
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+ try (writer) {
+ writer.append(LEAF_MANIFEST_WITH_MDV);
+ }
+
+ ManifestInfo summary = LEAF_MANIFEST_WITH_MDV.manifestInfo();
+ long leafRowsAssigned =
+ summary.addedRowsCount() + summary.existingRowsCount() +
summary.modifiedRowsCount();
+ assertThat(writer.assignedRowIds()).isEqualTo(leafRowsAssigned);
+
+ InputFile in = output.toInputFile();
+ for (TrackedFile file : List.of(readOne(writer, in), rawReadOne(in))) {
+ assertThat(file)
+ .usingComparator(V4TestComparators.trackedFileWithoutStatus())
+ .isEqualTo(LEAF_MANIFEST_WITH_MDV);
+ assertThat(file.tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Tracking uses the commit metadata")
+ .isEqualTo(
+ new TrackingStruct(
+ EntryStatus.ADDED,
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_SEQUENCE_NUMBER,
+ null, // not modified
+ COMMIT_FIRST_ROW_ID,
+ null, // no change bitmaps
+ null));
+ }
+ }
+
+ @Test
+ public void rootManifestFileExisting() throws IOException {
+ OutputFile output = new InMemoryOutputFile("manifest.parquet");
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(output),
+ SPECS,
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+ try (writer) {
+ writer.existing(LEAF_MANIFEST_WITH_MDV);
+ }
+
+ assertThat(writer.assignedRowIds()).isEqualTo(0L);
+
+ InputFile in = output.toInputFile();
+ for (TrackedFile file : List.of(readOne(writer, in), rawReadOne(in))) {
+ assertThat(file)
+ .usingComparator(V4TestComparators.trackedFileWithoutStatus())
+ .isEqualTo(LEAF_MANIFEST_WITH_MDV);
+ Tracking originalTracking = LEAF_MANIFEST_WITH_MDV.tracking();
+ assertThat(file.tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Tracking uses the original metadata")
+ .isEqualTo(
+ new TrackingStruct(
+ EntryStatus.EXISTING,
+ originalTracking.snapshotId(),
+ originalTracking.dataSequenceNumber(),
+ originalTracking.fileSequenceNumber(),
+ originalTracking.modifiedSnapshotId(),
+ originalTracking.firstRowId(),
+ null, // no change bitmaps
+ null));
+ }
+ }
+
+ @Test
+ public void rootManifestFileDelete() throws IOException {
+ OutputFile output = new InMemoryOutputFile("manifest.parquet");
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(output),
+ Map.of(),
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+ try (writer) {
+ writer.delete(LEAF_MANIFEST);
+ }
+
+ assertThat(writer.assignedRowIds()).isEqualTo(0L);
+
+ assertThat(read(writer, output.toInputFile())).isEmpty();
+
+ TrackedFile file = rawReadOne(output.toInputFile());
+ assertThat(file)
+ .usingComparator(V4TestComparators.trackedFileWithoutStatus())
+ .isEqualTo(LEAF_MANIFEST);
+ Tracking originalTracking = LEAF_MANIFEST.tracking();
+ assertThat(file.tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Deleted entry preserves original metadata, sets snapshot ID")
+ .isEqualTo(
+ new TrackingStruct(
+ EntryStatus.DELETED,
+ COMMIT_SNAPSHOT_ID,
+ originalTracking.dataSequenceNumber(),
+ originalTracking.fileSequenceNumber(),
+ originalTracking.modifiedSnapshotId(),
+ originalTracking.firstRowId(),
+ null, // no change bitmaps
+ null));
+ }
+
+ @Test
+ public void rootManifestFileReplace() throws IOException {
+ OutputFile output = new InMemoryOutputFile("manifest.parquet");
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(output),
+ Map.of(),
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+ try (writer) {
+ writer.replace(LEAF_MANIFEST_WITH_MDV);
+ }
+
+ assertThat(writer.assignedRowIds()).isEqualTo(0L);
+
+ assertThat(read(writer, output.toInputFile())).isEmpty();
+
+ TrackedFile file = rawReadOne(output.toInputFile());
+ assertThat(file)
+ .usingComparator(V4TestComparators.trackedFileWithoutStatus())
+ .isEqualTo(LEAF_MANIFEST_WITH_MDV);
+ Tracking originalTracking = LEAF_MANIFEST_WITH_MDV.tracking();
+ assertThat(file.tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Replaced entry preserves original metadata, sets snapshot ID")
+ .isEqualTo(
+ new TrackingStruct(
+ EntryStatus.REPLACED,
+ COMMIT_SNAPSHOT_ID,
+ originalTracking.dataSequenceNumber(),
+ originalTracking.fileSequenceNumber(),
+ originalTracking.modifiedSnapshotId(),
+ originalTracking.firstRowId(),
+ null, // no change bitmaps
+ null));
+ }
+
+ @Test
+ public void rootManifestFileWithDeletionVector() throws IOException {
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(new
InMemoryOutputFile("manifest.parquet")),
+ SPECS,
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+
+ try (writer) {
+ assertThatThrownBy(() -> writer.modify(LEAF_MANIFEST, DV_A))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Invalid manifest, data deletion vector must
be null");
+ }
+ }
+
+ @Test
+ public void rootMixedDataAndManifestFiles() throws IOException {
+ OutputFile output = new InMemoryOutputFile("manifest.parquet");
+ V4ManifestWriter writer =
+ V4ManifestWriter.createRoot(
+ EncryptionUtil.plainAsEncryptedOutput(output),
+ SPECS,
+ STATS_TYPE,
+ Map.of(),
+ SCHEMA.schemaId(),
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_FIRST_ROW_ID);
+ try (writer) {
+ writer.existing(FILE_A);
+ writer.append(FILE_B);
+ writer.existing(LEAF_MANIFEST_WITH_MDV);
+ }
+
+ assertThat(writer.assignedRowIds()).isEqualTo(FILE_B.recordCount());
+
+ List<TrackedFile> files = read(writer, output.toInputFile());
+ assertThat(files)
+ .usingComparatorForType(
+
V4TestComparators.trackedFileWithoutStatus(TRUNCATE_SPEC.partitionType()),
+ TrackedFile.class)
+ .containsExactly(FILE_A, FILE_B, LEAF_MANIFEST_WITH_MDV);
+ assertThat(files.get(0).tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Tracking should preserve FILE_A metadata")
+ .isEqualTo(FILE_A.tracking());
+ assertThat(files.get(1).tracking())
+ .usingComparator(V4TestComparators.TRACKING)
+ .as("Tracking uses the commit metadata for FILE_B")
+ .isEqualTo(
+ new TrackingStruct(
+ EntryStatus.ADDED,
+ COMMIT_SNAPSHOT_ID,
+ COMMIT_SEQUENCE_NUMBER,
+ COMMIT_SEQUENCE_NUMBER,
+ null, // not modified
+ COMMIT_FIRST_ROW_ID,
+ null, // no change bitmaps
+ null));
+ assertThat(files.get(2).tracking())
Review Comment:
Yes, that was an oversight.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]