gaborkaszab commented on code in PR #16867:
URL: https://github.com/apache/iceberg/pull/16867#discussion_r3481639790
##########
core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java:
##########
@@ -276,6 +282,156 @@ void testDVDeleteFileAdapterRejectsNullDeletionVector() {
.hasMessage("Cannot create DV delete file: no deletion vector");
}
+ @ParameterizedTest
+ @EnumSource(
+ value = FileContent.class,
+ names = {"DATA_MANIFEST", "DELETE_MANIFEST"})
+ void testManifestFileAdapterDelegation(FileContent contentType) {
+ ManifestInfo manifestInfo = createManifestInfo();
+ TrackedFile file =
+ manifestBuilder(contentType)
+ .manifestInfo(manifestInfo)
+ .keyMetadata(ByteBuffer.wrap(new byte[] {7, 8, 9}))
+ .build();
+ populateTrackingFields(file);
+
+ ManifestFile manifest = TrackedFileAdapters.asManifestFile(file);
+
+ ManifestContent expectedContent =
+ contentType == FileContent.DATA_MANIFEST ? ManifestContent.DATA :
ManifestContent.DELETES;
+ assertThat(manifest.path()).isEqualTo(MANIFEST_FILE_LOCATION);
+ assertThat(manifest.length()).isEqualTo(MANIFEST_FILE_SIZE);
+ assertThat(manifest.content()).isEqualTo(expectedContent);
+ assertThat(manifest.copy().content()).isEqualTo(expectedContent);
+ assertThat(manifest.sequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER);
+
assertThat(manifest.minSequenceNumber()).isEqualTo(manifestInfo.minSequenceNumber());
+ assertThat(manifest.snapshotId()).isEqualTo(SNAPSHOT_ID);
+
assertThat(manifest.addedFilesCount()).isEqualTo(manifestInfo.addedFilesCount());
+
assertThat(manifest.addedRowsCount()).isEqualTo(manifestInfo.addedRowsCount());
+
assertThat(manifest.existingFilesCount()).isEqualTo(manifestInfo.existingFilesCount());
+
assertThat(manifest.existingRowsCount()).isEqualTo(manifestInfo.existingRowsCount());
+
assertThat(manifest.deletedFilesCount()).isEqualTo(manifestInfo.deletedFilesCount());
+
assertThat(manifest.deletedRowsCount()).isEqualTo(manifestInfo.deletedRowsCount());
+ assertThat(manifest.firstRowId()).isEqualTo(FIRST_ROW_ID);
+ assertThat(manifest.keyMetadata()).isEqualTo(ByteBuffer.wrap(new byte[]
{7, 8, 9}));
+ assertThat(manifest.partitions()).isNull();
+ }
+
+ @Test
+ void testManifestFileAdapterPartitionSpecIdUnsupported() {
+ TrackedFile file = manifestFile(FileContent.DATA_MANIFEST,
createManifestInfo());
+
+ ManifestFile manifest = TrackedFileAdapters.asManifestFile(file);
+
+ assertThatThrownBy(manifest::partitionSpecId)
+ .isInstanceOf(UnsupportedOperationException.class)
+ .hasMessage("v4 manifests are not bound to a single partition spec");
+ }
+
+ @Test
+ void testManifestFileAdapterRejectsManifestWithDeletionVector() {
+ ManifestInfo manifestInfo =
+ ManifestInfoStruct.builder()
+ .addedFilesCount(3)
+ .existingFilesCount(5)
+ .deletedFilesCount(2)
+ .replacedFilesCount(0)
+ .addedRowsCount(300L)
+ .existingRowsCount(500L)
+ .deletedRowsCount(200L)
+ .replacedRowsCount(0L)
+ .minSequenceNumber(7L)
+ .dv(ByteBuffer.wrap(new byte[] {1, 2, 3}))
+ .dvCardinality(4L)
+ .build();
+ TrackedFile file = manifestFile(FileContent.DATA_MANIFEST, manifestInfo);
+
+ assertThatThrownBy(() -> TrackedFileAdapters.asManifestFile(file))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage(
+ "Cannot adapt manifest with a deletion vector to ManifestFile: %s",
+ MANIFEST_FILE_LOCATION);
+ }
+
+ @Test
+ void testManifestFileAdapterReturnsNullForUnsetNullableFields() {
+ TrackedFile file =
+
manifestBuilder(FileContent.DATA_MANIFEST).manifestInfo(createManifestInfo()).build();
+ // Inheritance fills the data sequence number, which the adapter requires;
first row ID and
+ // key metadata stay unset.
+ ((TrackingStruct) file.tracking()).set(DATA_SEQUENCE_NUMBER_ORDINAL,
DATA_SEQUENCE_NUMBER);
Review Comment:
instead of using the ordinal, we can use the `inheritFrom(some_manifest)`
method to do that.
##########
core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java:
##########
@@ -276,6 +282,156 @@ void testDVDeleteFileAdapterRejectsNullDeletionVector() {
.hasMessage("Cannot create DV delete file: no deletion vector");
}
+ @ParameterizedTest
+ @EnumSource(
+ value = FileContent.class,
+ names = {"DATA_MANIFEST", "DELETE_MANIFEST"})
+ void testManifestFileAdapterDelegation(FileContent contentType) {
+ ManifestInfo manifestInfo = createManifestInfo();
+ TrackedFile file =
+ manifestBuilder(contentType)
+ .manifestInfo(manifestInfo)
+ .keyMetadata(ByteBuffer.wrap(new byte[] {7, 8, 9}))
+ .build();
+ populateTrackingFields(file);
Review Comment:
Just FYI: in the follow-up for the TrackedFileBuilder PR, we decided to drop
`populateTrackingFields` and went to call the constructor of `TrackedFile`
instead using the `createTracking` (revived in the follow-up PR) method.
##########
core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java:
##########
@@ -276,6 +282,156 @@ void testDVDeleteFileAdapterRejectsNullDeletionVector() {
.hasMessage("Cannot create DV delete file: no deletion vector");
}
+ @ParameterizedTest
+ @EnumSource(
+ value = FileContent.class,
+ names = {"DATA_MANIFEST", "DELETE_MANIFEST"})
+ void testManifestFileAdapterDelegation(FileContent contentType) {
+ ManifestInfo manifestInfo = createManifestInfo();
+ TrackedFile file =
+ manifestBuilder(contentType)
+ .manifestInfo(manifestInfo)
+ .keyMetadata(ByteBuffer.wrap(new byte[] {7, 8, 9}))
+ .build();
+ populateTrackingFields(file);
+
+ ManifestFile manifest = TrackedFileAdapters.asManifestFile(file);
+
+ ManifestContent expectedContent =
+ contentType == FileContent.DATA_MANIFEST ? ManifestContent.DATA :
ManifestContent.DELETES;
+ assertThat(manifest.path()).isEqualTo(MANIFEST_FILE_LOCATION);
+ assertThat(manifest.length()).isEqualTo(MANIFEST_FILE_SIZE);
+ assertThat(manifest.content()).isEqualTo(expectedContent);
+ assertThat(manifest.copy().content()).isEqualTo(expectedContent);
+ assertThat(manifest.sequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER);
+
assertThat(manifest.minSequenceNumber()).isEqualTo(manifestInfo.minSequenceNumber());
+ assertThat(manifest.snapshotId()).isEqualTo(SNAPSHOT_ID);
+
assertThat(manifest.addedFilesCount()).isEqualTo(manifestInfo.addedFilesCount());
+
assertThat(manifest.addedRowsCount()).isEqualTo(manifestInfo.addedRowsCount());
+
assertThat(manifest.existingFilesCount()).isEqualTo(manifestInfo.existingFilesCount());
+
assertThat(manifest.existingRowsCount()).isEqualTo(manifestInfo.existingRowsCount());
+
assertThat(manifest.deletedFilesCount()).isEqualTo(manifestInfo.deletedFilesCount());
+
assertThat(manifest.deletedRowsCount()).isEqualTo(manifestInfo.deletedRowsCount());
+ assertThat(manifest.firstRowId()).isEqualTo(FIRST_ROW_ID);
+ assertThat(manifest.keyMetadata()).isEqualTo(ByteBuffer.wrap(new byte[]
{7, 8, 9}));
+ assertThat(manifest.partitions()).isNull();
+ }
+
+ @Test
+ void testManifestFileAdapterPartitionSpecIdUnsupported() {
+ TrackedFile file = manifestFile(FileContent.DATA_MANIFEST,
createManifestInfo());
+
+ ManifestFile manifest = TrackedFileAdapters.asManifestFile(file);
+
+ assertThatThrownBy(manifest::partitionSpecId)
+ .isInstanceOf(UnsupportedOperationException.class)
+ .hasMessage("v4 manifests are not bound to a single partition spec");
+ }
+
+ @Test
+ void testManifestFileAdapterRejectsManifestWithDeletionVector() {
+ ManifestInfo manifestInfo =
+ ManifestInfoStruct.builder()
+ .addedFilesCount(3)
+ .existingFilesCount(5)
+ .deletedFilesCount(2)
+ .replacedFilesCount(0)
+ .addedRowsCount(300L)
+ .existingRowsCount(500L)
+ .deletedRowsCount(200L)
+ .replacedRowsCount(0L)
+ .minSequenceNumber(7L)
+ .dv(ByteBuffer.wrap(new byte[] {1, 2, 3}))
+ .dvCardinality(4L)
+ .build();
+ TrackedFile file = manifestFile(FileContent.DATA_MANIFEST, manifestInfo);
+
+ assertThatThrownBy(() -> TrackedFileAdapters.asManifestFile(file))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage(
+ "Cannot adapt manifest with a deletion vector to ManifestFile: %s",
+ MANIFEST_FILE_LOCATION);
+ }
+
+ @Test
+ void testManifestFileAdapterReturnsNullForUnsetNullableFields() {
+ TrackedFile file =
+
manifestBuilder(FileContent.DATA_MANIFEST).manifestInfo(createManifestInfo()).build();
+ // Inheritance fills the data sequence number, which the adapter requires;
first row ID and
+ // key metadata stay unset.
+ ((TrackingStruct) file.tracking()).set(DATA_SEQUENCE_NUMBER_ORDINAL,
DATA_SEQUENCE_NUMBER);
+
+ ManifestFile manifest = TrackedFileAdapters.asManifestFile(file);
+
+ assertThat(manifest.firstRowId()).isNull();
+ assertThat(manifest.keyMetadata()).isNull();
+ }
+
+ @Test
+ void testManifestFileAdapterCopy() {
+ TrackedFile file =
+ manifestBuilder(FileContent.DATA_MANIFEST)
+ .manifestInfo(createManifestInfo())
+ .keyMetadata(ByteBuffer.wrap(new byte[] {7, 8, 9}))
+ .build();
+ populateTrackingFields(file);
+
+ ManifestFile original = TrackedFileAdapters.asManifestFile(file);
+ ManifestFile copy = original.copy();
+
+ assertThat(copy.path()).isEqualTo(original.path());
+ assertThat(copy.length()).isEqualTo(original.length());
+ assertThat(copy.content()).isEqualTo(original.content());
+ assertThat(copy.sequenceNumber()).isEqualTo(original.sequenceNumber());
+
assertThat(copy.minSequenceNumber()).isEqualTo(original.minSequenceNumber());
+ assertThat(copy.snapshotId()).isEqualTo(original.snapshotId());
+ assertThat(copy.addedFilesCount()).isEqualTo(original.addedFilesCount());
+ assertThat(copy.addedRowsCount()).isEqualTo(original.addedRowsCount());
+
assertThat(copy.existingFilesCount()).isEqualTo(original.existingFilesCount());
+
assertThat(copy.existingRowsCount()).isEqualTo(original.existingRowsCount());
+
assertThat(copy.deletedFilesCount()).isEqualTo(original.deletedFilesCount());
+ assertThat(copy.deletedRowsCount()).isEqualTo(original.deletedRowsCount());
+ assertThat(copy.firstRowId()).isEqualTo(original.firstRowId());
+ assertThat(copy.keyMetadata()).isEqualTo(original.keyMetadata());
+ assertThat(copy.partitions()).isNull();
+
assertThat(copy.keyMetadata().array()).isNotSameAs(original.keyMetadata().array());
+ }
+
+ @ParameterizedTest
+ @EnumSource(
+ value = FileContent.class,
+ mode = EnumSource.Mode.EXCLUDE,
+ names = {"DATA_MANIFEST", "DELETE_MANIFEST"})
+ void testManifestFileAdapterRejectsNonManifestContent(FileContent
contentType) {
+ TrackedFileStruct file = dummyTrackedFile(contentType);
+
+ assertThatThrownBy(() -> TrackedFileAdapters.asManifestFile(file))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid content type for ManifestFile: %s", contentType);
+ }
+
+ @Test
+ void testManifestFileAdapterRejectsNullDataSequenceNumber() {
+ // A freshly built manifest entry has no data sequence number until
inheritance fills it.
+ TrackedFile file =
+
manifestBuilder(FileContent.DATA_MANIFEST).manifestInfo(createManifestInfo()).build();
+
+ assertThatThrownBy(() -> TrackedFileAdapters.asManifestFile(file))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid data sequence number: null");
+ }
+
+ @Test
+ void testManifestFileAdapterRejectsNullSnapshotId() {
+ TrackedFile file = manifestFile(FileContent.DATA_MANIFEST,
createManifestInfo());
+ // Clear the snapshot ID to exercise the adapter's required-field guard.
Review Comment:
nit: don't think this comment is needed
##########
core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java:
##########
@@ -276,6 +282,156 @@ void testDVDeleteFileAdapterRejectsNullDeletionVector() {
.hasMessage("Cannot create DV delete file: no deletion vector");
}
+ @ParameterizedTest
+ @EnumSource(
+ value = FileContent.class,
+ names = {"DATA_MANIFEST", "DELETE_MANIFEST"})
+ void testManifestFileAdapterDelegation(FileContent contentType) {
+ ManifestInfo manifestInfo = createManifestInfo();
+ TrackedFile file =
+ manifestBuilder(contentType)
+ .manifestInfo(manifestInfo)
+ .keyMetadata(ByteBuffer.wrap(new byte[] {7, 8, 9}))
+ .build();
+ populateTrackingFields(file);
+
+ ManifestFile manifest = TrackedFileAdapters.asManifestFile(file);
+
+ ManifestContent expectedContent =
+ contentType == FileContent.DATA_MANIFEST ? ManifestContent.DATA :
ManifestContent.DELETES;
+ assertThat(manifest.path()).isEqualTo(MANIFEST_FILE_LOCATION);
+ assertThat(manifest.length()).isEqualTo(MANIFEST_FILE_SIZE);
+ assertThat(manifest.content()).isEqualTo(expectedContent);
+ assertThat(manifest.copy().content()).isEqualTo(expectedContent);
+ assertThat(manifest.sequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER);
+
assertThat(manifest.minSequenceNumber()).isEqualTo(manifestInfo.minSequenceNumber());
+ assertThat(manifest.snapshotId()).isEqualTo(SNAPSHOT_ID);
+
assertThat(manifest.addedFilesCount()).isEqualTo(manifestInfo.addedFilesCount());
+
assertThat(manifest.addedRowsCount()).isEqualTo(manifestInfo.addedRowsCount());
+
assertThat(manifest.existingFilesCount()).isEqualTo(manifestInfo.existingFilesCount());
+
assertThat(manifest.existingRowsCount()).isEqualTo(manifestInfo.existingRowsCount());
+
assertThat(manifest.deletedFilesCount()).isEqualTo(manifestInfo.deletedFilesCount());
+
assertThat(manifest.deletedRowsCount()).isEqualTo(manifestInfo.deletedRowsCount());
+ assertThat(manifest.firstRowId()).isEqualTo(FIRST_ROW_ID);
+ assertThat(manifest.keyMetadata()).isEqualTo(ByteBuffer.wrap(new byte[]
{7, 8, 9}));
+ assertThat(manifest.partitions()).isNull();
+ }
+
+ @Test
+ void testManifestFileAdapterPartitionSpecIdUnsupported() {
+ TrackedFile file = manifestFile(FileContent.DATA_MANIFEST,
createManifestInfo());
+
+ ManifestFile manifest = TrackedFileAdapters.asManifestFile(file);
+
+ assertThatThrownBy(manifest::partitionSpecId)
+ .isInstanceOf(UnsupportedOperationException.class)
+ .hasMessage("v4 manifests are not bound to a single partition spec");
+ }
+
+ @Test
+ void testManifestFileAdapterRejectsManifestWithDeletionVector() {
+ ManifestInfo manifestInfo =
+ ManifestInfoStruct.builder()
+ .addedFilesCount(3)
+ .existingFilesCount(5)
+ .deletedFilesCount(2)
+ .replacedFilesCount(0)
+ .addedRowsCount(300L)
+ .existingRowsCount(500L)
+ .deletedRowsCount(200L)
+ .replacedRowsCount(0L)
+ .minSequenceNumber(7L)
+ .dv(ByteBuffer.wrap(new byte[] {1, 2, 3}))
+ .dvCardinality(4L)
+ .build();
+ TrackedFile file = manifestFile(FileContent.DATA_MANIFEST, manifestInfo);
+
+ assertThatThrownBy(() -> TrackedFileAdapters.asManifestFile(file))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage(
+ "Cannot adapt manifest with a deletion vector to ManifestFile: %s",
+ MANIFEST_FILE_LOCATION);
+ }
+
+ @Test
+ void testManifestFileAdapterReturnsNullForUnsetNullableFields() {
+ TrackedFile file =
+
manifestBuilder(FileContent.DATA_MANIFEST).manifestInfo(createManifestInfo()).build();
+ // Inheritance fills the data sequence number, which the adapter requires;
first row ID and
+ // key metadata stay unset.
+ ((TrackingStruct) file.tracking()).set(DATA_SEQUENCE_NUMBER_ORDINAL,
DATA_SEQUENCE_NUMBER);
+
+ ManifestFile manifest = TrackedFileAdapters.asManifestFile(file);
+
+ assertThat(manifest.firstRowId()).isNull();
+ assertThat(manifest.keyMetadata()).isNull();
+ }
+
+ @Test
+ void testManifestFileAdapterCopy() {
+ TrackedFile file =
+ manifestBuilder(FileContent.DATA_MANIFEST)
+ .manifestInfo(createManifestInfo())
+ .keyMetadata(ByteBuffer.wrap(new byte[] {7, 8, 9}))
+ .build();
+ populateTrackingFields(file);
+
+ ManifestFile original = TrackedFileAdapters.asManifestFile(file);
+ ManifestFile copy = original.copy();
+
+ assertThat(copy.path()).isEqualTo(original.path());
+ assertThat(copy.length()).isEqualTo(original.length());
+ assertThat(copy.content()).isEqualTo(original.content());
+ assertThat(copy.sequenceNumber()).isEqualTo(original.sequenceNumber());
+
assertThat(copy.minSequenceNumber()).isEqualTo(original.minSequenceNumber());
+ assertThat(copy.snapshotId()).isEqualTo(original.snapshotId());
+ assertThat(copy.addedFilesCount()).isEqualTo(original.addedFilesCount());
+ assertThat(copy.addedRowsCount()).isEqualTo(original.addedRowsCount());
+
assertThat(copy.existingFilesCount()).isEqualTo(original.existingFilesCount());
+
assertThat(copy.existingRowsCount()).isEqualTo(original.existingRowsCount());
+
assertThat(copy.deletedFilesCount()).isEqualTo(original.deletedFilesCount());
+ assertThat(copy.deletedRowsCount()).isEqualTo(original.deletedRowsCount());
+ assertThat(copy.firstRowId()).isEqualTo(original.firstRowId());
+ assertThat(copy.keyMetadata()).isEqualTo(original.keyMetadata());
+ assertThat(copy.partitions()).isNull();
+
assertThat(copy.keyMetadata().array()).isNotSameAs(original.keyMetadata().array());
+ }
+
+ @ParameterizedTest
+ @EnumSource(
+ value = FileContent.class,
+ mode = EnumSource.Mode.EXCLUDE,
+ names = {"DATA_MANIFEST", "DELETE_MANIFEST"})
+ void testManifestFileAdapterRejectsNonManifestContent(FileContent
contentType) {
+ TrackedFileStruct file = dummyTrackedFile(contentType);
+
+ assertThatThrownBy(() -> TrackedFileAdapters.asManifestFile(file))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid content type for ManifestFile: %s", contentType);
+ }
+
+ @Test
+ void testManifestFileAdapterRejectsNullDataSequenceNumber() {
+ // A freshly built manifest entry has no data sequence number until
inheritance fills it.
Review Comment:
nit: probably the test is intuitive enough even without the comment
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
