rdblue commented on code in PR #16936:
URL: https://github.com/apache/iceberg/pull/16936#discussion_r4200417615
##########
core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java:
##########
@@ -671,6 +729,399 @@ void unknownSpecIdThrows() {
.hasMessageContaining("Cannot find partition spec for spec ID");
}
+ @Test
+ void dataTrackedFileAdapterHasNoTracking() {
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(DATA_FILE);
+
+ assertThat(result.tracking()).isNull();
+ assertWrappedDataFileMatchesFileFields(result, DATA_FILE);
+ assertThatThrownBy(result::formatVersion)
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessage("Format version is assigned at write time");
+ }
+
+ @Test
+ void dataTrackedFileAdapterFromAddedManifestEntry() {
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(addedEntry(DATA_FILE));
+
+ assertThat(result.tracking().status()).isEqualTo(EntryStatus.ADDED);
+ assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID);
+ assertThat(result.tracking().dataSequenceNumber()).isNull();
+ assertThat(result.tracking().fileSequenceNumber()).isNull();
+ assertThat(result.tracking().firstRowId()).isNull();
+ assertManifestPosition(result.tracking(), DATA_FILE);
+ assertWrappedDataFileMatchesFileFields(result, DATA_FILE);
+ }
+
+ @Test
+ void dataTrackedFileAdapterFromAddedManifestEntryWithNullSnapshotId() {
+ TrackedFile result =
+
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(newEntry().wrapAppend(null,
DATA_FILE));
+
+ assertThat(result.tracking().status()).isEqualTo(EntryStatus.ADDED);
+ assertThat(result.tracking().snapshotId()).isNull();
+ assertManifestPosition(result.tracking(), DATA_FILE);
+ assertWrappedDataFileMatchesFileFields(result, DATA_FILE);
+ }
+
+ @Test
+ void dataTrackedFileAdapterFromExistingManifestEntry() {
+ TrackedFile result =
+
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(existingEntry(DATA_FILE));
+
+ assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING);
+ assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID);
+
assertThat(result.tracking().dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER);
+
assertThat(result.tracking().fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER);
+ assertManifestPosition(result.tracking(), DATA_FILE);
+ assertWrappedDataFileMatchesFileFields(result, DATA_FILE);
+ }
+
+ @Test
+ void dataTrackedFileAdapterFromDeletedManifestEntry() {
+ TrackedFile result =
+
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(deletedEntry(DATA_FILE));
+
+ assertThat(result.tracking().status()).isEqualTo(EntryStatus.DELETED);
+ assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID);
+
assertThat(result.tracking().dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER);
+
assertThat(result.tracking().fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER);
+ assertManifestPosition(result.tracking(), DATA_FILE);
+ }
+
+ @Test
+ void dataTrackedFileAdapterReuse() {
+ TrackedFileAdapters.DataTrackedFile adapter =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA);
+
+ adapter.wrap(DATA_FILE);
+ assertThat(adapter.location()).isEqualTo(DATA_FILE_LOCATION);
+ assertThat(adapter.recordCount()).isEqualTo(100L);
+ assertThat(adapter.tracking()).isNull();
+
+ DataFile file2 =
+ new GenericDataFile(
+ UNPARTITIONED_SPEC.specId(),
+ "s3://bucket/data/file2.parquet",
+ FileFormat.PARQUET,
+ PartitionData.EMPTY,
+ 2048L,
+ new Metrics(200L, null, null, null, null),
+ null,
+ ImmutableList.of(0L),
+ null,
+ null);
+ assignManifestPosition(file2, "s3://bucket/table/manifest-2.parquet", 8L);
+ adapter.wrap(existingEntry(file2));
+ assertThat(adapter.location()).isEqualTo("s3://bucket/data/file2.parquet");
+ assertThat(adapter.recordCount()).isEqualTo(200L);
+ assertThat(adapter.tracking().status()).isEqualTo(EntryStatus.EXISTING);
+ assertManifestPosition(adapter.tracking(), file2);
+ assertThat(adapter.fileSizeInBytes()).isEqualTo(2048L);
+ assertThat(adapter.location()).isNotEqualTo(DATA_FILE.location());
+ }
+
+ @Test
+ void dataTrackedFileAdapterRejectsNullFile() {
+ TrackedFileAdapters.DataTrackedFile adapter =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA);
+ assertThatThrownBy(() -> adapter.wrap((DataFile) null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Invalid file: null");
+ }
+
+ @Test
+ void dataTrackedFileAdapterContentStats() {
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(DATA_FILE_WITH_METRICS);
+
+ ContentStats stats = result.contentStats();
+ assertThat(stats).isNotNull();
+
assertThat(stats.fieldStats()).extracting(FieldStats::fieldId).containsExactlyInAnyOrder(1,
2);
+
+ FieldStats<?> idStats = stats.statsFor(1);
+ assertThat(idStats.valueCount()).isEqualTo(100L);
+ assertThat(idStats.lowerBound()).isEqualTo(1);
+ assertThat(idStats.upperBound()).isEqualTo(1000);
+ }
+
+ @Test
+ void dataTrackedFileAdapterWithoutMetricsHasNoContentStats() {
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(DATA_FILE);
+
+ assertThat(result.contentStats()).isNull();
+ }
+
+ @Test
+ void dataTrackedFileAdapterKeepsPartitionTuple() {
+ DataFile partitioned =
+ new GenericDataFile(
+ PARTITIONED_SPEC.specId(),
+ DATA_FILE_LOCATION,
+ FileFormat.PARQUET,
+ PARTITION,
+ 1024L,
+ new Metrics(100L, null, null, null, null),
+ null,
+ ImmutableList.of(0L),
+ null,
+ null);
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(partitioned);
+
+ assertThat(result.partition().get(0, String.class)).isEqualTo("books");
+ assertThat(
+ TrackedFileAdapters.asDataFile(result, specsById(PARTITIONED_SPEC))
+ .partition()
+ .get(0, String.class))
+ .isEqualTo("books");
+ }
+
+ @Test
+ void dataFileDoubleWrapRoundTrip() {
+ DataFile source = DATA_FILE_WITH_METRICS;
+ Map<Integer, PartitionSpec> specs =
+ ImmutableMap.of(UNPARTITIONED_SPEC.specId(), UNPARTITIONED_SPEC);
+
+ TrackedFile tracked =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(existingEntry(source));
+
+ DataFile roundTripped = TrackedFileAdapters.asDataFile(tracked, specs);
+
+ assertThat(roundTripped.content()).isEqualTo(FileContent.DATA);
+ assertThat(roundTripped.location()).isEqualTo(source.location());
+ assertThat(roundTripped.format()).isEqualTo(source.format());
+ assertThat(roundTripped.recordCount()).isEqualTo(source.recordCount());
+
assertThat(roundTripped.fileSizeInBytes()).isEqualTo(source.fileSizeInBytes());
+ assertThat(roundTripped.specId()).isEqualTo(source.specId());
+ assertThat(roundTripped.partition()).isEqualTo(source.partition());
+ assertThat(roundTripped.sortOrderId()).isEqualTo(source.sortOrderId());
+ assertThat(roundTripped.splitOffsets()).isEqualTo(source.splitOffsets());
+ assertThat(roundTripped.keyMetadata()).isEqualTo(source.keyMetadata());
+
assertThat(roundTripped.dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER);
+
assertThat(roundTripped.fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER);
+ assertThat(roundTripped.firstRowId()).isNull();
+
assertThat(roundTripped.manifestLocation()).isEqualTo(source.manifestLocation());
+ assertThat(roundTripped.pos()).isEqualTo(source.pos());
+
assertThat(roundTripped.valueCounts()).containsAllEntriesOf(source.valueCounts());
+
assertThat(roundTripped.nullValueCounts()).containsAllEntriesOf(source.nullValueCounts());
+ assertThat(roundTripped.nanValueCounts()).isNull();
+
assertThat(roundTripped.lowerBounds()).containsAllEntriesOf(source.lowerBounds());
+
assertThat(roundTripped.upperBounds()).containsAllEntriesOf(source.upperBounds());
+ assertThat(roundTripped.columnSizes()).isNull();
+ }
+
+ @Test
+ void dataTrackedFileAdapterUnwrapsToOriginalTrackedFile() {
+ TrackedFile original = dummyTrackedFile(FileContent.DATA);
+ DataFile adapted = TrackedFileAdapters.asDataFile(original, UNPARTITIONED);
+
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(adapted);
+
+ assertThat(result).isSameAs(original);
+ }
+
+ @Test
+ void manifestTrackedFileAdapterUnwrapsToOriginalTrackedFile() {
+ TrackedFile original = dummyTrackedFile(FileContent.DATA_MANIFEST, 0);
+ ManifestFile adapted = TrackedFileAdapters.asManifestFile(original);
+
+ TrackedFile result = TrackedFileAdapters.forManifestFile().wrap(adapted);
+
+ assertThat(result).isSameAs(original);
+ assertThat(result.formatVersion()).isZero();
+ }
+
+ @Test
+ void dataManifestTrackedFileAdapter() {
+ ManifestFile manifest = writeManifestFile(ManifestContent.DATA);
+ TrackedFile result = TrackedFileAdapters.forManifestFile().wrap(manifest);
+
+ assertThat(result.contentType()).isEqualTo(FileContent.DATA_MANIFEST);
+ assertThat(result.formatVersion()).isZero();
+ assertThat(result.location()).isEqualTo(MANIFEST_LOCATION);
+ assertThat(result.fileFormat()).isEqualTo(FileFormat.AVRO);
+ assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING);
+ assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID);
+ assertThat(result.tracking().firstRowId()).isNull();
+ assertThat(result.recordCount()).isEqualTo(6L);
+ assertThat(result.manifestInfo()).isNotNull();
+ assertThat(result.manifestInfo().addedFilesCount()).isEqualTo(2);
+ assertThat(result.manifestInfo().existingFilesCount()).isEqualTo(3);
+ assertThat(result.manifestInfo().deletedFilesCount()).isEqualTo(1);
+ assertThat(result.manifestInfo().addedRowsCount()).isEqualTo(200L);
+ assertThat(result.manifestInfo().replacedFilesCount()).isEqualTo(0);
+ assertThat(result.manifestInfo().replacedRowsCount()).isEqualTo(0L);
+ assertThat(result.manifestInfo().modifiedFilesCount()).isEqualTo(0);
+ assertThat(result.manifestInfo().modifiedRowsCount()).isEqualTo(0L);
+ }
+
+ @Test
+ void deleteManifestTrackedFileAdapter() {
+ ManifestFile manifest = writeManifestFile(ManifestContent.DELETES);
+ TrackedFile result = TrackedFileAdapters.forManifestFile().wrap(manifest);
+
+ assertThat(result.contentType()).isEqualTo(FileContent.DELETE_MANIFEST);
+ assertThat(result.formatVersion()).isZero();
+ assertThat(result.recordCount()).isEqualTo(6L);
+ assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING);
+ assertThat(result.tracking().firstRowId()).isNull();
+ }
+
+ @Test
+ void manifestTrackedFileAdapterFailsWhenAddedFilesCountMissing() {
+ ManifestFile manifest = mock(ManifestFile.class);
+ when(manifest.path()).thenReturn(MANIFEST_LOCATION);
+ when(manifest.content()).thenReturn(ManifestContent.DATA);
+ when(manifest.addedFilesCount()).thenReturn(null);
+ when(manifest.existingFilesCount()).thenReturn(1);
+ when(manifest.deletedFilesCount()).thenReturn(0);
+ when(manifest.replacedFilesCount()).thenReturn(0);
+
+ assertThatThrownBy(() ->
TrackedFileAdapters.forManifestFile().wrap(manifest))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("missing added files count");
+ }
+
+ @Test
+ void manifestTrackedFileAdapterRejectsNullReplacedFilesCount() {
+ ManifestFile manifest = manifestWithCounts(1, 1, 0, null, 0);
+
+ assertThatThrownBy(() ->
TrackedFileAdapters.forManifestFile().wrap(manifest))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage(
+ "Cannot convert manifest %s: replaced files count must be 0 for v3
or earlier manifests, was null",
+ MANIFEST_LOCATION);
+ }
+
+ @Test
+ void manifestTrackedFileAdapterRejectsNonZeroReplacedFilesCount() {
+ ManifestFile manifest = manifestWithCounts(1, 1, 0, 1, 0);
+
+ assertThatThrownBy(() ->
TrackedFileAdapters.forManifestFile().wrap(manifest))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage(
+ "Cannot convert manifest %s: replaced files count must be 0 for v3
or earlier manifests, was 1",
+ MANIFEST_LOCATION);
+ }
+
+ @Test
+ void manifestTrackedFileAdapterRejectsNullModifiedFilesCount() {
+ ManifestFile manifest = manifestWithCounts(1, 1, 0, 0, null);
+
+ assertThatThrownBy(() ->
TrackedFileAdapters.forManifestFile().wrap(manifest))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage(
+ "Cannot convert manifest %s: modified files count must be 0 for v3
or earlier manifests, was null",
+ MANIFEST_LOCATION);
+ }
+
+ @Test
+ void manifestTrackedFileAdapterRejectsNonZeroModifiedFilesCount() {
+ ManifestFile manifest = manifestWithCounts(1, 1, 0, 0, 1);
+
+ assertThatThrownBy(() ->
TrackedFileAdapters.forManifestFile().wrap(manifest))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage(
+ "Cannot convert manifest %s: modified files count must be 0 for v3
or earlier manifests, was 1",
+ MANIFEST_LOCATION);
+ }
+
+ @Test
+ void manifestTrackedFileAdapterFailsWhenAddedRowsCountMissing() {
+ ManifestFile manifest = writeManifestFile(ManifestContent.DATA, 5L, 4L,
null);
+ TrackedFile tracked = TrackedFileAdapters.forManifestFile().wrap(manifest);
+
+ assertThatThrownBy(() -> tracked.manifestInfo().addedRowsCount())
+ .isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("null");
+ }
+
+ private static void assertWrappedDataFileMatchesFileFields(TrackedFile
result, DataFile file) {
+ assertThat(result.contentType()).isEqualTo(FileContent.DATA);
+ assertThat(result.location()).isEqualTo(file.location());
+ assertThat(result.fileFormat()).isEqualTo(file.format());
+ assertThat(result.recordCount()).isEqualTo(file.recordCount());
+ assertThat(result.fileSizeInBytes()).isEqualTo(file.fileSizeInBytes());
+ assertThat(result.specId()).isEqualTo(file.specId());
+ assertThat(result.sortOrderId()).isEqualTo(file.sortOrderId());
+ assertThat(result.keyMetadata()).isEqualTo(file.keyMetadata());
+ assertThat(result.splitOffsets()).isEqualTo(file.splitOffsets());
+ assertThat(result.manifestInfo()).isNull();
+ assertThat(result.deletionVector()).isNull();
+ assertThat(result.equalityIds()).isNull();
+ }
+
+ private static ManifestFile manifestWithCounts(
+ Integer addedFilesCount,
+ Integer existingFilesCount,
+ Integer deletedFilesCount,
+ Integer replacedFilesCount,
+ Integer modifiedFilesCount) {
+ ManifestFile manifest = mock(ManifestFile.class);
+ when(manifest.path()).thenReturn(MANIFEST_LOCATION);
+ when(manifest.content()).thenReturn(ManifestContent.DATA);
+ when(manifest.addedFilesCount()).thenReturn(addedFilesCount);
+ when(manifest.existingFilesCount()).thenReturn(existingFilesCount);
+ when(manifest.deletedFilesCount()).thenReturn(deletedFilesCount);
+ when(manifest.replacedFilesCount()).thenReturn(replacedFilesCount);
+ when(manifest.modifiedFilesCount()).thenReturn(modifiedFilesCount);
+ return manifest;
+ }
+
+ private static ManifestFile writeManifestFile(ManifestContent content) {
+ return writeManifestFile(content, 5L, 4L, 200L);
+ }
+
+ private static ManifestFile writeManifestFile(
+ ManifestContent content, long sequenceNumber, long minSequenceNumber) {
+ return writeManifestFile(content, sequenceNumber, minSequenceNumber, 200L);
Review Comment:
I'd prefer having constants defined at the top and to minimize these helper
methods since you have to look at how they work in order to see if a test is
valid or correct.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
