stevenzwu commented on code in PR #16936:
URL: https://github.com/apache/iceberg/pull/16936#discussion_r4319692470
##########
core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java:
##########
@@ -671,6 +733,312 @@ void unknownSpecIdThrows() {
.hasMessageContaining("Cannot find partition spec for spec ID");
}
+ @Test
+ void dataTrackedFileAdapterFromDataFile() {
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(DATA_FILE);
+
+ assertThat(result.tracking()).isNull();
+ assertWrappedDataFileMatchesFileFields(result, DATA_FILE);
+ assertThatThrownBy(result::formatVersion)
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessage("Format version is assigned at write time");
+ }
+
+ @Test
+ void dataTrackedFileAdapterFromExistingManifestEntry() {
+ TrackedFile result =
+ TrackedFileAdapters.forDataFile(TABLE_SCHEMA)
+ .wrap(
+ newEntry()
+ .wrapExisting(
+ SNAPSHOT_ID, DATA_SEQUENCE_NUMBER,
FILE_SEQUENCE_NUMBER, DATA_FILE));
+
+ assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING);
+ assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID);
+
assertThat(result.tracking().dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER);
+
assertThat(result.tracking().fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER);
+ assertThat(result.tracking().firstRowId()).isEqualTo(FIRST_ROW_ID);
+ assertManifestPosition(result.tracking(), DATA_FILE);
+ assertWrappedDataFileMatchesFileFields(result, DATA_FILE);
+ }
+
+ @Test
+ void dataTrackedFileAdapterReuse() {
+ TrackedFileAdapters.DataTrackedFile adapter =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA);
+
+ adapter.wrap(DATA_FILE);
+ assertWrappedDataFileMatchesFileFields(adapter, DATA_FILE);
+ assertThat(adapter.tracking()).isNull();
+
+ DataFile file2 =
+ new GenericDataFile(
+ UNPARTITIONED_SPEC.specId(),
+ "s3://bucket/data/file2.parquet",
+ FileFormat.PARQUET,
+ PartitionData.EMPTY,
+ 2048L,
+ new Metrics(200L, null, null, null, null),
+ null,
+ ImmutableList.of(0L),
+ null,
+ null);
+ assignManifestPosition(file2, "s3://bucket/table/manifest-2.parquet", 8L);
+ adapter.wrap(
+ newEntry().wrapExisting(SNAPSHOT_ID, DATA_SEQUENCE_NUMBER,
FILE_SEQUENCE_NUMBER, file2));
+ assertWrappedDataFileMatchesFileFields(adapter, file2);
+ assertThat(adapter.tracking().status()).isEqualTo(EntryStatus.EXISTING);
+ assertManifestPosition(adapter.tracking(), file2);
+ }
+
+ @Test
+ void dataTrackedFileAdapterRejectsNullFile() {
+ TrackedFileAdapters.DataTrackedFile adapter =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA);
+ assertThatThrownBy(() -> adapter.wrap((DataFile) null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Invalid file: null");
+ }
+
+ @Test
+ void dataTrackedFileAdapterContentStats() {
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(DATA_FILE_WITH_METRICS);
+
+ ContentStats stats = result.contentStats();
+ assertThat(stats).isNotNull();
+
assertThat(stats.fieldStats()).extracting(FieldStats::fieldId).containsExactlyInAnyOrder(1,
2);
+
+ FieldStats<?> idStats = stats.statsFor(1);
+ assertThat(idStats.valueCount()).isEqualTo(100L);
+ assertThat(idStats.lowerBound()).isEqualTo(1);
+ assertThat(idStats.upperBound()).isEqualTo(1000);
+ }
+
+ @Test
+ void dataTrackedFileAdapterWithoutMetricsHasNoContentStats() {
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(DATA_FILE);
+
+ assertThat(result.contentStats()).isNull();
+ }
+
+ @Test
+ void dataTrackedFileAdapterKeepsPartitionTuple() {
+ DataFile partitioned =
+ new GenericDataFile(
+ PARTITIONED_SPEC.specId(),
+ DATA_FILE_LOCATION,
+ FileFormat.PARQUET,
+ PARTITION,
+ 1024L,
+ new Metrics(100L, null, null, null, null),
+ null,
+ ImmutableList.of(0L),
+ null,
+ null);
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(partitioned);
+
+ assertThat(result.partition())
+ .usingComparator(Comparators.forType(PARTITIONED_SPEC.partitionType()))
+ .isEqualTo(PARTITION);
+ assertThat(TrackedFileAdapters.asDataFile(result,
specsById(PARTITIONED_SPEC)).partition())
+ .usingComparator(Comparators.forType(PARTITIONED_SPEC.partitionType()))
+ .isEqualTo(PARTITION);
+ }
+
+ @Test
+ void dataTrackedFileAdapterUnwrapsToOriginalTrackedFile() {
+ TrackedFile source = trackedFile(FileContent.DATA);
+ DataFile dataFile = TrackedFileAdapters.asDataFile(source, UNPARTITIONED);
+ TrackedFile roundTripped =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(dataFile);
+ assertThat(roundTripped).isSameAs(source);
+ }
+
+ @Test
+ void manifestTrackedFileAdapterUnwrapsToOriginalTrackedFile() {
+ TrackedFile original = trackedFile(FileContent.DATA_MANIFEST, 0);
+ ManifestFile adapted = TrackedFileAdapters.asManifestFile(original);
+ TrackedFile result = TrackedFileAdapters.forManifestFile().wrap(adapted);
+ assertThat(result).isSameAs(original);
+ assertThat(result.formatVersion()).isZero();
+ }
+
+ @Test
+ void dataManifestTrackedFileAdapter() {
+ ManifestFile manifest = writeManifestFile(ManifestContent.DATA);
+ TrackedFile result = TrackedFileAdapters.forManifestFile().wrap(manifest);
+
+ assertThat(result.contentType()).isEqualTo(FileContent.DATA_MANIFEST);
+ assertThat(result.formatVersion()).isZero();
+ assertThat(result.location()).isEqualTo(MANIFEST_LOCATION);
+ assertThat(result.fileFormat()).isEqualTo(FileFormat.AVRO);
+ assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING);
+ assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID);
+ assertThat(result.tracking().firstRowId()).isNull();
+ assertThat(result.recordCount())
+ .isEqualTo(ADDED_FILES_COUNT + EXISTING_FILES_COUNT +
DELETED_FILES_COUNT);
+ assertThat(result.manifestInfo()).isNotNull();
+
assertThat(result.manifestInfo().addedFilesCount()).isEqualTo(ADDED_FILES_COUNT);
+
assertThat(result.manifestInfo().existingFilesCount()).isEqualTo(EXISTING_FILES_COUNT);
+
assertThat(result.manifestInfo().deletedFilesCount()).isEqualTo(DELETED_FILES_COUNT);
+
assertThat(result.manifestInfo().addedRowsCount()).isEqualTo(ADDED_ROWS_COUNT);
+ assertThat(result.manifestInfo().replacedFilesCount()).isEqualTo(0);
+ assertThat(result.manifestInfo().replacedRowsCount()).isEqualTo(0L);
+ assertThat(result.manifestInfo().modifiedFilesCount()).isEqualTo(0);
+ assertThat(result.manifestInfo().modifiedRowsCount()).isEqualTo(0L);
+ }
+
+ @Test
+ void deleteManifestTrackedFileAdapter() {
+ ManifestFile manifest = writeManifestFile(ManifestContent.DELETES);
+ TrackedFile result = TrackedFileAdapters.forManifestFile().wrap(manifest);
+
+ assertThat(result.contentType()).isEqualTo(FileContent.DELETE_MANIFEST);
+ assertThat(result.formatVersion()).isZero();
+ assertThat(result.recordCount())
+ .isEqualTo(ADDED_FILES_COUNT + EXISTING_FILES_COUNT +
DELETED_FILES_COUNT);
Review Comment:
Embedded the construction in manifestTrackedFileAdapter and removed
newManifestFile, so the counts sit next to the assertions.
##########
core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java:
##########
@@ -577,6 +640,271 @@ void unknownSpecIdThrows() {
.hasMessageContaining("Cannot find partition spec for spec ID");
}
+ @Test
+ void dataTrackedFileAdapterFromDataFile() {
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(DATA_FILE);
+
+ assertThat(result.tracking()).isNull();
+ assertWrappedDataFileMatchesFileFields(result, DATA_FILE);
+ }
+
+ @Test
+ void dataTrackedFileAdapterFromExistingManifestEntry() {
+ TrackedFile result =
+ TrackedFileAdapters.forDataFile(TABLE_SCHEMA)
+ .wrap(
+ newEntry()
+ .wrapExisting(
+ SNAPSHOT_ID, DATA_SEQUENCE_NUMBER,
FILE_SEQUENCE_NUMBER, DATA_FILE));
+
+ assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING);
+ assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID);
+
assertThat(result.tracking().dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER);
+
assertThat(result.tracking().fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER);
+ assertThat(result.tracking().firstRowId()).isEqualTo(FIRST_ROW_ID);
+ assertManifestPosition(result.tracking(), DATA_FILE);
+ assertWrappedDataFileMatchesFileFields(result, DATA_FILE);
+ }
+
+ @Test
+ void dataTrackedFileAdapterReuse() {
+ TrackedFileAdapters.DataTrackedFile adapter =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA);
+
+ adapter.wrap(DATA_FILE);
+ assertWrappedDataFileMatchesFileFields(adapter, DATA_FILE);
+ assertThat(adapter.tracking()).isNull();
+
+ DataFile file2 =
+ new GenericDataFile(
+ UNPARTITIONED_SPEC.specId(),
+ "s3://bucket/data/file2.parquet",
+ FileFormat.PARQUET,
+ PartitionData.EMPTY,
+ 2048L,
+ new Metrics(200L, null, null, null, null),
+ null,
+ ImmutableList.of(0L),
+ null,
+ null);
+ assignManifestPosition(file2, "s3://bucket/table/manifest-2.parquet", 8L);
+ adapter.wrap(
+ newEntry().wrapExisting(SNAPSHOT_ID, DATA_SEQUENCE_NUMBER,
FILE_SEQUENCE_NUMBER, file2));
+ assertWrappedDataFileMatchesFileFields(adapter, file2);
+ assertThat(adapter.tracking().status()).isEqualTo(EntryStatus.EXISTING);
+ assertManifestPosition(adapter.tracking(), file2);
+ }
+
+ @Test
+ void dataTrackedFileAdapterRejectsNullFile() {
+ TrackedFileAdapters.DataTrackedFile adapter =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA);
+ assertThatThrownBy(() -> adapter.wrap((DataFile) null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Invalid file: null");
+ }
+
+ @Test
+ void dataTrackedFileAdapterContentStats() {
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(DATA_FILE_WITH_METRICS);
+
+ ContentStats stats = result.contentStats();
+ assertThat(stats).isNotNull();
+
assertThat(stats.fieldStats()).extracting(FieldStats::fieldId).containsExactlyInAnyOrder(1,
2);
+
+ FieldStats<?> idStats = stats.statsFor(1);
+ assertThat(idStats.valueCount()).isEqualTo(100L);
+ assertThat(idStats.lowerBound()).isEqualTo(1);
+ assertThat(idStats.upperBound()).isEqualTo(1000);
+ }
+
+ @Test
+ void dataTrackedFileAdapterWithoutMetricsHasNoContentStats() {
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(DATA_FILE);
+
+ assertThat(result.contentStats()).isNull();
+ }
+
+ @Test
+ void dataTrackedFileAdapterKeepsPartitionTuple() {
+ DataFile partitioned =
+ new GenericDataFile(
+ PARTITIONED_SPEC.specId(),
+ DATA_FILE_LOCATION,
+ FileFormat.PARQUET,
+ PARTITION,
+ 1024L,
+ new Metrics(100L, null, null, null, null),
+ null,
+ ImmutableList.of(0L),
+ null,
+ null);
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(partitioned);
+
+ assertThat(result.partition())
+ .usingComparator(Comparators.forType(PARTITIONED_SPEC.partitionType()))
+ .isEqualTo(PARTITION);
+ }
+
+ @Test
+ void dataTrackedFileAdapterUnwrapsToOriginalTrackedFile() {
Review Comment:
Moved only `dataTrackedFileAdapterUnwrapsToOriginalTrackedFile` next to
`dataTrackedFileAdapterRejectsNullFile`.
`manifestTrackedFileAdapterUnwrapsToOriginalTrackedFile` stays with the
other `ManifestTrackedFile` tests.
##########
core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java:
##########
@@ -577,6 +640,271 @@ void unknownSpecIdThrows() {
.hasMessageContaining("Cannot find partition spec for spec ID");
}
+ @Test
+ void dataTrackedFileAdapterFromDataFile() {
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(DATA_FILE);
+
+ assertThat(result.tracking()).isNull();
+ assertWrappedDataFileMatchesFileFields(result, DATA_FILE);
+ }
+
+ @Test
+ void dataTrackedFileAdapterFromExistingManifestEntry() {
+ TrackedFile result =
+ TrackedFileAdapters.forDataFile(TABLE_SCHEMA)
+ .wrap(
+ newEntry()
+ .wrapExisting(
+ SNAPSHOT_ID, DATA_SEQUENCE_NUMBER,
FILE_SEQUENCE_NUMBER, DATA_FILE));
+
+ assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING);
+ assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID);
+
assertThat(result.tracking().dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER);
+
assertThat(result.tracking().fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER);
+ assertThat(result.tracking().firstRowId()).isEqualTo(FIRST_ROW_ID);
+ assertManifestPosition(result.tracking(), DATA_FILE);
+ assertWrappedDataFileMatchesFileFields(result, DATA_FILE);
+ }
+
+ @Test
+ void dataTrackedFileAdapterReuse() {
+ TrackedFileAdapters.DataTrackedFile adapter =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA);
+
+ adapter.wrap(DATA_FILE);
+ assertWrappedDataFileMatchesFileFields(adapter, DATA_FILE);
+ assertThat(adapter.tracking()).isNull();
+
+ DataFile file2 =
+ new GenericDataFile(
+ UNPARTITIONED_SPEC.specId(),
+ "s3://bucket/data/file2.parquet",
+ FileFormat.PARQUET,
+ PartitionData.EMPTY,
+ 2048L,
+ new Metrics(200L, null, null, null, null),
+ null,
+ ImmutableList.of(0L),
+ null,
+ null);
+ assignManifestPosition(file2, "s3://bucket/table/manifest-2.parquet", 8L);
+ adapter.wrap(
+ newEntry().wrapExisting(SNAPSHOT_ID, DATA_SEQUENCE_NUMBER,
FILE_SEQUENCE_NUMBER, file2));
+ assertWrappedDataFileMatchesFileFields(adapter, file2);
+ assertThat(adapter.tracking().status()).isEqualTo(EntryStatus.EXISTING);
+ assertManifestPosition(adapter.tracking(), file2);
+ }
+
+ @Test
+ void dataTrackedFileAdapterRejectsNullFile() {
+ TrackedFileAdapters.DataTrackedFile adapter =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA);
+ assertThatThrownBy(() -> adapter.wrap((DataFile) null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Invalid file: null");
+ }
+
+ @Test
+ void dataTrackedFileAdapterContentStats() {
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(DATA_FILE_WITH_METRICS);
+
+ ContentStats stats = result.contentStats();
+ assertThat(stats).isNotNull();
+
assertThat(stats.fieldStats()).extracting(FieldStats::fieldId).containsExactlyInAnyOrder(1,
2);
+
+ FieldStats<?> idStats = stats.statsFor(1);
+ assertThat(idStats.valueCount()).isEqualTo(100L);
+ assertThat(idStats.lowerBound()).isEqualTo(1);
+ assertThat(idStats.upperBound()).isEqualTo(1000);
+ }
+
+ @Test
+ void dataTrackedFileAdapterWithoutMetricsHasNoContentStats() {
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(DATA_FILE);
+
+ assertThat(result.contentStats()).isNull();
+ }
+
+ @Test
+ void dataTrackedFileAdapterKeepsPartitionTuple() {
+ DataFile partitioned =
+ new GenericDataFile(
+ PARTITIONED_SPEC.specId(),
+ DATA_FILE_LOCATION,
+ FileFormat.PARQUET,
+ PARTITION,
+ 1024L,
+ new Metrics(100L, null, null, null, null),
+ null,
+ ImmutableList.of(0L),
+ null,
+ null);
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(partitioned);
+
+ assertThat(result.partition())
+ .usingComparator(Comparators.forType(PARTITIONED_SPEC.partitionType()))
+ .isEqualTo(PARTITION);
+ }
+
+ @Test
+ void dataTrackedFileAdapterUnwrapsToOriginalTrackedFile() {
+ TrackedFile original = trackedFile(FileContent.DATA);
+ DataFile adapted = TrackedFileAdapters.asDataFile(original, UNPARTITIONED);
+ TrackedFile roundTripped =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(adapted);
+ assertThat(roundTripped).isSameAs(original);
+ }
+
+ @Test
+ void manifestTrackedFileAdapterUnwrapsToOriginalTrackedFile() {
+ TrackedFile original = trackedFile(FileContent.DATA_MANIFEST);
+ ManifestFile adapted = TrackedFileAdapters.asManifestFile(original);
+ TrackedFile roundTripped =
TrackedFileAdapters.forManifestFile().wrap(adapted);
+ assertThat(roundTripped).isSameAs(original);
+ }
+
+ @ParameterizedTest
+ @EnumSource(ManifestContent.class)
+ void manifestTrackedFileAdapter(ManifestContent content) {
+ FileContent expectedContent =
+ content == ManifestContent.DATA ? FileContent.DATA_MANIFEST :
FileContent.DELETE_MANIFEST;
+ ManifestFile manifest = newManifestFile(content);
+ TrackedFile result = TrackedFileAdapters.forManifestFile().wrap(manifest);
+
+ assertThat(result.contentType()).isEqualTo(expectedContent);
+ assertThat(result.manifestInfo().formatVersion()).isZero();
+ assertThat(result.location()).isEqualTo(MANIFEST_LOCATION);
+ assertThat(result.fileFormat()).isEqualTo(FileFormat.AVRO);
+ assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING);
+ assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID);
+ assertThat(result.tracking().firstRowId()).isNull();
Review Comment:
A data manifest now uses `FIRST_ROW_ID`. A delete manifest stays null
because a delete manifest's `first_row_id` is always null. The assertion
follows the content.
##########
core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java:
##########
@@ -577,6 +640,271 @@ void unknownSpecIdThrows() {
.hasMessageContaining("Cannot find partition spec for spec ID");
}
+ @Test
+ void dataTrackedFileAdapterFromDataFile() {
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(DATA_FILE);
+
+ assertThat(result.tracking()).isNull();
+ assertWrappedDataFileMatchesFileFields(result, DATA_FILE);
+ }
+
+ @Test
+ void dataTrackedFileAdapterFromExistingManifestEntry() {
+ TrackedFile result =
+ TrackedFileAdapters.forDataFile(TABLE_SCHEMA)
+ .wrap(
+ newEntry()
Review Comment:
Inlined the construction at both call sites and removed the helper.
```java
new GenericManifestEntry<DataFile>(
ManifestEntry.getSchema(UNPARTITIONED_SPEC.partitionType()).asStruct())
.wrapExisting(SNAPSHOT_ID, DATA_SEQUENCE_NUMBER, FILE_SEQUENCE_NUMBER,
DATA_FILE)
```
The reuse test uses the same construction with `file2` (instead of
`DATA_FILE`).
##########
core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java:
##########
@@ -577,6 +640,271 @@ void unknownSpecIdThrows() {
.hasMessageContaining("Cannot find partition spec for spec ID");
}
+ @Test
+ void dataTrackedFileAdapterFromDataFile() {
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(DATA_FILE);
+
+ assertThat(result.tracking()).isNull();
+ assertWrappedDataFileMatchesFileFields(result, DATA_FILE);
+ }
+
+ @Test
+ void dataTrackedFileAdapterFromExistingManifestEntry() {
+ TrackedFile result =
+ TrackedFileAdapters.forDataFile(TABLE_SCHEMA)
+ .wrap(
+ newEntry()
+ .wrapExisting(
+ SNAPSHOT_ID, DATA_SEQUENCE_NUMBER,
FILE_SEQUENCE_NUMBER, DATA_FILE));
+
+ assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING);
+ assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID);
+
assertThat(result.tracking().dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER);
+
assertThat(result.tracking().fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER);
+ assertThat(result.tracking().firstRowId()).isEqualTo(FIRST_ROW_ID);
+ assertManifestPosition(result.tracking(), DATA_FILE);
+ assertWrappedDataFileMatchesFileFields(result, DATA_FILE);
+ }
+
+ @Test
+ void dataTrackedFileAdapterReuse() {
+ TrackedFileAdapters.DataTrackedFile adapter =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA);
+
+ adapter.wrap(DATA_FILE);
+ assertWrappedDataFileMatchesFileFields(adapter, DATA_FILE);
+ assertThat(adapter.tracking()).isNull();
+
+ DataFile file2 =
+ new GenericDataFile(
+ UNPARTITIONED_SPEC.specId(),
+ "s3://bucket/data/file2.parquet",
+ FileFormat.PARQUET,
+ PartitionData.EMPTY,
+ 2048L,
+ new Metrics(200L, null, null, null, null),
+ null,
+ ImmutableList.of(0L),
+ null,
+ null);
+ assignManifestPosition(file2, "s3://bucket/table/manifest-2.parquet", 8L);
+ adapter.wrap(
+ newEntry().wrapExisting(SNAPSHOT_ID, DATA_SEQUENCE_NUMBER,
FILE_SEQUENCE_NUMBER, file2));
+ assertWrappedDataFileMatchesFileFields(adapter, file2);
+ assertThat(adapter.tracking().status()).isEqualTo(EntryStatus.EXISTING);
+ assertManifestPosition(adapter.tracking(), file2);
+ }
+
+ @Test
+ void dataTrackedFileAdapterRejectsNullFile() {
+ TrackedFileAdapters.DataTrackedFile adapter =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA);
+ assertThatThrownBy(() -> adapter.wrap((DataFile) null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Invalid file: null");
+ }
+
+ @Test
+ void dataTrackedFileAdapterContentStats() {
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(DATA_FILE_WITH_METRICS);
+
+ ContentStats stats = result.contentStats();
+ assertThat(stats).isNotNull();
+
assertThat(stats.fieldStats()).extracting(FieldStats::fieldId).containsExactlyInAnyOrder(1,
2);
+
+ FieldStats<?> idStats = stats.statsFor(1);
+ assertThat(idStats.valueCount()).isEqualTo(100L);
+ assertThat(idStats.lowerBound()).isEqualTo(1);
+ assertThat(idStats.upperBound()).isEqualTo(1000);
+ }
+
+ @Test
+ void dataTrackedFileAdapterWithoutMetricsHasNoContentStats() {
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(DATA_FILE);
+
+ assertThat(result.contentStats()).isNull();
+ }
+
+ @Test
+ void dataTrackedFileAdapterKeepsPartitionTuple() {
+ DataFile partitioned =
+ new GenericDataFile(
+ PARTITIONED_SPEC.specId(),
+ DATA_FILE_LOCATION,
+ FileFormat.PARQUET,
+ PARTITION,
+ 1024L,
+ new Metrics(100L, null, null, null, null),
+ null,
+ ImmutableList.of(0L),
+ null,
+ null);
+ TrackedFile result =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(partitioned);
+
+ assertThat(result.partition())
+ .usingComparator(Comparators.forType(PARTITIONED_SPEC.partitionType()))
+ .isEqualTo(PARTITION);
+ }
+
+ @Test
+ void dataTrackedFileAdapterUnwrapsToOriginalTrackedFile() {
+ TrackedFile original = trackedFile(FileContent.DATA);
+ DataFile adapted = TrackedFileAdapters.asDataFile(original, UNPARTITIONED);
+ TrackedFile roundTripped =
TrackedFileAdapters.forDataFile(TABLE_SCHEMA).wrap(adapted);
+ assertThat(roundTripped).isSameAs(original);
+ }
+
+ @Test
+ void manifestTrackedFileAdapterUnwrapsToOriginalTrackedFile() {
+ TrackedFile original = trackedFile(FileContent.DATA_MANIFEST);
+ ManifestFile adapted = TrackedFileAdapters.asManifestFile(original);
+ TrackedFile roundTripped =
TrackedFileAdapters.forManifestFile().wrap(adapted);
+ assertThat(roundTripped).isSameAs(original);
+ }
+
+ @ParameterizedTest
+ @EnumSource(ManifestContent.class)
+ void manifestTrackedFileAdapter(ManifestContent content) {
+ FileContent expectedContent =
+ content == ManifestContent.DATA ? FileContent.DATA_MANIFEST :
FileContent.DELETE_MANIFEST;
+ ManifestFile manifest = newManifestFile(content);
+ TrackedFile result = TrackedFileAdapters.forManifestFile().wrap(manifest);
+
+ assertThat(result.contentType()).isEqualTo(expectedContent);
+ assertThat(result.manifestInfo().formatVersion()).isZero();
+ assertThat(result.location()).isEqualTo(MANIFEST_LOCATION);
+ assertThat(result.fileFormat()).isEqualTo(FileFormat.AVRO);
+ assertThat(result.tracking().status()).isEqualTo(EntryStatus.EXISTING);
+ assertThat(result.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID);
+ assertThat(result.tracking().firstRowId()).isNull();
+ assertThat(result.recordCount())
+ .isEqualTo(ADDED_FILES_COUNT + EXISTING_FILES_COUNT +
DELETED_FILES_COUNT);
Review Comment:
Embedded the `GenericManifestFile` construction in
`manifestTrackedFileAdapter` and removed `newManifestFile` helper.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
