rdblue commented on code in PR #18109:
URL: https://github.com/apache/iceberg/pull/18109#discussion_r4031389734
##########
core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java:
##########
@@ -44,486 +45,700 @@
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.transforms.Transforms;
-import org.apache.iceberg.types.Comparators;
import org.apache.iceberg.types.TypeUtil;
import org.apache.iceberg.types.Types;
-import org.apache.iceberg.util.LocationUtil;
import org.junit.jupiter.api.Named;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.FieldSource;
-import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.Mockito;
class TestV4ManifestReader {
+ private static final ManifestFile UNREAD_MANIFEST_FILE =
Mockito.mock(ManifestFile.class);
+
+ static {
+ Mockito.when(UNREAD_MANIFEST_FILE.path())
+ .thenReturn(FileFormat.PARQUET.addExtension("manifest"));
+ Mockito.when(UNREAD_MANIFEST_FILE.formatVersion()).thenReturn(4);
+
Mockito.when(UNREAD_MANIFEST_FILE.content()).thenReturn(ManifestContent.DATA);
+ }
+
private static final long SNAPSHOT_ID = 42L;
private static final int FORMAT_VERSION_V4 = 4;
private static final long RECORD_COUNT = 100L;
private static final long FILE_SIZE_IN_BYTES = 1024L;
- private static final String TABLE_LOCATION = "s3://bucket/db/table";
private static final DeletionVector DV = dv("s3://bucket/dv.puffin");
+ private static final Tracking ADDED_TRACKING =
TrackingBuilder.added(SNAPSHOT_ID).build();
+
+ private static final ManifestInfo MANIFEST_INFO =
+ new ManifestInfoStruct(49, 51, 0, 0, 4_900L, 5_100L, 0L, 0L, 1L, null,
null);
+
private static final Schema TABLE_SCHEMA =
new Schema(
optional(1, "id", Types.IntegerType.get()), optional(2, "data",
Types.StringType.get()));
- private static final PartitionSpec ID_PARTITIONING =
- PartitionSpec.builderFor(TABLE_SCHEMA).identity("id").build();
- private static final Types.StructType ID_PARTITION_TYPE =
ID_PARTITIONING.partitionType();
- private static final Types.StructType EMPTY_PARTITION =
Types.StructType.of();
- private static final PartitionData EMPTY_PARTITION_DATA = new
PartitionData(EMPTY_PARTITION);
+ private static final Schema LOCATION_ONLY_SCHEMA = new
Schema(TrackedFile.LOCATION);
+
+ private static final PartitionSpec ID_PARTITIONED =
+
PartitionSpec.builderFor(TABLE_SCHEMA).withSpecId(1).identity("id").build();
+ private static final Types.StructType ID_PARTITIONED_TYPE =
ID_PARTITIONED.partitionType();
private static final Map<Integer, PartitionSpec> ID_PARTITIONING_SPECS =
- ImmutableMap.of(ID_PARTITIONING.specId(), ID_PARTITIONING);
+ ImmutableMap.of(ID_PARTITIONED.specId(), ID_PARTITIONED);
+
+ private static final Types.StructType UNPARTITIONED_TYPE =
Types.StructType.of();
private static final Map<Integer, PartitionSpec> UNPARTITIONED_SPECS =
ImmutableMap.of(PartitionSpec.unpartitioned().specId(),
PartitionSpec.unpartitioned());
- private static final List<FileFormat> MANIFEST_FORMATS =
- ImmutableList.of(FileFormat.AVRO, FileFormat.PARQUET);
+ private static final MetricsConfig METRICS_CONFIG =
+ MetricsConfig.from(ImmutableMap.of(), TABLE_SCHEMA, null);
+ private static final Types.StructType STATS_TYPE =
+ StatsUtil.statsWriteSchema(TABLE_SCHEMA, METRICS_CONFIG);
+ private static final Types.StructType ID_ONLY_STATS_TYPE =
+ Types.StructType.of(STATS_TYPE.field("id"));
+ private static final Types.StructType DATA_ONLY_STATS_TYPE =
+ Types.StructType.of(STATS_TYPE.field("data"));
+
+ private static final FieldStatsStruct<Integer> ID_STATS =
+ new FieldStatsStruct<>(
+ STATS_TYPE.fieldType("id").asStructType(), 0, 99, true,
RECORD_COUNT, 0, 0, null);
+ private static final FieldStatsStruct<String> DATA_STATS =
+ new FieldStatsStruct<>(
+ STATS_TYPE.fieldType("data").asStructType(), "a", "z", false,
RECORD_COUNT, 20, 0, null);
+ private static final ContentStatsStruct CONTENT_STATS = new
ContentStatsStruct(STATS_TYPE);
+
+ static {
+ CONTENT_STATS.setStats(1, ID_STATS);
+ CONTENT_STATS.setStats(2, DATA_STATS);
+ }
- // a data file whose tracking carries every inheritable and change-tracking
value set
- private static final TrackedFile FILE_WITH_FULL_TRACKING =
- new TrackedFileStruct(
- new TrackingStruct(
- EntryStatus.ADDED,
- SNAPSHOT_ID,
- 5L, // data sequence number
- 6L, // file sequence number
- 7L, // dv snapshot id
- 8L, // first row id
- new byte[] {1, 2}, // deleted positions
- new byte[] {3, 4}), // replaced positions
- FileContent.DATA,
- FORMAT_VERSION_V4,
- "s3://bucket/file.parquet",
- FileFormat.PARQUET,
- RECORD_COUNT,
- FILE_SIZE_IN_BYTES,
- 0,
- EMPTY_PARTITION_DATA,
- null,
- null,
- null,
- null,
- null,
- null,
- null);
-
- // shared data files: FILE_A is in partition id=1, FILE_B in partition id=2.
Locations are stored
- // relative to the table location (the default), so the reader resolves them
against the table
- private static final TrackedFile FILE_A = dataFile("data-a.parquet",
partition(1));
- private static final TrackedFile FILE_B = dataFile("data-b.parquet",
partition(2));
- private static final TrackedFile EQ_DELETES_A =
deleteFile("eq-deletes-a.parquet", partition(1));
- private static final TrackedFile EQ_DELETES_B =
deleteFile("eq-deletes-b.parquet", partition(2));
+ private static final Comparator<TrackedFile> FILE_COMPARATOR =
+ V4TestComparators.trackedFileStatusOnly(ID_PARTITIONED_TYPE);
+ private static final Comparator<TrackedFile> UNPARTITIONED_FILE_COMPARATOR =
+ V4TestComparators.trackedFileStatusOnly(UNPARTITIONED_TYPE);
+
+ // shared data files: FILE_A is in partition id=1, FILE_B in partition id=2
+ private static final TrackedFile UNPARTITIONED_FILE =
+ unpartitionedFileWithoutStats("s3://bucket/table/unpartitioned.parquet");
+ private static final TrackedFile FILE_A =
+
idPartitionedDataFileWithoutStats("s3://bucket/table/id=1/file-a.parquet",
idPartition(1));
+ private static final TrackedFile FILE_B =
+
idPartitionedDataFileWithoutStats("s3://bucket/table/id=2/file-b.parquet",
idPartition(2));
+ private static final TrackedFile EQ_DELETES_A =
+ idPartitionedDeleteFileWithoutStats(
+ "s3://bucket/table/id=1/eq-deletes-a.parquet", idPartition(1));
+ private static final TrackedFile EQ_DELETES_B =
+ idPartitionedDeleteFileWithoutStats(
+ "s3://bucket/table/id=2/eq-deletes-b.parquet", idPartition(2));
private static final TrackedFile DATA_MANIFEST_REF =
- manifestRef(FileContent.DATA_MANIFEST, "data-leaf.parquet");
+ manifestRef(FileContent.DATA_MANIFEST,
"s3://bucket/table/data-leaf.parquet");
private static final TrackedFile DELETE_MANIFEST_REF =
- manifestRef(FileContent.DELETE_MANIFEST, "delete-leaf.parquet");
+ manifestRef(FileContent.DELETE_MANIFEST,
"s3://bucket/table/delete-leaf.parquet");
+
+ private static final List<FileFormat> MANIFEST_FORMATS =
+ ImmutableList.of(FileFormat.AVRO, FileFormat.PARQUET);
- private final InMemoryFileIO io = new InMemoryFileIO();
+ private static final InMemoryFileIO IO = new InMemoryFileIO();
@ParameterizedTest
@FieldSource("MANIFEST_FORMATS")
- public void readsWrittenFile(FileFormat format) throws IOException {
+ public void readDataFile(FileFormat format) throws IOException {
TrackedFile file =
new TrackedFileStruct(
- addedTracking(),
+ ADDED_TRACKING,
FileContent.DATA,
FORMAT_VERSION_V4,
"s3://bucket/data/file.parquet",
FileFormat.PARQUET,
RECORD_COUNT,
FILE_SIZE_IN_BYTES,
- ID_PARTITIONING.specId(),
- partition(7),
- null,
- 1, // sort order id
+ ID_PARTITIONED.specId(),
+ idPartition(7),
+ CONTENT_STATS,
+ SortOrder.unsorted().orderId(),
DV,
- null,
- ByteBuffer.wrap(new byte[] {1, 2, 3}),
+ null, // manifest info
+ ByteBuffer.wrap(new byte[] {1, 2, 3}), // key metadata
ImmutableList.of(50L, 100L),
- null);
+ null); // equality field IDs
- ManifestFile manifest = writeManifest(format, ID_PARTITION_TYPE,
ImmutableList.of(file));
+ ManifestFile manifest = writeManifest(format, ID_PARTITIONED_TYPE, file);
- TrackedFile actual = Iterables.getOnlyElement(read(manifest,
ID_PARTITIONING_SPECS));
+ V4ManifestReader.Builder builder =
+ V4ManifestReader.builder(manifest, IO, TABLE_SCHEMA,
ID_PARTITIONING_SPECS)
+ .metricsConfig(METRICS_CONFIG);
+ TrackedFile actual = readOne(builder);
- // compare with tracking reduced to status: the reader fills
status-independent tracking
- // fields (row position, sequence numbers via inheritance) that the
written file does not have
- Types.StructType comparisonType =
- TypeUtil.replaceFieldTypes(
- TrackedFile.schema(ID_PARTITION_TYPE, Types.StructType.of()),
- ImmutableMap.of(
- TrackedFile.TRACKING.fieldId(),
Types.StructType.of(Tracking.STATUS)))
- .asStruct();
- assertThat((StructLike) actual)
- .usingComparator(Comparators.forType(comparisonType))
- .isEqualTo(file);
+ assertThat(actual).usingComparator(FILE_COMPARATOR).isEqualTo(file);
}
@ParameterizedTest
@FieldSource("MANIFEST_FORMATS")
- public void equalityDeleteRoundTrip(FileFormat format) throws IOException {
+ public void readForScanPlanningDoesNotCopyStats(FileFormat format) throws
IOException {
+ TrackedFile file =
+ new TrackedFileStruct(
+ ADDED_TRACKING,
+ FileContent.DATA,
+ FORMAT_VERSION_V4,
+ "s3://bucket/data/file.parquet",
+ FileFormat.PARQUET,
+ RECORD_COUNT,
+ FILE_SIZE_IN_BYTES,
+ ID_PARTITIONED.specId(),
+ idPartition(7),
+ CONTENT_STATS,
+ SortOrder.unsorted().orderId(),
+ DV,
+ null, // manifest info
+ ByteBuffer.wrap(new byte[] {1, 2, 3}), // key metadata
+ ImmutableList.of(50L, 100L),
+ null); // equality field IDs
+
+ ManifestFile manifest = writeManifest(format, ID_PARTITIONED_TYPE, file);
+
+ V4ManifestReader.Builder builder =
+ V4ManifestReader.builder(manifest, IO, TABLE_SCHEMA,
ID_PARTITIONING_SPECS)
+ .forScanPlanning()
+ .metricsConfig(METRICS_CONFIG);
+ TrackedFile actual = readOne(builder);
+
+
assertThat(actual).usingComparator(FILE_COMPARATOR).isEqualTo(file.copyWithoutStats());
+ }
+
+ @ParameterizedTest
+ @FieldSource("MANIFEST_FORMATS")
+ public void readForScanPlanningCopiesRequestedStats(FileFormat format)
throws IOException {
+ int idFieldId = TABLE_SCHEMA.findField("id").fieldId();
+ TrackedFile file =
+ new TrackedFileStruct(
+ ADDED_TRACKING,
+ FileContent.DATA,
+ FORMAT_VERSION_V4,
+ "s3://bucket/data/file.parquet",
+ FileFormat.PARQUET,
+ RECORD_COUNT,
+ FILE_SIZE_IN_BYTES,
+ ID_PARTITIONED.specId(),
+ idPartition(7),
+ CONTENT_STATS,
+ SortOrder.unsorted().orderId(),
+ DV,
+ null, // manifest info
+ ByteBuffer.wrap(new byte[] {1, 2, 3}), // key metadata
+ ImmutableList.of(50L, 100L),
+ null); // equality field IDs
+
+ ManifestFile manifest = writeManifest(format, ID_PARTITIONED_TYPE, file);
+
+ V4ManifestReader.Builder builder =
+ V4ManifestReader.builder(manifest, IO, TABLE_SCHEMA,
ID_PARTITIONING_SPECS)
+ .forScanPlanning()
+ .metricsConfig(METRICS_CONFIG)
+ .projectStats(idFieldId);
+ TrackedFile actual = readOne(builder);
+
+ assertThat(actual)
+ .usingComparator(FILE_COMPARATOR)
+ .isEqualTo(file.copyWithStats(Set.of(idFieldId)));
+ }
+
+ @ParameterizedTest
+ @FieldSource("MANIFEST_FORMATS")
+ public void readEqualityDelete(FileFormat format) throws IOException {
TrackedFile delete =
new TrackedFileStruct(
- addedTracking(),
+ ADDED_TRACKING,
FileContent.EQUALITY_DELETES,
FORMAT_VERSION_V4,
"s3://bucket/eq-delete.parquet",
FileFormat.PARQUET,
RECORD_COUNT,
FILE_SIZE_IN_BYTES,
- 0,
- EMPTY_PARTITION_DATA,
- null,
- null,
- null,
- null,
- null,
- null,
+ ID_PARTITIONED.specId(),
+ idPartition(7),
+ CONTENT_STATS,
+ SortOrder.unsorted().orderId(),
+ null, // dv
+ null, // manifest info
+ ByteBuffer.wrap(new byte[] {1, 2, 3}), // key metadata
+ null, // split offsets
ImmutableList.of(1, 2));
- ManifestFile manifest = writeManifest(format, EMPTY_PARTITION,
ImmutableList.of(delete));
+ ManifestFile manifest = writeManifest(format, ID_PARTITIONED_TYPE, delete);
- TrackedFile actual = Iterables.getOnlyElement(read(manifest,
UNPARTITIONED_SPECS));
- assertThat(actual.contentType()).isEqualTo(FileContent.EQUALITY_DELETES);
- assertThat(actual.equalityIds()).containsExactly(1, 2);
+ V4ManifestReader.Builder builder =
+ V4ManifestReader.builder(manifest, IO, TABLE_SCHEMA,
ID_PARTITIONING_SPECS)
+ .metricsConfig(METRICS_CONFIG);
+ TrackedFile actual = readOne(builder);
+
+ assertThat(actual).usingComparator(FILE_COMPARATOR).isEqualTo(delete);
}
@ParameterizedTest
@FieldSource("MANIFEST_FORMATS")
- public void statusFiltering(FileFormat format) throws IOException {
- List<TrackedFile> files =
- ImmutableList.of(
- fileWithStatus(EntryStatus.ADDED, "s3://bucket/added.parquet"),
- fileWithStatus(EntryStatus.EXISTING,
"s3://bucket/existing.parquet"),
- fileWithStatus(EntryStatus.MODIFIED,
"s3://bucket/modified.parquet"),
- fileWithStatus(EntryStatus.DELETED, "s3://bucket/deleted.parquet"),
- fileWithStatus(EntryStatus.REPLACED,
"s3://bucket/replaced.parquet"));
-
- ManifestFile manifest = writeManifest(format, EMPTY_PARTITION, files);
-
- try (V4ManifestReader reader =
- V4ManifestReader.builder(manifest, io, UNPARTITIONED_SPECS,
TABLE_LOCATION).build()) {
- assertThat(reader)
- .extracting(file -> file.tracking().status())
- .containsExactly(EntryStatus.ADDED, EntryStatus.EXISTING,
EntryStatus.MODIFIED);
- }
+ public void readManifestFile(FileFormat format) throws IOException {
+ TrackedFile manifestRef =
+ new TrackedFileStruct(
+ ADDED_TRACKING,
+ FileContent.DATA_MANIFEST,
+ FORMAT_VERSION_V4,
+ "s3://bucket/leaf-manifest.parquet",
+ FileFormat.PARQUET,
+ RECORD_COUNT,
+ FILE_SIZE_IN_BYTES,
+ null, // spec id
+ null, // partition
+ CONTENT_STATS,
+ null, // sort order id
+ null, // dv
+ MANIFEST_INFO,
+ ByteBuffer.wrap(new byte[] {1, 2, 3}), // key metadata
+ null, // split offsets
+ ImmutableList.of(1, 2));
- try (V4ManifestReader reader =
- V4ManifestReader.builder(manifest, io, UNPARTITIONED_SPECS,
TABLE_LOCATION)
- .includeAll()
- .build()) {
- assertThat(reader)
- .extracting(file -> file.tracking().status())
- .containsExactly(
- EntryStatus.ADDED,
- EntryStatus.EXISTING,
- EntryStatus.MODIFIED,
- EntryStatus.DELETED,
- EntryStatus.REPLACED);
- }
+ ManifestFile manifest = writeManifest(format, ID_PARTITIONED_TYPE,
manifestRef);
+
+ V4ManifestReader.Builder builder =
+ V4ManifestReader.builder(manifest, IO, TABLE_SCHEMA,
ID_PARTITIONING_SPECS)
+ .metricsConfig(METRICS_CONFIG);
+ TrackedFile actual = readOne(builder);
+
+ assertThat(actual).usingComparator(FILE_COMPARATOR).isEqualTo(manifestRef);
}
@ParameterizedTest
@FieldSource("MANIFEST_FORMATS")
- public void manifestLocationAndPosition(FileFormat format) throws
IOException {
+ public void statusFilter(FileFormat format) throws IOException {
List<TrackedFile> files =
ImmutableList.of(
- dataFile("s3://bucket/a.parquet", EMPTY_PARTITION_DATA),
- dataFile("s3://bucket/b.parquet", EMPTY_PARTITION_DATA),
- dataFile("s3://bucket/c.parquet", EMPTY_PARTITION_DATA));
+ unpartitionedFileWithStatus(EntryStatus.ADDED,
"s3://bucket/added.parquet"),
+ unpartitionedFileWithStatus(EntryStatus.MODIFIED,
"s3://bucket/modified.parquet"),
+ unpartitionedFileWithStatus(EntryStatus.DELETED,
"s3://bucket/deleted.parquet"),
+ unpartitionedFileWithStatus(EntryStatus.EXISTING,
"s3://bucket/existing.parquet"),
+ unpartitionedFileWithStatus(EntryStatus.REPLACED,
"s3://bucket/replaced.parquet"));
+
+ ManifestFile manifest = writeManifest(format, UNPARTITIONED_TYPE, files);
+
+ List<TrackedFile> liveFiles =
+ read(
+ V4ManifestReader.builder(manifest, IO, TABLE_SCHEMA,
UNPARTITIONED_SPECS)
+ .metricsConfig(METRICS_CONFIG));
+ assertThat(liveFiles)
+ .usingComparatorForType(FILE_COMPARATOR, TrackedFile.class)
+ .containsExactly(files.get(0), files.get(1), files.get(3));
+
+ List<TrackedFile> allFiles =
+ read(
+ V4ManifestReader.builder(manifest, IO, TABLE_SCHEMA,
UNPARTITIONED_SPECS)
+ .metricsConfig(METRICS_CONFIG)
+ .includeAll());
+ assertThat(allFiles)
+ .usingComparatorForType(FILE_COMPARATOR, TrackedFile.class)
+ .containsExactlyElementsOf(files);
+ }
+
+ @ParameterizedTest
+ @FieldSource("MANIFEST_FORMATS")
+ public void inheritanceManifestLocationAndPosition(FileFormat format) throws
IOException {
+ List<TrackedFile> files =
+ ImmutableList.of(FILE_A, FILE_B, DATA_MANIFEST_REF,
DELETE_MANIFEST_REF);
+
+ ManifestFile manifest = writeManifest(format, UNPARTITIONED_TYPE, files);
- ManifestFile manifest = writeManifest(format, EMPTY_PARTITION, files);
+ V4ManifestReader.Builder builder =
+ V4ManifestReader.builder(manifest, IO, TABLE_SCHEMA,
ID_PARTITIONING_SPECS)
+ .metricsConfig(METRICS_CONFIG);
+ List<TrackedFile> read = read(builder);
- List<TrackedFile> read = read(manifest, UNPARTITIONED_SPECS);
assertThat(read)
.allSatisfy(
file ->
assertThat(file.tracking().manifestLocation()).isEqualTo(manifest.path()));
- assertThat(read).extracting(file ->
file.tracking().manifestPos()).containsExactly(0L, 1L, 2L);
+ assertThat(read)
+ .extracting(file -> file.tracking().manifestPos())
+ .containsExactly(0L, 1L, 2L, 3L);
}
- @ParameterizedTest(name = "{0} / {1}")
- @MethodSource("selectiveReadModes")
- public void selectiveReadReturnsOnlyRequestedFields(
- FileFormat format, Consumer<V4ManifestReader.Builder> configureRead)
throws IOException {
- List<TrackedFile> files =
- ImmutableList.of(
- dataFile("s3://bucket/live.parquet", EMPTY_PARTITION_DATA),
- fileWithStatus(EntryStatus.DELETED, "s3://bucket/deleted.parquet"),
- fileWithStatus(EntryStatus.REPLACED,
"s3://bucket/replaced.parquet"));
+ @Test
+ public void projectionFullByDefault() {
+ Types.StructType readSchema =
+ V4ManifestReader.builder(UNREAD_MANIFEST_FILE, IO, TABLE_SCHEMA,
ID_PARTITIONING_SPECS)
+ .metricsConfig(METRICS_CONFIG)
+ .filter(Expressions.equal("id", 5)) // does not cause stats to be
filtered
+ .build()
+ .readSchema()
+ .asStruct();
- ManifestFile manifest = writeManifest(format, EMPTY_PARTITION, files);
+ Types.StructType expected =
+ TypeUtil.replaceFieldTypes(
+ TrackedFile.schema(ID_PARTITIONED_TYPE, STATS_TYPE),
+ ImmutableMap.of(TrackedFile.TRACKING.fieldId(),
TrackingStruct.BASE_TYPE))
+ .asStruct();
- V4ManifestReader.Builder builder =
- V4ManifestReader.builder(manifest, io, UNPARTITIONED_SPECS,
TABLE_LOCATION);
- configureRead.accept(builder);
- try (V4ManifestReader reader = builder.build()) {
- TrackedFile actual = Iterables.getOnlyElement(reader);
-
- // the requested field is read
- assertThat(actual.location()).isEqualTo("s3://bucket/live.parquet");
-
- // the reader always projects the fields it consumes internally, even
though the caller
- // selected only location: content type and status (liveness filtering
keeps only the live
- // entry), and manifest position (from row_position)
- assertThat(actual.contentType()).isEqualTo(FileContent.DATA);
- assertThat(actual.tracking().status()).isEqualTo(EntryStatus.ADDED);
- assertThat(actual.tracking().manifestPos()).isEqualTo(0L);
-
- // every field the caller did not request and the reader does not
require is omitted;
- // content stats in particular (the largest projection) is not read
- assertThat(actual.contentStats()).isNull();
- assertThat(actual.fileFormat()).isNull();
- assertThat(actual.recordCount()).isEqualTo(-1L);
- assertThat(actual.fileSizeInBytes()).isEqualTo(-1L);
- assertThat(actual.specId()).isNull();
- assertThat(actual.partition()).isNull();
- assertThat(actual.sortOrderId()).isNull();
- assertThat(actual.deletionVector()).isNull();
- assertThat(actual.keyMetadata()).isNull();
- assertThat(actual.splitOffsets()).isNull();
- assertThat(actual.equalityIds()).isNull();
- }
+ assertThat(readSchema)
+ .as("No projection configuration should project the full table
manifest schema")
+ .isEqualTo(expected);
}
- private static Stream<Arguments> selectiveReadModes() {
- Map<String, Consumer<V4ManifestReader.Builder>> modes =
- ImmutableMap.of(
- "project",
- builder -> builder.project(new Schema(TrackedFile.LOCATION)),
- "select",
- builder -> builder.select("location"),
- "case-insensitive select",
- builder -> builder.select("LOCATION").caseSensitive(false));
- return MANIFEST_FORMATS.stream()
- .flatMap(
- format ->
- modes.entrySet().stream()
- .map(mode -> Arguments.of(format, Named.of(mode.getKey(),
mode.getValue()))));
+ @Test
+ public void projectionDependsOnMetricsConfig() {
+ MetricsConfig metricsWithoutID =
+ MetricsConfig.from(
+ ImmutableMap.of(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"id", "none"),
+ TABLE_SCHEMA,
+ null);
+
+ Types.StructType readSchema =
+ V4ManifestReader.builder(UNREAD_MANIFEST_FILE, IO, TABLE_SCHEMA,
ID_PARTITIONING_SPECS)
+ .metricsConfig(metricsWithoutID)
+ .build()
+ .readSchema()
+ .asStruct();
+
+ Types.StructType expected =
+ TypeUtil.replaceFieldTypes(
+ TrackedFile.schema(ID_PARTITIONED_TYPE, DATA_ONLY_STATS_TYPE),
+ ImmutableMap.of(TrackedFile.TRACKING.fieldId(),
TrackingStruct.BASE_TYPE))
+ .asStruct();
+
+ assertThat(readSchema)
+ .as("Scan planning configuration should automatically prune tracking
and stats")
Review Comment:
Looks like this was copied from the test below.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
