joyhaldar commented on code in PR #17711: URL: https://github.com/apache/iceberg/pull/17711#discussion_r3813937250
########## data/src/test/java/org/apache/iceberg/data/ReadFormatModelTests.java: ########## @@ -0,0 +1,2414 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.data; + +import static org.apache.iceberg.MetadataColumns.DELETE_FILE_PATH; +import static org.apache.iceberg.MetadataColumns.DELETE_FILE_POS; +import static org.apache.iceberg.TestBase.SCHEMA; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assumptions.assumeThat; +import static org.junit.jupiter.api.Assumptions.assumeFalse; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.MetadataColumns; +import org.apache.iceberg.MetricsConfig; +import org.apache.iceberg.MetricsModes; +import org.apache.iceberg.MetricsModes.MetricsMode; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.StructLike; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.TestTables; +import org.apache.iceberg.deletes.EqualityDeleteWriter; +import org.apache.iceberg.deletes.PositionDelete; +import org.apache.iceberg.deletes.PositionDeleteWriter; +import org.apache.iceberg.encryption.EncryptedFiles; +import org.apache.iceberg.encryption.EncryptedOutputFile; +import org.apache.iceberg.encryption.EncryptionKeyMetadata; +import org.apache.iceberg.exceptions.NotFoundException; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.Literal; +import org.apache.iceberg.formats.FileWriterBuilder; +import org.apache.iceberg.formats.FormatModelRegistry; +import org.apache.iceberg.formats.ReadBuilder; +import org.apache.iceberg.inmemory.InMemoryFileIO; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.DataWriter; +import org.apache.iceberg.io.DeleteSchemaUtil; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.mapping.MappingUtil; +import org.apache.iceberg.mapping.NameMapping; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.types.Comparators; +import org.apache.iceberg.types.Conversions; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.TypeUtil; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.FieldSource; + +public abstract class ReadFormatModelTests<T> { Review Comment: Done Péter. ########## data/src/test/java/org/apache/iceberg/data/BaseFormatModelTests.java: ########## @@ -18,226 +18,38 @@ */ package org.apache.iceberg.data; -import static org.apache.iceberg.MetadataColumns.DELETE_FILE_PATH; -import static org.apache.iceberg.MetadataColumns.DELETE_FILE_POS; -import static org.apache.iceberg.TestBase.SCHEMA; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.assertj.core.api.Assumptions.assumeThat; -import static org.junit.jupiter.api.Assumptions.assumeFalse; -import java.io.File; import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.Collection; -import java.util.Comparator; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.UUID; import java.util.function.BiConsumer; -import java.util.function.BiFunction; import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.FileFormat; -import org.apache.iceberg.MetadataColumns; -import org.apache.iceberg.MetricsConfig; -import org.apache.iceberg.MetricsModes; -import org.apache.iceberg.MetricsModes.MetricsMode; -import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; -import org.apache.iceberg.StructLike; -import org.apache.iceberg.TableProperties; -import org.apache.iceberg.TestTables; import org.apache.iceberg.deletes.EqualityDeleteWriter; -import org.apache.iceberg.deletes.PositionDelete; -import org.apache.iceberg.deletes.PositionDeleteWriter; import org.apache.iceberg.encryption.EncryptedFiles; import org.apache.iceberg.encryption.EncryptedOutputFile; import org.apache.iceberg.encryption.EncryptingFileIO; -import org.apache.iceberg.encryption.EncryptionKeyMetadata; import org.apache.iceberg.encryption.EncryptionManager; import org.apache.iceberg.encryption.EncryptionTestHelpers; import org.apache.iceberg.encryption.NativeEncryptionKeyMetadata; import org.apache.iceberg.exceptions.AlreadyExistsException; -import org.apache.iceberg.exceptions.NotFoundException; -import org.apache.iceberg.exceptions.ValidationException; -import org.apache.iceberg.expressions.Expression; -import org.apache.iceberg.expressions.Expressions; -import org.apache.iceberg.expressions.Literal; import org.apache.iceberg.formats.FileWriterBuilder; import org.apache.iceberg.formats.FormatModelRegistry; -import org.apache.iceberg.formats.ReadBuilder; -import org.apache.iceberg.inmemory.InMemoryFileIO; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.DataWriter; -import org.apache.iceberg.io.DeleteSchemaUtil; import org.apache.iceberg.io.InputFile; -import org.apache.iceberg.mapping.MappingUtil; -import org.apache.iceberg.mapping.NameMapping; -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; -import org.apache.iceberg.relocated.com.google.common.collect.Lists; -import org.apache.iceberg.relocated.com.google.common.collect.Maps; -import org.apache.iceberg.types.Comparators; -import org.apache.iceberg.types.Conversions; -import org.apache.iceberg.types.Type; -import org.apache.iceberg.types.TypeUtil; -import org.apache.iceberg.types.Types; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.FieldSource; -public abstract class BaseFormatModelTests<T> { - - protected abstract Class<T> engineType(); - - protected abstract Object engineSchema(Schema schema); - - protected abstract T convertToEngine(Record record, Schema schema); - - protected abstract void assertEquals(Schema schema, List<T> expected, List<T> actual); - - protected abstract Object convertConstantToEngine(Type type, Object value); - - protected boolean supportsBatchReads() { - return false; - } - - @TempDir private File tableDir; - - /** - * Returns the set of type IDs that this engine cannot handle. The base implementation returns an - * empty collection; engines override to declare which top-level columns should be excluded from - * test schemas. - */ - protected Collection<Type.TypeID> unsupportedTypeIds() { - return Set.of(); - } - - private Schema filterUnsupported(Schema schema) { - Collection<Type.TypeID> unsupportedTypeIds = unsupportedTypeIds(); - if (unsupportedTypeIds.isEmpty()) { - return schema; - } - - List<Types.NestedField> kept = - schema.columns().stream() - .filter( - col -> - TypeUtil.find(col.type(), t -> unsupportedTypeIds.contains(t.typeId())) == null) - .toList(); - return kept.size() == schema.columns().size() ? schema : new Schema(kept); - } - - private Schema supportedSchema(DataGenerator generator) { - Schema filtered = filterUnsupported(generator.schema()); - assumeThat(filtered.columns()) - .as("All columns of %s are unsupported by this engine", generator) - .isNotEmpty(); - return filtered; - } - - private static List<Record> project(List<Record> records, Schema targetSchema) { - if (records.isEmpty()) { - return records; - } - - return records.stream() - .map( - source -> { - Record result = GenericRecord.create(targetSchema); - for (Types.NestedField col : targetSchema.columns()) { - result.setField(col.name(), source.getField(col.name())); - } - - return result; - }) - .toList(); - } - - static final String FEATURE_FILTER = "filter"; - static final String FEATURE_CASE_SENSITIVE = "caseSensitive"; - static final String FEATURE_SPLIT = "split"; - static final String FEATURE_READER_DEFAULT = "readerDefault"; - static final String FEATURE_REUSE_CONTAINERS = "reuseContainers"; - static final String FEATURE_COLUMN_LEVEL_METRICS = "columnLevelMetrics"; - static final String FEATURE_COLUMN_METRICS_TRUNCATE_BINARY = "columnMetricsTruncateBinary"; - static final String FEATURE_NATIVE_ENCRYPTION = "nativeEncryption"; - static final String FEATURE_AES_STREAM_ENCRYPTION = "aesStreamEncryption"; - static final String FEATURE_VARIANT = "variant"; - - private static final Map<FileFormat, String[]> MISSING_FEATURES = - Map.of( - FileFormat.AVRO, - new String[] { - FEATURE_FILTER, - FEATURE_CASE_SENSITIVE, - FEATURE_SPLIT, - FEATURE_COLUMN_LEVEL_METRICS, - FEATURE_COLUMN_METRICS_TRUNCATE_BINARY, - FEATURE_NATIVE_ENCRYPTION, - FEATURE_VARIANT - }, - FileFormat.ORC, - new String[] { - FEATURE_REUSE_CONTAINERS, - FEATURE_COLUMN_METRICS_TRUNCATE_BINARY, - FEATURE_READER_DEFAULT, - FEATURE_AES_STREAM_ENCRYPTION, - FEATURE_NATIVE_ENCRYPTION, - FEATURE_VARIANT - }, - FileFormat.PARQUET, - new String[] {FEATURE_AES_STREAM_ENCRYPTION}); - - private static final FileFormat[] FILE_FORMATS = FileFormatTestSupport.formats(); - - private static final List<Arguments> FORMAT_AND_GENERATOR = - Arrays.stream(FILE_FORMATS) - .flatMap( - format -> - Arrays.stream(DataGenerators.ALL) - .filter(generator -> supportsGenerator(format, generator)) - .map(generator -> Arguments.of(format, generator))) - .toList(); - - private InMemoryFileIO fileIO; - private EncryptedOutputFile encryptedFile; - - @BeforeEach - void before() { - this.fileIO = new InMemoryFileIO(); - this.encryptedFile = - EncryptedFiles.encryptedOutput( - fileIO.newOutputFile("test-file-" + UUID.randomUUID()), EncryptionKeyMetadata.EMPTY); - } - - @AfterEach - void after() { - try { - fileIO.deleteFile(encryptedFile.encryptingOutputFile()); - } catch (NotFoundException ignored) { - // ignore if the file is not created - } - - this.encryptedFile = null; - if (fileIO != null) { - fileIO.close(); - } - - TestTables.clearTables(); - } +public abstract class BaseFormatModelTests<T> extends ReadFormatModelTests<T> { Review Comment: Done Péter. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
