anoopj commented on code in PR #17541: URL: https://github.com/apache/iceberg/pull/17541#discussion_r4158433983
########## core/src/test/java/org/apache/iceberg/TestFilePlanner.java: ########## @@ -0,0 +1,677 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.tuple; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.function.UnaryOperator; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.inmemory.InMemoryFileIO; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.FileAppender; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.metrics.DefaultMetricsContext; +import org.apache.iceberg.metrics.ScanMetrics; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.transforms.Transforms; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.LocationUtil; +import org.apache.iceberg.util.ThreadPools; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.FieldSource; + +class TestFilePlanner { Review Comment: Revised test plan: | Group | Test | Purpose | |---|---|---| | 1. Layouts (parallel execution) | `rootWithOnlyDataFiles` | Root of only DATA entries yields one task per file, no deletes | | | `rootWithOnlyLeafManifests` | Root of only DATA_MANIFEST entries expands leaves; all leaf files emitted | | | `rootWithMixedDataAndLeafManifests` | Union of root data files and expanded leaf files | | 2. Filtering + case sensitivity | `filterSkipsAndMatchesRootDataFilesAndManifests` | A root-level filter both skips and matches DATA files and DATA_MANIFEST refs | | | `filterSkipsAndMatchesLeafDataFiles` | A filter forwarded into an opened leaf skips and matches data files there | | | `filterHonorsCaseSensitivity` | Same filter with a differently-cased column; verifies the setting is forwarded | | 3. File-to-task conversion | `colocatedDvAttachedAsPositionDeleteFile` | A DATA entry with a colocated DV yields a task carrying the DV as a POSITION_DELETES DeleteFile with correct fields | | | `residualForMixedPartitionedAndUnpartitioned` | On a manifest mixing partitioned and unpartitioned files, with part of the filter removed by partition pruning, the correct residual is attached per file | | | `ignoreResidualsProducesAlwaysTrueResidual` | With `ignoreResiduals`, the task residual is `alwaysTrue` (built on the residual test) | | 4. Table location | `tableLocationForwardedToRootAndLeafReaders` | Relative file paths resolve against the table location for both a root data file and a leaf data file | | 5. Lazy evaluation | `closingWithoutIteratingOpensOnlyRoot` | Custom FileIO records opens; building then closing the plan without iterating opens only the root | | | `leafOpenedOnlyWhenPrecedingLeafExhausted` | Serial, iterating far enough to drain leaf 1 does not open leaf 2 | | 6. Negative cases | `deleteManifestInRootUnsupported` | DELETE_MANIFEST in the root fails | | | `v3ManifestLeafUnsupported` | A non-v4 (v3) manifest leaf fails | | 7. Scan metrics | `scanMetricsForComplexFilteredPlan` | Built on the most complex filtering test: result data files, result DVs, total/scanned sizes, skipped data files, scanned and skipped data manifests, and colocated DVs counted as DVs (not as indexed delete files) | -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
