rshdhere commented on code in PR #18925:
URL: https://github.com/apache/iceberg/pull/18925#discussion_r4573707973


##########
core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java:
##########
@@ -583,7 +597,189 @@ private void wrapInternal(DataFile newFile) {
           newFile.content());
 
       this.file = newFile;
-      this.stats = hasContentStats(newFile) ? statsWrapper.wrap(newFile) : 
null;
+      this.stats = wrapStats(fileWithPartitionSourceStats());
+    }
+
+    private ContentStats wrapStats(DataFile dataFile) {
+      return hasContentStats(dataFile) ? statsWrapper.wrap(dataFile) : null;
+    }
+
+    private DataFile fileWithPartitionSourceStats() {
+      PartitionSpec spec = specsById.get(file.specId());
+      Preconditions.checkArgument(
+          spec != null, "Cannot find partition spec for spec ID: %s", 
file.specId());
+
+      Set<Integer> partitionSourceIds = Sets.newHashSet();
+      Set<Integer> floatingPointSourceIds = Sets.newHashSet();
+      for (PartitionField field : spec.fields()) {
+        if (field.transform().isVoid()) {
+          continue;
+        }
+
+        Preconditions.checkArgument(
+            field.transform().preservesOrder(),
+            "Cannot convert data file %s: partition field does not preserve 
order: %s",
+            file.location(),
+            field);
+        Type sourceType = tableSchema.findType(field.sourceId());
+        Preconditions.checkArgument(
+            sourceType != null,
+            "Cannot convert data file %s: partition source column was dropped: 
%s",
+            file.location(),
+            field);
+
+        partitionSourceIds.add(field.sourceId());
+        if (StatsUtil.isFloatingPoint(sourceType)) {
+          floatingPointSourceIds.add(field.sourceId());
+        }
+      }
+
+      if (partitionSourceIds.isEmpty()) {
+        return file;
+      }
+
+      Map<Integer, Long> valueCounts =
+          fillFromPartition(file.valueCounts(), partitionSourceIds, id -> 
file.recordCount());
+      Map<Integer, Long> nullValueCounts =
+          fillFromPartition(file.nullValueCounts(), partitionSourceIds, 
this::nullValueCount);
+      Map<Integer, Long> nanValueCounts =
+          fillFromPartition(file.nanValueCounts(), floatingPointSourceIds, 
this::nanValueCount);
+      Map<Integer, ByteBuffer> lowerBounds =
+          fillFromPartition(file.lowerBounds(), partitionSourceIds, id -> 
boundsFor(id).lower());
+      Map<Integer, ByteBuffer> upperBounds =
+          fillFromPartition(file.upperBounds(), partitionSourceIds, id -> 
boundsFor(id).upper());
+
+      return new Delegates.DelegatingDataFile(file) {
+        @Override
+        public Map<Integer, Long> valueCounts() {
+          return valueCounts;
+        }
+
+        @Override
+        public Map<Integer, Long> nullValueCounts() {
+          return nullValueCounts;
+        }
+
+        @Override
+        public Map<Integer, Long> nanValueCounts() {
+          return nanValueCounts;
+        }
+
+        @Override
+        public Map<Integer, ByteBuffer> lowerBounds() {
+          return lowerBounds;
+        }
+
+        @Override
+        public Map<Integer, ByteBuffer> upperBounds() {
+          return upperBounds;
+        }
+      };
+    }
+
+    // a null or NaN partition value means every row's source value is null or 
NaN
+    private long nullValueCount(int sourceColumnId) {
+      return partitionValueFor(sourceColumnId) == null ? file.recordCount() : 
0L;
+    }
+
+    private long nanValueCount(int sourceColumnId) {
+      return NaNUtil.isNaN(partitionValueFor(sourceColumnId)) ? 
file.recordCount() : 0L;
+    }
+
+    private StatsUtil.PartitionSourceBounds<ByteBuffer> boundsFor(int 
sourceColumnId) {
+      if (useColumnBounds(sourceColumnId)) {
+        return new StatsUtil.PartitionSourceBounds<>(
+            file.lowerBounds().get(sourceColumnId), 
file.upperBounds().get(sourceColumnId));
+      }
+
+      Type type = tableSchema.findType(sourceColumnId);
+      StatsUtil.PartitionSourceBounds<?> bounds = 
narrowestBoundsFor(sourceColumnId);
+      return new StatsUtil.PartitionSourceBounds<>(
+          Conversions.toByteBuffer(type, bounds.lower()),
+          Conversions.toByteBuffer(type, bounds.upper()));
+    }
+
+    private boolean useColumnBounds(int columnId) {

Review Comment:
   Since string and binary bounds are never tight anyway (Stats Tightness in 
the spec), a truncated column bound is still a valid bound: the lower is a 
prefix of the real min and the upper is incremented past the real max. Could we 
keep the narrower of the column bounds and the partition bounds here instead of 
dropping the column bounds, the same way `narrowestBoundsFor` already does 
across partition fields?
   
   With `truncate[1]` on a string column, a file with column bounds `apple` / 
`avocado` and partition value `a` comes out as `a` / `b`, so the v4 stats end 
up looser than the v3 stats they came from. With identity the partition bounds 
would still win since they're exact.
   
   Test I ran locally against this PR (fails with `but was: "a"`):
   
   ```java
   @Test
   void dataTrackedFileAdapterKeepsNarrowerColumnBoundsForTruncatedString() {
     Schema schema = new Schema(optional(1, "name", Types.StringType.get()));
     PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("name", 
1).build();
     Metrics columnBounds =
         new Metrics(
             RECORD_COUNT,
             null, // columnSizes
             ImmutableMap.of(1, RECORD_COUNT),
             ImmutableMap.of(1, 0L),
             null, // nanValueCounts
             ImmutableMap.of(1, 
Conversions.toByteBuffer(Types.StringType.get(), "apple")),
             ImmutableMap.of(1, 
Conversions.toByteBuffer(Types.StringType.get(), "avocado")));
     DataFile file = dataFile(spec, Row.of("a"), columnBounds);
   
     ContentStats stats =
         TrackedFileAdapters.forDataFile(schema, 
specsById(spec)).wrap(file).contentStats();
   
     FieldStats<Object> nameStats = stats.statsFor(1);
     assertThat(nameStats.lowerBound().toString()).isEqualTo("apple");
     assertThat(nameStats.upperBound().toString()).isEqualTo("avocado");
   }
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to