amogh-jahagirdar commented on code in PR #18925:
URL: https://github.com/apache/iceberg/pull/18925#discussion_r4574999579
##########
core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java:
##########
@@ -583,7 +597,189 @@ private void wrapInternal(DataFile newFile) {
newFile.content());
this.file = newFile;
- this.stats = hasContentStats(newFile) ? statsWrapper.wrap(newFile) :
null;
+ this.stats = wrapStats(fileWithPartitionSourceStats());
+ }
+
+ private ContentStats wrapStats(DataFile dataFile) {
+ return hasContentStats(dataFile) ? statsWrapper.wrap(dataFile) : null;
+ }
+
+ private DataFile fileWithPartitionSourceStats() {
+ PartitionSpec spec = specsById.get(file.specId());
+ Preconditions.checkArgument(
+ spec != null, "Cannot find partition spec for spec ID: %s",
file.specId());
+
+ Set<Integer> partitionSourceIds = Sets.newHashSet();
+ Set<Integer> floatingPointSourceIds = Sets.newHashSet();
+ for (PartitionField field : spec.fields()) {
+ if (field.transform().isVoid()) {
+ continue;
+ }
+
+ Preconditions.checkArgument(
+ field.transform().preservesOrder(),
+ "Cannot convert data file %s: partition field does not preserve
order: %s",
+ file.location(),
+ field);
+ Type sourceType = tableSchema.findType(field.sourceId());
+ Preconditions.checkArgument(
+ sourceType != null,
+ "Cannot convert data file %s: partition source column was dropped:
%s",
+ file.location(),
+ field);
+
+ partitionSourceIds.add(field.sourceId());
+ if (StatsUtil.isFloatingPoint(sourceType)) {
+ floatingPointSourceIds.add(field.sourceId());
+ }
+ }
+
+ if (partitionSourceIds.isEmpty()) {
+ return file;
+ }
+
+ Map<Integer, Long> valueCounts =
+ fillFromPartition(file.valueCounts(), partitionSourceIds, id ->
file.recordCount());
+ Map<Integer, Long> nullValueCounts =
+ fillFromPartition(file.nullValueCounts(), partitionSourceIds,
this::nullValueCount);
+ Map<Integer, Long> nanValueCounts =
+ fillFromPartition(file.nanValueCounts(), floatingPointSourceIds,
this::nanValueCount);
+ Map<Integer, ByteBuffer> lowerBounds =
+ fillFromPartition(file.lowerBounds(), partitionSourceIds, id ->
boundsFor(id).lower());
+ Map<Integer, ByteBuffer> upperBounds =
+ fillFromPartition(file.upperBounds(), partitionSourceIds, id ->
boundsFor(id).upper());
+
+ return new Delegates.DelegatingDataFile(file) {
+ @Override
+ public Map<Integer, Long> valueCounts() {
+ return valueCounts;
+ }
+
+ @Override
+ public Map<Integer, Long> nullValueCounts() {
+ return nullValueCounts;
+ }
+
+ @Override
+ public Map<Integer, Long> nanValueCounts() {
+ return nanValueCounts;
+ }
+
+ @Override
+ public Map<Integer, ByteBuffer> lowerBounds() {
+ return lowerBounds;
+ }
+
+ @Override
+ public Map<Integer, ByteBuffer> upperBounds() {
+ return upperBounds;
+ }
+ };
+ }
+
+ // a null or NaN partition value means every row's source value is null or
NaN
+ private long nullValueCount(int sourceColumnId) {
+ return partitionValueFor(sourceColumnId) == null ? file.recordCount() :
0L;
+ }
+
+ private long nanValueCount(int sourceColumnId) {
+ return NaNUtil.isNaN(partitionValueFor(sourceColumnId)) ?
file.recordCount() : 0L;
+ }
+
+ private StatsUtil.PartitionSourceBounds<ByteBuffer> boundsFor(int
sourceColumnId) {
+ if (useColumnBounds(sourceColumnId)) {
+ return new StatsUtil.PartitionSourceBounds<>(
+ file.lowerBounds().get(sourceColumnId),
file.upperBounds().get(sourceColumnId));
+ }
+
+ Type type = tableSchema.findType(sourceColumnId);
+ StatsUtil.PartitionSourceBounds<?> bounds =
narrowestBoundsFor(sourceColumnId);
+ return new StatsUtil.PartitionSourceBounds<>(
+ Conversions.toByteBuffer(type, bounds.lower()),
+ Conversions.toByteBuffer(type, bounds.upper()));
+ }
+
+ private boolean useColumnBounds(int columnId) {
Review Comment:
Yeah this is right. For context for string/binary I went for the most
conservative/simplest approach which is just to use the partition value always
to derive the stats but you're right that for string/binary we could even
further have narrower bounds (if the file has a truncate transform on a string
column with a width lower than what the string/binary may have been truncated
to via metrics configuration). In any combination of truncation , we can always
just pick the narrowest and still be able to re-derive the tuple.
I'd even go further just to generalize the logic to always pick the
narrowest across both stats (if they exist) and the partition value rather than
bifurcate situations where we use the column stats or not (regardless of data
type).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]