pvary commented on code in PR #15906:
URL: https://github.com/apache/iceberg/pull/15906#discussion_r3092993505
##########
data/src/test/java/org/apache/iceberg/data/BaseFormatModelTests.java:
##########
@@ -719,4 +981,120 @@ private static String splitSizeProperty(FileFormat
fileFormat) {
"No split size property defined for format: " + fileFormat);
};
}
+
+ private void assertValueAndNullCounts(
+ FileFormat fileFormat,
+ Schema schema,
+ List<Record> genericRecords,
+ Map<Integer, Long> valueCounts,
+ Map<Integer, Long> nullValueCounts) {
+ assumeSupports(fileFormat, FEATURE_COLUMN_LEVEL_METRICS);
+ for (Types.NestedField field : schema.columns()) {
+ if (field.type().isPrimitiveType()) {
+ assertThat(valueCounts).containsKey(field.fieldId());
+ assertThat(nullValueCounts).containsKey(field.fieldId());
+
+ long nullCount =
+ genericRecords.stream().filter(r -> r.getField(field.name()) ==
null).count();
+
+
assertThat(valueCounts.get(field.fieldId())).isEqualTo(genericRecords.size());
+ assertThat(nullValueCounts.get(field.fieldId())).isEqualTo(nullCount);
+ }
+ }
+ }
+
+ private void assertBounds(
+ FileFormat fileFormat,
+ Schema schema,
+ List<Record> genericRecords,
+ Map<Integer, ByteBuffer> lowerBounds,
+ Map<Integer, ByteBuffer> upperBounds) {
+ assumeSupports(fileFormat, FEATURE_COLUMN_LEVEL_METRICS);
+ for (Types.NestedField field : schema.columns()) {
+ if (field.type().isPrimitiveType()) {
+ assertThat(lowerBounds).containsKey(field.fieldId());
+ assertThat(upperBounds).containsKey(field.fieldId());
+
+ ByteBuffer lowerBuffer = lowerBounds.get(field.fieldId());
+ ByteBuffer upperBuffer = upperBounds.get(field.fieldId());
+
+ Comparator<Object> cmp =
Comparators.forType(field.type().asPrimitiveType());
+
+ Object[] minMax = computeMinMax(genericRecords, field, cmp);
+ Object expectedMin = minMax[0];
+ Object expectedMax = minMax[1];
+
+ if (expectedMin != null) {
+ assertThat(lowerBuffer).isNotNull();
+ Object actualLower = Conversions.fromByteBuffer(field.type(),
lowerBuffer);
+ assertThat(cmp.compare(actualLower, expectedMin)).isEqualTo(0);
+ }
+
+ if (expectedMax != null) {
+ assertThat(upperBuffer).isNotNull();
+ Object actualUpper = Conversions.fromByteBuffer(field.type(),
upperBuffer);
+ assertThat(cmp.compare(actualUpper, expectedMax)).isEqualTo(0);
+ }
+ }
+ }
+ }
+
+ private static Object[] computeMinMax(
+ List<Record> records, Types.NestedField field, Comparator<Object> cmp) {
+ Object min = null;
+ Object max = null;
+ for (Record record : records) {
+ Object value = record.getField(field.name());
+ if (value == null) {
+ continue;
+ }
+
+ if (value instanceof Float && ((Float) value).isNaN()) {
+ continue;
+ }
+
+ if (value instanceof Double && ((Double) value).isNaN()) {
+ continue;
+ }
+
+ if (min == null || cmp.compare(value, min) < 0) {
+ min = value;
+ }
+
+ if (max == null || cmp.compare(value, max) > 0) {
+ max = value;
+ }
+ }
+
+ return new Object[] {min, max};
+ }
+
+ private void assertBoundsNull(
+ FileFormat fileFormat,
+ Schema schema,
+ Map<Integer, ByteBuffer> lowerBounds,
+ Map<Integer, ByteBuffer> upperBounds) {
+ assumeSupports(fileFormat, FEATURE_COLUMN_LEVEL_METRICS);
Review Comment:
Should we just return fast and not check the bounds instead of skipping the
whole test
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
