rdblue commented on code in PR #17159:
URL: https://github.com/apache/iceberg/pull/17159#discussion_r3598797763
##########
core/src/test/java/org/apache/iceberg/TestStatsUtil.java:
##########
@@ -18,511 +18,449 @@
*/
package org.apache.iceberg;
-import static org.apache.iceberg.FieldStatistic.AVG_VALUE_SIZE_IN_BYTES;
-import static org.apache.iceberg.FieldStatistic.LOWER_BOUND;
-import static org.apache.iceberg.FieldStatistic.NAN_VALUE_COUNT;
-import static org.apache.iceberg.FieldStatistic.NULL_VALUE_COUNT;
-import static org.apache.iceberg.FieldStatistic.TIGHT_BOUNDS;
-import static org.apache.iceberg.FieldStatistic.UPPER_BOUND;
-import static org.apache.iceberg.FieldStatistic.VALUE_COUNT;
-import static org.apache.iceberg.types.Types.NestedField.optional;
-import static org.apache.iceberg.types.Types.NestedField.required;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.util.List;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.stream.IntStream;
+import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.FieldSource;
public class TestStatsUtil {
- // the reserved field IDs from the reserved field ID space as defined in
- // https://iceberg.apache.org/spec/#reserved-field-ids
- private static final int RESERVED_FIELD_IDS_START = Integer.MAX_VALUE - 200;
+ @Test
+ public void testToBaseIdWithData() {
+ assertThat(StatsUtil.toBaseId(0)).isEqualTo(10_000);
+ assertThat(StatsUtil.toBaseId(1)).isEqualTo(10_200);
+ assertThat(StatsUtil.toBaseId(100)).isEqualTo(30_000);
+ assertThat(StatsUtil.toBaseId(999_949)).isEqualTo(199_999_800);
+ }
@Test
- public void statsIdsForTableColumns() {
- int offset = 0;
- for (int id = 0; id < StatsUtil.MAX_DATA_FIELD_ID; id++) {
- int statsFieldId = StatsUtil.statsFieldIdForField(id);
- int expected = StatsUtil.STATS_SPACE_FIELD_ID_START_FOR_DATA_FIELDS +
offset;
- assertThat(statsFieldId).as("at pos %s", id).isEqualTo(expected);
- offset += StatsUtil.NUM_SUPPORTED_STATS_PER_COLUMN;
- assertThat(StatsUtil.fieldIdForStatsField(statsFieldId)).as("at pos %s",
id).isEqualTo(id);
- }
+ public void testToBaseIdWithMetadata() {
+
assertThat(StatsUtil.toBaseId(MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId()))
+ .isEqualTo(9_000);
+
assertThat(StatsUtil.toBaseId(MetadataColumns.ROW_ID.fieldId())).isEqualTo(9_200);
+ }
- // also verify hardcoded field IDs from docs
- int fieldId = 0;
- int statsFieldId = 10_000;
-
assertThat(StatsUtil.statsFieldIdForField(fieldId)).isEqualTo(statsFieldId);
-
assertThat(StatsUtil.fieldIdForStatsField(statsFieldId)).isEqualTo(fieldId);
-
- fieldId = 1;
- statsFieldId = 10_200;
-
assertThat(StatsUtil.statsFieldIdForField(fieldId)).isEqualTo(statsFieldId);
-
assertThat(StatsUtil.fieldIdForStatsField(statsFieldId)).isEqualTo(fieldId);
-
- fieldId = 2;
- statsFieldId = 10_400;
-
assertThat(StatsUtil.statsFieldIdForField(fieldId)).isEqualTo(statsFieldId);
-
assertThat(StatsUtil.fieldIdForStatsField(statsFieldId)).isEqualTo(fieldId);
-
- fieldId = 5;
- statsFieldId = 11_000;
-
assertThat(StatsUtil.statsFieldIdForField(fieldId)).isEqualTo(statsFieldId);
-
assertThat(StatsUtil.fieldIdForStatsField(statsFieldId)).isEqualTo(fieldId);
-
- fieldId = 100;
- statsFieldId = 30_000;
-
assertThat(StatsUtil.statsFieldIdForField(fieldId)).isEqualTo(statsFieldId);
-
assertThat(StatsUtil.fieldIdForStatsField(statsFieldId)).isEqualTo(fieldId);
-
- fieldId = StatsUtil.MAX_DATA_FIELD_ID;
- statsFieldId = StatsUtil.MAX_DATA_STATS_FIELD_ID;
-
assertThat(StatsUtil.statsFieldIdForField(fieldId)).isEqualTo(statsFieldId);
-
assertThat(StatsUtil.fieldIdForStatsField(statsFieldId)).isEqualTo(fieldId);
-
- fieldId = -1;
- statsFieldId = -1;
-
assertThat(StatsUtil.statsFieldIdForField(fieldId)).isEqualTo(statsFieldId);
-
assertThat(StatsUtil.fieldIdForStatsField(statsFieldId)).isEqualTo(fieldId);
+ @Test
+ public void testBaseIdUnassigned() {
+ assertThat(StatsUtil.toBaseId(999_950)).isEqualTo(-1);
+ assertThat(StatsUtil.toBaseId(-1)).isEqualTo(-1);
+
assertThat(StatsUtil.toBaseId(MetadataColumns.FILE_PATH.fieldId())).isEqualTo(-1);
}
@Test
- public void statsIdsOverflowForTableColumns() {
- // pick 100 random IDs that are > MAX_FIELD_ID and <
RESERVED_FIELD_IDS_START as going over
- // the entire ID range takes too long
- int invalidFieldId = -1;
- for (int i = 0; i < 100; i++) {
- int id =
- ThreadLocalRandom.current()
- .nextInt(StatsUtil.MAX_DATA_FIELD_ID + 1,
RESERVED_FIELD_IDS_START);
- assertThat(StatsUtil.statsFieldIdForField(id)).as("at pos %s",
id).isEqualTo(invalidFieldId);
- }
+ public void testToFieldIdWithData() {
+ assertThat(StatsUtil.toFieldId(10_000)).isEqualTo(0);
+ assertThat(StatsUtil.toFieldId(10_200)).isEqualTo(1);
+ assertThat(StatsUtil.toFieldId(30_000)).isEqualTo(100);
+ assertThat(StatsUtil.toFieldId(199_999_800)).isEqualTo(999_949);
+ }
- assertThat(StatsUtil.fieldIdForStatsField(-1)).isEqualTo(invalidFieldId);
- assertThat(StatsUtil.fieldIdForStatsField(0)).isEqualTo(invalidFieldId);
- assertThat(StatsUtil.fieldIdForStatsField(200)).isEqualTo(invalidFieldId);
-
assertThat(StatsUtil.fieldIdForStatsField(5_000)).isEqualTo(invalidFieldId);
-
assertThat(StatsUtil.fieldIdForStatsField(8_600)).isEqualTo(invalidFieldId);
-
assertThat(StatsUtil.fieldIdForStatsField(10_001)).isEqualTo(invalidFieldId);
-
assertThat(StatsUtil.fieldIdForStatsField(10_201)).isEqualTo(invalidFieldId);
-
assertThat(StatsUtil.fieldIdForStatsField(10_500)).isEqualTo(invalidFieldId);
-
assertThat(StatsUtil.fieldIdForStatsField(10_900)).isEqualTo(invalidFieldId);
-
- // stats field IDs at or above the exclusive upper bound are invalid
-
assertThat(StatsUtil.fieldIdForStatsField(StatsUtil.STATS_SPACE_FIELD_ID_END))
- .isEqualTo(invalidFieldId);
-
assertThat(StatsUtil.fieldIdForStatsField(StatsUtil.STATS_SPACE_FIELD_ID_END +
200))
- .isEqualTo(invalidFieldId);
-
assertThat(StatsUtil.fieldIdForStatsField(Integer.MAX_VALUE)).isEqualTo(invalidFieldId);
-
- // field ID just past MAX_DATA_FIELD_ID is invalid
- assertThat(StatsUtil.statsFieldIdForField(StatsUtil.MAX_DATA_FIELD_ID + 1))
- .isEqualTo(invalidFieldId);
+ @Test
+ public void testToFieldIdWithMetadata() {
+ assertThat(StatsUtil.toFieldId(9_000))
+ .isEqualTo(MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId());
+
assertThat(StatsUtil.toFieldId(9_200)).isEqualTo(MetadataColumns.ROW_ID.fieldId());
}
@Test
- public void statsIdsForMetadataColumns() {
- int fieldId = MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId();
- int statsFieldId = 9_000;
-
assertThat(StatsUtil.statsFieldIdForField(fieldId)).isEqualTo(statsFieldId);
-
assertThat(StatsUtil.fieldIdForStatsField(statsFieldId)).isEqualTo(fieldId);
-
- fieldId = MetadataColumns.ROW_ID.fieldId();
- statsFieldId = 9_200;
-
assertThat(StatsUtil.statsFieldIdForField(fieldId)).isEqualTo(statsFieldId);
-
assertThat(StatsUtil.fieldIdForStatsField(statsFieldId)).isEqualTo(fieldId);
-
- // reserved metadata fields with IDs below/above the reserved stats range
have no stats
- int invalidFieldId = -1;
-
assertThat(StatsUtil.fieldIdForStatsField(8_800)).isEqualTo(invalidFieldId);
-
assertThat(StatsUtil.fieldIdForStatsField(9_400)).isEqualTo(invalidFieldId);
-
assertThat(StatsUtil.fieldIdForStatsField(9_600)).isEqualTo(invalidFieldId);
-
assertThat(StatsUtil.fieldIdForStatsField(9_800)).isEqualTo(invalidFieldId);
- assertThat(IntStream.range(RESERVED_FIELD_IDS_START, Integer.MAX_VALUE))
- .filteredOn(id -> !StatsUtil.SUPPORTED_METADATA_FIELD_IDS.contains(id))
- .allSatisfy(
- id ->
- assertThat(StatsUtil.statsFieldIdForField(id))
- .as("at pos %s", id)
- .isEqualTo(invalidFieldId));
+ public void testToFieldIdOutsideRange() {
+ assertThatThrownBy(() -> StatsUtil.toFieldId(200_000_000))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid stats field ID: 200000000");
+ assertThatThrownBy(() -> StatsUtil.toFieldId(200_000_001))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid stats field ID: 200000001");
+ assertThatThrownBy(() -> StatsUtil.toFieldId(8_800))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid stats field ID: 8800");
+ assertThatThrownBy(() -> StatsUtil.toFieldId(8_801))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid stats field ID: 8801");
}
@Test
- public void contentStatsForSimpleSchema() {
- Types.NestedField intField = required(0, "i", Types.IntegerType.get());
- Types.NestedField floatField = required(2, "f", Types.FloatType.get());
- Types.NestedField stringField = required(4, "s", Types.StringType.get());
- Types.NestedField booleanField = required(6, "b", Types.BooleanType.get());
- Types.NestedField uuidField = required(StatsUtil.MAX_DATA_FIELD_ID, "u",
Types.UUIDType.get());
- Schema schema = new Schema(intField, floatField, stringField,
booleanField, uuidField);
- Schema expectedStatsSchema =
- new Schema(
- optional(
- 146,
- "content_stats",
- Types.StructType.of(
- optional(10000, "i",
FieldStatistic.fieldStatsFor(intField, 10000)),
- optional(10400, "f",
FieldStatistic.fieldStatsFor(floatField, 10400)),
- optional(10800, "s",
FieldStatistic.fieldStatsFor(stringField, 10800)),
- optional(11200, "b",
FieldStatistic.fieldStatsFor(booleanField, 11200)),
- optional(
- StatsUtil.MAX_DATA_STATS_FIELD_ID,
- "u",
- FieldStatistic.fieldStatsFor(
- uuidField, StatsUtil.MAX_DATA_STATS_FIELD_ID)))));
- Schema statsSchema = new Schema(StatsUtil.contentStatsFor(schema));
-
assertThat(statsSchema.asStruct()).isEqualTo(expectedStatsSchema.asStruct());
+ public void testToFieldIdReservedMetadataRange() {
+ // 9,000 to 10,000 (exclusive) is reserved for metadata column stats
+ assertThatThrownBy(() -> StatsUtil.toFieldId(9_400))
+ .isInstanceOf(UnsupportedOperationException.class)
+ .hasMessage("Unsupported metadata stats field ID: 9400");
}
@Test
- public void contentStatsForComplexSchema() {
- Types.NestedField listElement = optional(3, "element",
Types.IntegerType.get());
- Types.NestedField structInt = optional(7, "int", Types.IntegerType.get());
- Types.NestedField structString = optional(8, "string",
Types.StringType.get());
- Types.NestedField mapKey = required(22, "key", Types.IntegerType.get());
- Types.NestedField mapValue = optional(24, "value", Types.StringType.get());
- Types.NestedField variantField = required(30, "variant",
Types.VariantType.get());
- Types.NestedField uuidField = required(StatsUtil.MAX_DATA_FIELD_ID, "u",
Types.UUIDType.get());
- Schema schema =
- new Schema(
- required(0, "i", Types.IntegerType.get()),
- required(2, "list", Types.ListType.ofOptional(3,
Types.IntegerType.get())),
- required(
- 6,
- "simple_struct",
- Types.StructType.of(
- optional(7, "int", Types.IntegerType.get()),
- optional(8, "string", Types.StringType.get()))),
- required(
- 20,
- "b",
- Types.MapType.ofOptional(22, 24, Types.IntegerType.get(),
Types.StringType.get())),
- variantField,
- uuidField);
- Schema expectedStatsSchema =
- new Schema(
- optional(
- 146,
- "content_stats",
- Types.StructType.of(
- optional(
- 10000,
- "i",
- FieldStatistic.fieldStatsFor(
- required(0, "i", Types.IntegerType.get()), 10000)),
- optional(
- 10600, "list.element",
FieldStatistic.fieldStatsFor(listElement, 10600)),
- optional(
- 11400, "simple_struct.int",
FieldStatistic.fieldStatsFor(structInt, 11400)),
- optional(
- 11600,
- "simple_struct.string",
- FieldStatistic.fieldStatsFor(structString, 11600)),
- optional(14400, "b.key",
FieldStatistic.fieldStatsFor(mapKey, 14400)),
- optional(14800, "b.value",
FieldStatistic.fieldStatsFor(mapValue, 14800)),
- optional(16000, "variant",
FieldStatistic.fieldStatsFor(variantField, 16000)),
- optional(
- StatsUtil.MAX_DATA_STATS_FIELD_ID,
- "u",
- FieldStatistic.fieldStatsFor(
- uuidField, StatsUtil.MAX_DATA_STATS_FIELD_ID)))));
- Schema statsSchema = new Schema(StatsUtil.contentStatsFor(schema));
-
assertThat(statsSchema.asStruct()).isEqualTo(expectedStatsSchema.asStruct());
+ public void testStatOffset() {
+ assertThat(StatsUtil.statOffset(10_000)).isEqualTo(0);
+ assertThat(StatsUtil.statOffset(10_201)).isEqualTo(1);
+ assertThat(StatsUtil.statOffset(10_211)).isEqualTo(11);
+ assertThat(StatsUtil.statOffset(10_399)).isEqualTo(199);
+ assertThat(StatsUtil.statOffset(10_400)).isEqualTo(0);
+ assertThat(StatsUtil.statOffset(199_999_999)).isEqualTo(199);
}
@Test
- public void contentStatsChildNamesAreUnique() {
- Schema schema =
- new Schema(
- required(1, "a", Types.StructType.of(required(2, "x",
Types.IntegerType.get()))),
- required(3, "b", Types.StructType.of(required(4, "x",
Types.IntegerType.get()))));
+ public void testStatOffsetOutsideRange() {
+ assertThatThrownBy(() -> StatsUtil.statOffset(200_000_000))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid stats field ID: 200000000");
+ assertThatThrownBy(() -> StatsUtil.statOffset(200_000_001))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid stats field ID: 200000001");
+ assertThatThrownBy(() -> StatsUtil.statOffset(8_800))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid stats field ID: 8800");
+ assertThatThrownBy(() -> StatsUtil.statOffset(8_801))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid stats field ID: 8801");
+ }
- Types.StructType contentStats =
StatsUtil.contentStatsFor(schema).type().asStructType();
+ private static final List<Type> FIXED_WIDTH_TYPES =
+ List.of(
+ Types.BooleanType.get(),
+ Types.IntegerType.get(),
+ Types.LongType.get(),
+ Types.DateType.get(),
+ Types.TimeType.get(),
+ Types.TimestampType.withoutZone(),
+ Types.TimestampType.withZone(),
+ Types.TimestampNanoType.withoutZone(),
+ Types.TimestampNanoType.withZone(),
+ Types.DecimalType.of(9, 2),
+ Types.UUIDType.get(),
+ Types.FixedType.ofLength(16));
+
+ @ParameterizedTest
+ @FieldSource("FIXED_WIDTH_TYPES")
+ public void testFixedWidthPrimitiveStruct(Type type) {
+ // fixed-width, non-floating-point types track bounds (including tight
bounds) but have no NaN
+ // count or average value size
+ Types.StructType expected =
+ Types.StructType.of(
+ Types.NestedField.optional(30_001, "lower_bound", type),
+ Types.NestedField.optional(30_002, "upper_bound", type),
+ Types.NestedField.optional(30_003, "tight_bounds",
Types.BooleanType.get()),
+ Types.NestedField.optional(30_004, "value_count",
Types.LongType.get()),
+ Types.NestedField.optional(30_005, "null_value_count",
Types.LongType.get()));
+
+ Types.StructType actual =
+ StatsUtil.fieldStatsStruct(true, type, 30_000,
MetricsModes.Full.get());
+
+ assertSameStructure(expected, actual);
+ }
-
assertThat(contentStats.fields().stream().map(Types.NestedField::name).toList())
- .doesNotHaveDuplicates()
- .containsExactly("a.x", "b.x");
+ private static final List<Type> VARIABLE_WIDTH_TYPES =
+ List.of(Types.StringType.get(), Types.BinaryType.get());
+
+ @ParameterizedTest
+ @FieldSource("VARIABLE_WIDTH_TYPES")
+ public void testVariableWidthPrimitiveStruct(Type type) {
+ // variable-width types track bounds (including tight bounds) and an
average value size, but no
+ // NaN count
+ Types.StructType expected =
+ Types.StructType.of(
+ Types.NestedField.optional(30_001, "lower_bound", type),
+ Types.NestedField.optional(30_002, "upper_bound", type),
+ Types.NestedField.optional(30_003, "tight_bounds",
Types.BooleanType.get()),
+ Types.NestedField.optional(30_004, "value_count",
Types.LongType.get()),
+ Types.NestedField.optional(30_005, "null_value_count",
Types.LongType.get()),
+ Types.NestedField.optional(30_007, "avg_value_size_in_bytes",
Types.IntegerType.get()));
+
+ Types.StructType actual =
+ StatsUtil.fieldStatsStruct(true, type, 30_000,
MetricsModes.Full.get());
+
+ assertSameStructure(expected, actual);
}
- @Test
- public void contentStatsSkipsFieldsOutsideStatsRange() {
- Types.NestedField validField = required(0, "i", Types.IntegerType.get());
- Types.NestedField outOfRangeField =
- required(StatsUtil.MAX_DATA_FIELD_ID + 1, "out_of_range",
Types.IntegerType.get());
- Schema schema = new Schema(validField, outOfRangeField);
- Schema expectedStatsSchema =
- new Schema(
- optional(
- 146,
- "content_stats",
- Types.StructType.of(
- optional(10000, "i",
FieldStatistic.fieldStatsFor(validField, 10000)))));
- Schema statsSchema = new Schema(StatsUtil.contentStatsFor(schema));
-
assertThat(statsSchema.asStruct()).isEqualTo(expectedStatsSchema.asStruct());
+ private static final List<Type> GEO_TYPES =
+ List.of(
+ Types.GeometryType.crs84(),
+ Types.GeometryType.of("srid:3857"),
+ Types.GeographyType.crs84(),
+ Types.GeographyType.of("srid:4269"));
+
+ @ParameterizedTest
+ @FieldSource("GEO_TYPES")
+ public void testGeoStruct(Type type) {
+ // geometry and geography use bounding-box structs for their bounds, do
not track tight bounds,
+ // and record an average value size
+ Types.StructType lowerBound =
+ Types.StructType.of(
+ Types.NestedField.required(30_010, "x", Types.DoubleType.get()),
+ Types.NestedField.required(30_011, "y", Types.DoubleType.get()),
+ Types.NestedField.optional(30_012, "z", Types.DoubleType.get()),
+ Types.NestedField.optional(30_013, "m", Types.DoubleType.get()));
+ Types.StructType upperBound =
+ Types.StructType.of(
+ Types.NestedField.required(30_014, "x", Types.DoubleType.get()),
+ Types.NestedField.required(30_015, "y", Types.DoubleType.get()),
+ Types.NestedField.optional(30_016, "z", Types.DoubleType.get()),
+ Types.NestedField.optional(30_017, "m", Types.DoubleType.get()));
+ Types.StructType expected =
+ Types.StructType.of(
+ Types.NestedField.optional(30_001, "lower_bound", lowerBound),
+ Types.NestedField.optional(30_002, "upper_bound", upperBound),
+ Types.NestedField.optional(30_004, "value_count",
Types.LongType.get()),
+ Types.NestedField.optional(30_005, "null_value_count",
Types.LongType.get()),
+ Types.NestedField.optional(30_007, "avg_value_size_in_bytes",
Types.IntegerType.get()));
+
+ Types.StructType actual =
+ StatsUtil.fieldStatsStruct(true, type, 30_000,
MetricsModes.Full.get());
+
+ assertSameStructure(expected, actual);
}
- @Test
- public void conditionalFieldInclusionForInteger() {
- assertThat(
- fieldStatsNames(
- FieldStatistic.fieldStatsFor(required(1, "x",
Types.IntegerType.get()), 10000)))
- .containsExactly(
- LOWER_BOUND.fieldName(),
- UPPER_BOUND.fieldName(),
- TIGHT_BOUNDS.fieldName(),
- VALUE_COUNT.fieldName())
- .doesNotContain(
- NULL_VALUE_COUNT.fieldName(),
- NAN_VALUE_COUNT.fieldName(),
- AVG_VALUE_SIZE_IN_BYTES.fieldName());
+ private static final List<Type> FLOATING_POINT_TYPES =
+ List.of(Types.FloatType.get(), Types.DoubleType.get());
+
+ @ParameterizedTest
+ @FieldSource("FLOATING_POINT_TYPES")
+ public void testFloatingPointStruct(Type type) {
+ // floating-point types track bounds (including tight bounds) and a NaN
count, but have no
+ // average value size
+ Types.StructType expected =
+ Types.StructType.of(
+ Types.NestedField.optional(30_001, "lower_bound", type),
+ Types.NestedField.optional(30_002, "upper_bound", type),
+ Types.NestedField.optional(30_003, "tight_bounds",
Types.BooleanType.get()),
+ Types.NestedField.optional(30_004, "value_count",
Types.LongType.get()),
+ Types.NestedField.optional(30_005, "null_value_count",
Types.LongType.get()),
+ Types.NestedField.optional(30_006, "nan_value_count",
Types.LongType.get()));
+
+ Types.StructType actual =
+ StatsUtil.fieldStatsStruct(true, type, 30_000,
MetricsModes.Full.get());
+
+ assertSameStructure(expected, actual);
+ }
- assertThat(
- fieldStatsNames(
- FieldStatistic.fieldStatsFor(optional(1, "x",
Types.IntegerType.get()), 10000)))
- .containsExactly(
- LOWER_BOUND.fieldName(),
- UPPER_BOUND.fieldName(),
- TIGHT_BOUNDS.fieldName(),
- VALUE_COUNT.fieldName(),
- NULL_VALUE_COUNT.fieldName())
- .doesNotContain(NAN_VALUE_COUNT.fieldName(),
AVG_VALUE_SIZE_IN_BYTES.fieldName());
+ private static final Types.StructType POINT =
+ Types.StructType.of(
+ Types.NestedField.required(1, "x", Types.FloatType.get()),
+ Types.NestedField.required(2, "y", Types.FloatType.get()));
+
+ private static final List<Type> NESTED_TYPES =
+ List.of(
+ Types.ListType.ofRequired(1, Types.IntegerType.get()),
+ Types.ListType.ofOptional(1, POINT),
+ Types.MapType.ofRequired(1, 2, Types.StringType.get(),
Types.StringType.get()),
+ Types.MapType.ofOptional(1, 2, Types.StringType.get(), POINT));
+
+ @ParameterizedTest
+ @FieldSource("NESTED_TYPES")
+ public void testNestedTypesHaveNoStats(Type type) {
+ // list and map types are not tracked and produce no stats struct
+ assertThat(StatsUtil.fieldStatsStruct(true, type, 30_000,
MetricsModes.Full.get())).isNull();
}
@Test
- public void conditionalFieldInclusionForFloatAndDouble() {
- assertThat(
- fieldStatsNames(
- FieldStatistic.fieldStatsFor(required(1, "x",
Types.FloatType.get()), 10000)))
- .containsExactly(
- LOWER_BOUND.fieldName(),
- UPPER_BOUND.fieldName(),
- TIGHT_BOUNDS.fieldName(),
- VALUE_COUNT.fieldName(),
- NAN_VALUE_COUNT.fieldName())
- .doesNotContain(NULL_VALUE_COUNT.fieldName(),
AVG_VALUE_SIZE_IN_BYTES.fieldName());
-
- assertThat(
- fieldStatsNames(
- FieldStatistic.fieldStatsFor(optional(1, "x",
Types.DoubleType.get()), 10000)))
- .containsExactly(
- LOWER_BOUND.fieldName(),
- UPPER_BOUND.fieldName(),
- TIGHT_BOUNDS.fieldName(),
- VALUE_COUNT.fieldName(),
- NULL_VALUE_COUNT.fieldName(),
- NAN_VALUE_COUNT.fieldName());
+ public void testRequiredField() {
+ // a required column does not produce a null_value_count field
+ Type type = Types.IntegerType.get();
+ Types.StructType expected =
+ Types.StructType.of(
+ Types.NestedField.optional(30_001, "lower_bound", type),
+ Types.NestedField.optional(30_002, "upper_bound", type),
+ Types.NestedField.optional(30_003, "tight_bounds",
Types.BooleanType.get()),
+ Types.NestedField.optional(30_004, "value_count",
Types.LongType.get()));
+
+ Types.StructType actual =
+ StatsUtil.fieldStatsStruct(false, type, 30_000,
MetricsModes.Full.get());
+
+ assertSameStructure(expected, actual);
}
@Test
- public void conditionalFieldInclusionForString() {
- assertThat(
- fieldStatsNames(
- FieldStatistic.fieldStatsFor(required(1, "x",
Types.StringType.get()), 10000)))
- .containsExactly(
- LOWER_BOUND.fieldName(),
- UPPER_BOUND.fieldName(),
- TIGHT_BOUNDS.fieldName(),
- VALUE_COUNT.fieldName(),
- AVG_VALUE_SIZE_IN_BYTES.fieldName())
- .doesNotContain(NULL_VALUE_COUNT.fieldName(),
NAN_VALUE_COUNT.fieldName());
-
+ public void testStringNoneMode() {
+ // none mode produces no stats struct
assertThat(
- fieldStatsNames(
- FieldStatistic.fieldStatsFor(optional(1, "x",
Types.StringType.get()), 10000)))
- .containsExactly(
- LOWER_BOUND.fieldName(),
- UPPER_BOUND.fieldName(),
- TIGHT_BOUNDS.fieldName(),
- VALUE_COUNT.fieldName(),
- NULL_VALUE_COUNT.fieldName(),
- AVG_VALUE_SIZE_IN_BYTES.fieldName());
+ StatsUtil.fieldStatsStruct(
+ true, Types.StringType.get(), 30_000, MetricsModes.None.get()))
+ .isNull();
}
@Test
- public void conditionalFieldInclusionForBinary() {
- assertThat(
- fieldStatsNames(
- FieldStatistic.fieldStatsFor(optional(1, "x",
Types.BinaryType.get()), 10000)))
- .containsExactly(
- LOWER_BOUND.fieldName(),
- UPPER_BOUND.fieldName(),
- TIGHT_BOUNDS.fieldName(),
- VALUE_COUNT.fieldName(),
- NULL_VALUE_COUNT.fieldName(),
- AVG_VALUE_SIZE_IN_BYTES.fieldName())
- .doesNotContain(NAN_VALUE_COUNT.fieldName());
+ public void testStringCountsMode() {
+ // counts mode drops the bounds (and tight_bounds) but keeps counts and
the value size
+ Types.StructType expected =
+ Types.StructType.of(
+ Types.NestedField.optional(30_004, "value_count",
Types.LongType.get()),
+ Types.NestedField.optional(30_005, "null_value_count",
Types.LongType.get()),
+ Types.NestedField.optional(30_007, "avg_value_size_in_bytes",
Types.IntegerType.get()));
+
+ Types.StructType actual =
+ StatsUtil.fieldStatsStruct(true, Types.StringType.get(), 30_000,
MetricsModes.Counts.get());
+
+ assertSameStructure(expected, actual);
+ }
- assertThat(
- fieldStatsNames(
- FieldStatistic.fieldStatsFor(required(1, "x",
Types.BinaryType.get()), 10000)))
- .containsExactly(
- LOWER_BOUND.fieldName(),
- UPPER_BOUND.fieldName(),
- TIGHT_BOUNDS.fieldName(),
- VALUE_COUNT.fieldName(),
- AVG_VALUE_SIZE_IN_BYTES.fieldName())
- .doesNotContain(NULL_VALUE_COUNT.fieldName(),
NAN_VALUE_COUNT.fieldName());
+ private static final List<MetricsModes.MetricsMode> BOUNDS_MODES =
+ List.of(MetricsModes.Truncate.withLength(16), MetricsModes.Full.get());
+
+ @ParameterizedTest
+ @FieldSource("BOUNDS_MODES")
+ public void testStringBoundsModes(MetricsModes.MetricsMode mode) {
+ // truncate and full both retain the full bounds struct
+ Type string = Types.StringType.get();
+ Types.StructType expected =
+ Types.StructType.of(
+ Types.NestedField.optional(30_001, "lower_bound", string),
+ Types.NestedField.optional(30_002, "upper_bound", string),
+ Types.NestedField.optional(30_003, "tight_bounds",
Types.BooleanType.get()),
+ Types.NestedField.optional(30_004, "value_count",
Types.LongType.get()),
+ Types.NestedField.optional(30_005, "null_value_count",
Types.LongType.get()),
+ Types.NestedField.optional(30_007, "avg_value_size_in_bytes",
Types.IntegerType.get()));
+
+ Types.StructType actual = StatsUtil.fieldStatsStruct(true, string, 30_000,
mode);
+
+ assertSameStructure(expected, actual);
}
@Test
- public void conditionalFieldInclusionForGeometry() {
- Types.StructType requiredStats =
- FieldStatistic.fieldStatsFor(required(1, "x",
Types.GeometryType.crs84()), 10000);
- assertThat(fieldStatsNames(requiredStats))
- .containsExactly(LOWER_BOUND.fieldName(), UPPER_BOUND.fieldName(),
VALUE_COUNT.fieldName())
- .doesNotContain(
- TIGHT_BOUNDS.fieldName(),
- NULL_VALUE_COUNT.fieldName(),
- NAN_VALUE_COUNT.fieldName(),
- AVG_VALUE_SIZE_IN_BYTES.fieldName());
- assertGeoBoundStructs(requiredStats, 10000);
-
- Types.StructType optionalStats =
- FieldStatistic.fieldStatsFor(optional(1, "x",
Types.GeometryType.crs84()), 10000);
- assertThat(fieldStatsNames(optionalStats))
- .containsExactly(
- LOWER_BOUND.fieldName(),
- UPPER_BOUND.fieldName(),
- VALUE_COUNT.fieldName(),
- NULL_VALUE_COUNT.fieldName())
- .doesNotContain(
- TIGHT_BOUNDS.fieldName(),
- NAN_VALUE_COUNT.fieldName(),
- AVG_VALUE_SIZE_IN_BYTES.fieldName());
- assertGeoBoundStructs(optionalStats, 10000);
+ public void testContentStatsReadSchema() {
Review Comment:
I'll add a test, but the majority of many of these cases aren't relevant
because the field types are tested directly.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
