rdblue commented on code in PR #18604:
URL: https://github.com/apache/iceberg/pull/18604#discussion_r4462230804
##########
core/src/test/java/org/apache/iceberg/TestStatsUtil.java:
##########
@@ -564,6 +572,293 @@ public void testContentStatsWriteSchemaOmitsListAndMap() {
assertThat(actual.fields()).isEmpty();
}
+ @Test
+ void identityBounds() {
+ assertThat(sourceBounds(Types.StringType.get(), Transforms.identity(),
"books"))
+ .isEqualTo(new StatsUtil.PartitionSourceBounds<>("books", "books"));
+ }
+
+ @Test
+ void identityNaN() {
+ assertThat(sourceBounds(Types.FloatType.get(), Transforms.identity(),
Float.NaN)).isNull();
+ assertThat(sourceBounds(Types.DoubleType.get(), Transforms.identity(),
Double.NaN)).isNull();
+ }
+
+ private static Stream<Partition> truncatedNumbers() {
+ Type decimal = Types.DecimalType.of(9, 2);
+ return Stream.of(
+ partition(Types.IntegerType.get(), Transforms.truncate(10),
20).withExpectedBounds(20, 29),
+ partition(Types.IntegerType.get(), Transforms.truncate(10), -10)
+ .withExpectedBounds(-10, -1),
+ partition(Types.LongType.get(), Transforms.truncate(100), 12_300L)
+ .withExpectedBounds(12_300L, 12_399L),
+ partition(decimal, Transforms.truncate(50), new BigDecimal("12.00"))
+ .withExpectedBounds(new BigDecimal("12.00"), new
BigDecimal("12.49")),
+ partition(decimal, Transforms.truncate(50), new BigDecimal("-0.50"))
+ .withExpectedBounds(new BigDecimal("-0.50"), new
BigDecimal("-0.01")));
+ }
+
+ @ParameterizedTest
+ @MethodSource("truncatedNumbers")
+ void truncateNumberBounds(Partition partition) {
+ assertThat(sourceBounds(partition)).isEqualTo(partition.expected());
+ }
+
+ @Test
+ void truncateNumberNearMax() {
+ assertThat(sourceBounds(Types.IntegerType.get(), Transforms.truncate(10),
2_147_483_640))
+ .isEqualTo(new StatsUtil.PartitionSourceBounds<>(2_147_483_640,
Integer.MAX_VALUE));
+ assertThat(sourceBounds(Types.LongType.get(), Transforms.truncate(100),
Long.MAX_VALUE - 7))
+ .isEqualTo(new StatsUtil.PartitionSourceBounds<>(Long.MAX_VALUE - 7,
Long.MAX_VALUE));
+ assertThat(
+ sourceBounds(
+ Types.DecimalType.of(9, 2), Transforms.truncate(7), new
BigDecimal("9999999.94")))
+ .isEqualTo(
+ new StatsUtil.PartitionSourceBounds<>(
+ new BigDecimal("9999999.94"), new BigDecimal("9999999.99")));
+ }
+
+ @Test
+ void truncateNumberAtMinWraps() {
+ assertThat(sourceBounds(Types.IntegerType.get(), Transforms.truncate(10),
2_147_483_646))
+ .isEqualTo(new StatsUtil.PartitionSourceBounds<>(Integer.MIN_VALUE,
Integer.MIN_VALUE + 7));
+ assertThat(
+ sourceBounds(Types.LongType.get(), Transforms.truncate(10),
9_223_372_036_854_775_806L))
+ .isEqualTo(new StatsUtil.PartitionSourceBounds<>(Long.MIN_VALUE,
Long.MIN_VALUE + 7));
+ }
+
+ @Test
+ void truncateNumberAtMinWithPowerOfTwoWidth() {
+ assertThat(sourceBounds(Types.IntegerType.get(), Transforms.truncate(8),
Integer.MIN_VALUE))
+ .isEqualTo(new StatsUtil.PartitionSourceBounds<>(Integer.MIN_VALUE,
Integer.MIN_VALUE + 7));
+ assertThat(sourceBounds(Types.LongType.get(), Transforms.truncate(8),
Long.MIN_VALUE))
+ .isEqualTo(new StatsUtil.PartitionSourceBounds<>(Long.MIN_VALUE,
Long.MIN_VALUE + 7));
+ }
+
+ @Test
+ void truncateStringBounds() {
+ assertThat(sourceBounds(Types.StringType.get(), Transforms.truncate(3),
"abc"))
+ .isEqualTo(new StatsUtil.PartitionSourceBounds<>("abc", "abd"));
+ }
+
+ @Test
+ void truncateShortString() {
+ assertThat(sourceBounds(Types.StringType.get(), Transforms.truncate(3),
"ab"))
+ .isEqualTo(new StatsUtil.PartitionSourceBounds<>("ab", "ab"));
+ }
+
+ @Test
+ void truncateShortStringWithSurrogatePair() {
+ String value = "ab" + new String(Character.toChars(0x1F4B0));
+
+ assertThat(sourceBounds(Types.StringType.get(), Transforms.truncate(4),
value))
+ .isEqualTo(new StatsUtil.PartitionSourceBounds<>(value, value));
+ }
+
+ @Test
+ void truncateStringAtMaxCodePoint() {
+ String maxCodePoint = new
String(Character.toChars(Character.MAX_CODE_POINT));
+
+ assertThat(sourceBounds(Types.StringType.get(), Transforms.truncate(1),
maxCodePoint))
+ .isEqualTo(new StatsUtil.PartitionSourceBounds<>(maxCodePoint, null));
+ }
+
+ @Test
+ void truncateBinaryBounds() {
+ ByteBuffer prefix = ByteBuffer.wrap(new byte[] {1, 2});
+
+ assertThat(sourceBounds(Types.BinaryType.get(), Transforms.truncate(2),
prefix))
+ .isEqualTo(
+ new StatsUtil.PartitionSourceBounds<>(prefix, ByteBuffer.wrap(new
byte[] {1, 3})));
+ }
+
+ @Test
+ void truncateShortBinary() {
+ ByteBuffer value = ByteBuffer.wrap(new byte[] {1});
+
+ assertThat(sourceBounds(Types.BinaryType.get(), Transforms.truncate(2),
value))
+ .isEqualTo(new StatsUtil.PartitionSourceBounds<>(value, value));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"identity", "truncate[2]"})
+ void binaryBoundsDoNotSharePosition(String transform) {
+ StatsUtil.PartitionSourceBounds<?> bounds =
+ sourceBounds(
+ Types.BinaryType.get(),
+ Transforms.fromString(transform),
+ ByteBuffer.wrap(new byte[] {1}));
+
+ ((ByteBuffer) bounds.lower()).get();
+
+ assertThat((ByteBuffer) bounds.upper()).isEqualTo(ByteBuffer.wrap(new
byte[] {1}));
+ }
+
+ @Test
+ void truncateBinaryAtMaxBytes() {
+ ByteBuffer prefix = ByteBuffer.wrap(new byte[] {(byte) 0xFF, (byte) 0xFF});
+
+ assertThat(sourceBounds(Types.BinaryType.get(), Transforms.truncate(2),
prefix))
+ .isEqualTo(new StatsUtil.PartitionSourceBounds<>(prefix, null));
+ }
+
+ private static Stream<Partition> timePartitionValues() {
+ Type date = Types.DateType.get();
+ Type timestamp = Types.TimestampType.withZone();
+ Type timestampNano = Types.TimestampNanoType.withZone();
+ return Stream.of(
+ partition(date, Transforms.year(), years("2023-01-01"))
+ .withExpectedBounds(days("2023-01-01"), days("2023-12-31")),
+ partition(date, Transforms.month(), months("2023-02-01"))
+ .withExpectedBounds(days("2023-02-01"), days("2023-02-28")),
+ partition(date, Transforms.day(), days("2023-02-10"))
+ .withExpectedBounds(days("2023-02-10"), days("2023-02-10")),
+ partition(timestamp, Transforms.year(), years("2023-01-01"))
+ .withExpectedBounds(
+ micros("2023-01-01T00:00:00Z"),
micros("2023-12-31T23:59:59.999999Z")),
+ partition(timestamp, Transforms.month(), months("2023-02-01"))
+ .withExpectedBounds(
+ micros("2023-02-01T00:00:00Z"),
micros("2023-02-28T23:59:59.999999Z")),
+ partition(timestamp, Transforms.day(), days("2023-02-10"))
+ .withExpectedBounds(
+ micros("2023-02-10T00:00:00Z"),
micros("2023-02-10T23:59:59.999999Z")),
+ partition(timestamp, Transforms.hour(), hours("2023-02-10T05:00:00Z"))
+ .withExpectedBounds(
+ micros("2023-02-10T05:00:00Z"),
micros("2023-02-10T05:59:59.999999Z")),
+ partition(timestampNano, Transforms.hour(),
hours("2023-02-10T05:00:00Z"))
+ .withExpectedBounds(
+ nanos("2023-02-10T05:00:00Z"),
nanos("2023-02-10T05:59:59.999999999Z")));
+ }
+
+ @ParameterizedTest
+ @MethodSource("timePartitionValues")
+ void timeBounds(Partition partition) {
+ assertThat(sourceBounds(partition)).isEqualTo(partition.expected());
+ }
+
+ private static Stream<Partition> timePartitionValuesAtOrBefore1970() {
+ Type timestamp = Types.TimestampType.withZone();
+ return Stream.of(
+ partition(timestamp, Transforms.year(), 0)
Review Comment:
The `day` transforms are fine though.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
