Fokko commented on code in PR #4407:
URL: https://github.com/apache/iceberg-python/pull/4407#discussion_r4619290166


##########
pyiceberg/transforms.py:
##########
@@ -1030,6 +1035,88 @@ def pyarrow_transform(self, source: IcebergType) -> 
"Callable[[pa.Array], pa.Arr
         return lambda arr: pa.nulls(len(arr), type=arr.type)
 
 
+def _integer_bounds(source: IcebergType) -> tuple[int, int]:
+    """Return the minimum value and the size of the range of an integer 
type."""
+    if isinstance(source, IntegerType):
+        return -(2**31), 2**32
+    return -(2**63), 2**64
+
+
+def _wrap_integer(value: int, source: IcebergType) -> int:
+    """Wrap an integer into the range of the source type, like Java's two's 
complement arithmetic."""
+    minimum, modulus = _integer_bounds(source)
+    return (value - minimum) % modulus + minimum
+
+
+def _truncate_integer_projection(
+    name: str,
+    pred: BoundLiteralPredicate,
+    transform: Callable[[Any | None], Any | None],
+    source: IcebergType,
+    width: int,
+) -> UnboundPredicate | None:
+    """Project a predicate on a truncate partition of an integer column.
+
+    The values from the minimum of the source type truncate to a partition 
value at the top of the
+    type's range, which does not sort with the other partition values. A range 
that can match those
+    values therefore also matches that partition.
+    """
+    minimum, _ = _integer_bounds(source)
+    floor_mod = minimum % width
+    if floor_mod == 0 or not isinstance(pred.literal, LongLiteral):
+        return _truncate_number(name, pred, transform)
+
+    wrapped_minimum = _wrap_integer(minimum - floor_mod, source)
+    # The first partition value after the block that holds the minimum of the 
source type
+    next_block_start = minimum - floor_mod + width
+    boundary = pred.literal.value
+
+    if isinstance(pred, BoundGreaterThan):
+        if boundary + 1 < next_block_start:
+            return Or(GreaterThanOrEqual(Reference(name), next_block_start), 
EqualTo(Reference(name), wrapped_minimum))
+        return _truncate_number(name, pred, transform)
+    elif isinstance(pred, BoundGreaterThanOrEqual):
+        if boundary < next_block_start:
+            return Or(GreaterThanOrEqual(Reference(name), next_block_start), 
EqualTo(Reference(name), wrapped_minimum))
+        return _truncate_number(name, pred, transform)
+
+    projected = _truncate_number(name, pred, transform)
+    if isinstance(pred, (BoundLessThan, BoundLessThanOrEqual)) and projected 
is not None:
+        return Or(projected, EqualTo(Reference(name), wrapped_minimum))

Review Comment:
   Nit: when the boundary falls in the block that holds the minimum (for 
example `x < -2**31 + 100` with `truncate[1000]`), `_truncate_number` projects 
to `LessThanOrEqual(name, wrapped_minimum)`, which keeps almost every 
partition. Only the wrapped partition can match there, so this could return 
`EqualTo(Reference(name), wrapped_minimum)` alone. That is the case 
`test_count_truncate_partitions_near_the_minimum` exercises.



##########
pyiceberg/transforms.py:
##########
@@ -1030,6 +1035,88 @@ def pyarrow_transform(self, source: IcebergType) -> 
"Callable[[pa.Array], pa.Arr
         return lambda arr: pa.nulls(len(arr), type=arr.type)
 
 
+def _integer_bounds(source: IcebergType) -> tuple[int, int]:
+    """Return the minimum value and the size of the range of an integer 
type."""
+    if isinstance(source, IntegerType):
+        return -(2**31), 2**32
+    return -(2**63), 2**64
+
+
+def _wrap_integer(value: int, source: IcebergType) -> int:
+    """Wrap an integer into the range of the source type, like Java's two's 
complement arithmetic."""
+    minimum, modulus = _integer_bounds(source)
+    return (value - minimum) % modulus + minimum
+
+
+def _truncate_integer_projection(
+    name: str,
+    pred: BoundLiteralPredicate,
+    transform: Callable[[Any | None], Any | None],
+    source: IcebergType,
+    width: int,
+) -> UnboundPredicate | None:
+    """Project a predicate on a truncate partition of an integer column.
+
+    The values from the minimum of the source type truncate to a partition 
value at the top of the
+    type's range, which does not sort with the other partition values. A range 
that can match those
+    values therefore also matches that partition.
+    """
+    minimum, _ = _integer_bounds(source)
+    floor_mod = minimum % width
+    if floor_mod == 0 or not isinstance(pred.literal, LongLiteral):
+        return _truncate_number(name, pred, transform)
+
+    wrapped_minimum = _wrap_integer(minimum - floor_mod, source)
+    # The first partition value after the block that holds the minimum of the 
source type
+    next_block_start = minimum - floor_mod + width
+    boundary = pred.literal.value
+
+    if isinstance(pred, BoundGreaterThan):
+        if boundary + 1 < next_block_start:
+            return Or(GreaterThanOrEqual(Reference(name), next_block_start), 
EqualTo(Reference(name), wrapped_minimum))

Review Comment:
   Nit: the `EqualTo` here is redundant. `wrapped_minimum` is always greater 
than `next_block_start` (they differ by the size of the type's range minus the 
width), so `GreaterThanOrEqual(Reference(name), next_block_start)` already 
includes it. Returning just that keeps the projection an `UnboundPredicate`, so 
the residual evaluator can still use it to prune. Same for the 
`BoundGreaterThanOrEqual` branch below.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to