kevinjqliu commented on code in PR #3547:
URL: https://github.com/apache/iceberg-python/pull/3547#discussion_r3449427409
##########
tests/expressions/test_evaluator.py:
##########
@@ -1152,6 +1152,48 @@ def test_strict_some_nulls(strict_data_file_schema:
Schema, strict_data_file_2:
assert not should_read, "Should not match: equal on some nulls column"
+def test_strict_not_equal_and_not_in_with_mixed_nulls_and_matching_bounds() ->
None:
+ schema = Schema(NestedField(1, "x", IntegerType(), required=False))
+ data_file = DataFile.from_args(
+ file_path="file.parquet",
+ file_format=FileFormat.PARQUET,
+ partition={},
+ record_count=2,
+ file_size_in_bytes=1,
+ value_counts={1: 2},
+ null_value_counts={1: 1},
+ nan_value_counts=None,
+ lower_bounds={1: to_bytes(IntegerType(), 5)},
+ upper_bounds={1: to_bytes(IntegerType(), 5)},
+ )
+
+ should_read = _StrictMetricsEvaluator(schema, NotEqualTo("x",
5)).eval(data_file)
+ assert should_read == ROWS_MIGHT_NOT_MATCH, "Should not match: bounds
prove the non-null value is 5"
+
+ should_read = _StrictMetricsEvaluator(schema, NotIn("x", {5,
6})).eval(data_file)
+ assert should_read == ROWS_MIGHT_NOT_MATCH, "Should not match: bounds
prove the non-null value is 5"
+
+
+def test_strict_not_equal_and_not_in_with_all_nulls() -> None:
+ schema = Schema(NestedField(1, "x", IntegerType(), required=False))
+ data_file = DataFile.from_args(
+ file_path="file.parquet",
+ file_format=FileFormat.PARQUET,
+ partition={},
+ record_count=2,
+ file_size_in_bytes=1,
+ value_counts={1: 2},
+ null_value_counts={1: 2},
+ nan_value_counts=None,
+ )
+
+ should_read = _StrictMetricsEvaluator(schema, NotEqualTo("x",
5)).eval(data_file)
Review Comment:
`value_count = 2` and `null_count = 2`, so all values are null. That means
every row matches `NotEqualTo("x", 5)` / `NotIn("x", {5, 6})`.
##########
tests/expressions/test_evaluator.py:
##########
@@ -1198,6 +1240,50 @@ def test_strict_not_nan(strict_data_file_schema: Schema,
strict_data_file_1: Dat
assert not should_read, "Should not match: null values are not nan"
[email protected]("field_type", [FloatType(), DoubleType()])
+def
test_strict_not_equal_and_not_in_with_mixed_nans_and_matching_bounds(field_type:
PrimitiveType) -> None:
+ schema = Schema(NestedField(1, "x", field_type, required=False))
+ data_file = DataFile.from_args(
+ file_path="file.parquet",
+ file_format=FileFormat.PARQUET,
+ partition={},
+ record_count=2,
+ file_size_in_bytes=1,
+ value_counts={1: 2},
+ null_value_counts={1: 0},
+ nan_value_counts={1: 1},
+ lower_bounds={1: to_bytes(field_type, 5.0)},
+ upper_bounds={1: to_bytes(field_type, 5.0)},
+ )
+
+ should_read = _StrictMetricsEvaluator(schema, NotEqualTo("x",
5.0)).eval(data_file)
Review Comment:
`value_count = 2` and `nan_count = 1`, so there is 1 non-NaN value. Bounds
`[5.0..5.0]` mean the non-NaN value is `5.0`, so `NotEqualTo("x", 5.0)` /
`NotIn("x", {5.0, 6.0})` cannot match every row.
##########
tests/expressions/test_evaluator.py:
##########
@@ -1198,6 +1240,50 @@ def test_strict_not_nan(strict_data_file_schema: Schema,
strict_data_file_1: Dat
assert not should_read, "Should not match: null values are not nan"
[email protected]("field_type", [FloatType(), DoubleType()])
+def
test_strict_not_equal_and_not_in_with_mixed_nans_and_matching_bounds(field_type:
PrimitiveType) -> None:
+ schema = Schema(NestedField(1, "x", field_type, required=False))
+ data_file = DataFile.from_args(
+ file_path="file.parquet",
+ file_format=FileFormat.PARQUET,
+ partition={},
+ record_count=2,
+ file_size_in_bytes=1,
+ value_counts={1: 2},
+ null_value_counts={1: 0},
+ nan_value_counts={1: 1},
+ lower_bounds={1: to_bytes(field_type, 5.0)},
+ upper_bounds={1: to_bytes(field_type, 5.0)},
+ )
+
+ should_read = _StrictMetricsEvaluator(schema, NotEqualTo("x",
5.0)).eval(data_file)
+ assert should_read == ROWS_MIGHT_NOT_MATCH, "Should not match: bounds
prove the non-NaN value is 5.0"
+
+ should_read = _StrictMetricsEvaluator(schema, NotIn("x", {5.0,
6.0})).eval(data_file)
+ assert should_read == ROWS_MIGHT_NOT_MATCH, "Should not match: bounds
prove the non-NaN value is 5.0"
+
+
[email protected]("field_type", [FloatType(), DoubleType()])
+def test_strict_not_equal_and_not_in_with_all_nans(field_type: PrimitiveType)
-> None:
+ schema = Schema(NestedField(1, "x", field_type, required=False))
+ data_file = DataFile.from_args(
+ file_path="file.parquet",
+ file_format=FileFormat.PARQUET,
+ partition={},
+ record_count=2,
+ file_size_in_bytes=1,
+ value_counts={1: 2},
+ null_value_counts={1: 0},
+ nan_value_counts={1: 2},
+ )
+
+ should_read = _StrictMetricsEvaluator(schema, NotEqualTo("x",
5.0)).eval(data_file)
Review Comment:
`value_count = 2` and `nan_count = 2`, so all values are NaN. That means
every row matches `NotEqualTo("x", 5.0)` / `NotIn("x", {5.0, 6.0})`.
##########
tests/expressions/test_evaluator.py:
##########
@@ -1152,6 +1152,48 @@ def test_strict_some_nulls(strict_data_file_schema:
Schema, strict_data_file_2:
assert not should_read, "Should not match: equal on some nulls column"
+def test_strict_not_equal_and_not_in_with_mixed_nulls_and_matching_bounds() ->
None:
+ schema = Schema(NestedField(1, "x", IntegerType(), required=False))
+ data_file = DataFile.from_args(
+ file_path="file.parquet",
+ file_format=FileFormat.PARQUET,
+ partition={},
+ record_count=2,
+ file_size_in_bytes=1,
+ value_counts={1: 2},
+ null_value_counts={1: 1},
+ nan_value_counts=None,
+ lower_bounds={1: to_bytes(IntegerType(), 5)},
+ upper_bounds={1: to_bytes(IntegerType(), 5)},
+ )
+
+ should_read = _StrictMetricsEvaluator(schema, NotEqualTo("x",
5)).eval(data_file)
+ assert should_read == ROWS_MIGHT_NOT_MATCH, "Should not match: bounds
prove the non-null value is 5"
Review Comment:
`value_count = 2` and `null_count = 1`, so there is 1 non-null value. Bounds
`[5..5]` mean the non-null value is `5`, so `NotEqualTo("x", 5)` / `NotIn("x",
{5, 6})` cannot match every row.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
