devanbenz commented on issue #26169:
URL: https://github.com/apache/datafusion/issues/26169#issuecomment-6125269091
Taking a look at Q17 without `filter_pushdown`:
```
SELECT
sum(l_extendedprice) / 7.0 AS avg_yearly
FROM
lineitem,
part
WHERE
p_partkey = l_partkey
AND p_brand = 'Brand#23'
AND p_container = 'MED BOX'
AND l_quantity < (
SELECT
0.2 * avg(l_quantity)
FROM
lineitem
WHERE
l_partkey = p_partkey)
ProjectionExec: expr=[CAST(sum(lineitem.l_extendedprice)@0 AS Float64) / 7
as avg_yearly], metrics=[output_rows=1, elapsed_compute=2.29µs,
output_bytes=8.0 B, output_batches=1, expr_0_eval_time=2.00µs]
AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice)],
metrics=[output_rows=1, elapsed_compute=5.25µs, output_bytes=16.0 B,
output_batches=1, agg_expr_0_arguments_time=2.58µs,
agg_expr_0_evaluate_time=83ns, agg_expr_0_merge_time=545ns, output_rows_skew=0%]
CoalescePartitionsExec, metrics=[output_rows=14, elapsed_compute=9.04µs,
output_bytes=224.0 B, output_batches=14]
AggregateExec: mode=Partial, gby=[],
aggr=[sum(lineitem.l_extendedprice)], metrics=[output_rows=14,
elapsed_compute=40.58µs, output_bytes=224.0 B, output_batches=14,
agg_expr_0_arguments_time=4.33µs, agg_expr_0_state_time=6.54µs,
agg_expr_0_update_time=16.38µs, output_rows_skew=0%]
HashJoinExec: mode=Partitioned, join_type=LeftSemi,
on=[(p_partkey@2, l_partkey@1)], filter=CAST(l_quantity@0 AS Decimal128(30,
15)) < Float64(0.2) * avg(lineitem.l_quantity)@1,
projection=[l_extendedprice@1], metrics=[output_rows=53.84 K,
elapsed_compute=3.99s, output_bytes=1792.0 KB, output_batches=14,
build_mem_used=41.9 MB, array_map_created_count=0, build_input_batches=84,
build_input_rows=601.0 K, input_batches=2.46 K, input_rows=20.05 M,
build_time=3.92s, join_time=74.09ms, avg_fanout=3000.26% (601.0 K/20.03 K),
output_rows_skew=0.06%, probe_hit_rate=0.1% (20.03 K/20.00 M)]
HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@0,
l_partkey@0)], projection=[l_quantity@2, l_extendedprice@3, p_partkey@0],
metrics=[output_rows=601.0 K, elapsed_compute=3.31s, output_bytes=26.2 MB,
output_batches=84, build_mem_used=3.2 MB, array_map_created_count=0,
build_input_batches=14, build_input_rows=20.03 K, input_batches=73.25 K,
input_rows=600.0 M, build_time=9.89ms, join_time=3.31s, avg_fanout=100% (601.0
K/601.0 K), output_rows_skew=0.05%, probe_hit_rate=0.1% (601.0 K/600.0 M)]
RepartitionExec: partitioning=Hash([p_partkey@0], 14),
input_partitions=14, metrics=[output_rows=20.03 K, elapsed_compute=37.39µs,
output_bytes=896.0 KB, output_batches=14, spill_count=0, spilled_bytes=0.0 B,
spilled_rows=0, fetch_time=274.15ms, repartition_time=772.50µs,
send_time=48.87µs, output_rows_skew=0.06%]
FilterExec: p_brand@1 = Brand#23 AND p_container@2 = MED BOX,
projection=[p_partkey@0], metrics=[output_rows=20.03 K,
elapsed_compute=82.92ms, output_bytes=896.0 KB, output_batches=14,
selectivity=0.1% (20.03 K/20.00 M)]
DataSourceExec: file_groups={14 groups:
[[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/part.parquet:0..49193775],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/part.parquet:49193775..98387550],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/part.parquet:98387550..147581325],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/part.parquet:147581325..196775100],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/part.parquet:196775100..245968875],
...]}, projection=[p_partkey, p_brand, p_container], file_type=parquet,
predicate=p_brand@3 = Brand#23 AND p_container@6 = MED BOX,
pruning_predicate=p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <=
Brand#23 AND Brand#23 <= p_brand_max@1 AND p_container_null_count@6 !=
row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5,
required_guarantees=[p_brand in (Brand#23), p_container in (MED BOX)],
metrics=[output_rows=20.00 M, el
apsed_compute=144.64ms, output_bytes=777.7 MB, output_batches=2.48 K,
files_ranges_pruned_statistics=14 total → 14 matched,
row_groups_pruned_statistics=191 total → 191 matched,
page_index_pages_pruned=1.15 K total → 1.15 K matched,
page_index_rows_pruned=20.00 M total → 20.00 M matched,
limit_pruned_row_groups=0 total → 0 matched, batches_split=0,
bytes_processed=656.8 MB, bytes_scanned=141.5 MB, file_open_errors=0,
file_scan_errors=0, files_opened=14, files_processed=14,
num_predicate_creation_errors=0, predicate_evaluation_errors=0,
pushdown_rows_matched=0, pushdown_rows_pruned=0,
row_groups_pruned_dynamic_filter=0, predicate_cache_inner_records=0,
predicate_cache_records=0, bloom_filter_eval_time=8.53µs,
metadata_load_time=1.84ms, page_index_eval_time=2.02ms,
row_pushdown_eval_time=42ns, statistics_eval_time=667.32µs,
time_elapsed_opening=7.06ms, time_elapsed_processing=153.55ms,
time_elapsed_scanning_total=266.43ms, time_elapsed_scanning_until_data=12.78ms,
output_r
ows_skew=0.13%, scan_efficiency_ratio=21.55% (148.4 M/688.7 M)]
RepartitionExec: partitioning=Hash([l_partkey@0], 14),
input_partitions=14, metrics=[output_rows=600.0 M, elapsed_compute=32.65ms,
output_bytes=22.4 GB, output_batches=73.25 K, spill_count=0, spilled_bytes=0.0
B, spilled_rows=0, fetch_time=13.90s, repartition_time=3.17s, send_time=3.84s,
output_rows_skew=0%]
DataSourceExec: file_groups={14 groups:
[[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/lineitem.parquet:0..1938998887],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/lineitem.parquet:1938998887..3877997774],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/lineitem.parquet:3877997774..5816996661],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/lineitem.parquet:5816996661..7755995548],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/lineitem.parquet:7755995548..9694994435],
...]}, projection=[l_partkey, l_quantity, l_extendedprice], file_type=parquet,
predicate=DynamicFilter [ CASE hash_repartition % 14 WHEN 0 THEN l_partkey@1 >=
4880 AND l_partkey@1 <= 19999654 AND hash_lookup WHEN 1 THEN l_partkey@1 >=
2409 AND l_partkey@1 <= 19978138 AND hash_lookup WHEN 2 THEN l_partkey@1 >=
12919 AND l_partkey@1 <= 19966033 AND hash_lookup WHEN 3 THEN l_partkey@1 >=
6079 AND l_partkey@1 <= 19988248 AND hash
_lookup WHEN 4 THEN l_partkey@1 >= 11200 AND l_partkey@1 <= 19983248 AND
hash_lookup WHEN 5 THEN l_partkey@1 >= 21574 AND l_partkey@1 <= 19986826 AND
hash_lookup WHEN 6 THEN l_partkey@1 >= 12554 AND l_partkey@1 <= 19994996 AND
hash_lookup WHEN 7 THEN l_partkey@1 >= 10687 AND l_partkey@1 <= 19968087 AND
hash_lookup WHEN 8 THEN l_partkey@1 >= 11256 AND l_partkey@1 <= 19999539 AND
hash_lookup WHEN 9 THEN l_partkey@1 >= 26396 AND l_partkey@1 <= 19999987 AND
hash_lookup WHEN 10 THEN l_partkey@1 >= 26889 AND l_partkey@1 <= 19942847 AND
hash_lookup WHEN 11 THEN l_partkey@1 >= 2425 AND l_partkey@1 <= 19981418 AND
hash_lookup WHEN 12 THEN l_partkey@1 >= 7682 AND l_partkey@1 <= 19973698 AND
hash_lookup WHEN 13 THEN l_partkey@1 >= 20870 AND l_partkey@1 <= 19992676 AND
hash_lookup ELSE false END ], dynamic_rg_pruning=eligible,
pruning_predicate=l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >=
4880 AND l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19999654
AND l_partke
y_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1407/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0 >= 2409 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19978138 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1456/1048576 buckets) OR
l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 12919 AND
l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19966033 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1400/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0 >= 6079 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19988248 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1465/1048576 buckets) OR
l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 11200 AND
l_partkey_null_count@1 != row
_count@2 AND l_partkey_min@3 <= 19983248 AND l_partkey_null_count@1 !=
row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1396/1048576
buckets) OR l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 21574
AND l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19986826 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1489/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0 >= 12554 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19994996 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1422/1048576 buckets) OR
l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 10687 AND
l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19968087 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1397/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_p
artkey_max@0 >= 11256 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19999539 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1444/1048576 buckets) OR
l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 26396 AND
l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19999987 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1473/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0 >= 26889 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19942847 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1395/1048576 buckets) OR
l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 2425 AND
l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19981418 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1460/1048576 buc
kets) OR l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 7682 AND
l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19973698 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1382/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0 >= 20870 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19992676 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1427/1048576 buckets) OR
false, required_guarantees=[], metrics=[output_rows=600.0 M,
elapsed_compute=10.43s, output_bytes=22.5 GB, output_batches=75.83 K,
files_ranges_pruned_statistics=14 total → 14 matched,
row_groups_pruned_statistics=5.23 K total → 5.23 K matched,
page_index_pages_pruned=31.39 K total → 31.39 K matched,
page_index_rows_pruned=600.0 M total → 600.0 M matched,
limit_pruned_row_groups=0 total → 0 matched, batches_split=0,
bytes_processed=25.3 GB, bytes_sc
anned=8.8 GB, file_open_errors=0, file_scan_errors=0, files_opened=14,
files_processed=14, num_predicate_creation_errors=0,
predicate_evaluation_errors=0, pushdown_rows_matched=0, pushdown_rows_pruned=0,
row_groups_pruned_dynamic_filter=0, predicate_cache_inner_records=0,
predicate_cache_records=0, bloom_filter_eval_time=82.07µs,
metadata_load_time=1.69ms, page_index_eval_time=20.66ms,
row_pushdown_eval_time=42ns, statistics_eval_time=7.48ms,
time_elapsed_opening=1.05s, time_elapsed_processing=11.44s,
time_elapsed_scanning_total=23.30s, time_elapsed_scanning_until_data=28.56ms,
output_rows_skew=0%, scan_efficiency_ratio=34.82% (9.45 B/27.15 B)]
ProjectionExec: expr=[CAST(0.2 * CAST(avg(lineitem.l_quantity)@1
AS Float64) AS Decimal128(30, 15)) as Float64(0.2) * avg(lineitem.l_quantity),
l_partkey@0 as l_partkey], metrics=[output_rows=20.00 M,
elapsed_compute=120.90ms, output_bytes=38.6 GB, output_batches=2.45 K,
expr_0_eval_time=120.33ms, expr_1_eval_time=212.94µs]
AggregateExec:
mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey],
aggr=[avg(lineitem.l_quantity)],
metrics=[output_rows=20.00 M, elapsed_compute=46.66s,
output_bytes=114.8 GB, output_batches=2.45 K,
spill_count=0, spilled_bytes=0.0 B, spilled_rows=0,
agg_expr_0_arguments_time=33.56ms,
agg_expr_0_evaluate_time=168.91ms,
agg_expr_0_merge_time=11.39s, agg_expr_0_state_time=14ns,
aggregate_arguments_time=56.75ms, aggregation_time=11.40s,
emitting_time=169.74ms,
time_calculating_group_ids=34.98s,
output_rows_skew=0%]
RepartitionExec: partitioning=Hash([l_partkey@0], 14),
input_partitions=14, metrics=[output_rows=600.0 M, elapsed_compute=48.99ms,
output_bytes=17.9 GB, output_batches=73.25 K, spill_count=0, spilled_bytes=0.0
B, spilled_rows=0, fetch_time=11.56s, repartition_time=2.84s, send_time=9.62s,
output_rows_skew=0%]
AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey],
aggr=[avg(lineitem.l_quantity)], metrics=[output_rows=600.0 M,
elapsed_compute=318.23ms, output_bytes=18.7 GB, output_batches=75.83 K,
spill_count=0, spilled_bytes=0.0 B, spilled_rows=0, early_emit_count=0,
skipped_aggregation_rows=598.5 M, agg_expr_0_arguments_time=15.03ms,
agg_expr_0_convert_to_state_time=185.98ms, agg_expr_0_state_time=12.00µs,
agg_expr_0_update_time=7.63ms, aggregate_arguments_time=22.69ms,
aggregation_time=7.65ms, emitting_time=40.41µs,
time_calculating_group_ids=70.96ms, output_rows_skew=0%,
reduction_factor=99.75% (1.49 M/1.49 M)]
DataSourceExec: file_groups={14 groups:
[[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/lineitem.parquet:0..1938998887],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/lineitem.parquet:1938998887..3877997774],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/lineitem.parquet:3877997774..5816996661],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/lineitem.parquet:5816996661..7755995548],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/lineitem.parquet:7755995548..9694994435],
...]}, projection=[l_partkey, l_quantity], file_type=parquet,
predicate=DynamicFilter [ CASE hash_repartition % 14 WHEN 0 THEN l_partkey@1 >=
4880 AND l_partkey@1 <= 19999654 AND hash_lookup WHEN 1 THEN l_partkey@1 >=
2409 AND l_partkey@1 <= 19978138 AND hash_lookup WHEN 2 THEN l_partkey@1 >=
12919 AND l_partkey@1 <= 19966033 AND hash_lookup WHEN 3 THEN l_partkey@1 >=
6079 AND l_partkey@1 <= 19988248 AND hash_lookup WHEN
4 THEN l_partkey@1 >= 11200 AND l_partkey@1 <= 19983248 AND hash_lookup WHEN 5
THEN l_partkey@1 >= 21574 AND l_partkey@1 <= 19986826 AND hash_lookup WHEN 6
THEN l_partkey@1 >= 12554 AND l_partkey@1 <= 19994996 AND hash_lookup WHEN 7
THEN l_partkey@1 >= 10687 AND l_partkey@1 <= 19968087 AND hash_lookup WHEN 8
THEN l_partkey@1 >= 11256 AND l_partkey@1 <= 19999539 AND hash_lookup WHEN 9
THEN l_partkey@1 >= 26396 AND l_partkey@1 <= 19999987 AND hash_lookup WHEN 10
THEN l_partkey@1 >= 26889 AND l_partkey@1 <= 19942847 AND hash_lookup WHEN 11
THEN l_partkey@1 >= 2425 AND l_partkey@1 <= 19981418 AND hash_lookup WHEN 12
THEN l_partkey@1 >= 7682 AND l_partkey@1 <= 19973698 AND hash_lookup WHEN 13
THEN l_partkey@1 >= 20870 AND l_partkey@1 <= 19992676 AND hash_lookup ELSE
false END ], dynamic_rg_pruning=eligible,
pruning_predicate=l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >=
4880 AND l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19999654
AND l_partkey_null_count@
1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0,
1407/1048576 buckets) OR l_partkey_null_count@1 != row_count@2 AND
l_partkey_max@0 >= 2409 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19978138 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1456/1048576 buckets) OR
l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 12919 AND
l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19966033 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1400/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0 >= 6079 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19988248 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1465/1048576 buckets) OR
l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 11200 AND
l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19983248 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1396/1048576 buckets) OR
l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 21574 AND
l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19986826 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1489/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0 >= 12554 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19994996 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1422/1048576 buckets) OR
l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 10687 AND
l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19968087 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1397/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0
>= 11256 AND l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <=
>19999539 AND l_partkey_null_count@1 != row_count@2 AND
>KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1444/1048576 buckets) OR
>l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 26396 AND
>l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19999987 AND
>l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
>l_partkey_max@0, 1473/1048576 buckets) OR l_partkey_null_count@1 !=
>row_count@2 AND l_partkey_max@0 >= 26889 AND l_partkey_null_count@1 !=
>row_count@2 AND l_partkey_min@3 <= 19942847 AND l_partkey_null_count@1 !=
>row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0,
>1395/1048576 buckets) OR l_partkey_null_count@1 != row_count@2 AND
>l_partkey_max@0 >= 2425 AND l_partkey_null_count@1 != row_count@2 AND
>l_partkey_min@3 <= 19981418 AND l_partkey_null_count@1 != row_count@2 AND
>KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1460/1048576 buckets) OR
>l_pa
rtkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 7682 AND
l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19973698 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1382/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0 >= 20870 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19992676 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1427/1048576 buckets) OR
false, required_guarantees=[], metrics=[output_rows=600.0 M,
elapsed_compute=5.76s, output_bytes=13.6 GB, output_batches=75.83 K,
files_ranges_pruned_statistics=14 total → 14 matched,
row_groups_pruned_statistics=5.23 K total → 5.23 K matched,
page_index_pages_pruned=31.39 K total → 31.39 K matched,
page_index_rows_pruned=600.0 M total → 600.0 M matched,
limit_pruned_row_groups=0 total → 0 matched, batches_split=0,
bytes_processed=25.3 GB, bytes_scanned=4.8 GB,
file_open_errors=0, file_scan_errors=0, files_opened=14, files_processed=14,
num_predicate_creation_errors=0, predicate_evaluation_errors=0,
pushdown_rows_matched=0, pushdown_rows_pruned=0,
row_groups_pruned_dynamic_filter=0, predicate_cache_inner_records=0,
predicate_cache_records=0, bloom_filter_eval_time=77.57µs,
metadata_load_time=8.50ms, page_index_eval_time=21.07ms,
row_pushdown_eval_time=42ns, statistics_eval_time=6.58ms,
time_elapsed_opening=1.01s, time_elapsed_processing=6.77s,
time_elapsed_scanning_total=63.89s, time_elapsed_scanning_until_data=20.93ms,
output_rows_skew=0%, scan_efficiency_ratio=19.13% (5.19 B/27.15 B)]
```
`AggregateExec` is taking up most of the query compute time.
`time_calculating_group_ids=34.98s` So looks like without the pushdown we need
to group every part key for 600m rows. Looking at the joins, they discard most
of it -- so for this plan, the join ordering seems fine?
After enabling pushdown, the `AggregateExec` takes very little cpu time. All
the CPU time goes in to the `DynamicFilter` within the scan:
```
ProjectionExec: expr=[CAST(sum(lineitem.l_extendedprice)@0 AS Float64) / 7
as avg_yearly], metrics=[output_rows=1, elapsed_compute=6.83µs,
output_bytes=8.0 B, output_batches=1, expr_0_eval_time=3.00µs]
AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice)],
metrics=[output_rows=1, elapsed_compute=9.83µs, output_bytes=16.0 B,
output_batches=1, agg_expr_0_arguments_time=1.96µs,
agg_expr_0_evaluate_time=42ns, agg_expr_0_merge_time=298ns, output_rows_skew=0%]
CoalescePartitionsExec, metrics=[output_rows=14,
elapsed_compute=16.54µs, output_bytes=224.0 B, output_batches=14]
AggregateExec: mode=Partial, gby=[],
aggr=[sum(lineitem.l_extendedprice)], metrics=[output_rows=14,
elapsed_compute=35.04µs, output_bytes=224.0 B, output_batches=14,
agg_expr_0_arguments_time=3.21µs, agg_expr_0_state_time=5.04µs,
agg_expr_0_update_time=13.42µs, output_rows_skew=0%]
HashJoinExec: mode=Partitioned, join_type=LeftSemi,
on=[(p_partkey@2, l_partkey@1)], filter=CAST(l_quantity@0 AS Decimal128(30,
15)) < Float64(0.2) * avg(lineitem.l_quantity)@1,
projection=[l_extendedprice@1], metrics=[output_rows=53.84 K,
elapsed_compute=44.26ms, output_bytes=1792.0 KB, output_batches=14,
build_mem_used=41.9 MB, array_map_created_count=0, build_input_batches=84,
build_input_rows=601.0 K, input_batches=28, input_rows=73.87 K,
build_time=20.92ms, join_time=23.33ms, avg_fanout=3000.26% (601.0 K/20.03 K),
output_rows_skew=0.06%, probe_hit_rate=100% (20.03 K/20.03 K)]
HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@0,
l_partkey@0)], projection=[l_quantity@2, l_extendedprice@3, p_partkey@0],
metrics=[output_rows=601.0 K, elapsed_compute=7.46ms, output_bytes=26.2 MB,
output_batches=84, build_mem_used=3.2 MB, array_map_created_count=0,
build_input_batches=14, build_input_rows=20.03 K, input_batches=84,
input_rows=601.0 K, build_time=530.09µs, join_time=6.93ms, avg_fanout=100%
(601.0 K/601.0 K), output_rows_skew=0.05%, probe_hit_rate=100% (601.0 K/601.0
K)]
RepartitionExec: partitioning=Hash([p_partkey@0], 14),
input_partitions=14, metrics=[output_rows=20.03 K, elapsed_compute=26.61µs,
output_bytes=896.0 KB, output_batches=14, spill_count=0, spilled_bytes=0.0 B,
spilled_rows=0, fetch_time=313.42ms, repartition_time=560.08µs,
send_time=547.64µs, output_rows_skew=0.06%]
DataSourceExec: file_groups={14 groups:
[[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/part.parquet:0..49193775],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/part.parquet:49193775..98387550],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/part.parquet:98387550..147581325],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/part.parquet:147581325..196775100],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/part.parquet:196775100..245968875],
...]}, projection=[p_partkey], file_type=parquet, predicate=p_brand@3 =
Brand#23 AND p_container@6 = MED BOX, pruning_predicate=p_brand_null_count@2 !=
row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND
p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND
MED BOX <= p_container_max@5, required_guarantees=[p_brand in (Brand#23),
p_container in (MED BOX)], metrics=[output_rows=20.03 K,
elapsed_compute=213.99ms,
output_bytes=11.9 MB, output_batches=191, files_ranges_pruned_statistics=14
total → 14 matched, row_groups_pruned_statistics=191 total → 191 matched,
page_index_pages_pruned=1.15 K total → 1.15 K matched,
page_index_rows_pruned=20.00 M total → 20.00 M matched,
limit_pruned_row_groups=0 total → 0 matched, batches_split=0,
bytes_processed=656.8 MB, bytes_scanned=141.5 MB, file_open_errors=0,
file_scan_errors=0, files_opened=14, files_processed=14,
num_predicate_creation_errors=0, predicate_evaluation_errors=0,
pushdown_rows_matched=20.03 K, pushdown_rows_pruned=19.98 M,
row_groups_pruned_dynamic_filter=0, predicate_cache_inner_records=0,
predicate_cache_records=0, bloom_filter_eval_time=8.74µs,
metadata_load_time=4.41ms, page_index_eval_time=1.62ms,
row_pushdown_eval_time=32.04ms, statistics_eval_time=2.54ms,
time_elapsed_opening=20.90ms, time_elapsed_processing=239.15ms,
time_elapsed_scanning_total=293.78ms, time_elapsed_scanning_until_data=28.32ms,
output_rows_skew=0.17%,
scan_efficiency_ratio=21.54% (148.4 M/688.7 M)]
RepartitionExec: partitioning=Hash([l_partkey@0], 14),
input_partitions=14, metrics=[output_rows=601.0 K, elapsed_compute=59.53µs,
output_bytes=26.2 MB, output_batches=84, spill_count=0, spilled_bytes=0.0 B,
spilled_rows=0, fetch_time=40.90s, repartition_time=18.44ms, send_time=24.44ms,
output_rows_skew=0.05%]
DataSourceExec: file_groups={14 groups:
[[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/lineitem.parquet:0..1938998887],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/lineitem.parquet:1938998887..3877997774],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/lineitem.parquet:3877997774..5816996661],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/lineitem.parquet:5816996661..7755995548],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/lineitem.parquet:7755995548..9694994435],
...]}, projection=[l_partkey, l_quantity, l_extendedprice], file_type=parquet,
predicate=DynamicFilter [ CASE hash_repartition % 14 WHEN 0 THEN l_partkey@1 >=
4880 AND l_partkey@1 <= 19999654 AND hash_lookup WHEN 1 THEN l_partkey@1 >=
2409 AND l_partkey@1 <= 19978138 AND hash_lookup WHEN 2 THEN l_partkey@1 >=
12919 AND l_partkey@1 <= 19966033 AND hash_lookup WHEN 3 THEN l_partkey@1 >=
6079 AND l_partkey@1 <= 19988248 AND hash
_lookup WHEN 4 THEN l_partkey@1 >= 11200 AND l_partkey@1 <= 19983248 AND
hash_lookup WHEN 5 THEN l_partkey@1 >= 21574 AND l_partkey@1 <= 19986826 AND
hash_lookup WHEN 6 THEN l_partkey@1 >= 12554 AND l_partkey@1 <= 19994996 AND
hash_lookup WHEN 7 THEN l_partkey@1 >= 10687 AND l_partkey@1 <= 19968087 AND
hash_lookup WHEN 8 THEN l_partkey@1 >= 11256 AND l_partkey@1 <= 19999539 AND
hash_lookup WHEN 9 THEN l_partkey@1 >= 26396 AND l_partkey@1 <= 19999987 AND
hash_lookup WHEN 10 THEN l_partkey@1 >= 26889 AND l_partkey@1 <= 19942847 AND
hash_lookup WHEN 11 THEN l_partkey@1 >= 2425 AND l_partkey@1 <= 19981418 AND
hash_lookup WHEN 12 THEN l_partkey@1 >= 7682 AND l_partkey@1 <= 19973698 AND
hash_lookup WHEN 13 THEN l_partkey@1 >= 20870 AND l_partkey@1 <= 19992676 AND
hash_lookup ELSE false END ], dynamic_rg_pruning=eligible,
pruning_predicate=l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >=
4880 AND l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19999654
AND l_partke
y_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1407/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0 >= 2409 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19978138 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1456/1048576 buckets) OR
l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 12919 AND
l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19966033 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1400/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0 >= 6079 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19988248 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1465/1048576 buckets) OR
l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 11200 AND
l_partkey_null_count@1 != row
_count@2 AND l_partkey_min@3 <= 19983248 AND l_partkey_null_count@1 !=
row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1396/1048576
buckets) OR l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 21574
AND l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19986826 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1489/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0 >= 12554 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19994996 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1422/1048576 buckets) OR
l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 10687 AND
l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19968087 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1397/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_p
artkey_max@0 >= 11256 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19999539 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1444/1048576 buckets) OR
l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 26396 AND
l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19999987 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1473/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0 >= 26889 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19942847 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1395/1048576 buckets) OR
l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 2425 AND
l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19981418 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1460/1048576 buc
kets) OR l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 7682 AND
l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19973698 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1382/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0 >= 20870 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19992676 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1427/1048576 buckets) OR
false, required_guarantees=[], metrics=[output_rows=601.0 K,
elapsed_compute=37.61s, output_bytes=22.9 MB, output_batches=5.23 K,
files_ranges_pruned_statistics=14 total → 14 matched,
row_groups_pruned_statistics=5.23 K total → 5.23 K matched,
page_index_pages_pruned=31.39 K total → 31.39 K matched,
page_index_rows_pruned=600.0 M total → 600.0 M matched,
limit_pruned_row_groups=0 total → 0 matched, batches_split=0,
bytes_processed=25.3 GB, bytes_sca
nned=8.8 GB, file_open_errors=0, file_scan_errors=0, files_opened=14,
files_processed=14, num_predicate_creation_errors=0,
predicate_evaluation_errors=0, pushdown_rows_matched=601.0 K,
pushdown_rows_pruned=599.4 M, row_groups_pruned_dynamic_filter=0,
predicate_cache_inner_records=600.0 M, predicate_cache_records=601.0 K,
bloom_filter_eval_time=81.94µs, metadata_load_time=277.65µs,
page_index_eval_time=18.19ms, row_pushdown_eval_time=28.98s,
statistics_eval_time=3.54ms, time_elapsed_opening=993.42ms,
time_elapsed_processing=38.56s, time_elapsed_scanning_total=40.04s,
time_elapsed_scanning_until_data=122.47ms, output_rows_skew=0%,
scan_efficiency_ratio=34.82% (9.45 B/27.15 B)]
ProjectionExec: expr=[CAST(0.2 * CAST(avg(lineitem.l_quantity)@1
AS Float64) AS Decimal128(30, 15)) as Float64(0.2) * avg(lineitem.l_quantity),
l_partkey@0 as l_partkey], metrics=[output_rows=20.03 K,
elapsed_compute=249.33µs, output_bytes=537.0 KB, output_batches=14,
expr_0_eval_time=242.16µs, expr_1_eval_time=1.00µs]
AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as
l_partkey], aggr=[avg(lineitem.l_quantity)], metrics=[output_rows=20.03 K,
elapsed_compute=2.24ms, output_bytes=537.0 KB, output_batches=14,
spill_count=0, spilled_bytes=0.0 B, spilled_rows=0,
agg_expr_0_arguments_time=24.75µs, agg_expr_0_evaluate_time=253.00µs,
agg_expr_0_merge_time=451.00µs, agg_expr_0_state_time=14ns,
aggregate_arguments_time=37.71µs, aggregation_time=457.34µs,
emitting_time=263.42µs, time_calculating_group_ids=1.42ms,
output_rows_skew=0.06%]
RepartitionExec: partitioning=Hash([l_partkey@0], 14),
input_partitions=14, metrics=[output_rows=248.1 K, elapsed_compute=53.02µs,
output_bytes=10.5 MB, output_batches=42, spill_count=0, spilled_bytes=0.0 B,
spilled_rows=0, fetch_time=34.71s, repartition_time=4.28ms, send_time=788.38µs,
output_rows_skew=0.06%]
AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey],
aggr=[avg(lineitem.l_quantity)], metrics=[output_rows=248.1 K,
elapsed_compute=49.26ms, output_bytes=39.1 MB, output_batches=42,
spill_count=0, spilled_bytes=0.0 B, spilled_rows=0, early_emit_count=0,
skipped_aggregation_rows=0, agg_expr_0_arguments_time=1.27ms,
agg_expr_0_convert_to_state_time=14ns, agg_expr_0_state_time=22.75µs,
agg_expr_0_update_time=10.61ms, aggregate_arguments_time=1.96ms,
aggregation_time=11.20ms, emitting_time=43.42µs,
time_calculating_group_ids=33.32ms, output_rows_skew=0%,
reduction_factor=41.27% (248.1 K/601.0 K)]
DataSourceExec: file_groups={14 groups:
[[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/lineitem.parquet:0..1938998887],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/lineitem.parquet:1938998887..3877997774],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/lineitem.parquet:3877997774..5816996661],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/lineitem.parquet:5816996661..7755995548],
[Users/devan/Documents/OSS/polars-2.0-benchmark/data/tpch-100/lineitem.parquet:7755995548..9694994435],
...]}, projection=[l_partkey, l_quantity], file_type=parquet,
predicate=DynamicFilter [ CASE hash_repartition % 14
WHEN 0 THEN l_partkey@1 >= 4880 AND l_partkey@1 <= 19999654 AND hash_lookup
WHEN 1 THEN l_partkey@1 >= 2409 AND l_partkey@1 <= 19978138 AND hash_lookup
WHEN 2 THEN l_partkey@1 >= 12919 AND l_partkey@1 <= 19966033 AND hash_lookup
WHEN 3 THEN l_partkey@1 >= 6079 AND l_partkey@1 <= 19988248 AND hash_lookup
WHEN 4 THEN l_partkey@1 >= 11200 AND l_partkey@1 <= 19983248 AND hash_lookup
WHEN 5 THEN l_partkey@1 >= 21574 AND l_partkey@1 <= 19986826 AND hash_lookup
WHEN 6 THEN l_partkey@1 >= 12554 AND l_partkey@1 <= 19994996 AND hash_lookup
WHEN 7 THEN l_partkey@1 >= 10687 AND l_partkey@1 <= 19968087 AND hash_lookup
WHEN 8 THEN l_partkey@1 >= 11256 AND l_partkey@1 <= 19999539 AND hash_lookup
WHEN 9 THEN l_partkey@1 >= 26396 AND l_partkey@1 <= 19999987 AND hash_lookup
WHEN 10 THEN l_partkey@1 >= 26889 AND l_partkey@1 <= 19942847 AND hash_lookup
WHEN 11 THEN l_partkey@1 >= 2425 AND l_partkey@1 <= 19981418 AND hash_lookup
WHEN 12 THEN l_partkey@1 >= 7682 AND l_partkey@1 <= 19973698 AND hash_lookup
WHEN 13 THEN l_partkey@1 >= 20870 AND l_partkey@1 <= 19992676 AND hash_lookup
ELSE false END ], dynamic_rg_pruning=eligible,
pruning_predicate=l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >=
4880 AND l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19999654
AND l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1407/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0 >= 2409 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19978138 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1456/1048576 buckets) OR
l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 12919 AND
l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19966033 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1400/1048576 bu
ckets) OR l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 6079
AND l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19988248 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1465/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0 >= 11200 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19983248 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1396/1048576 buckets) OR
l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 21574 AND
l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19986826 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1489/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0 >= 12554 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19994996 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BI
TMAP(l_partkey_min@3, l_partkey_max@0, 1422/1048576 buckets) OR
l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 10687 AND
l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19968087 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1397/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0 >= 11256 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19999539 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1444/1048576 buckets) OR
l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 26396 AND
l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19999987 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1473/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0 >= 26889 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19942847 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1395/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0 >= 2425 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19981418 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1460/1048576 buckets) OR
l_partkey_null_count@1 != row_count@2 AND l_partkey_max@0 >= 7682 AND
l_partkey_null_count@1 != row_count@2 AND l_partkey_min@3 <= 19973698 AND
l_partkey_null_count@1 != row_count@2 AND KEY_RANGE_BITMAP(l_partkey_min@3,
l_partkey_max@0, 1382/1048576 buckets) OR l_partkey_null_count@1 != row_count@2
AND l_partkey_max@0 >= 20870 AND l_partkey_null_count@1 != row_count@2 AND
l_partkey_min@3 <= 19992676 AND l_partkey_null_count@1 != row_count@2 AND
KEY_RANGE_BITMAP(l_partkey_min@3, l_partkey_max@0, 1427/1048576 buckets) OR
false, required_guarantees=[], metrics=[output_rows=601.0 K,
elapsed_compute=32.08s, output
_bytes=13.8 MB, output_batches=5.23 K, files_ranges_pruned_statistics=14 total
→ 14 matched, row_groups_pruned_statistics=5.23 K total → 5.23 K matched,
page_index_pages_pruned=31.39 K total → 31.39 K matched,
page_index_rows_pruned=600.0 M total → 600.0 M matched,
limit_pruned_row_groups=0 total → 0 matched, batches_split=0,
bytes_processed=25.3 GB, bytes_scanned=4.8 GB, file_open_errors=0,
file_scan_errors=0, files_opened=14, files_processed=14,
num_predicate_creation_errors=0, predicate_evaluation_errors=0,
pushdown_rows_matched=601.0 K, pushdown_rows_pruned=599.4 M,
row_groups_pruned_dynamic_filter=0, predicate_cache_inner_records=600.0 M,
predicate_cache_records=601.0 K, bloom_filter_eval_time=77.24µs,
metadata_load_time=360.15µs, page_index_eval_time=17.48ms,
row_pushdown_eval_time=27.24s, statistics_eval_time=818.03µs,
time_elapsed_opening=957.84ms, time_elapsed_processing=33.03s,
time_elapsed_scanning_total=33.75s, time_elapsed_scanning_until_data=106.55ms,
outp
ut_rows_skew=0%, scan_efficiency_ratio=19.13% (5.19 B/27.15 B)]
```
Enabling pushdown shaves off a bit of time.
- pushdown off: 6.71s
- pushdown on: 5.57s
- Polars: 1.42s
`PhysicalExpr::evaluate` taking up most of the flamegraph.
<img width="1703" height="190" alt="Image"
src="https://github.com/user-attachments/assets/2ab8afae-3947-4a8e-b8a9-128b955c2c92";
/>
`filter_array` memory movement:
<img width="1728" height="202" alt="Image"
src="https://github.com/user-attachments/assets/207fbd70-62e3-4869-abee-204b55e449d0";
/>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
