This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
from c8b130ad82a [fix](variable) support set ON and ALL to variable (#58287)
add a13241b76c7 [feature](reader) Optimize Complex Type Column Reading
with Column Pruning (#57204)
No new revisions were added by this update.
Summary of changes:
be/src/http/action/debug_point_action.cpp | 1 -
be/src/olap/iterators.h | 4 +-
be/src/olap/rowset/beta_rowset_reader.cpp | 4 +
be/src/olap/rowset/rowset_reader_context.h | 5 +
be/src/olap/rowset/segment_v2/column_reader.cpp | 361 +-
be/src/olap/rowset/segment_v2/column_reader.h | 70 +-
be/src/olap/rowset/segment_v2/segment.cpp | 16 +
be/src/olap/tablet_reader.cpp | 3 +
be/src/olap/tablet_reader.h | 4 +
be/src/olap/tablet_schema.cpp | 19 +
be/src/olap/tablet_schema.h | 10 +
be/src/runtime/descriptors.cpp | 58 +-
be/src/runtime/descriptors.h | 14 +
be/src/vec/columns/column_map.cpp | 1 +
be/src/vec/data_types/data_type_factory.cpp | 3 +-
be/src/vec/exec/format/orc/vorc_reader.cpp | 377 +-
be/src/vec/exec/format/orc/vorc_reader.h | 21 +-
be/src/vec/exec/format/parquet/schema_desc.cpp | 87 +-
be/src/vec/exec/format/parquet/schema_desc.h | 30 +-
.../exec/format/parquet/vparquet_column_reader.cpp | 207 +-
.../exec/format/parquet/vparquet_column_reader.h | 101 +-
.../exec/format/parquet/vparquet_group_reader.cpp | 27 +-
.../exec/format/parquet/vparquet_group_reader.h | 8 +-
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 50 +-
be/src/vec/exec/format/parquet/vparquet_reader.h | 6 +-
.../table/hive/hive_orc_nested_column_utils.cpp | 168 +
.../table/hive/hive_orc_nested_column_utils.h | 43 +
.../hive/hive_parquet_nested_column_utils.cpp | 171 +
.../table/hive/hive_parquet_nested_column_utils.h | 41 +
be/src/vec/exec/format/table/hive_reader.cpp | 271 +-
be/src/vec/exec/format/table/hive_reader.h | 14 +
.../iceberg/iceberg_orc_nested_column_utils.cpp | 168 +
.../iceberg/iceberg_orc_nested_column_utils.h | 44 +
.../iceberg_parquet_nested_column_utils.cpp | 172 +
.../iceberg/iceberg_parquet_nested_column_utils.h | 42 +
be/src/vec/exec/format/table/iceberg_reader.cpp | 162 +-
be/src/vec/exec/format/table/iceberg_reader.h | 7 +
.../format/table/nested_column_access_helper.h | 82 +
.../vec/exec/format/table/table_format_reader.cpp | 21 +-
be/src/vec/exec/format/table/table_format_reader.h | 36 +
be/src/vec/exec/scan/file_scanner.cpp | 6 +-
be/src/vec/exec/scan/file_scanner.h | 2 +-
be/src/vec/exec/scan/olap_scanner.cpp | 22 +-
be/src/vec/functions/function.cpp | 3 +
...4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc | Bin 0 -> 2277 bytes
...4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc | Bin 0 -> 2334 bytes
...2c4c8-e0bd-4db6-b550-6323977446a4.metadata.json | 145 +
...66011-3129-4bfb-9b4b-c1510c566d73.metadata.json | 176 +
.../6a876742-85c2-4f84-ae24-292482fa9288-m0.avro | Bin 0 -> 8175 bytes
...212-1-6a876742-85c2-4f84-ae24-292482fa9288.avro | Bin 0 -> 4264 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4068 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4020 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4043 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4082 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4067 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4047 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4077 bytes
...aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet | Bin 0 -> 4147 bytes
...38ed9-31af-46c8-9274-506c7f2709c9.metadata.json | 145 +
...4732d-f823-4900-8185-3eb3cb23e4a9.metadata.json | 176 +
.../51fb6058-7fc2-4791-96a9-f3111933728a-m0.avro | Bin 0 -> 8783 bytes
...552-1-51fb6058-7fc2-4791-96a9-f3111933728a.avro | Bin 0 -> 4264 bytes
...a144c37-16a4-47c6-96db-0007175b5c90-0-00001.orc | Bin 0 -> 7181 bytes
...20949-0948-4f33-bcf4-5b6bcb0cd400.metadata.json | 548 ++
...ddc20-6aa8-4864-9d15-546dd5a6bae4.metadata.json | 579 ++
.../d9aed717-61ef-422b-b044-a16ca31f9e86-m0.avro | 1 +
...228-1-d9aed717-61ef-422b-b044-a16ca31f9e86.avro | Bin 0 -> 4261 bytes
...35f-d581-40e4-8d56-a929aded99e4-0-00001.parquet | Bin 0 -> 21586 bytes
...ae7dc-be1a-410b-ae74-7cf04ad10765.metadata.json | 548 ++
...56265-e89e-4c9f-a0fb-7f3075d4e8f9.metadata.json | 579 ++
.../bee22dff-a1d3-420d-bfba-c1b4193ab0df-m0.avro | 5 +
...002-1-bee22dff-a1d3-420d-bfba-c1b4193ab0df.avro | Bin 0 -> 4262 bytes
.../test_data/nested_user_profiles_orc/_SUCCESS} | 0
...f23-05d1-4043-a533-b155ef52b720-c000.snappy.orc | Bin 0 -> 8396 bytes
.../nested_user_profiles_parquet/_SUCCESS} | 0
...1a03-4efc-ab51-557e9369a1f9-c000.snappy.parquet | Bin 0 -> 21157 bytes
.../olap/rowset/segment_v2/column_reader_test.cpp | 220 +
.../exec/format/parquet/parquet_thrift_test.cpp | 12 +-
.../hive/hive_reader_create_column_ids_test.cpp | 1193 +++
.../exec/format/table/hive/hive_reader_test.cpp | 740 ++
.../iceberg_reader_create_column_ids_test.cpp | 1184 +++
.../format/table/iceberg/iceberg_reader_test.cpp | 733 ++
contrib/apache-orc | 2 +-
.../org/apache/doris/analysis/AccessPathInfo.java | 38 +
.../org/apache/doris/analysis/SlotDescriptor.java | 43 +
.../org/apache/doris/datasource/FileScanNode.java | 15 +-
.../doris/datasource/hive/HMSExternalTable.java | 40 +
.../doris/datasource/hive/source/HiveScanNode.java | 37 +-
.../org/apache/doris/nereids/StatementContext.java | 9 +
.../glue/translator/PhysicalPlanTranslator.java | 5 +-
.../glue/translator/PlanTranslatorContext.java | 18 +
.../doris/nereids/jobs/executor/Rewriter.java | 6 +
.../org/apache/doris/nereids/rules/RuleSet.java | 2 +
.../org/apache/doris/nereids/rules/RuleType.java | 8 +
.../doris/nereids/rules/analysis/BindRelation.java | 9 +-
.../nereids/rules/analysis/ExpressionAnalyzer.java | 10 +
.../nereids/rules/analysis/NormalizeAggregate.java | 3 +-
.../rules/expression/ExpressionNormalization.java | 2 +
.../rules/expression/ExpressionRuleType.java | 1 +
.../expression/rules/NormalizeStructElement.java | 66 +
.../rewrite/AccessPathExpressionCollector.java | 511 ++
.../rules/rewrite/AccessPathPlanCollector.java | 176 +
.../nereids/rules/rewrite/NestedColumnPruning.java | 486 ++
.../nereids/rules/rewrite/PushDownProject.java | 412 ++
.../nereids/rules/rewrite/SlotTypeReplacer.java | 728 ++
.../rules/rewrite/VariantSubPathPruning.java | 3 +-
.../org/apache/doris/nereids/trees/TreeNode.java | 2 +-
.../trees/expressions/PreferPushDownProject.java | 41 +
.../nereids/trees/expressions/SlotReference.java | 56 +-
.../expressions/functions/scalar/ArrayFirst.java | 6 +
.../expressions/functions/scalar/ArrayLast.java | 6 +
.../expressions/functions/scalar/ArrayMap.java | 3 +-
.../expressions/functions/scalar/ElementAt.java | 4 +-
.../functions/scalar/HighOrderFunction.java | 3 +-
.../functions/scalar/MapContainsEntry.java | 3 +-
.../functions/scalar/MapContainsKey.java | 3 +-
.../functions/scalar/MapContainsValue.java | 3 +-
.../expressions/functions/scalar/MapEntries.java | 3 +-
.../expressions/functions/scalar/MapKeys.java | 3 +-
.../expressions/functions/scalar/MapValues.java | 3 +-
.../functions/scalar/StructElement.java | 9 +-
.../trees/expressions/functions/table/File.java | 8 +-
.../trees/expressions/functions/table/Hdfs.java | 9 +-
.../trees/expressions/functions/table/Local.java | 9 +-
.../trees/expressions/functions/table/S3.java | 9 +-
.../expressions/visitor/ScalarFunctionVisitor.java | 14 +-
.../trees/plans/logical/LogicalFileScan.java | 82 +-
.../trees/plans/logical/LogicalHudiScan.java | 22 +-
.../trees/plans/logical/LogicalOlapScan.java | 24 +-
.../trees/plans/logical/LogicalTVFRelation.java | 23 +-
.../plans/logical/SupportPruneNestedColumn.java | 24 +
.../logical/SupportPruneNestedColumnFormats.java | 42 +
.../org/apache/doris/nereids/types/ArrayType.java | 2 +-
.../org/apache/doris/nereids/types/MapType.java | 2 +-
.../doris/nereids/types/NestedColumnPrunable.java | 22 +
.../apache/doris/nereids/types/StructField.java | 3 +-
.../org/apache/doris/nereids/types/StructType.java | 10 +-
.../apache/doris/planner/MaterializationNode.java | 2 +
.../org/apache/doris/planner/OlapScanNode.java | 3 +
.../java/org/apache/doris/planner/PlanNode.java | 113 +-
.../java/org/apache/doris/qe/SessionVariable.java | 9 +
.../rules/rewrite/PruneNestedColumnTest.java | 652 ++
.../apache/doris/utframe/TestWithFeService.java | 4 +-
gensrc/proto/descriptors.proto | 45 +
gensrc/thrift/Descriptors.thrift | 48 +-
.../complex_types/test_pruned_columns.out | 31 +
regression-test/data/nereids_arith_p0/decimal.out | 7600 +++++++++----------
regression-test/data/nereids_arith_p0/integer.out | 7644 ++++++++++----------
regression-test/data/nereids_arith_p0/string.out | 4332 +++++------
.../complex_types/test_pruned_columns.groovy | 88 +
150 files changed, 23515 insertions(+), 10085 deletions(-)
create mode 100644
be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.cpp
create mode 100644
be/src/vec/exec/format/table/hive/hive_orc_nested_column_utils.h
create mode 100644
be/src/vec/exec/format/table/hive/hive_parquet_nested_column_utils.cpp
create mode 100644
be/src/vec/exec/format/table/hive/hive_parquet_nested_column_utils.h
create mode 100644
be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.cpp
create mode 100644
be/src/vec/exec/format/table/iceberg/iceberg_orc_nested_column_utils.h
create mode 100644
be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.cpp
create mode 100644
be/src/vec/exec/format/table/iceberg/iceberg_parquet_nested_column_utils.h
create mode 100644 be/src/vec/exec/format/table/nested_column_access_helper.h
create mode 100644
be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/00000-0-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc
create mode 100644
be/test/exec/test_data/complex_user_profiles_iceberg_orc/data/00001-1-e4897963-0081-4127-bebe-35dc7dc1edeb-0-00001.orc
create mode 100644
be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/00000-aa52c4c8-e0bd-4db6-b550-6323977446a4.metadata.json
create mode 100644
be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/00001-88a66011-3129-4bfb-9b4b-c1510c566d73.metadata.json
create mode 100644
be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/6a876742-85c2-4f84-ae24-292482fa9288-m0.avro
create mode 100644
be/test/exec/test_data/complex_user_profiles_iceberg_orc/metadata/snap-2197982135324349212-1-6a876742-85c2-4f84-ae24-292482fa9288.avro
create mode 100644
be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00000-0-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
create mode 100644
be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00001-1-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
create mode 100644
be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00002-2-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
create mode 100644
be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00003-3-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
create mode 100644
be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00004-4-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
create mode 100644
be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00005-5-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
create mode 100644
be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00006-6-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
create mode 100644
be/test/exec/test_data/complex_user_profiles_iceberg_parquet/data/00007-7-a0022aad-d3b6-4e73-b181-f0a09aac7034-0-00001.parquet
create mode 100644
be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/00000-30938ed9-31af-46c8-9274-506c7f2709c9.metadata.json
create mode 100644
be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/00001-b1b4732d-f823-4900-8185-3eb3cb23e4a9.metadata.json
create mode 100644
be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/51fb6058-7fc2-4791-96a9-f3111933728a-m0.avro
create mode 100644
be/test/exec/test_data/complex_user_profiles_iceberg_parquet/metadata/snap-1689809712605583552-1-51fb6058-7fc2-4791-96a9-f3111933728a.avro
create mode 100644
be/test/exec/test_data/nested_user_profiles_iceberg_orc/data/00000-8-5a144c37-16a4-47c6-96db-0007175b5c90-0-00001.orc
create mode 100644
be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/00000-daa20949-0948-4f33-bcf4-5b6bcb0cd400.metadata.json
create mode 100644
be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/00001-5c6ddc20-6aa8-4864-9d15-546dd5a6bae4.metadata.json
create mode 100644
be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/d9aed717-61ef-422b-b044-a16ca31f9e86-m0.avro
create mode 100644
be/test/exec/test_data/nested_user_profiles_iceberg_orc/metadata/snap-514179801967160228-1-d9aed717-61ef-422b-b044-a16ca31f9e86.avro
create mode 100644
be/test/exec/test_data/nested_user_profiles_iceberg_parquet/data/00000-9-a7e0135f-d581-40e4-8d56-a929aded99e4-0-00001.parquet
create mode 100644
be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/00000-ed5ae7dc-be1a-410b-ae74-7cf04ad10765.metadata.json
create mode 100644
be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/00001-a1156265-e89e-4c9f-a0fb-7f3075d4e8f9.metadata.json
create mode 100644
be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/bee22dff-a1d3-420d-bfba-c1b4193ab0df-m0.avro
create mode 100644
be/test/exec/test_data/nested_user_profiles_iceberg_parquet/metadata/snap-6622777947381868002-1-bee22dff-a1d3-420d-bfba-c1b4193ab0df.avro
copy be/{src/pch/pch.cc =>
test/exec/test_data/nested_user_profiles_orc/_SUCCESS} (100%)
create mode 100644
be/test/exec/test_data/nested_user_profiles_orc/part-00000-62614f23-05d1-4043-a533-b155ef52b720-c000.snappy.orc
copy be/{src/pch/pch.cc =>
test/exec/test_data/nested_user_profiles_parquet/_SUCCESS} (100%)
create mode 100644
be/test/exec/test_data/nested_user_profiles_parquet/part-00000-64a7a390-1a03-4efc-ab51-557e9369a1f9-c000.snappy.parquet
create mode 100644 be/test/olap/rowset/segment_v2/column_reader_test.cpp
create mode 100644
be/test/vec/exec/format/table/hive/hive_reader_create_column_ids_test.cpp
create mode 100644 be/test/vec/exec/format/table/hive/hive_reader_test.cpp
create mode 100644
be/test/vec/exec/format/table/iceberg/iceberg_reader_create_column_ids_test.cpp
create mode 100644
be/test/vec/exec/format/table/iceberg/iceberg_reader_test.cpp
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/analysis/AccessPathInfo.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/NormalizeStructElement.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/NestedColumnPruning.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownProject.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PreferPushDownProject.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/SupportPruneNestedColumn.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/SupportPruneNestedColumnFormats.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/types/NestedColumnPrunable.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
create mode 100644
regression-test/data/datatype_p0/complex_types/test_pruned_columns.out
create mode 100644
regression-test/suites/datatype_p0/complex_types/test_pruned_columns.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
