This is an automated email from the ASF dual-hosted git repository.
lihaopeng pushed a change to branch topn-lazy-materialize-poc
in repository https://gitbox.apache.org/repos/asf/doris.git
discard 17283e04319 fe
discard 1bef8bfc7a5 Support topn-lazy-materialize
add 8340c7d93e5 [fix](compaction)init _cumulative_compaction_policies to
avoid out_of_range (#49767)
add 12d5da146d9 [Feature](dictionary) Support Dictionary Table (#45926)
add 799728cbd25 Support topn-lazy-materialize
add d5df73bc71e fe
add 2cbb1d855b0 do code format
add f32bb79ec84 fix compile error
add 58b67778601 fix test failed
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (17283e04319)
\
N -- N -- N refs/heads/topn-lazy-materialize-poc (58b67778601)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
be/src/exec/rowid_fetcher.cpp | 2 +-
be/src/http/action/dictionary_status_action.cpp | 45 ++
.../src/http/action/dictionary_status_action.h | 23 +-
be/src/olap/olap_server.cpp | 1 +
be/src/pipeline/exec/dict_sink_operator.cpp | 184 +++++
be/src/pipeline/exec/dict_sink_operator.h | 93 +++
be/src/pipeline/exec/operator.cpp | 2 +
be/src/pipeline/pipeline_fragment_context.cpp | 10 +
be/src/runtime/descriptors.cpp | 12 +
be/src/runtime/descriptors.h | 6 +
be/src/runtime/exec_env.h | 4 +
be/src/runtime/exec_env_init.cpp | 9 +-
be/src/service/backend_service.cpp | 11 +
be/src/service/backend_service.h | 3 +
be/src/service/http_service.cpp | 6 +
be/src/service/internal_service.cpp | 30 +
be/src/service/internal_service.h | 14 +
be/src/udf/udf.h | 8 +
be/src/vec/columns/column.h | 1 +
be/src/vec/columns/column_string.h | 4 +
be/src/vec/exec/scan/file_scanner.cpp | 110 ++-
be/src/vec/exec/scan/file_scanner.h | 4 +-
be/src/vec/exec/scan/scanner.h | 2 +-
be/src/vec/exprs/vectorized_fn_call.cpp | 5 +
be/src/vec/functions/complex_dict_hash_map.h | 105 +++
.../vec/functions/complex_hash_map_dictionary.cpp | 230 ++++++
be/src/vec/functions/complex_hash_map_dictionary.h | 104 +++
be/src/vec/functions/dictionary.cpp | 148 ++++
be/src/vec/functions/dictionary.h | 212 ++++++
be/src/vec/functions/dictionary_factory.cpp | 64 ++
be/src/vec/functions/dictionary_factory.h | 151 ++++
be/src/vec/functions/dictionary_util.h | 78 ++
be/src/vec/functions/function.h | 14 +
be/src/vec/functions/function_cast.h | 6 -
be/src/vec/functions/function_dict_get.cpp | 114 +++
be/src/vec/functions/function_dict_get_many.cpp | 132 ++++
be/src/vec/functions/function_ip.h | 6 -
be/src/vec/functions/ip_address_dictionary.cpp | 330 +++++++++
be/src/vec/functions/ip_address_dictionary.h | 98 +++
be/src/vec/functions/simple_function_factory.h | 4 +
be/src/vec/runtime/ip_address_cidr.h | 19 +-
be/src/vec/runtime/ipv4_value.h | 13 +
be/test/olap/id_manager_test.cpp | 2 +-
be/test/testutil/mock/mock_in_expr.cpp | 2 +-
be/test/testutil/run_all_tests.cpp | 4 +-
.../dictionary/dictionary_get_nullable_test.cpp | 293 ++++++++
.../exec/dictionary/dictionary_ip_trie_test.cpp | 76 ++
.../vec/exec/dictionary/dictionary_status_test.cpp | 109 +++
.../vec/exec/dictionary/dictionary_util_test.cpp | 159 ++++
.../exec/dictionary/dictionary_version_test.cpp | 161 +++++
.../vec/exec/format/parquet/parquet_read_lines.cpp | 5 +-
be/test/vec/exec/orc/orc_read_lines.cpp | 50 +-
.../function_complex_hash_map_dict_test.cpp | 135 ++++
.../vec/function/function_dict_get_many_test.cpp | 89 +++
be/test/vec/function/function_dict_get_test.cpp | 89 +++
.../vec/function/function_hash_map_dict_test.cpp | 68 ++
be/test/vec/function/function_hash_map_dict_test.h | 122 ++++
be/test/vec/function/function_ip_dict_test.cpp | 131 ++++
be/test/vec/function/function_ip_dict_test.h | 251 +++++++
docker/runtime/doris-compose/Dockerfile | 2 +-
docker/runtime/doris-compose/Readme.md | 72 +-
docker/runtime/doris-compose/command.py | 13 +-
docker/runtime/doris-compose/requirements.txt | 5 +-
.../main/java/org/apache/doris/common/Config.java | 32 +-
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 5 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 25 +-
.../org/apache/doris/analysis/RedirectStatus.java | 2 +-
.../doris/catalog/BuiltinScalarFunctions.java | 4 +
.../main/java/org/apache/doris/catalog/Env.java | 21 +
.../java/org/apache/doris/catalog/OlapTable.java | 20 +-
.../org/apache/doris/catalog/ScalarFunction.java | 10 +
.../main/java/org/apache/doris/catalog/Table.java | 1 +
.../java/org/apache/doris/catalog/TableIf.java | 4 +-
.../apache/doris/datasource/InternalCatalog.java | 10 +-
.../apache/doris/datasource/hive/HMSDlaTable.java | 2 +
.../doris/datasource/hive/HMSExternalTable.java | 19 +
.../apache/doris/datasource/hive/HiveDlaTable.java | 5 +
.../apache/doris/datasource/hive/HudiDlaTable.java | 5 +
.../doris/datasource/hive/IcebergDlaTable.java | 9 +
.../datasource/iceberg/IcebergExternalTable.java | 12 +
.../datasource/paimon/PaimonExternalTable.java | 11 +
.../org/apache/doris/dictionary/Dictionary.java | 438 +++++++++++
.../DictionaryDistribution.java} | 30 +-
.../apache/doris/dictionary/DictionaryManager.java | 800 +++++++++++++++++++++
.../LayoutType.java} | 12 +-
.../org/apache/doris/job/base/AbstractJob.java | 3 +
.../java/org/apache/doris/job/common/JobType.java | 1 +
.../apache/doris/job/executor/TaskProcessor.java | 2 +-
.../doris/job/extensions/insert/InsertTask.java | 31 +-
.../apache/doris/job/extensions/mtmv/MTMVTask.java | 8 +-
.../job/manager/TaskDisruptorGroupManager.java | 11 +-
.../apache/doris/job/scheduler/JobScheduler.java | 7 +-
.../org/apache/doris/job/task/AbstractTask.java | 8 +
.../org/apache/doris/journal/JournalEntity.java | 24 +
.../doris/mtmv/MTMVMaxTimestampSnapshot.java | 5 +
.../org/apache/doris/mtmv/MTMVRelatedTableIf.java | 19 +-
.../apache/doris/mtmv/MTMVSnapshotIdSnapshot.java | 5 +
.../java/org/apache/doris/mtmv/MTMVSnapshotIf.java | 5 +
.../apache/doris/mtmv/MTMVTimestampSnapshot.java | 5 +
.../org/apache/doris/mtmv/MTMVVersionSnapshot.java | 5 +
.../org/apache/doris/nereids/NereidsPlanner.java | 11 +-
.../org/apache/doris/nereids/StatementContext.java | 29 +
.../nereids/analyzer/UnboundDictionarySink.java | 134 ++++
.../nereids/analyzer/UnboundTableSinkCreator.java | 9 +
.../glue/translator/ExpressionTranslator.java | 69 ++
.../glue/translator/PhysicalPlanTranslator.java | 122 +++-
.../doris/nereids/parser/LogicalPlanBuilder.java | 147 ++++
.../properties/DistributionSpecAllSingleton.java} | 18 +-
.../nereids/properties/PhysicalProperties.java | 5 +-
.../nereids/properties/RequestPropertyDeriver.java | 8 +
.../org/apache/doris/nereids/rules/RuleSet.java | 2 +
.../org/apache/doris/nereids/rules/RuleType.java | 2 +
.../doris/nereids/rules/analysis/BindSink.java | 90 ++-
.../nereids/rules/analysis/CollectRelation.java | 9 +-
...icalDictionarySinkToPhysicalDictionarySink.java | 47 ++
.../nereids/rules/rewrite/ExprIdRewriter.java | 9 +
.../expressions/functions/scalar/DictGet.java | 138 ++++
.../expressions/functions/scalar/DictGetMany.java | 163 +++++
.../expressions/visitor/ScalarFunctionVisitor.java | 10 +
.../apache/doris/nereids/trees/plans/PlanType.java | 9 +
.../plans/commands/CreateDictionaryCommand.java | 78 ++
.../plans/commands/DropDictionaryCommand.java | 72 ++
.../plans/commands/ExplainDictionaryCommand.java | 88 +++
.../plans/commands/ShowDictionariesCommand.java | 108 +++
.../plans/commands/info/CreateDictionaryInfo.java | 287 ++++++++
.../commands/info/DictionaryColumnDefinition.java | 104 +++
.../commands/insert/DictionaryInsertExecutor.java | 94 +++
.../insert/InsertIntoDictionaryCommand.java | 80 +++
.../commands/insert/InsertIntoTableCommand.java | 57 +-
.../trees/plans/commands/insert/InsertUtils.java | 9 +-
.../commands/refresh/RefreshDictionaryCommand.java | 61 ++
.../trees/plans/distribute/DistributePlanner.java | 18 +
.../BackendDistributedPlanWorkerManager.java | 23 +
.../worker/DistributedPlanWorkerManager.java | 6 +
.../distribute/worker/job/UnassignedAllBEJob.java | 154 ++++
.../worker/job/UnassignedJobBuilder.java | 7 +
.../trees/plans/logical/LogicalDictionarySink.java | 153 ++++
.../plans/physical/PhysicalDictionarySink.java | 160 +++++
.../trees/plans/physical/PhysicalDistribute.java | 5 +-
.../plans/physical/PhysicalOlapTableSink.java | 2 +
.../trees/plans/physical/PhysicalTableSink.java | 2 +-
.../trees/plans/visitor/CommandVisitor.java | 25 +
.../nereids/trees/plans/visitor/SinkVisitor.java | 15 +
.../CreateDictionaryPersistInfo.java} | 24 +-
.../DictionaryDecreaseVersionInfo.java} | 24 +-
.../DictionaryIncreaseVersionInfo.java} | 24 +-
.../DictionaryPersistInfo.java} | 31 +-
.../doris/persist/DropDictionaryPersistInfo.java | 60 ++
.../java/org/apache/doris/persist/EditLog.java | 37 +
.../org/apache/doris/persist/OperationType.java | 6 +
.../org/apache/doris/persist/gson/GsonUtils.java | 4 +-
.../doris/persist/meta/MetaPersistMethod.java | 6 +
.../doris/persist/meta/PersistMetaModules.java | 2 +-
.../org/apache/doris/planner/DictionarySink.java | 125 ++++
.../org/apache/doris/planner/OlapScanNode.java | 13 +-
.../org/apache/doris/planner/PlanFragment.java | 16 +-
.../java/org/apache/doris/planner/ScanNode.java | 5 +
.../main/java/org/apache/doris/qe/Coordinator.java | 54 +-
.../org/apache/doris/rpc/BackendServiceClient.java | 15 +
.../org/apache/doris/rpc/BackendServiceProxy.java | 32 +
.../org/apache/doris/system/SystemInfoService.java | 17 +
.../org/apache/doris/common/GenericPoolTest.java | 6 +
.../apache/doris/utframe/MockedBackendFactory.java | 6 +
gensrc/proto/internal_service.proto | 29 +
gensrc/thrift/BackendService.thrift | 13 +
gensrc/thrift/DataSinks.thrift | 19 +
gensrc/thrift/Descriptors.thrift | 4 +
gensrc/thrift/Types.thrift | 9 +-
.../test_dict_load_and_get_hash_map.out | Bin 0 -> 2620 bytes
.../test_dict_load_and_get_ip_trie.out | Bin 0 -> 159 bytes
.../test_dict_nullable_key.out | Bin 0 -> 152 bytes
regression-test/data/dictionary_p0/test_ddl.out | Bin 0 -> 225 bytes
.../data/dictionary_p0/test_dict_get_many.out | Bin 0 -> 263 bytes
.../data/dictionary_p0/test_dict_get_nullable.out | Bin 0 -> 404 bytes
.../dictionary_p0/test_dictionary_fe_failover.out | Bin 0 -> 173 bytes
.../data/dictionary_p0/test_refresh_dict.out | Bin 0 -> 717 bytes
.../data/dictionary_p0/test_trade_demo.out | Bin 0 -> 5381 bytes
.../upgrade/test_dictionary_restart.out | Bin 0 -> 115 bytes
.../org/apache/doris/regression/suite/Suite.groovy | 49 ++
.../test_dict_load_and_get_hash_map.groovy | 249 +++++++
.../test_dict_load_and_get_ip_trie.groovy | 100 +++
.../test_dict_nullable_key.groovy | 112 +++
.../dictionary_p0/test_create_drop_sync.groovy | 108 +++
.../suites/dictionary_p0/test_ddl.groovy | 320 +++++++++
.../suites/dictionary_p0/test_dict_get.groovy | 85 +++
.../suites/dictionary_p0/test_dict_get_many.groovy | 63 ++
.../dictionary_p0/test_dict_get_nullable.groovy | 93 +++
.../test_dictionary_fe_failover.groovy | 157 ++++
.../suites/dictionary_p0/test_refresh_dict.groovy | 256 +++++++
.../suites/dictionary_p0/test_trade_demo.groovy | 310 ++++++++
.../suites/dictionary_p0/upgrade/load.groovy | 129 ++++
.../upgrade/test_dictionary_restart.groovy | 39 +
192 files changed, 10822 insertions(+), 339 deletions(-)
create mode 100644 be/src/http/action/dictionary_status_action.cpp
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/DistributedPlanWorkerManager.java
=> be/src/http/action/dictionary_status_action.h (66%)
create mode 100644 be/src/pipeline/exec/dict_sink_operator.cpp
create mode 100644 be/src/pipeline/exec/dict_sink_operator.h
create mode 100644 be/src/vec/functions/complex_dict_hash_map.h
create mode 100644 be/src/vec/functions/complex_hash_map_dictionary.cpp
create mode 100644 be/src/vec/functions/complex_hash_map_dictionary.h
create mode 100644 be/src/vec/functions/dictionary.cpp
create mode 100644 be/src/vec/functions/dictionary.h
create mode 100644 be/src/vec/functions/dictionary_factory.cpp
create mode 100644 be/src/vec/functions/dictionary_factory.h
create mode 100644 be/src/vec/functions/dictionary_util.h
create mode 100644 be/src/vec/functions/function_dict_get.cpp
create mode 100644 be/src/vec/functions/function_dict_get_many.cpp
create mode 100644 be/src/vec/functions/ip_address_dictionary.cpp
create mode 100644 be/src/vec/functions/ip_address_dictionary.h
create mode 100644 be/test/vec/exec/dictionary/dictionary_get_nullable_test.cpp
create mode 100644 be/test/vec/exec/dictionary/dictionary_ip_trie_test.cpp
create mode 100644 be/test/vec/exec/dictionary/dictionary_status_test.cpp
create mode 100644 be/test/vec/exec/dictionary/dictionary_util_test.cpp
create mode 100644 be/test/vec/exec/dictionary/dictionary_version_test.cpp
create mode 100644 be/test/vec/function/function_complex_hash_map_dict_test.cpp
create mode 100644 be/test/vec/function/function_dict_get_many_test.cpp
create mode 100644 be/test/vec/function/function_dict_get_test.cpp
create mode 100644 be/test/vec/function/function_hash_map_dict_test.cpp
create mode 100644 be/test/vec/function/function_hash_map_dict_test.h
create mode 100644 be/test/vec/function/function_ip_dict_test.cpp
create mode 100644 be/test/vec/function/function_ip_dict_test.h
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/dictionary/Dictionary.java
copy
fe/fe-core/src/main/java/org/apache/doris/{nereids/trees/plans/distribute/worker/DistributedPlanWorkerManager.java
=> dictionary/DictionaryDistribution.java} (53%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/dictionary/DictionaryManager.java
copy fe/fe-core/src/main/java/org/apache/doris/{mtmv/MTMVSnapshotIf.java =>
dictionary/LayoutType.java} (80%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundDictionarySink.java
copy fe/fe-core/src/main/java/org/apache/doris/{mtmv/MTMVSnapshotIf.java =>
nereids/properties/DistributionSpecAllSingleton.java} (60%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalDictionarySinkToPhysicalDictionarySink.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGet.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DictGetMany.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateDictionaryCommand.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropDictionaryCommand.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExplainDictionaryCommand.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowDictionariesCommand.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateDictionaryInfo.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DictionaryColumnDefinition.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/DictionaryInsertExecutor.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoDictionaryCommand.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshDictionaryCommand.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedAllBEJob.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDictionarySink.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDictionarySink.java
copy
fe/fe-core/src/main/java/org/apache/doris/{nereids/trees/plans/distribute/worker/DistributedPlanWorkerManager.java
=> persist/CreateDictionaryPersistInfo.java} (54%)
copy
fe/fe-core/src/main/java/org/apache/doris/{nereids/trees/plans/distribute/worker/DistributedPlanWorkerManager.java
=> persist/DictionaryDecreaseVersionInfo.java} (54%)
copy
fe/fe-core/src/main/java/org/apache/doris/{nereids/trees/plans/distribute/worker/DistributedPlanWorkerManager.java
=> persist/DictionaryIncreaseVersionInfo.java} (54%)
copy
fe/fe-core/src/main/java/org/apache/doris/{nereids/trees/plans/distribute/worker/DistributedPlanWorkerManager.java
=> persist/DictionaryPersistInfo.java} (51%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/persist/DropDictionaryPersistInfo.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/planner/DictionarySink.java
create mode 100644
regression-test/data/dictionary_p0/dictionary_load_and_get/test_dict_load_and_get_hash_map.out
create mode 100644
regression-test/data/dictionary_p0/dictionary_load_and_get/test_dict_load_and_get_ip_trie.out
create mode 100644
regression-test/data/dictionary_p0/dictionary_load_and_get/test_dict_nullable_key.out
create mode 100644 regression-test/data/dictionary_p0/test_ddl.out
create mode 100644 regression-test/data/dictionary_p0/test_dict_get_many.out
create mode 100644
regression-test/data/dictionary_p0/test_dict_get_nullable.out
create mode 100644
regression-test/data/dictionary_p0/test_dictionary_fe_failover.out
create mode 100644 regression-test/data/dictionary_p0/test_refresh_dict.out
create mode 100644 regression-test/data/dictionary_p0/test_trade_demo.out
create mode 100644
regression-test/data/dictionary_p0/upgrade/test_dictionary_restart.out
create mode 100644
regression-test/suites/dictionary_p0/dictionary_load_and_get/test_dict_load_and_get_hash_map.groovy
create mode 100644
regression-test/suites/dictionary_p0/dictionary_load_and_get/test_dict_load_and_get_ip_trie.groovy
create mode 100644
regression-test/suites/dictionary_p0/dictionary_load_and_get/test_dict_nullable_key.groovy
create mode 100644
regression-test/suites/dictionary_p0/test_create_drop_sync.groovy
create mode 100644 regression-test/suites/dictionary_p0/test_ddl.groovy
create mode 100644 regression-test/suites/dictionary_p0/test_dict_get.groovy
create mode 100644
regression-test/suites/dictionary_p0/test_dict_get_many.groovy
create mode 100644
regression-test/suites/dictionary_p0/test_dict_get_nullable.groovy
create mode 100644
regression-test/suites/dictionary_p0/test_dictionary_fe_failover.groovy
create mode 100644
regression-test/suites/dictionary_p0/test_refresh_dict.groovy
create mode 100644 regression-test/suites/dictionary_p0/test_trade_demo.groovy
create mode 100644 regression-test/suites/dictionary_p0/upgrade/load.groovy
create mode 100644
regression-test/suites/dictionary_p0/upgrade/test_dictionary_restart.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
