This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a change to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from fef6c7df5b8 [Fix](Variant) fix variant partial update with row store
enabled (#36793)
new f04c185cc6c [Refactor](meta) Delete useless classes and write methods
(#36894)
new 5b68de56cd1 [refactor](nereids) New distribute planner (#36531)
new 018ff0ae94c [feature](insert)support external hive truncate table DDL
(#36801)
new a89a43c2a97 [Fix](hive-writer) Fixed the issue where
`uncompletedMpuPendingUploads` did not remove objects correctly. (#36905)
new 05f694a9405 [Fix](multi-catalog) Fix the transaction is not removed in
abnormal situations by removing transaction in finally block. (#36705)
new 15f162f6905 [Fix](variant) ignore serialization of nothing type
(#36997)
new 13e2a19bbb5 [chore](profile) rm useless code of profile (#36915)
new fee6a24ef64 [chore](upgrade) turn off fallback_to_original_planner
when upgrade (#37005)
new c0591389cc7 [enhancement](cloud) batching get visible version from
MetaService (#34615)
new 039d4f722e3 [fix](regression test) fix unstable single compaction test
p2 (#36881)
new 1cbc322d0c5 [improvement](jdbc catalog) Modify the maximum number of
connections in the connection pool to 30 by default (#36720)
new 74eaf05cb85 [fix](statistics)Escape fetch partition stats sql. (#36941)
new 05ca6c66f51 [fix](mtmv)fix mtmv dead lock (#37009)
new b2f90a1ce4f [Fix](autoinc) try fix concurrent load problem with auto
inc column (#36421)
new 66ca7e94c83 [bugfix](testcase)add java error log output (#35998)
new 82e4633053f Revert "[feature](cloud) support file cache only cache
index pages (#36273) (#36797)
new 9255c366ec1 (cloud-merge) Get fileCacheSize by RPC always (#36857)
new 3666c75576f [performance](load) do not copy input_block in memtable
(#36939)
new 900976cf44a Revert "[enhancement](compaction) optimizing memory usage
for compaction (#36492)" (#37032)
new 848fda6afd3 [fix](cgroup memory) Correct cgroup mem info cache (#36966)
new 3df52cbf761 [improvement](segmentcache) limit segment cache by memory
or segment num (#37026)
The 21 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/cloud/cloud_base_compaction.cpp | 10 -
be/src/cloud/cloud_cumulative_compaction.cpp | 13 +-
be/src/common/config.cpp | 10 +-
be/src/common/config.h | 8 -
be/src/olap/base_compaction.cpp | 10 -
be/src/olap/base_tablet.h | 5 -
be/src/olap/compaction.cpp | 15 +-
be/src/olap/compaction.h | 2 -
be/src/olap/cumulative_compaction.cpp | 15 +-
be/src/olap/iterators.h | 15 +-
be/src/olap/lru_cache.cpp | 5 +
be/src/olap/lru_cache.h | 4 +-
be/src/olap/memtable.cpp | 14 +-
be/src/olap/merger.cpp | 67 +---
be/src/olap/merger.h | 6 +-
be/src/olap/rowset/rowset_meta.h | 15 -
be/src/olap/rowset/segcompaction.cpp | 2 +-
be/src/olap/rowset/segment_v2/column_reader.cpp | 4 +-
.../rowset/segment_v2/indexed_column_reader.cpp | 3 +-
be/src/olap/segment_loader.h | 13 +-
be/src/olap/tablet_reader.h | 2 -
be/src/runtime/exec_env_init.cpp | 8 +-
be/src/util/cgroup_util.cpp | 2 +-
be/src/util/mem_info.cpp | 8 +-
be/src/vec/core/block.cpp | 25 +-
be/src/vec/core/block.h | 5 +-
be/src/vec/data_types/data_type_object.cpp | 14 +-
be/src/vec/olap/vertical_block_reader.cpp | 18 +-
be/src/vec/olap/vertical_block_reader.h | 3 +-
be/src/vec/olap/vertical_merge_iterator.cpp | 29 +-
be/src/vec/olap/vertical_merge_iterator.h | 25 +-
be/src/vec/sink/autoinc_buffer.cpp | 150 +++++----
be/src/vec/sink/autoinc_buffer.h | 35 ++-
be/test/olap/base_compaction_test.cpp | 84 ------
be/test/olap/rowid_conversion_test.cpp | 6 +-
be/test/testutil/run_all_tests.cpp | 2 +-
be/test/vec/olap/vertical_compaction_test.cpp | 14 +-
cloud/src/meta-service/meta_service.cpp | 3 +
cloud/src/meta-service/meta_service_txn.cpp | 6 +-
.../apache/doris/jdbc/JdbcDataSourceConfig.java | 2 +-
.../java/org/apache/doris/common/io/DeepCopy.java | 1 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 1 +
.../main/java/org/apache/doris/alter/Alter.java | 4 +-
.../apache/doris/analysis/TruncateTableStmt.java | 3 -
.../java/org/apache/doris/backup/BackupMeta.java | 1 +
.../doris/catalog/AutoIncrementGenerator.java | 5 +-
.../apache/doris/catalog/CatalogRecycleBin.java | 2 +
.../main/java/org/apache/doris/catalog/Column.java | 30 +-
.../java/org/apache/doris/catalog/ColumnStats.java | 26 +-
.../org/apache/doris/catalog/DistributionInfo.java | 1 +
.../main/java/org/apache/doris/catalog/Env.java | 17 +-
.../apache/doris/catalog/HashDistributionInfo.java | 1 +
.../org/apache/doris/catalog/JdbcResource.java | 2 +-
.../apache/doris/catalog/ListPartitionInfo.java | 1 +
.../apache/doris/catalog/MaterializedIndex.java | 11 +-
.../java/org/apache/doris/catalog/OlapTable.java | 18 +-
.../java/org/apache/doris/catalog/Partition.java | 9 +
.../org/apache/doris/catalog/PartitionInfo.java | 20 +-
.../doris/catalog/RandomDistributionInfo.java | 1 +
.../apache/doris/catalog/RangePartitionInfo.java | 1 +
.../java/org/apache/doris/catalog/Replica.java | 22 +-
.../apache/doris/catalog/SinglePartitionInfo.java | 1 +
.../main/java/org/apache/doris/catalog/Tablet.java | 12 +-
.../org/apache/doris/catalog/TempPartitions.java | 24 +-
.../apache/doris/cloud/CacheHotspotManager.java | 39 ++-
.../apache/doris/cloud/catalog/CloudPartition.java | 33 +-
.../apache/doris/cloud/catalog/CloudReplica.java | 17 +-
.../cloud/datasource/CloudInternalCatalog.java | 4 -
.../transaction/CloudGlobalTransactionMgr.java | 4 +-
.../src/main/java/org/apache/doris/common/Id.java | 7 +-
.../doris/common/NereidsSqlCacheManager.java | 9 +-
.../java/org/apache/doris/common/TreeNode.java | 25 ++
.../doris/common/profile/ExecutionProfile.java | 23 --
.../org/apache/doris/common/profile/Profile.java | 10 +
.../doris/common/profile/SummaryProfile.java | 13 +
.../org/apache/doris/datasource/CatalogIf.java | 3 +
.../apache/doris/datasource/ExternalCatalog.java | 23 ++
.../doris/datasource/hive/HMSCachedClient.java | 2 +
.../doris/datasource/hive/HMSTransaction.java | 18 ++
.../doris/datasource/hive/HiveMetadataOps.java | 16 +
.../hive/PostgreSQLJdbcHMSCachedClient.java | 4 +
.../datasource/hive/ThriftHMSCachedClient.java | 19 +-
.../datasource/iceberg/IcebergMetadataOps.java | 5 +
.../datasource/operations/ExternalMetadataOps.java | 8 +
.../org/apache/doris/nereids/NereidsPlanner.java | 60 +++-
.../org/apache/doris/nereids/SqlCacheContext.java | 1 -
.../doris/nereids/parser/LogicalPlanBuilder.java | 3 +
.../properties/ChildrenPropertiesRegulator.java | 52 ++--
.../functions/table/TableValuedFunction.java | 4 +
.../doris/nereids/trees/plans/PlaceholderId.java | 7 +-
.../trees/plans/commands/ExplainCommand.java | 1 +
.../plans/distribute/BucketSpecifyInstances.java} | 27 +-
.../plans/distribute/DefaultSpecifyInstances.java} | 27 +-
.../trees/plans/distribute/DistributePlanner.java | 66 ++++
.../trees/plans/distribute/DistributedPlan.java | 57 ++++
.../trees/plans/distribute/FragmentIdMapping.java | 71 +++++
.../plans/distribute/NereidsSpecifyInstances.java | 59 ++++
.../plans/distribute/PipelineDistributedPlan.java | 72 +++++
.../BackendDistributedPlanWorkerManager.java | 63 ++++
.../plans/distribute/worker/BackendWorker.java | 74 +++++
.../distribute/worker/DistributedPlanWorker.java} | 33 +-
.../worker/DistributedPlanWorkerManager.java} | 26 +-
.../worker/LoadBalanceScanWorkerSelector.java | 336 +++++++++++++++++++++
.../distribute/worker/ScanWorkerSelector.java | 62 ++++
.../plans/distribute/worker/WorkerScanRanges.java} | 28 +-
.../trees/plans/distribute/worker/Workload.java} | 25 +-
.../worker/job/AbstractUnassignedJob.java | 74 +++++
.../worker/job/AbstractUnassignedScanJob.java | 202 +++++++++++++
.../plans/distribute/worker/job/AssignedJob.java} | 34 +--
.../distribute/worker/job/AssignedJobBuilder.java | 64 ++++
.../distribute/worker/job/BucketScanSource.java | 148 +++++++++
.../worker/job/CustomAssignmentJob.java} | 26 +-
.../distribute/worker/job/DefaultScanSource.java | 111 +++++++
.../worker/job/LocalShuffleAssignedJob.java | 43 +++
.../plans/distribute/worker/job/ScanRange.java} | 24 +-
.../plans/distribute/worker/job/ScanRanges.java | 109 +++++++
.../plans/distribute/worker/job/ScanSource.java} | 34 +--
.../plans/distribute/worker/job/Splittable.java | 69 +++++
.../distribute/worker/job/StaticAssignedJob.java | 106 +++++++
.../UnassignedGatherScanMultiRemoteTablesJob.java | 87 ++++++
.../plans/distribute/worker/job/UnassignedJob.java | 52 ++++
.../worker/job/UnassignedJobBuilder.java | 266 ++++++++++++++++
.../worker/job/UnassignedQueryConstantJob.java | 50 +++
.../job/UnassignedScanBucketOlapTableJob.java | 310 +++++++++++++++++++
.../job/UnassignedScanSingleOlapTableJob.java | 82 +++++
.../job/UnassignedScanSingleRemoteTableJob.java | 53 ++++
.../worker/job/UnassignedShuffleJob.java | 124 ++++++++
.../worker/job/UnassignedSpecifyInstancesJob.java | 49 +++
.../worker/job/UninstancedScanSource.java} | 29 +-
.../distribute/worker/job/WorkerScanSource.java} | 28 +-
.../org/apache/doris/nereids/util/JoinUtils.java | 22 +-
.../java/org/apache/doris/nereids/util/Utils.java | 14 +
.../apache/doris/persist/BackendIdsUpdateInfo.java | 63 ----
.../java/org/apache/doris/persist/ClusterInfo.java | 114 -------
.../org/apache/doris/planner/DataGenScanNode.java | 4 +
.../org/apache/doris/planner/OlapScanNode.java | 37 ++-
.../org/apache/doris/planner/PlanFragment.java | 25 ++
.../java/org/apache/doris/planner/PlanNode.java | 25 ++
.../java/org/apache/doris/planner/ScanNode.java | 9 +-
.../java/org/apache/doris/qe/ConnectContext.java | 6 +
.../main/java/org/apache/doris/qe/Coordinator.java | 51 ++--
.../org/apache/doris/qe/NereidsCoordinator.java | 194 ++++++++++++
.../java/org/apache/doris/qe/SessionVariable.java | 54 +++-
.../java/org/apache/doris/qe/StmtExecutor.java | 12 +-
.../org/apache/doris/qe/cache/CacheAnalyzer.java | 11 +-
.../apache/doris/service/FrontendServiceImpl.java | 10 +
.../apache/doris/statistics/AnalysisManager.java | 2 +-
.../PartitionColumnStatisticCacheLoader.java | 2 +-
.../doris/statistics/StatisticsRepository.java | 6 +-
.../main/java/org/apache/doris/system/Backend.java | 10 -
.../doris/transaction/HiveTransactionManager.java | 7 +-
.../transaction/IcebergTransactionManager.java | 7 +-
.../apache/doris/transaction/TabletCommitInfo.java | 11 +-
.../org/apache/doris/catalog/ColumnStatTest.java | 21 +-
.../java/org/apache/doris/catalog/ColumnTest.java | 18 +-
.../org/apache/doris/catalog/JdbcResourceTest.java | 4 +-
.../doris/catalog/MaterializedIndexTest.java | 6 +-
.../doris/catalog/RangePartitionInfoTest.java | 6 +-
.../java/org/apache/doris/catalog/ReplicaTest.java | 12 +-
.../java/org/apache/doris/catalog/TabletTest.java | 6 +-
.../apache/doris/catalog/TempPartitionTest.java | 4 +-
.../doris/datasource/TestHMSCachedClient.java | 3 +
.../org/apache/doris/qe/HmsQueryCacheTest.java | 14 +-
gensrc/proto/cloud.proto | 3 +
gensrc/thrift/FrontendService.thrift | 1 +
.../unique/test_unique_auto_inc_concurrent.out | 10 +
.../hive/ddl/test_hive_truncate_table.out | 24 ++
.../distribute/colocate_union_numbers.out | 10 +
.../nereids_syntax_p0/distribute/local_shuffle.out | 36 +++
.../prune_bucket_with_bucket_shuffle_join.out | 5 +
.../distribute/query_constant.out | 12 +
.../data/nereids_syntax_p0/distribute/shuffle.out | 8 +
.../distribute/shuffle_left_join.out | 9 +
regression-test/data/variant_p0/load.out | 12 +-
.../data/variant_p0/test_sub_path_pruning.out | 10 +-
.../compaction_width_array_column.groovy | 137 ---------
.../test_single_compaction_fault_injection.groovy | 68 +++--
.../test_single_replica_compaction.groovy | 72 +++--
.../spark_connector/spark_connector.groovy | 10 +-
.../spark_connector/spark_connector_arrow.groovy | 10 +-
.../spark_connector_read_type.groovy | 10 +-
.../unique/test_unique_auto_inc_concurrent.groovy | 59 ++++
.../hive/ddl/test_hive_truncate_table.groovy | 100 ++++++
.../test_forbid_unknown_col_stats.groovy | 2 +-
.../distribute/colocate_union_numbers.groovy | 70 +++++
.../distribute/local_shuffle.groovy | 187 ++++++++++++
.../prune_bucket_with_bucket_shuffle_join.groovy | 98 ++++++
.../distribute/query_constant.groovy | 38 ++-
.../nereids_syntax_p0/distribute/shuffle.groovy | 35 +--
.../distribute/shuffle_left_join.groovy | 117 +++++++
.../suites/statistics/test_partition_stats.groovy | 27 ++
regression-test/suites/variant_p0/load.groovy | 8 +-
.../suites/variant_p0/test_sub_path_pruning.groovy | 20 +-
193 files changed, 5016 insertions(+), 1483 deletions(-)
delete mode 100644 be/test/olap/base_compaction_test.cpp
copy
fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java =>
nereids/trees/plans/distribute/BucketSpecifyInstances.java} (51%)
copy
fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java =>
nereids/trees/plans/distribute/DefaultSpecifyInstances.java} (51%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributePlanner.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/DistributedPlan.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/FragmentIdMapping.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/NereidsSpecifyInstances.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/PipelineDistributedPlan.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/BackendDistributedPlanWorkerManager.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/BackendWorker.java
copy
fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java =>
nereids/trees/plans/distribute/worker/DistributedPlanWorker.java} (52%)
copy
fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java =>
nereids/trees/plans/distribute/worker/DistributedPlanWorkerManager.java} (51%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/LoadBalanceScanWorkerSelector.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/ScanWorkerSelector.java
copy
fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java =>
nereids/trees/plans/distribute/worker/WorkerScanRanges.java} (52%)
copy
fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java =>
nereids/trees/plans/distribute/worker/Workload.java} (51%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedJob.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AbstractUnassignedScanJob.java
copy
fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java =>
nereids/trees/plans/distribute/worker/job/AssignedJob.java} (51%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/AssignedJobBuilder.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/BucketScanSource.java
copy
fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java =>
nereids/trees/plans/distribute/worker/job/CustomAssignmentJob.java} (51%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/DefaultScanSource.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/LocalShuffleAssignedJob.java
copy
fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java =>
nereids/trees/plans/distribute/worker/job/ScanRange.java} (51%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/ScanRanges.java
copy
fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java =>
nereids/trees/plans/distribute/worker/job/ScanSource.java} (52%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/Splittable.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/StaticAssignedJob.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedGatherScanMultiRemoteTablesJob.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJob.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedJobBuilder.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedQueryConstantJob.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanBucketOlapTableJob.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanSingleOlapTableJob.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedScanSingleRemoteTableJob.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedShuffleJob.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedSpecifyInstancesJob.java
copy
fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java =>
nereids/trees/plans/distribute/worker/job/UninstancedScanSource.java} (52%)
copy
fe/fe-core/src/main/java/org/apache/doris/{catalog/SinglePartitionInfo.java =>
nereids/trees/plans/distribute/worker/job/WorkerScanSource.java} (51%)
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/persist/BackendIdsUpdateInfo.java
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/persist/ClusterInfo.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/qe/NereidsCoordinator.java
create mode 100644
regression-test/data/data_model_p0/unique/test_unique_auto_inc_concurrent.out
create mode 100644
regression-test/data/external_table_p0/hive/ddl/test_hive_truncate_table.out
create mode 100644
regression-test/data/nereids_syntax_p0/distribute/colocate_union_numbers.out
create mode 100644
regression-test/data/nereids_syntax_p0/distribute/local_shuffle.out
create mode 100644
regression-test/data/nereids_syntax_p0/distribute/prune_bucket_with_bucket_shuffle_join.out
create mode 100644
regression-test/data/nereids_syntax_p0/distribute/query_constant.out
create mode 100644
regression-test/data/nereids_syntax_p0/distribute/shuffle.out
create mode 100644
regression-test/data/nereids_syntax_p0/distribute/shuffle_left_join.out
delete mode 100644
regression-test/suites/compaction/compaction_width_array_column.groovy
create mode 100644
regression-test/suites/data_model_p0/unique/test_unique_auto_inc_concurrent.groovy
create mode 100644
regression-test/suites/external_table_p0/hive/ddl/test_hive_truncate_table.groovy
create mode 100644
regression-test/suites/nereids_syntax_p0/distribute/colocate_union_numbers.groovy
create mode 100644
regression-test/suites/nereids_syntax_p0/distribute/local_shuffle.groovy
create mode 100644
regression-test/suites/nereids_syntax_p0/distribute/prune_bucket_with_bucket_shuffle_join.groovy
copy
fe/fe-core/src/main/java/org/apache/doris/catalog/SinglePartitionInfo.java =>
regression-test/suites/nereids_syntax_p0/distribute/query_constant.groovy (51%)
copy
fe/fe-core/src/main/java/org/apache/doris/catalog/SinglePartitionInfo.java =>
regression-test/suites/nereids_syntax_p0/distribute/shuffle.groovy (51%)
create mode 100644
regression-test/suites/nereids_syntax_p0/distribute/shuffle_left_join.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
