This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a change to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
from 5f1b888a24f add 2.1.0-rc04 tag
new 06a6477275a [test](regression-test) move test_alter_user.groovy to run
nonConcurrent, for it has set global operation (#29772)
new f79ec8ea7e9 [test](regression-test) fix case bug
suites/export/test_array_export.groovy (#29783)
new fd4795dacee [opt](Nereids) add graph sql function and one arg truncate
(#29864)
new ee66f1563e5 [fix](Nereids) fix rf push down union (#29847)
new 41875a0bf5e [fix](move-memtable) check segment id in add_segment
(#29898)
new 88eab1b4b97 [doc](hight-concurrent-point-query) Improve and supplement
hight-concurrent-point-query documentation (#29396)
new e7b221ba66e [fix](be-ut) Fix unstable test cases (#29896)
new 19987354329 [Improvement](function) enable ipv6_num_to_string function
to support handling of IPv6 type (#29886)
new 12f936558eb [fix](doc) spell errors fixes for debug-point-action
(#28152)
new 1e225b56abb [fix](doc)Added english translation for monitoring Metric
description page (#28435)
new e417128fb95 [bug](bitmap) should return error status when execute
failed (#29841)
new d47adbb81f0 [Fix](nereids) Fix cte rewrite by mv failure and
predicates compensation by mistake (#29820)
new b3e37b3efa1 [unit test](statistics)Add unit test case for auto
analyze. #29904
new 9d3a017706a [fix](doriswriter)Fix the problem that specifying multiple
loadurls does not take effect #29865
new 1dc0c74ad98 [improvement](statistics)Stop analyze quickly after user
close auto analyze. #29809
new 8b4ffcc8f73 [typo](docs) fix typo of outfile and export md (#29804)
new e1a12cf2228 [improvement](auth)Not allowed to operate internal_schema
database (#29790)
new 6598b4f7c87 [fix](http) fix exception when querying map data through
http #29686
new 7b301195378 [improve](multi-table-load) pause job when can not find
table #29870
new d127527af39 [feat](meta) Reuse HMS statistics analyzed by Spark engine
for Analyze Task. (#28525)
new 56f0dc77fe3 [doc](unique key) update description for unique key in
data model doc (#28902)
new 91e546cfe34 [deps](hadoop) upgrade hadoop deps to 3.3.4.6 (#29908)
new c92648cb273 [ut](meta) added unit test for frontend service impl
(#28455)
new 115815739ce [bugfix](fe) add check for leg/lead function params
(#29617)
new 0b16938b7f4 [Fix](Nereids) Fix datatype length wrong when string
contains chinese (#29885)
new bbfc3d037e1 [doc](auto-inc) Add user oriented doc for auto increment
column (#29230)
new a69ce49b07d [fix](Nereids) adjust min/max stats for cast function if
types are comparable (#28166)
new 7c493b08c5d [refactor](dialect) make http sql converter plugin and
audit loader as builtin plugin (#29692)
new e3a1138da73 [fix](migrate disk) fix tablet disk migration timeout too
large (#29895)
new c599cf311dc [fix](migrate) migrate check old tablet had deleted
(#29909)
new 612186e657d [doc](sql-dialect)(audit) add doc for sql dialect and
audit plugin (#29775)
new e5c8192d478 [ut](stats) Added tests for JDBC analysis tasks (#28591)
new d48c8a1dcea [test](ut) added UT cases for show partitions external
table (#29565)
new a974e968410 [community](tool) add a tool to pick pr from one branch to
another (#29764)
new 620cfc3cd7c [fix](move-memtable) set idle timeout equal to load
timeout (#29839)
new a836f418542 [enhance](serde)update slice reserve and deduce slice back
usage #29879
new 5e697990a88 [bugfix](timeout) serving_blocks_num may cause timeout,
try to fix it (#29912)
new 25428bd7fb2 [fix](kerberos) fix BE kerberos ccache renew, optimize
kerbero options (#29291)
new 7309061db45 [pipelineX](improvement) Adjust local exchange strategy
(#29915)
new 615d94bbc72 [log](insertadd log in parse insert into values data
(#29903)
new 74e4486c653 [fix](partition) Add more log for single replica load when
partition id eq 0 (#28707)
new 2bb83f76210 [fix](schema cache) adjust the destruction order of
_tablet_schema_cache and storage engine (#29923)
new 4efc8647de0 [deps](hadoop) update hadoop on BE side to 3.3.6 (#29939)
new 8ca807578f7 [fix](migrate disk) fix migrate disk lost data during
publish version (#29887)
new 4b4fd1a290d [improvement](log) add txn log (#28875)
new 9773fef4a14 [fix](class-loader) fix class loader conflict on BE side
(#29942)
new fd59a3e3a86 [chore](Fix) Fix uninitilized buffer in read_cluster_id()
(#29949)
new c8845c9e07a [opt](scanner) Improve the efficiency of TOPN opt (#29937)
new fd66ce09289 [enhancement](docs) Clarify the requirement for JDK exact
version 8 in install guide (#29921)
new 7a574df9fc4 [fix](pipelineX) fix multi be may be missing profiles
#29914
new 54d23e0f8e7 [typo](docs) fix default value display exception (#29859)
new b47e289560b [refactor](Nereids): avoid ConnectContext.get() ASAP to
improve proformance (#29952)
new 3f20e28f3d5 [doc](jvm) Correct the jvm parameters of be (#29958)
new b7b8e593921 [opt](scanner) use buffered queue to avoid acquiring locks
frequently (#29938)
new 97955da749f [enhancement](fe-memory) support label num threshold to
reduce fe memory consumption (#22889)
new e35b26f4fc5 [feature](auditlog)Add runtime cpu time/peak memory metric
(#29925)
new 05a65b9f81a [improve](join) remove join probe dependency of wait rf
publish finish #29792
new f6dc6ea13ba [improvement](catalog) Escape characters for columns in
recovery predicate pushdown in SQL (#29854)
new be893d792cd [fix](jni) fix jni_reader function name get_nex_block to
get_next_block (#29943)
new 9282ff8e568 [asf] remove some collaborators (#29967)
new ffc6f58e850 [pipelineX](fix) Fix incorrect partition number (#29963)
new e09118eb9c4 extract agg in struct info node (#29853)
new 9e30a67a2ab [Improve](topn opt) avoid crash when rpc returned row
contains duplicated row entry (#29872)
new b99b9efda80 [fix](Nereids) top-n with top project should hit top-n opt
(#29971)
new 94bfa6e46de [fix](Nereids): fix wrong api calling (#29977)
new c9cf9ab8411 [pipelineX](improvement) Improve data distribution for
streaming agg (#29969)
new d5dcdf3e077 [Improve](array) support array_enumerate_uniq and
array_suffle for nereids (#29936)
new 8433169e5ee [test](regression-test) fix case bug, add 'order by' to
make it stable (#29981)
new 0ea5f0f5de4 [fix](Nereids): fix offset in PlanTranslator (#29789)
new 43597afe2ce [bugfix](core) writer status is read and write
concurrently and will core by use after free (#29982)
new 66513d57f99 [feature](function) support ip function named
ipv6_cidr_to_range(addr, cidr) (#29812)
new 0afb4561cdc [chore](ci) replace clickbench pipeline to performance
pipeline (#29929)
new c7d3b833c5b [fix](ci) Update .asf.yaml (#30004)
new ece5f8e86c2 [pipelineX](fix) Fix input data distribution for distinct
streaming agg (#29980)
new 2bf8c51baaf [test](regression) Add debug level log of editlog for p0
p1 (#29992)
new fd5d986239b [fix](regression-test) fix err log limit test global
impact for setting param (#29993)
new f7e12605a2e [fix](Nereids): fix NPE InferPredicates (#29978)
new 779ed259724 [fix](scan) crashing caused by unlocked reading of tablet
(#30005)
new ce2e84e9a6c [fix](nereids)need validate auto partition columns in DDL
(#29985)
new 8240c4e225d [Enhance](tools) change SSB perf tests to standardize
behavior (#30002)
new 2297a5b6fce [doc](statistics)Update docs for statistics (#29926)
new 07de535c4cc [fix](Nereids) should not fold constant when do ordinal
group by (#29976)
new 4d6de1f181c [opt](invert index) optimize the code structure of
inverted index queries (#29771)
new e1bcdc35fdd (fix)[group-commit] Fix some group commit case (#30008)
new 22978726e3b [opt](nereids) if column stats are unknown, 10-20
table-join optimization use cascading instead of dphyp (#29902)
new 3c502f64446 [fix](case) add sync after streamload (#30009)
new 08db92a49a6 [fix](regression) spare .testfile to make disk checker
happy when injecting fault (#30012)
new f53d2c28cbb [improvement](catalog) fix jdbc mysql catalog to_date fun
pushdown (#29900)
new 75cafa8672b [enhancement](jdbc catalog) Enhance function pushdown of
Jdbc Oracle Catalog (#29972)
new 55a0f900035 [fix](jobs)fix create task int job (#30011)
new d3bf23d70da [chore](removelogs) remove debug query timeout logs
(#30006)
new 6ce9c67383d [fix](Nereids) struct type coercion call wrong api (#30018)
new d11e797d4cf [fix](decimal) fix wrong decimal overflow caused by
uninitialized nested column of null value (#29960)
new 4bf4239d7aa [feature](Nereids): optimize logical group expression in
dphyp (#30000)
new 74991c4af2d [bugfix](paimon)support native and jni to read paimon for
minio/cos #29933
new 168afdb965c [fix](disk balance) Change disk rebalance unpick time to
configurable (#28949)
new 2916745cf22 [Chore](arm) Support auto-SIMD on arm and refactor
compilation docs (#29956)
new 4e41e1d7976 [feat](Nereids) persist constraint in table (#29767)
The 98 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.asf.yaml | 2 -
be/CMakeLists.txt | 10 +
be/src/agent/task_worker_pool.cpp | 12 +-
be/src/common/config.cpp | 15 +-
be/src/common/config.h | 19 +-
be/src/exec/rowid_fetcher.cpp | 15 +-
be/src/io/fs/hdfs_file_system.cpp | 17 +-
be/src/io/fs/hdfs_file_system.h | 10 +-
be/src/io/fs/local_file_writer.cpp | 6 +-
be/src/io/hdfs_builder.cpp | 64 +-
be/src/io/hdfs_builder.h | 6 +-
be/src/olap/delta_writer.cpp | 10 +-
be/src/olap/parallel_scanner_builder.cpp | 5 +-
.../inverted_index/query/conjunction_query.cpp | 15 +-
.../inverted_index/query/conjunction_query.h | 32 +-
.../inverted_index/query/disjunction_query.cpp | 10 +-
.../inverted_index/query/disjunction_query.h | 27 +-
.../inverted_index/query/phrase_prefix_query.cpp | 27 +-
.../inverted_index/query/phrase_prefix_query.h | 31 +-
.../inverted_index/query/phrase_query.cpp | 4 +
.../segment_v2/inverted_index/query/phrase_query.h | 23 +-
.../inverted_index/query/prefix_query.cpp | 4 +-
.../segment_v2/inverted_index/query/prefix_query.h | 6 +-
.../query/{conjunction_query.h => query.h} | 48 +-
.../inverted_index/query/query_factory.h | 53 ++
.../inverted_index/query/regexp_query.cpp | 19 +-
.../segment_v2/inverted_index/query/regexp_query.h | 18 +-
.../rowset/segment_v2/inverted_index_reader.cpp | 100 +--
.../olap/rowset/segment_v2/inverted_index_reader.h | 27 +-
be/src/olap/tablet.h | 4 +-
be/src/olap/task/engine_publish_version_task.cpp | 28 +-
be/src/olap/task/engine_storage_migration_task.cpp | 28 +-
be/src/olap/task/engine_storage_migration_task.h | 3 +-
be/src/olap/txn_manager.cpp | 41 +-
.../distinct_streaming_aggregation_sink_operator.h | 9 +
be/src/pipeline/exec/hashjoin_build_sink.h | 4 +
be/src/pipeline/exec/hashjoin_probe_operator.h | 4 +
be/src/pipeline/exec/scan_operator.cpp | 13 +
.../exec/streaming_aggregation_sink_operator.h | 2 +-
be/src/pipeline/exec/union_source_operator.cpp | 6 +
be/src/pipeline/pipeline_task.cpp | 16 +-
.../local_exchange/local_exchange_sink_operator.h | 27 +-
.../pipeline_x/local_exchange/local_exchanger.cpp | 22 +-
be/src/pipeline/pipeline_x/operator.h | 13 +-
.../pipeline_x/pipeline_x_fragment_context.cpp | 13 +-
.../pipeline_x/pipeline_x_fragment_context.h | 1 +
be/src/pipeline/pipeline_x/pipeline_x_task.cpp | 10 +-
be/src/runtime/fragment_mgr.cpp | 3 +
be/src/runtime/group_commit_mgr.cpp | 3 +-
be/src/runtime/load_stream.cpp | 12 +-
be/src/runtime/memory/mem_tracker.h | 8 +
be/src/runtime/memory/mem_tracker_limiter.cpp | 6 +
be/src/runtime/plan_fragment_executor.cpp | 17 +-
be/src/runtime/query_context.cpp | 21 +
be/src/runtime/query_context.h | 8 +
be/src/runtime/query_statistics.cpp | 16 +-
be/src/runtime/query_statistics.h | 24 +-
be/src/runtime/runtime_query_statistics_mgr.cpp | 32 +-
be/src/runtime/runtime_query_statistics_mgr.h | 15 +-
be/src/service/internal_service.cpp | 4 +-
be/src/util/debug_points.h | 14 +
be/src/vec/common/ipv6_to_binary.h | 69 +++
be/src/vec/core/field.h | 3 +-
be/src/vec/data_types/data_type.h | 70 ++-
.../vec/data_types/serde/data_type_array_serde.cpp | 6 +-
be/src/vec/exec/format/avro/avro_jni_reader.cpp | 2 +-
be/src/vec/exec/format/jni_reader.cpp | 2 +-
be/src/vec/exec/format/table/hudi_jni_reader.cpp | 2 +-
.../exec/format/table/max_compute_jni_reader.cpp | 2 +-
be/src/vec/exec/format/table/paimon_reader.cpp | 2 +-
be/src/vec/exec/jni_connector.cpp | 2 +-
be/src/vec/exec/jni_connector.h | 2 +-
be/src/vec/exec/scan/pip_scanner_context.h | 92 ++-
be/src/vec/exec/scan/scanner_context.cpp | 3 -
be/src/vec/exec/scan/scanner_context.h | 6 +-
be/src/vec/exec/scan/scanner_scheduler.cpp | 12 -
be/src/vec/exec/scan/vscan_node.cpp | 5 -
be/src/vec/exec/scan/vscanner.h | 6 +-
be/src/vec/exec/vunion_node.cpp | 6 +
be/src/vec/functions/function_bitmap.cpp | 14 +-
be/src/vec/functions/function_bitmap_variadic.cpp | 4 +-
be/src/vec/functions/function_cast.h | 100 ++-
be/src/vec/functions/function_coalesce.cpp | 2 +-
be/src/vec/functions/function_encryption.cpp | 5 +-
be/src/vec/functions/function_hex.cpp | 2 +-
be/src/vec/functions/function_ifnull.h | 3 +-
be/src/vec/functions/function_ip.cpp | 1 +
be/src/vec/functions/function_ip.h | 137 +++++
be/src/vec/functions/function_quantile_state.cpp | 6 +-
be/src/vec/functions/function_reverse.h | 5 +-
be/src/vec/functions/function_string.cpp | 11 +-
be/src/vec/functions/function_string.h | 8 +-
be/src/vec/functions/like.cpp | 2 +-
be/src/vec/sink/load_stream_stub.cpp | 5 +-
be/src/vec/sink/load_stream_stub.h | 2 +-
be/src/vec/sink/writer/async_result_writer.cpp | 6 +-
be/src/vec/sink/writer/async_result_writer.h | 5 +-
be/src/vec/sink/writer/vtablet_writer_v2.cpp | 5 +-
be/test/agent/task_worker_pool_test.cpp | 8 +-
be/test/runtime/load_stream_test.cpp | 2 +-
docs/en/docs/admin-manual/config/be-config.md | 10 -
docs/en/docs/admin-manual/config/fe-config.md | 4 +-
.../http-actions/fe/debug-point-action.md | 4 +-
.../maint-monitor/monitor-metrics/metrics.md | 313 +++++++++-
docs/en/docs/advanced/auto-increment.md | 493 +++++++++++++++
docs/en/docs/advanced/time-zone.md | 7 +-
docs/en/docs/data-table/data-model.md | 37 +-
docs/en/docs/data-table/index/inverted-index.md | 12 +
docs/en/docs/ecosystem/audit-plugin.md | 40 +-
.../ecosystem/udf/java-user-defined-function.md | 2 +-
docs/en/docs/get-starting/quick-start.md | 34 +-
.../docs/install/source-install/compilation-arm.md | 659 ++++++++++----------
docs/en/docs/install/standard-deployment.md | 14 +-
docs/en/docs/lakehouse/multi-catalog/jdbc.md | 6 +-
docs/en/docs/lakehouse/multi-catalog/paimon.md | 32 +-
docs/en/docs/lakehouse/sql-dialect.md | 71 +++
.../query-async-materialized-view.md | 1 -
.../hight-concurrent-point-query.md | 56 +-
docs/en/docs/query-acceleration/statistics.md | 10 +-
.../sql-functions/json-functions/json-insert.md | 2 +-
.../Manipulation/EXPORT.md | 2 +-
.../Data-Manipulation-Statements/OUTFILE.md | 4 +-
docs/sidebars.json | 4 +-
docs/zh-CN/docs/admin-manual/config/be-config.md | 10 -
docs/zh-CN/docs/admin-manual/config/fe-config.md | 8 +-
docs/zh-CN/docs/advanced/auto-increment.md | 490 +++++++++++++++
docs/zh-CN/docs/advanced/time-zone.md | 7 +-
docs/zh-CN/docs/data-table/data-model.md | 32 +-
docs/zh-CN/docs/data-table/index/inverted-index.md | 14 +-
docs/zh-CN/docs/ecosystem/audit-plugin.md | 41 +-
.../ecosystem/udf/java-user-defined-function.md | 2 +-
.../docs/install/source-install/compilation-arm.md | 670 +++++++++++----------
docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md | 6 +-
docs/zh-CN/docs/lakehouse/multi-catalog/paimon.md | 32 +-
docs/zh-CN/docs/lakehouse/sql-dialect.md | 70 +++
.../query-async-materialized-view.md | 1 -
.../hight-concurrent-point-query.md | 59 +-
docs/zh-CN/docs/query-acceleration/statistics.md | 12 +-
.../sql-functions/json-functions/json-insert.md | 2 +-
.../sql-functions/json-functions/json-replace.md | 2 +-
.../Manipulation/EXPORT.md | 2 +-
.../Data-Manipulation-Statements/OUTFILE.md | 4 +-
.../doriswriter/DorisStreamLoadObserver.java | 10 +-
.../org/apache/doris/catalog/PrimitiveType.java | 2 -
.../main/java/org/apache/doris/common/Config.java | 22 +-
.../org/apache/doris/common/FeMetaVersion.java | 4 +-
.../doris/analysis/AlterDatabasePropertyStmt.java | 3 +-
.../doris/analysis/AlterDatabaseQuotaStmt.java | 3 +-
.../apache/doris/analysis/AlterDatabaseRename.java | 3 +-
.../org/apache/doris/analysis/AlterTableStmt.java | 2 +
.../org/apache/doris/analysis/ArrayLiteral.java | 16 +
.../java/org/apache/doris/analysis/CastExpr.java | 2 +-
.../org/apache/doris/analysis/CreateDbStmt.java | 3 +-
.../org/apache/doris/analysis/CreateTableStmt.java | 3 +-
.../org/apache/doris/analysis/DecimalLiteral.java | 2 +-
.../java/org/apache/doris/analysis/DropDbStmt.java | 3 +-
.../org/apache/doris/analysis/DropTableStmt.java | 3 +-
.../main/java/org/apache/doris/analysis/Expr.java | 27 +-
.../org/apache/doris/analysis/FloatLiteral.java | 2 +-
.../org/apache/doris/analysis/InlineViewRef.java | 2 +-
.../doris/analysis/InsertOverwriteTableStmt.java | 3 +
.../java/org/apache/doris/analysis/IntLiteral.java | 2 +-
.../org/apache/doris/analysis/LargeIntLiteral.java | 2 +-
.../org/apache/doris/analysis/LiteralExpr.java | 12 +
.../apache/doris/analysis/NativeInsertStmt.java | 39 +-
.../NumericLiteralExpr.java} | 26 +-
.../java/org/apache/doris/analysis/SlotRef.java | 31 +-
.../org/apache/doris/analysis/StructLiteral.java | 7 +
.../apache/doris/analysis/TruncateTableStmt.java | 3 +-
.../doris/catalog/BuiltinScalarFunctions.java | 8 +
.../main/java/org/apache/doris/catalog/Env.java | 1 +
.../doris/catalog/InternalSchemaInitializer.java | 82 ++-
.../main/java/org/apache/doris/catalog/Table.java | 20 +-
.../org/apache/doris/catalog/TableAttributes.java | 53 ++
.../java/org/apache/doris/catalog/TableIf.java | 34 +-
.../doris/catalog/constraint/Constraint.java | 31 +-
.../catalog/constraint/ForeignKeyConstraint.java | 12 +-
.../catalog/constraint/PrimaryKeyConstraint.java | 9 +-
.../doris/catalog/constraint/TableIdentifier.java | 26 +-
.../doris/catalog/constraint/UniqueConstraint.java | 8 +-
.../doris/catalog/external/ExternalTable.java | 10 +
.../doris/catalog/external/HMSExternalTable.java | 49 +-
.../java/org/apache/doris/clone/Rebalancer.java | 3 +-
.../doris/common/profile/ExecutionProfile.java | 33 +-
.../util/InternalDatabaseUtil.java} | 42 +-
.../doris/datasource/ExternalSchemaCache.java | 5 +
.../paimon/PaimonFileExternalCatalog.java | 15 +
.../property/constants/PaimonProperties.java | 1 +
.../org/apache/doris/httpv2/rest/LoadAction.java | 46 +-
.../doris/job/extensions/insert/InsertJob.java | 7 +-
.../org/apache/doris/journal/JournalEntity.java | 7 +
.../org/apache/doris/load/StreamLoadRecordMgr.java | 6 +-
.../org/apache/doris/load/loadv2/BulkLoadJob.java | 4 +-
.../org/apache/doris/load/loadv2/LoadManager.java | 73 ++-
.../doris/load/routineload/RoutineLoadJob.java | 2 +-
.../doris/load/routineload/RoutineLoadManager.java | 50 +-
.../load/routineload/RoutineLoadScheduler.java | 17 +-
.../org/apache/doris/load/sync/SyncChecker.java | 1 +
.../org/apache/doris/load/sync/SyncJobManager.java | 67 ++-
.../org/apache/doris/nereids/StatementContext.java | 24 +-
.../org/apache/doris/nereids/cost/CostModelV1.java | 2 +-
.../nereids/exceptions/MetaNotFoundException.java | 74 +++
.../glue/translator/PhysicalPlanTranslator.java | 30 +-
.../glue/translator/PlanTranslatorContext.java | 4 +
.../java/org/apache/doris/nereids/jobs/Job.java | 5 +
.../nereids/jobs/cascades/CostAndEnforcerJob.java | 5 -
.../jobs/cascades/OptimizeGroupExpressionJob.java | 7 +-
.../doris/nereids/jobs/executor/Optimizer.java | 16 +-
.../joinorder/hypergraph/node/StructInfoNode.java | 62 ++
.../hypergraph/receiver/PlanReceiver.java | 144 +----
.../org/apache/doris/nereids/metrics/Event.java | 7 +-
.../doris/nereids/minidump/MinidumpUtils.java | 36 +-
.../org/apache/doris/nereids/parser/Dialect.java | 26 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 9 +-
.../processor/post/AddOffsetIntoDistribute.java | 41 ++
.../nereids/processor/post/PlanPostProcessors.java | 1 +
.../processor/post/RuntimeFilterGenerator.java | 11 +
.../doris/nereids/processor/post/TopNScanOpt.java | 8 +
.../org/apache/doris/nereids/rules/RuleSet.java | 9 +-
.../ResolveOrdinalInOrderByAndGroupBy.java | 12 +-
.../mv/AbstractMaterializedViewAggregateRule.java | 28 +-
.../mv/AbstractMaterializedViewRule.java | 215 ++-----
.../rules/exploration/mv/HyperGraphComparator.java | 35 +-
.../mv/LogicalCompatibilityContext.java | 8 +-
.../exploration/mv/MaterializedViewUtils.java | 18 +
.../nereids/rules/exploration/mv/Predicates.java | 178 +++++-
.../nereids/rules/exploration/mv/StructInfo.java | 32 +-
.../expression/rules/FoldConstantRuleOnFE.java | 2 +-
.../doris/nereids/rules/rewrite/CTEInline.java | 7 +-
.../nereids/rules/rewrite/InferPredicates.java | 4 +-
.../rules/rewrite/PredicatePropagation.java | 5 +-
.../doris/nereids/stats/ExpressionEstimation.java | 64 +-
.../doris/nereids/stats/StatsCalculator.java | 7 +
...pv6NumToString.java => ArrayEnumerateUniq.java} | 47 +-
.../scalar/{Truncate.java => ArrayShuffle.java} | 52 +-
.../scalar/{Ipv6NumToString.java => G.java} | 37 +-
.../{Ipv6NumToString.java => IPv6CIDRToRange.java} | 37 +-
.../functions/scalar/Ipv6NumToString.java | 5 +-
.../expressions/functions/scalar/Truncate.java | 18 +-
.../trees/expressions/functions/window/Lag.java | 16 +-
.../trees/expressions/functions/window/Lead.java | 16 +-
.../functions/window/WindowFunction.java | 18 +-
.../trees/expressions/literal/DecimalLiteral.java | 2 +-
.../expressions/literal/DecimalV3Literal.java | 2 +-
.../trees/expressions/literal/DoubleLiteral.java | 2 +-
.../trees/expressions/literal/FloatLiteral.java | 2 +-
...egerLikeLiteral.java => FractionalLiteral.java} | 20 +-
.../expressions/literal/IntegerLikeLiteral.java | 2 +-
...IntegerLikeLiteral.java => NumericLiteral.java} | 20 +-
.../expressions/literal/StringLikeLiteral.java | 5 +-
.../expressions/visitor/ScalarFunctionVisitor.java | 20 +
.../org/apache/doris/nereids/trees/plans/Plan.java | 2 +-
.../trees/plans/commands/AddConstraintCommand.java | 15 +-
.../trees/plans/commands/DeleteFromCommand.java | 10 +
.../plans/commands/DropConstraintCommand.java | 9 +-
.../plans/commands/InsertIntoTableCommand.java | 11 +-
.../commands/InsertOverwriteTableCommand.java | 3 +
.../plans/commands/ShowConstraintsCommand.java | 2 +-
.../trees/plans/commands/info/AlterMTMVInfo.java | 4 +-
.../trees/plans/commands/info/CreateMTMVInfo.java | 4 +-
.../trees/plans/commands/info/CreateTableInfo.java | 32 +-
.../trees/plans/commands/info/DropMTMVInfo.java | 4 +-
.../trees/plans/commands/info/RefreshMTMVInfo.java | 4 +-
.../nereids/trees/plans/logical/LogicalJoin.java | 7 +
.../trees/plans/physical/PhysicalDistribute.java | 30 +-
.../trees/plans/physical/PhysicalHashJoin.java | 14 -
.../trees/plans/physical/PhysicalLimit.java | 5 +
.../trees/plans/physical/PhysicalProject.java | 71 ++-
.../plans/visitor/ExpressionLineageReplacer.java | 25 +-
.../doris/nereids/util/TypeCoercionUtils.java | 4 +-
.../java/org/apache/doris/nereids/util/Utils.java | 12 +
.../apache/doris/persist/AlterConstraintLog.java | 61 ++
.../java/org/apache/doris/persist/EditLog.java | 22 +-
.../org/apache/doris/persist/OperationType.java | 3 +
.../org/apache/doris/persist/gson/GsonUtils.java | 11 +
.../org/apache/doris/planner/MysqlScanNode.java | 3 +-
.../external/jdbc/JdbcFunctionPushDownRule.java | 30 +-
.../doris/planner/external/jdbc/JdbcScanNode.java | 42 +-
.../doris/planner/external/odbc/OdbcScanNode.java | 2 +-
.../planner/external/paimon/PaimonScanNode.java | 17 +-
.../java/org/apache/doris/plugin/AuditPlugin.java | 2 +
.../java/org/apache/doris/plugin/PluginMgr.java | 26 +-
.../doris/plugin/{ => audit}/AuditEvent.java | 9 +-
.../doris/plugin/audit/AuditLoaderPlugin.java | 252 ++++++++
.../{qe => plugin/audit}/AuditLogBuilder.java | 7 +-
.../doris/plugin/audit/AuditStreamLoader.java | 182 ++++++
.../doris/plugin/{ => audit}/LoadAuditEvent.java | 2 +-
.../plugin/{ => audit}/StreamLoadAuditEvent.java | 2 +-
.../dialect}/HttpDialectConverterPlugin.java | 80 +--
.../doris/plugin/dialect}/HttpDialectUtils.java | 6 +-
.../org/apache/doris/qe/AuditEventProcessor.java | 2 +-
.../java/org/apache/doris/qe/AuditLogHelper.java | 40 +-
.../java/org/apache/doris/qe/ConnectContext.java | 2 +-
.../java/org/apache/doris/qe/ConnectProcessor.java | 9 +-
.../main/java/org/apache/doris/qe/Coordinator.java | 3 +
.../java/org/apache/doris/qe/GlobalVariable.java | 23 +-
.../java/org/apache/doris/qe/SessionVariable.java | 510 ++++++++--------
.../java/org/apache/doris/qe/StmtExecutor.java | 23 +-
.../WorkloadRuntimeStatusMgr.java | 3 +-
.../apache/doris/service/FrontendServiceImpl.java | 11 +
.../apache/doris/statistics/HMSAnalysisTask.java | 41 +-
.../doris/statistics/StatisticsAutoCollector.java | 26 +-
.../doris/tablefunction/MetadataGenerator.java | 2 +-
.../doris/transaction/DatabaseTransactionMgr.java | 72 ++-
.../doris/analysis/ShowPartitionsStmtTest.java | 186 +++++-
.../catalog/constraint/ConstraintPersistTest.java | 305 ++++++++++
.../apache/doris/load/loadv2/LoadManagerTest.java | 46 +-
.../load/routineload/RoutineLoadManagerTest.java | 43 +-
.../apache/doris/load/sync/SyncJobManagerTest.java | 36 ++
.../joinorder/hypergraph/CompareOuterJoinTest.java | 6 +-
.../jobs/joinorder/hypergraph/InferJoinTest.java | 6 +-
.../joinorder/hypergraph/InferPredicateTest.java | 6 +-
.../joinorder/hypergraph/PullupExpressionTest.java | 6 +-
.../exploration/mv/HyperGraphAggTest.java} | 54 +-
.../exploration/mv/HyperGraphComparatorTest.java | 4 +-
.../nereids/stats/ExpressionEstimationTest.java | 71 +++
.../doris/nereids/stats/FilterEstimationTest.java | 4 +-
.../org/apache/doris/nereids/util/UtilsTest.java} | 26 +-
.../org/apache/doris/plugin/DialectPluginTest.java | 2 +-
.../apache/doris/plugin}/HttpDialectUtilsTest.java | 12 +-
.../org/apache/doris/plugin}/SimpleHttpServer.java | 2 +-
.../apache/doris/plugin/TestDialectPlugin1.java | 2 +-
.../apache/doris/qe/AuditEventProcessorTest.java | 5 +-
.../doris/service/FrontendServiceImplTest.java | 64 ++
.../doris/statistics/AnalysisManagerTest.java | 53 ++
.../doris/statistics/JdbcAnalysisTaskTest.java | 157 +++++
.../doris/statistics/OlapAnalysisTaskTest.java | 2 +-
.../statistics/StatisticsAutoCollectorTest.java | 71 +++
.../transaction/DatabaseTransactionMgrTest.java | 12 +-
.../apache/doris/utframe/TestWithFeService.java | 11 +
.../doris/plugin/audit/AuditLoaderPlugin.java | 2 +-
fe_plugins/http-dialect-converter/pom.xml | 119 ----
.../src/main/assembly/plugin.conf | 22 -
.../src/main/assembly/plugin.properties | 23 -
.../src/main/assembly/zip.xml | 43 --
fe_plugins/pom.xml | 1 -
gensrc/proto/internal_service.proto | 1 +
gensrc/script/doris_builtins_functions.py | 12 +-
.../test_always_nullable_window_function.out | 40 +-
.../test_array_zip_array_enumerate_uniq.out | 12 +
.../jdbc/test_mysql_jdbc_catalog.out | 25 +-
.../paimon/paimon_base_filesystem.out | 17 +
.../test_migrate_disk_with_publish_version.out | 11 +
regression-test/data/mtmv_p0/test_build_mtmv.out | 3 +
.../mv/agg_with_roll_up/aggregate_with_roll_up.out | 36 ++
.../aggregate_without_roll_up.out | 28 +
.../inner_join_dphyp.out} | 16 +
.../outer_join_dphyp.out} | 44 ++
.../nereids_rules_p0/mv/join/inner/inner_join.out | 16 +
.../mv/join/left_outer/outer_join.out | 44 ++
.../data/nereids_syntax_p0/group_by_constant.out | 3 +
.../shape/query11.out | 6 +-
.../nereids_tpcds_shape_sf1000_p0/shape/query4.out | 24 +-
.../nereids_tpcds_shape_sf1000_p0/shape/query5.out | 26 +-
.../shape/query54.out | 24 +-
.../shape/query58.out | 117 ++--
.../shape/query64.out | 128 ++--
.../shape/query71.out | 8 +-
.../shape/query74.out | 6 +-
.../noStatsRfPrune/query54.out | 4 +-
.../noStatsRfPrune/query64.out | 165 +++--
.../no_stats_shape/query11.out | 6 +-
.../no_stats_shape/query4.out | 8 +-
.../no_stats_shape/query54.out | 20 +-
.../no_stats_shape/query64.out | 165 +++--
.../no_stats_shape/query74.out | 6 +-
.../rf_prune/query4.out | 16 +-
.../rf_prune/query5.out | 20 +-
.../rf_prune/query54.out | 20 +-
.../rf_prune/query58.out | 12 +-
.../rf_prune/query64.out | 92 +--
.../rf_prune/query71.out | 8 +-
.../nereids_tpcds_shape_sf100_p0/shape/query11.out | 6 +-
.../nereids_tpcds_shape_sf100_p0/shape/query4.out | 24 +-
.../nereids_tpcds_shape_sf100_p0/shape/query5.out | 26 +-
.../nereids_tpcds_shape_sf100_p0/shape/query54.out | 22 +-
.../nereids_tpcds_shape_sf100_p0/shape/query58.out | 12 +-
.../nereids_tpcds_shape_sf100_p0/shape/query64.out | 100 +--
.../nereids_tpcds_shape_sf100_p0/shape/query71.out | 8 +-
.../nereids_tpcds_shape_sf100_p0/shape/query74.out | 6 +-
.../sql_functions/case_function/case_null1.csv | 4 +-
.../test_ipv6_cidr_to_range_function.out | 32 +
.../test_ipv6_num_to_string_function.out | 11 +
regression-test/pipeline/p0/conf/be.conf | 4 +-
regression-test/pipeline/p0/conf/fe.conf | 2 +
regression-test/pipeline/p1/conf/fe.conf | 2 +
.../suites/account_p0/test_alter_user.groovy | 2 +-
.../test_always_nullable_window_function.groovy | 8 +-
.../test_array_zip_array_enumerate_uniq.groovy | 32 +-
.../suites/ddl_p0/test_alter_view.groovy | 4 +-
.../suites/export/test_array_export.groovy | 2 +-
.../jdbc/test_clickhouse_jdbc_catalog.groovy | 4 +-
.../jdbc/test_mysql_jdbc_catalog.groovy | 19 +-
.../jdbc/test_oracle_jdbc_catalog.groovy | 6 +
.../paimon/paimon_base_filesystem.groovy | 66 ++
.../test_load_stream_fault_injection.groovy | 20 +-
.../operate_internal_schema.groovy | 126 ++++
.../data/multi_table_load_invalid_table.csv | 1 +
.../routine_load/test_routine_load_error.groovy | 94 +++
.../test_stream_load_err_log_limit.groovy | 2 +-
.../test_migrate_disk_with_publish_version.groovy | 84 +++
.../suites/mtmv_p0/test_build_mtmv.groovy | 59 +-
.../agg_with_roll_up/aggregate_with_roll_up.groovy | 90 ++-
.../aggregate_without_roll_up.groovy | 107 +++-
.../inner_join_dphyp.groovy} | 38 +-
.../outer_join_dphyp.groovy} | 86 ++-
.../mv/join/inner/inner_join.groovy | 35 +-
.../mv/join/left_outer/outer_join.groovy | 82 ++-
.../nereids_syntax_p0/group_by_constant.groovy | 2 +
.../suites/nereids_syntax_p0/set_operation.groovy | 1 +
.../test_ipv6_cidr_to_range_function.groovy | 85 +++
.../test_ipv6_num_to_string_function.groovy | 54 ++
.../suites/statistics/analyze_stats.groovy | 249 +++++---
thirdparty/CHANGELOG.md | 8 +
tools/pick_pr.sh | 108 ++++
tools/ssb-tools/bin/run-ssb-flat-queries.sh | 15 +-
tools/ssb-tools/bin/run-ssb-queries.sh | 13 +-
417 files changed, 9800 insertions(+), 3730 deletions(-)
copy be/src/olap/rowset/segment_v2/inverted_index/query/{conjunction_query.h
=> query.h} (52%)
create mode 100644
be/src/olap/rowset/segment_v2/inverted_index/query/query_factory.h
create mode 100644 be/src/vec/common/ipv6_to_binary.h
create mode 100644 docs/en/docs/advanced/auto-increment.md
create mode 100644 docs/en/docs/lakehouse/sql-dialect.md
create mode 100644 docs/zh-CN/docs/advanced/auto-increment.md
create mode 100644 docs/zh-CN/docs/lakehouse/sql-dialect.md
copy fe/fe-core/src/main/java/org/apache/doris/{plugin/AuditPlugin.java =>
analysis/NumericLiteralExpr.java} (61%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/catalog/TableAttributes.java
copy
fe/fe-core/src/main/java/org/apache/doris/{catalog/constraint/Constraint.java
=> common/util/InternalDatabaseUtil.java} (53%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/MetaNotFoundException.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/AddOffsetIntoDistribute.java
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{Ipv6NumToString.java
=> ArrayEnumerateUniq.java} (60%)
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{Truncate.java
=> ArrayShuffle.java} (56%)
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{Ipv6NumToString.java
=> G.java} (61%)
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{Ipv6NumToString.java
=> IPv6CIDRToRange.java} (56%)
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/{IntegerLikeLiteral.java
=> FractionalLiteral.java} (73%)
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/{IntegerLikeLiteral.java
=> NumericLiteral.java} (73%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/persist/AlterConstraintLog.java
rename fe/fe-core/src/main/java/org/apache/doris/plugin/{ =>
audit}/AuditEvent.java (97%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
rename fe/fe-core/src/main/java/org/apache/doris/{qe =>
plugin/audit}/AuditLogBuilder.java (97%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditStreamLoader.java
rename fe/fe-core/src/main/java/org/apache/doris/plugin/{ =>
audit}/LoadAuditEvent.java (99%)
rename fe/fe-core/src/main/java/org/apache/doris/plugin/{ =>
audit}/StreamLoadAuditEvent.java (99%)
rename
{fe_plugins/http-dialect-converter/src/main/java/org/apache/doris/plugin/dialect/http
=>
fe/fe-core/src/main/java/org/apache/doris/plugin/dialect}/HttpDialectConverterPlugin.java
(56%)
rename
{fe_plugins/http-dialect-converter/src/main/java/org/apache/doris/plugin/dialect/http
=>
fe/fe-core/src/main/java/org/apache/doris/plugin/dialect}/HttpDialectUtils.java
(98%)
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/catalog/constraint/ConstraintPersistTest.java
copy
fe/fe-core/src/test/java/org/apache/doris/nereids/{jobs/joinorder/hypergraph/InferPredicateTest.java
=> rules/exploration/mv/HyperGraphAggTest.java} (57%)
copy fe/fe-core/src/{main/java/org/apache/doris/plugin/AuditPlugin.java =>
test/java/org/apache/doris/nereids/util/UtilsTest.java} (61%)
rename
{fe_plugins/http-dialect-converter/src/test/java/org/apache/doris/plugin/dialect/http
=> fe/fe-core/src/test/java/org/apache/doris/plugin}/HttpDialectUtilsTest.java
(90%)
rename
{fe_plugins/http-dialect-converter/src/test/java/org/apache/doris/plugin/dialect/http
=> fe/fe-core/src/test/java/org/apache/doris/plugin}/SimpleHttpServer.java
(98%)
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/statistics/JdbcAnalysisTaskTest.java
delete mode 100644 fe_plugins/http-dialect-converter/pom.xml
delete mode 100755
fe_plugins/http-dialect-converter/src/main/assembly/plugin.conf
delete mode 100755
fe_plugins/http-dialect-converter/src/main/assembly/plugin.properties
delete mode 100644 fe_plugins/http-dialect-converter/src/main/assembly/zip.xml
create mode 100644
regression-test/data/external_table_p2/paimon/paimon_base_filesystem.out
create mode 100644
regression-test/data/migrate_p0/test_migrate_disk_with_publish_version.out
copy regression-test/data/nereids_rules_p0/mv/join/{inner/inner_join.out =>
dphyp_inner/inner_join_dphyp.out} (95%)
copy regression-test/data/nereids_rules_p0/mv/join/{left_outer/outer_join.out
=> dphyp_outer/outer_join_dphyp.out} (82%)
create mode 100644
regression-test/data/query_p0/sql_functions/ip_functions/test_ipv6_cidr_to_range_function.out
create mode 100644
regression-test/data/query_p0/sql_functions/ip_functions/test_ipv6_num_to_string_function.out
create mode 100644
regression-test/suites/external_table_p2/paimon/paimon_base_filesystem.groovy
create mode 100644
regression-test/suites/internal_schema_p0/operate_internal_schema.groovy
create mode 100644
regression-test/suites/load_p0/routine_load/data/multi_table_load_invalid_table.csv
create mode 100644
regression-test/suites/load_p0/routine_load/test_routine_load_error.groovy
create mode 100644
regression-test/suites/migrate_p0/test_migrate_disk_with_publish_version.groovy
copy regression-test/suites/nereids_rules_p0/mv/join/{inner/inner_join.groovy
=> dphyp_inner/inner_join_dphyp.groovy} (94%)
copy
regression-test/suites/nereids_rules_p0/mv/join/{left_outer/outer_join.groovy
=> dphyp_outer/outer_join_dphyp.groovy} (85%)
create mode 100644
regression-test/suites/query_p0/sql_functions/ip_functions/test_ipv6_cidr_to_range_function.groovy
create mode 100644
regression-test/suites/query_p0/sql_functions/ip_functions/test_ipv6_num_to_string_function.groovy
create mode 100755 tools/pick_pr.sh
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
