This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 62c7d0a4215 [Fix](point query) add query options for short circuit
queries (#32530) (#32684)
62c7d0a4215 is described below
commit 62c7d0a4215be2c606f7ae43ef6930cf958ceb2f
Author: lihangyu <[email protected]>
AuthorDate: Fri Mar 22 18:03:18 2024 +0800
[Fix](point query) add query options for short circuit queries (#32530)
(#32684)
Some options like `be_exec_version` needed for functions
---
be/src/runtime/runtime_state.h | 2 ++
be/src/service/point_query_executor.cpp | 18 ++++++++++++++----
be/src/service/point_query_executor.h | 2 +-
.../java/org/apache/doris/analysis/PrepareStmt.java | 16 ++++++++++++++++
.../java/org/apache/doris/planner/OriginalPlanner.java | 2 ++
.../main/java/org/apache/doris/planner/Planner.java | 6 ++++++
.../main/java/org/apache/doris/qe/PointQueryExec.java | 10 ++++++++++
gensrc/proto/internal_service.proto | 6 ++++++
.../suites/point_query_p0/test_point_query.groovy | 16 ++++++++++++++++
9 files changed, 73 insertions(+), 5 deletions(-)
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index d031f36fbed..79b82b94a11 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -485,6 +485,8 @@ public:
_query_mem_tracker = tracker;
}
+ void set_query_options(const TQueryOptions& query_options) {
_query_options = query_options; }
+
bool enable_profile() const {
return _query_options.__isset.enable_profile &&
_query_options.enable_profile;
}
diff --git a/be/src/service/point_query_executor.cpp
b/be/src/service/point_query_executor.cpp
index e5883d177bc..b39edb6d69c 100644
--- a/be/src/service/point_query_executor.cpp
+++ b/be/src/service/point_query_executor.cpp
@@ -20,6 +20,7 @@
#include <fmt/format.h>
#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/Exprs_types.h>
+#include <gen_cpp/PaloInternalService_types.h>
#include <gen_cpp/internal_service.pb.h>
#include <stdlib.h>
@@ -51,9 +52,10 @@ namespace doris {
Reusable::~Reusable() {}
constexpr static int s_preallocted_blocks_num = 32;
Status Reusable::init(const TDescriptorTable& t_desc_tbl, const
std::vector<TExpr>& output_exprs,
- size_t block_size) {
+ const TQueryOptions& query_options, size_t block_size) {
SCOPED_MEM_COUNT_BY_HOOK(&_mem_size);
_runtime_state = RuntimeState::create_unique();
+ _runtime_state->set_query_options(query_options);
RETURN_IF_ERROR(DescriptorTbl::create(_runtime_state->obj_pool(),
t_desc_tbl, &_desc_tbl));
_runtime_state->set_desc_tbl(_desc_tbl);
_block_pool.resize(block_size);
@@ -183,13 +185,21 @@ Status PointQueryExecutor::init(const
PTabletKeyLookupRequest* request,
reinterpret_cast<const
uint8_t*>(request->output_expr().data()), &len, false,
&t_output_exprs));
_reusable = reusable_ptr;
+ TQueryOptions t_query_options;
+ len = request->query_options().size();
+ if (request->has_query_options()) {
+ RETURN_IF_ERROR(deserialize_thrift_msg(
+ reinterpret_cast<const
uint8_t*>(request->query_options().data()), &len, false,
+ &t_query_options));
+ }
if (uuid != 0) {
// could be reused by requests after, pre allocte more blocks
- RETURN_IF_ERROR(
- reusable_ptr->init(t_desc_tbl, t_output_exprs.exprs,
s_preallocted_blocks_num));
+ RETURN_IF_ERROR(reusable_ptr->init(t_desc_tbl,
t_output_exprs.exprs, t_query_options,
+ s_preallocted_blocks_num));
LookupConnectionCache::instance()->add(uuid, reusable_ptr);
} else {
- RETURN_IF_ERROR(reusable_ptr->init(t_desc_tbl,
t_output_exprs.exprs, 1));
+ RETURN_IF_ERROR(
+ reusable_ptr->init(t_desc_tbl, t_output_exprs.exprs,
t_query_options, 1));
}
}
_tablet =
StorageEngine::instance()->tablet_manager()->get_tablet(request->tablet_id());
diff --git a/be/src/service/point_query_executor.h
b/be/src/service/point_query_executor.h
index 3e21b1d4b9a..90ebcc1d5cc 100644
--- a/be/src/service/point_query_executor.h
+++ b/be/src/service/point_query_executor.h
@@ -72,7 +72,7 @@ public:
}
Status init(const TDescriptorTable& t_desc_tbl, const std::vector<TExpr>&
output_exprs,
- size_t block_size = 1);
+ const TQueryOptions& query_options, size_t block_size = 1);
std::unique_ptr<vectorized::Block> get_block();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/PrepareStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/PrepareStmt.java
index 1545ea67efb..2e116ae80e9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PrepareStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PrepareStmt.java
@@ -23,6 +23,7 @@ import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TDescriptorTable;
import org.apache.doris.thrift.TExpr;
import org.apache.doris.thrift.TExprList;
+import org.apache.doris.thrift.TQueryOptions;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
@@ -47,6 +48,7 @@ public class PrepareStmt extends StatementBase {
// outputExprs are heavy work
private ByteString serializedDescTable;
private ByteString serializedOutputExpr;
+ private ByteString serializedQueryOptions;
private TDescriptorTable descTable;
private UUID id;
@@ -134,6 +136,16 @@ public class PrepareStmt extends StatementBase {
}
}
+ public void cacheSerializedQueryOptions(TQueryOptions queryOptions) {
+ try {
+ serializedQueryOptions = ByteString.copyFrom(
+ new TSerializer().serialize(queryOptions));
+ } catch (TException e) {
+ LOG.warn("failed to serilize queryOptions , {}", e.getMessage());
+ Preconditions.checkState(false, e.getMessage());
+ }
+ }
+
public ByteString getSerializedDescTable() {
return serializedDescTable;
}
@@ -142,6 +154,10 @@ public class PrepareStmt extends StatementBase {
return serializedOutputExpr;
}
+ public ByteString getSerializedQueryOptions() {
+ return serializedQueryOptions;
+ }
+
@Override
public void analyze(Analyzer analyzer) throws UserException {
if (inner instanceof SelectStmt) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
index db327bddcb9..569a5f3a4eb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
@@ -97,6 +97,7 @@ public class OriginalPlanner extends Planner {
public void plan(StatementBase queryStmt, TQueryOptions queryOptions)
throws UserException {
+ this.queryOptions = queryOptions;
createPlanFragments(queryStmt, analyzer, queryOptions);
}
@@ -299,6 +300,7 @@ public class OriginalPlanner extends Planner {
// Cache them for later request better performance
analyzer.getPrepareStmt().cacheSerializedDescriptorTable(olapScanNode.getDescTable());
analyzer.getPrepareStmt().cacheSerializedOutputExprs(rootFragment.getOutputExprs());
+
analyzer.getPrepareStmt().cacheSerializedQueryOptions(queryOptions);
}
} else if (selectStmt.isTwoPhaseReadOptEnabled()) {
// Optimize query like `SELECT ... FROM <tbl> WHERE ... ORDER
BY ... LIMIT ...`
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java
index 22495e792ff..186286e9da6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java
@@ -46,6 +46,8 @@ public abstract class Planner {
protected boolean isBlockQuery = false;
+ protected TQueryOptions queryOptions;
+
public abstract List<ScanNode> getScanNodes();
public abstract void plan(StatementBase queryStmt,
@@ -118,6 +120,10 @@ public abstract class Planner {
return isBlockQuery;
}
+ public TQueryOptions getQueryOptions() {
+ return queryOptions;
+ }
+
public abstract DescriptorTable getDescTable();
public abstract List<RuntimeFilter> getRuntimeFilters();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExec.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExec.java
index cf61d33b9a9..4b0302cc403 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExec.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExec.java
@@ -39,6 +39,7 @@ import org.apache.doris.rpc.TCustomProtocolFactory;
import org.apache.doris.system.Backend;
import org.apache.doris.thrift.TExpr;
import org.apache.doris.thrift.TExprList;
+import org.apache.doris.thrift.TQueryOptions;
import org.apache.doris.thrift.TResultBatch;
import org.apache.doris.thrift.TScanRangeLocations;
import org.apache.doris.thrift.TStatusCode;
@@ -70,8 +71,10 @@ public class PointQueryExec implements CoordInterface {
// ByteString serialized for prepared statement
private ByteString serializedDescTable;
private ByteString serializedOutputExpr;
+ private ByteString serializedQueryOptions;
private ArrayList<Expr> outputExprs;
private DescriptorTable descriptorTable;
+ private TQueryOptions queryOptions;
private long tabletID = 0;
private long timeoutMs = Config.point_query_timeout_ms; // default 10s
@@ -108,6 +111,7 @@ public class PointQueryExec implements CoordInterface {
this.equalPredicats = planRoot.getPointQueryEqualPredicates();
this.descriptorTable = planRoot.getDescTable();
this.outputExprs = fragment.getOutputExprs();
+ this.queryOptions = planner.getQueryOptions();
PrepareStmt prepareStmt = analyzer == null ? null :
analyzer.getPrepareStmt();
if (prepareStmt != null && prepareStmt.getPreparedType() ==
PrepareStmt.PreparedType.FULL_PREPARED) {
@@ -116,6 +120,7 @@ public class PointQueryExec implements CoordInterface {
this.serializedDescTable = prepareStmt.getSerializedDescTable();
this.serializedOutputExpr = prepareStmt.getSerializedOutputExprs();
this.isBinaryProtocol = prepareStmt.isBinaryProtocol();
+ this.serializedQueryOptions =
prepareStmt.getSerializedQueryOptions();
} else {
// TODO
// planner.getDescTable().toThrift();
@@ -244,12 +249,17 @@ public class PointQueryExec implements CoordInterface {
serializedOutputExpr = ByteString.copyFrom(
new TSerializer().serialize(exprList));
}
+ if (serializedQueryOptions == null) {
+ serializedQueryOptions = ByteString.copyFrom(
+ new TSerializer().serialize(queryOptions));
+ }
InternalService.PTabletKeyLookupRequest.Builder requestBuilder
= InternalService.PTabletKeyLookupRequest.newBuilder()
.setTabletId(tabletID)
.setDescTbl(serializedDescTable)
.setOutputExpr(serializedOutputExpr)
+ .setQueryOptions(serializedQueryOptions)
.setIsBinaryRow(isBinaryProtocol);
if (cacheID != null) {
InternalService.UUID.Builder uuidBuilder =
InternalService.UUID.newBuilder();
diff --git a/gensrc/proto/internal_service.proto
b/gensrc/proto/internal_service.proto
index 8b4795c4b7e..0230180796a 100644
--- a/gensrc/proto/internal_service.proto
+++ b/gensrc/proto/internal_service.proto
@@ -287,6 +287,12 @@ message PTabletKeyLookupRequest {
optional bytes output_expr = 5;
// return binary mysql row format if true
optional bool is_binary_row = 6;
+
+ // For cloud
+ // version to read
+ optional int64 version = 7;
+ // serilized from TQueryOptions
+ optional bytes query_options = 8;
}
message PTabletKeyLookupResponse {
diff --git a/regression-test/suites/point_query_p0/test_point_query.groovy
b/regression-test/suites/point_query_p0/test_point_query.groovy
index f7df5ab820f..653c151d89c 100644
--- a/regression-test/suites/point_query_p0/test_point_query.groovy
+++ b/regression-test/suites/point_query_p0/test_point_query.groovy
@@ -256,6 +256,22 @@ suite("test_point_query") {
qt_sql """select /*+ SET_VAR(enable_nereids_planner=false) */
* from ${tableName} where customer_key = 0"""
}
}
+ sql "DROP TABLE IF EXISTS test_ODS_EBA_LLREPORT";
+ sql """
+ CREATE TABLE `test_ODS_EBA_LLREPORT` (
+ `RPTNO` VARCHAR(20) NOT NULL ,
+ `A_ENTTYP` VARCHAR(6) NULL ,
+ `A_INTIME` DATETIME NULL
+ ) ENGINE=OLAP
+ UNIQUE KEY(`RPTNO`)
+ DISTRIBUTED BY HASH(`RPTNO`) BUCKETS 3
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "store_row_column" = "true"
+ );
+ """
+ sql "insert into test_ODS_EBA_LLREPORT(RPTNO) values('567890')"
+ sql "select /*+ SET_VAR(enable_nereids_planner=false) */
substr(RPTNO,2,5) from test_ODS_EBA_LLREPORT where RPTNO = '567890'"
} finally {
set_be_config.call("disable_storage_row_cache", "true")
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
