This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 89cf8d3f9b0 [refactor](nereids)use nereids to implement
TableQueryPlanAction (#39627)
89cf8d3f9b0 is described below
commit 89cf8d3f9b06d36a747b5a08372d1331a286395c
Author: starocean999 <40539150+starocean...@users.noreply.github.com>
AuthorDate: Thu Sep 12 14:07:57 2024 +0800
[refactor](nereids)use nereids to implement TableQueryPlanAction (#39627)
---
.../doris/httpv2/rest/TableQueryPlanAction.java | 252 ++++++++++++---------
.../org/apache/doris/nereids/trees/TreeNode.java | 17 ++
.../doris/http/TableQueryPlanActionTest.java | 18 ++
3 files changed, 179 insertions(+), 108 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java
index 6188b502692..8e03fef07f5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java
@@ -17,11 +17,7 @@
package org.apache.doris.httpv2.rest;
-import org.apache.doris.analysis.InlineViewRef;
-import org.apache.doris.analysis.SelectStmt;
import org.apache.doris.analysis.StatementBase;
-import org.apache.doris.analysis.TableName;
-import org.apache.doris.analysis.TableRef;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Table;
@@ -33,20 +29,31 @@ import org.apache.doris.common.util.NetUtils;
import org.apache.doris.httpv2.entity.ResponseEntityBuilder;
import org.apache.doris.httpv2.rest.manager.HttpUtils;
import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.NereidsPlanner;
+import org.apache.doris.nereids.analyzer.UnboundRelation;
+import org.apache.doris.nereids.glue.LogicalPlanAdapter;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.plans.commands.Command;
+import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink;
+import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
+import org.apache.doris.nereids.util.RelationUtil;
import org.apache.doris.planner.PlanFragment;
-import org.apache.doris.planner.Planner;
import org.apache.doris.planner.ScanNode;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.GlobalVariable;
-import org.apache.doris.qe.OriginStatement;
-import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.qe.SessionVariable;
import org.apache.doris.thrift.TDataSink;
import org.apache.doris.thrift.TDataSinkType;
import org.apache.doris.thrift.TMemoryScratchSink;
import org.apache.doris.thrift.TNetworkAddress;
import org.apache.doris.thrift.TPaloScanRange;
import org.apache.doris.thrift.TPlanFragment;
-import org.apache.doris.thrift.TQueryOptions;
import org.apache.doris.thrift.TQueryPlanInfo;
import org.apache.doris.thrift.TScanRangeLocations;
import org.apache.doris.thrift.TTabletVersionInfo;
@@ -163,120 +170,149 @@ public class TableQueryPlanAction extends
RestBaseController {
*/
private void handleQuery(ConnectContext context, String requestDb, String
requestTable, String sql,
Map<String, Object> result) throws DorisHttpException {
- // use SE to resolve sql
- StmtExecutor stmtExecutor = new StmtExecutor(context, new
OriginStatement(sql, 0), false);
+ List<StatementBase> stmts = null;
+ SessionVariable sessionVariable = context.getSessionVariable();
+ boolean needSetParallelResultSinkToFalse = false;
try {
- TQueryOptions tQueryOptions =
context.getSessionVariable().toThrift();
- // Conduct Planner create SingleNodePlan#createPlanFragments
- tQueryOptions.num_nodes = 1;
- // analyze sql
- stmtExecutor.analyze(tQueryOptions);
- } catch (Exception e) {
- throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST,
e.getMessage());
- }
- // the parsed logical statement
- StatementBase query = stmtExecutor.getParsedStmt();
- // only process select semantic
- if (!(query instanceof SelectStmt)) {
- throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST,
- "Select statement needed, but found [" + sql + " ]");
- }
- SelectStmt stmt = (SelectStmt) query;
- // just only process sql like `select * from table where <predicate>`,
only support executing scan semantic
- if (stmt.hasAggInfo() || stmt.hasAnalyticInfo()
- || stmt.hasOrderByClause() || stmt.hasOffset() ||
stmt.hasLimit() || stmt.isExplain()) {
- throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST,
- "only support single table filter-prune-scan, but found [
" + sql + "]");
- }
- // process only one table by one http query
- List<TableRef> fromTables = stmt.getTableRefs();
- if (fromTables.size() != 1) {
- throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST,
- "Select statement must have only one table");
- }
+ try {
+ if (!sessionVariable.enableParallelResultSink()) {
+ sessionVariable.setParallelResultSink(true);
+ needSetParallelResultSinkToFalse = true;
+ }
+ stmts = new NereidsParser().parseSQL(sql,
context.getSessionVariable());
+ } catch (Exception e) {
+ throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST,
e.getMessage());
+ }
+ // the parsed logical statement
+ StatementBase query = stmts.get(0);
+ if (!(query instanceof LogicalPlanAdapter)) {
+ throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST,
+ "Select statement needed, but found [" + sql + " ]");
+ }
+ LogicalPlan parsedPlan = ((LogicalPlanAdapter)
query).getLogicalPlan();
+ // only process select semantic
+ if (parsedPlan instanceof Command) {
+ throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST,
+ "Select statement needed, but found [" + sql + " ]");
+ }
- TableRef fromTable = fromTables.get(0);
- if (fromTable instanceof InlineViewRef) {
- throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST,
- "Select statement must not embed another statement");
- }
- // check consistent http requested resource with sql referenced
- // if consistent in this way, can avoid check privilege
- TableName tableAndDb = fromTables.get(0).getName();
- int lower = GlobalVariable.lowerCaseTableNames;
- //Determine whether table names are case-sensitive
- if (lower == 0) {
- if (!(tableAndDb.getDb().equals(requestDb) &&
tableAndDb.getTbl().equals(requestTable))) {
+ if
(!parsedPlan.collectToList(LogicalSubQueryAlias.class::isInstance).isEmpty()) {
throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST,
- "requested database and table must consistent with
sql: request [ "
- + requestDb + "." + requestTable + "]" + "and
sql [" + tableAndDb.toString() + "]");
+ "Select statement must not embed another statement");
}
- } else {
- if (!(tableAndDb.getDb().equalsIgnoreCase(requestDb)
- && tableAndDb.getTbl().equalsIgnoreCase(requestTable))) {
+
+ List<UnboundRelation> unboundRelations =
parsedPlan.collectToList(UnboundRelation.class::isInstance);
+ if (unboundRelations.size() != 1) {
throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST,
- "requested database and table must consistent with
sql: request [ "
- + requestDb + "." + requestTable + "]" + "and
sql [" + tableAndDb.toString() + "]");
+ "Select statement must have only one table");
}
- }
- // acquired Planner to get PlanNode and fragment templates
- Planner planner = stmtExecutor.planner();
- // acquire ScanNode to obtain pruned tablet
- // in this way, just retrieve only one scannode
- List<ScanNode> scanNodes = planner.getScanNodes();
- if (scanNodes.size() != 1) {
- throw new
DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR,
- "Planner should plan just only one ScanNode but found [ "
+ scanNodes.size() + "]");
- }
- List<TScanRangeLocations> scanRangeLocations =
scanNodes.get(0).getScanRangeLocations(0);
- // acquire the PlanFragment which the executable template
- List<PlanFragment> fragments = planner.getFragments();
- if (fragments.size() != 1) {
- throw new
DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR,
- "Planner should plan just only one PlanFragment but found
[ " + fragments.size() + "]");
- }
+ // check consistent http requested resource with sql referenced
+ // if consistent in this way, can avoid check privilege
+ List<String> tableQualifier =
RelationUtil.getQualifierName(context,
+ unboundRelations.get(0).getNameParts());
+ if (tableQualifier.size() != 3) {
+ throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST,
+ "can't find table " + String.join(",",
tableQualifier));
+ }
+ String dbName = tableQualifier.get(1);
+ String tableName = tableQualifier.get(2);
+
+ if (GlobalVariable.lowerCaseTableNames == 0) {
+ if (!(dbName.equals(requestDb) &&
tableName.equals(requestTable))) {
+ throw new
DorisHttpException(HttpResponseStatus.BAD_REQUEST,
+ "requested database and table must consistent with
sql: request [ "
+ + requestDb + "." + requestTable + "]" +
"and sql [" + dbName
+ + "." + tableName + "]");
+ }
+ } else {
+ if (!(dbName.equalsIgnoreCase(requestDb)
+ && tableName.equalsIgnoreCase(requestTable))) {
+ throw new
DorisHttpException(HttpResponseStatus.BAD_REQUEST,
+ "requested database and table must consistent with
sql: request [ "
+ + requestDb + "." + requestTable + "]" +
"and sql [" + dbName
+ + "." + tableName + "]");
+ }
+ }
+ NereidsPlanner nereidsPlanner = new
NereidsPlanner(context.getStatementContext());
+ LogicalPlan rewrittenPlan = (LogicalPlan)
nereidsPlanner.planWithLock(parsedPlan,
+ PhysicalProperties.GATHER,
ExplainCommand.ExplainLevel.REWRITTEN_PLAN);
+ if (!rewrittenPlan.allMatch(planTreeNode -> planTreeNode
instanceof LogicalOlapScan
+ || planTreeNode instanceof LogicalFilter || planTreeNode
instanceof LogicalProject
+ || planTreeNode instanceof LogicalResultSink)) {
+ throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST,
+ "only support single table filter-prune-scan, but
found [ " + sql + "]");
+ }
+ NereidsPlanner planner = new
NereidsPlanner(context.getStatementContext());
+ try {
+ planner.plan(query, context.getSessionVariable().toThrift());
+ } catch (Exception ex) {
+ throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST,
+ "only support single table filter-prune-scan, but
found [ " + sql + "]");
+ }
- TQueryPlanInfo tQueryPlanInfo = new TQueryPlanInfo();
+ // acquire ScanNode to obtain pruned tablet
+ // in this way, just retrieve only one scannode
+ List<ScanNode> scanNodes = planner.getScanNodes();
+ if (scanNodes.size() != 1) {
+ throw new
DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR,
+ "Planner should plan just only one ScanNode but found
[ " + scanNodes.size() + "]");
+ }
+ List<TScanRangeLocations> scanRangeLocations =
scanNodes.get(0).getScanRangeLocations(0);
+ // acquire the PlanFragment which the executable template
+ List<PlanFragment> fragments = planner.getFragments();
+ if (fragments.size() != 1) {
+ throw new
DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR,
+ "Planner should plan just only one PlanFragment but
found [ " + fragments.size() + "]");
+ }
+ TQueryPlanInfo tQueryPlanInfo = new TQueryPlanInfo();
- // acquire TPlanFragment
- TPlanFragment tPlanFragment = fragments.get(0).toThrift();
- // set up TMemoryScratchSink
- TDataSink tDataSink = new TDataSink();
- tDataSink.type = TDataSinkType.MEMORY_SCRATCH_SINK;
- tDataSink.memory_scratch_sink = new TMemoryScratchSink();
- tPlanFragment.output_sink = tDataSink;
- tQueryPlanInfo.plan_fragment = tPlanFragment;
- tQueryPlanInfo.desc_tbl = query.getAnalyzer().getDescTbl().toThrift();
- // set query_id
- UUID uuid = UUID.randomUUID();
- tQueryPlanInfo.query_id = new TUniqueId(uuid.getMostSignificantBits(),
uuid.getLeastSignificantBits());
+ // acquire TPlanFragment
+ TPlanFragment tPlanFragment = fragments.get(0).toThrift();
+ // set up TMemoryScratchSink
+ TDataSink tDataSink = new TDataSink();
+ tDataSink.type = TDataSinkType.MEMORY_SCRATCH_SINK;
+ tDataSink.memory_scratch_sink = new TMemoryScratchSink();
+ tPlanFragment.output_sink = tDataSink;
- Map<Long, TTabletVersionInfo> tabletInfo = new HashMap<>();
- // acquire resolved tablet distribution
- Map<String, Node> tabletRoutings =
assemblePrunedPartitions(scanRangeLocations);
- tabletRoutings.forEach((tabletId, node) -> {
- long tablet = Long.parseLong(tabletId);
- tabletInfo.put(tablet, new TTabletVersionInfo(tablet,
node.version, 0L /*version hash*/, node.schemaHash));
- });
- tQueryPlanInfo.tablet_info = tabletInfo;
+ tQueryPlanInfo.plan_fragment = tPlanFragment;
+ tQueryPlanInfo.desc_tbl = planner.getDescTable().toThrift();
+ // set query_id
+ UUID uuid = UUID.randomUUID();
+ tQueryPlanInfo.query_id = new
TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits());
- // serialize TQueryPlanInfo and encode plan with Base64 to string in
order to translate by json format
- TSerializer serializer;
- String opaquedQueryPlan;
- try {
- serializer = new TSerializer();
- byte[] queryPlanStream = serializer.serialize(tQueryPlanInfo);
- opaquedQueryPlan =
Base64.getEncoder().encodeToString(queryPlanStream);
- } catch (TException e) {
- throw new
DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR,
- "TSerializer failed to serialize PlanFragment, reason [ "
+ e.getMessage() + " ]");
+ Map<Long, TTabletVersionInfo> tabletInfo = new HashMap<>();
+ // acquire resolved tablet distribution
+ Map<String, Node> tabletRoutings =
assemblePrunedPartitions(scanRangeLocations);
+ tabletRoutings.forEach((tabletId, node) -> {
+ long tablet = Long.parseLong(tabletId);
+ tabletInfo.put(tablet, new TTabletVersionInfo(tablet,
node.version,
+ 0L /*version hash*/, node.schemaHash));
+ });
+ tQueryPlanInfo.tablet_info = tabletInfo;
+
+ // serialize TQueryPlanInfo and encode plan with Base64 to string
in order to translate by json format
+ TSerializer serializer;
+ String opaquedQueryPlan;
+ try {
+ serializer = new TSerializer();
+ byte[] queryPlanStream = serializer.serialize(tQueryPlanInfo);
+ opaquedQueryPlan =
Base64.getEncoder().encodeToString(queryPlanStream);
+ } catch (TException e) {
+ throw new
DorisHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR,
+ "TSerializer failed to serialize PlanFragment, reason
[ " + e.getMessage() + " ]");
+ }
+ result.put("partitions", tabletRoutings);
+ result.put("opaqued_query_plan", opaquedQueryPlan);
+ result.put("status", 200);
+ } finally {
+ if (needSetParallelResultSinkToFalse) {
+ sessionVariable.setParallelResultSink(false);
+ }
}
- result.put("partitions", tabletRoutings);
- result.put("opaqued_query_plan", opaquedQueryPlan);
- result.put("status", 200);
+
}
/**
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java
index 5900584b861..2c5decb9f9a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java
@@ -229,6 +229,23 @@ public interface TreeNode<NODE_TYPE extends
TreeNode<NODE_TYPE>> {
return false;
}
+ /**
+ * iterate top down and test predicate if all matched. Top-down traverse
implicitly.
+ * @param predicate predicate
+ * @return true if all predicate return true
+ */
+ default boolean allMatch(Predicate<TreeNode<NODE_TYPE>> predicate) {
+ if (!predicate.test(this)) {
+ return false;
+ }
+ for (NODE_TYPE child : children()) {
+ if (!child.allMatch(predicate)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
/**
* Collect the nodes that satisfied the predicate.
*/
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/http/TableQueryPlanActionTest.java
b/fe/fe-core/src/test/java/org/apache/doris/http/TableQueryPlanActionTest.java
index 219a48f2a6a..23549e0c98c 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/http/TableQueryPlanActionTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/http/TableQueryPlanActionTest.java
@@ -173,4 +173,22 @@ public class TableQueryPlanActionTest extends
DorisHttpTestCase {
String exception = (String) jsonObject.get("data");
Assert.assertTrue(exception.contains("table type is not OLAP"));
}
+
+ @Test
+ public void testHasAggFailure() throws IOException {
+ RequestBody body = RequestBody.create(
+ "{ \"sql\" : \" select k1,k2 from " + DB_NAME + "." +
TABLE_NAME + " group by k1, k2 \" }", JSON);
+ Request request = new Request.Builder()
+ .post(body)
+ .addHeader("Authorization", rootAuth)
+ .url(URI + PATH_URI)
+ .build();
+ Response response = networkClient.newCall(request).execute();
+ Assert.assertNotNull(response.body());
+ String respStr = response.body().string();
+ Assert.assertNotNull(respStr);
+ JSONObject jsonObject = (JSONObject) JSONValue.parse(respStr);
+ String exception = jsonObject.get("data").toString();
+ Assert.assertTrue(exception.contains("only support single table
filter-prune-scan"));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
