This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e34e72dd51 [feature](Nereids) show cost and execution time for each
plan (#17123)
e34e72dd51 is described below
commit e34e72dd51fa18140c6560f27e29e7a2add9059f
Author: 谢健 <jianx...@gmail.com>
AuthorDate: Tue Feb 28 18:59:57 2023 +0800
[feature](Nereids) show cost and execution time for each plan (#17123)
1. Show cost in optimized plan
2. show plan time schedule time and so on in profile
---
.../org/apache/doris/analysis/ShowQueryProfileStmt.java | 7 +++++++
.../org/apache/doris/common/util/ProfileManager.java | 16 ++++++++++++++++
.../doris/httpv2/controller/QueryProfileController.java | 4 +++-
.../java/org/apache/doris/nereids/NereidsPlanner.java | 14 ++++++++++----
.../main/java/org/apache/doris/nereids/memo/Memo.java | 4 ++--
.../java/org/apache/doris/nereids/memo/RankTest.java | 2 +-
6 files changed, 39 insertions(+), 8 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java
index b36cd30159..e8f8b6df65 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java
@@ -44,6 +44,13 @@ public class ShowQueryProfileStmt extends ShowStmt {
.addColumn(new Column("EndTime",
ScalarType.createVarchar(128)))
.addColumn(new Column("TotalTime",
ScalarType.createVarchar(128)))
.addColumn(new Column("QueryState",
ScalarType.createVarchar(128)))
+ .addColumn(new Column("TraceId",
ScalarType.createVarchar(128)))
+ .addColumn(new Column("AnalysisTime",
ScalarType.createVarchar(128)))
+ .addColumn(new Column("PlanTime",
ScalarType.createVarchar(128)))
+ .addColumn(new Column("ScheduleTime",
ScalarType.createVarchar(128)))
+ .addColumn(new Column("FetchResultTime",
ScalarType.createVarchar(128)))
+ .addColumn(new Column("WriteResultTime",
ScalarType.createVarchar(128)))
+ .addColumn(new Column("WaitAndFetchResultTime",
ScalarType.createVarchar(128)))
.build();
public static final ShowResultSetMetaData META_DATA_FRAGMENTS =
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
index 8d0c825e88..0d4c6d49b8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
@@ -79,6 +79,12 @@ public class ProfileManager {
public static final String PARALLEL_FRAGMENT_EXEC_INSTANCE = "Parallel
Fragment Exec Instance Num";
public static final String TRACE_ID = "Trace ID";
+ public static final String ANALYSIS_TIME = "Analysis Time";
+ public static final String FETCH_RESULT_TIME = "Fetch Result Time";
+ public static final String PLAN_TIME = "Plan Time";
+ public static final String SCHEDULE_TIME = "Schedule Time";
+ public static final String WRITE_RESULT_TIME = "Write Result Time";
+ public static final String WAIT_FETCH_RESULT_TIME = "Wait and Fetch Result
Time";
public enum ProfileType {
QUERY,
@@ -88,6 +94,9 @@ public class ProfileManager {
public static final List<String> PROFILE_HEADERS =
Collections.unmodifiableList(
Arrays.asList(JOB_ID, QUERY_ID, USER, DEFAULT_DB, SQL_STATEMENT,
QUERY_TYPE,
START_TIME, END_TIME, TOTAL_TIME, QUERY_STATE, TRACE_ID));
+ public static final List<String> EXECUTION_HEADERS =
Collections.unmodifiableList(
+ Arrays.asList(ANALYSIS_TIME, PLAN_TIME, SCHEDULE_TIME,
FETCH_RESULT_TIME,
+ WRITE_RESULT_TIME, WAIT_FETCH_RESULT_TIME));
private class ProfileElement {
public ProfileElement(RuntimeProfile profile) {
@@ -146,6 +155,10 @@ public class ProfileManager {
for (String header : PROFILE_HEADERS) {
element.infoStrings.put(header,
summaryProfile.getInfoString(header));
}
+ RuntimeProfile executionProfile =
summaryProfile.getChildList().get(0).first;
+ for (String header : EXECUTION_HEADERS) {
+ element.infoStrings.put(header,
executionProfile.getInfoString(header));
+ }
MultiProfileTreeBuilder builder = new MultiProfileTreeBuilder(profile);
try {
@@ -215,6 +228,9 @@ public class ProfileManager {
for (String str : PROFILE_HEADERS) {
row.add(infoStrings.get(str));
}
+ for (String str : EXECUTION_HEADERS) {
+ row.add(infoStrings.get(str));
+ }
result.add(row);
}
} finally {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/QueryProfileController.java
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/QueryProfileController.java
index 53744f4fc8..a75b1c7071 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/QueryProfileController.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/controller/QueryProfileController.java
@@ -69,7 +69,9 @@ public class QueryProfileController extends BaseController {
private void addFinishedQueryInfo(Map<String, Object> result) {
List<List<String>> finishedQueries =
ProfileManager.getInstance().getAllQueries();
- List<String> columnHeaders =
Lists.newLinkedList(ProfileManager.PROFILE_HEADERS);
+ List<String> columnHeaders = Lists.newLinkedList();
+ columnHeaders.addAll(ProfileManager.PROFILE_HEADERS);
+ columnHeaders.addAll(ProfileManager.EXECUTION_HEADERS);
int jobIdIndex = -1;
int queryIdIndex = -1;
int queryTypeIndex = -1;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index d19c176228..8456f6c2db 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -21,6 +21,7 @@ import org.apache.doris.analysis.DescriptorTable;
import org.apache.doris.analysis.ExplainOptions;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.common.NereidsException;
+import org.apache.doris.common.Pair;
import org.apache.doris.nereids.CascadesContext.Lock;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
@@ -76,6 +77,8 @@ public class NereidsPlanner extends Planner {
private Plan analyzedPlan;
private Plan rewrittenPlan;
private Plan optimizedPlan;
+ // The cost of optimized plan
+ private double cost = 0;
public NereidsPlanner(StatementContext statementContext) {
this.statementContext = statementContext;
@@ -261,12 +264,16 @@ public class NereidsPlanner extends Planner {
private PhysicalPlan chooseNthPlan(Group rootGroup, PhysicalProperties
physicalProperties, int nthPlan) {
if (nthPlan <= 1) {
+ cost = rootGroup.getLowestCostPlan(physicalProperties).orElseThrow(
+ () -> new AnalysisException("lowestCostPlans with
physicalProperties("
+ + physicalProperties + ") doesn't exist in root
group")).first;
return chooseBestPlan(rootGroup, physicalProperties);
}
Memo memo = cascadesContext.getMemo();
- long id = memo.rank(nthPlan);
- return memo.unrank(id);
+ Pair<Long, Double> idCost = memo.rank(nthPlan);
+ cost = idCost.second;
+ return memo.unrank(idCost.first);
}
private PhysicalPlan chooseBestPlan(Group rootGroup, PhysicalProperties
physicalProperties)
@@ -276,7 +283,6 @@ public class NereidsPlanner extends Planner {
() -> new AnalysisException("lowestCostPlans with
physicalProperties("
+ physicalProperties + ") doesn't exist in root
group")).second;
List<PhysicalProperties> inputPropertiesList =
groupExpression.getInputPropertiesList(physicalProperties);
-
List<Plan> planChildren = Lists.newArrayList();
for (int i = 0; i < groupExpression.arity(); i++) {
planChildren.add(chooseBestPlan(groupExpression.child(i),
inputPropertiesList.get(i)));
@@ -310,7 +316,7 @@ public class NereidsPlanner extends Planner {
case REWRITTEN_PLAN:
return rewrittenPlan.treeString();
case OPTIMIZED_PLAN:
- return optimizedPlan.treeString();
+ return "cost = " + cost + "\n" + optimizedPlan.treeString();
case ALL_PLAN:
return "========== PARSED PLAN ==========\n"
+ parsedPlan.treeString() + "\n\n"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
index 304a2da025..b27d9a1f59 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
@@ -719,7 +719,7 @@ public class Memo {
*
* In unrank() function, we will extract the actual physical function
according the unique ID
*/
- public long rank(long n) {
+ public Pair<Long, Double> rank(long n) {
double threshold = 0.000000001;
Preconditions.checkArgument(n > 0, "the n %d must be greater than 0 in
nthPlan", n);
List<Pair<Long, Double>> plans = rankGroup(root,
PhysicalProperties.GATHER);
@@ -737,7 +737,7 @@ public class Memo {
pq.poll();
}
}
- return pq.peek().first;
+ return pq.peek();
}
private List<Pair<Long, Double>> rankGroup(Group group, PhysicalProperties
prop) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/RankTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/RankTest.java
index 0eb91caf3a..3deb2a9638 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/RankTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/RankTest.java
@@ -55,7 +55,7 @@ public class RankTest extends TPCHTestBase {
.optimize()
.getCascadesContext()
.getMemo();
- PhysicalPlan plan1 = memo.unrank(memo.rank(1));
+ PhysicalPlan plan1 = memo.unrank(memo.rank(1).first);
PhysicalPlan plan2 = PlanChecker.from(connectContext)
.analyze(field.get(null).toString())
.rewrite()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
