This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 784aa53a47 [Fix](multi catalog statistics)Improve external table
statistics collection (#22224) (#22343)
784aa53a47 is described below
commit 784aa53a47b225d7d2718e7274fda790023d49b2
Author: Jibing-Li <64681310+jibing...@users.noreply.github.com>
AuthorDate: Fri Jul 28 23:08:14 2023 +0800
[Fix](multi catalog statistics)Improve external table statistics collection
(#22224) (#22343)
---
.../org/apache/doris/analysis/AnalyzeTblStmt.java | 8 +
.../org/apache/doris/analysis/ShowAnalyzeStmt.java | 1 +
.../doris/analysis/ShowAnalyzeTaskStatus.java | 3 +-
.../java/org/apache/doris/qe/ShowExecutor.java | 2 +
.../java/org/apache/doris/qe/StmtExecutor.java | 11 +-
.../org/apache/doris/statistics/AnalysisInfo.java | 14 +-
.../doris/statistics/AnalysisInfoBuilder.java | 10 +-
.../apache/doris/statistics/AnalysisManager.java | 138 ++++++++-----
.../doris/statistics/AnalysisTaskExecutor.java | 4 -
.../doris/statistics/AnalysisTaskWrapper.java | 8 +-
.../apache/doris/statistics/BaseAnalysisTask.java | 17 +-
.../apache/doris/statistics/HMSAnalysisTask.java | 33 +++-
.../org/apache/doris/statistics/HistogramTask.java | 1 +
.../apache/doris/statistics/MVAnalysisTask.java | 1 +
.../apache/doris/statistics/OlapAnalysisTask.java | 1 +
.../apache/doris/statistics/StatisticsCache.java | 6 +-
.../doris/statistics/util/StatisticsUtil.java | 2 +
.../doris/statistics/AnalysisManagerTest.java | 73 +++++++
.../hive/test_hive_analyze_db.out | 20 --
.../hive/test_hive_statistic.out | 57 ------
.../hive/test_hive_statistic_cache.out | 58 ------
.../hive/test_hive_analyze_db.groovy | 157 ++++++++++++++-
.../hive/test_hive_statistic.groovy | 215 +++++++++++++++++++--
.../hive/test_hive_statistic_cache.groovy | 191 ++++++++++++++++--
24 files changed, 779 insertions(+), 252 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
index ddf2c417c4..5ca1e7a870 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
@@ -85,6 +85,7 @@ public class AnalyzeTblStmt extends AnalyzeStmt {
private final TableName tableName;
private List<String> columnNames;
private List<String> partitionNames;
+ private boolean isAllColumns;
// after analyzed
private long dbId;
@@ -99,6 +100,7 @@ public class AnalyzeTblStmt extends AnalyzeStmt {
this.partitionNames = partitionNames == null ? null :
partitionNames.getPartitionNames();
this.columnNames = columnNames;
this.analyzeProperties = properties;
+ this.isAllColumns = columnNames == null;
}
public AnalyzeTblStmt(AnalyzeProperties analyzeProperties, TableName
tableName, List<String> columnNames, long dbId,
@@ -108,6 +110,7 @@ public class AnalyzeTblStmt extends AnalyzeStmt {
this.columnNames = columnNames;
this.dbId = dbId;
this.table = table;
+ this.isAllColumns = columnNames == null;
}
@Override
@@ -129,6 +132,7 @@ public class AnalyzeTblStmt extends AnalyzeStmt {
DatabaseIf db = catalog.getDbOrAnalysisException(dbName);
dbId = db.getId();
table = db.getTableOrAnalysisException(tblName);
+ isAllColumns = columnNames == null;
check();
}
@@ -296,4 +300,8 @@ public class AnalyzeTblStmt extends AnalyzeStmt {
public Database getDb() throws AnalysisException {
return
analyzer.getEnv().getInternalCatalog().getDbOrAnalysisException(dbId);
}
+
+ public boolean isAllColumns() {
+ return isAllColumns;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
index 73bf77b23c..3a0aa05a7d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
@@ -64,6 +64,7 @@ public class ShowAnalyzeStmt extends ShowStmt {
.add("message")
.add("last_exec_time_in_ms")
.add("state")
+ .add("progress")
.add("schedule_type")
.build();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeTaskStatus.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeTaskStatus.java
index 03d304f393..927a56d19d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeTaskStatus.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeTaskStatus.java
@@ -33,7 +33,8 @@ public class ShowAnalyzeTaskStatus extends ShowStmt {
.addColumn(new Column("task_id",
ScalarType.createVarchar(100)))
.addColumn(new Column("col_name",
ScalarType.createVarchar(1000)))
.addColumn(new Column("message",
ScalarType.createVarchar(1000)))
- .addColumn(new Column("last_exec_time_in_ms",
ScalarType.createVarchar(1000)))
+ .addColumn(new Column("last_state_change_time",
ScalarType.createVarchar(1000)))
+ .addColumn(new Column("time_cost_in_ms",
ScalarType.createVarchar(1000)))
.addColumn(new Column("state",
ScalarType.createVarchar(1000))).build();
private final long jobId;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 7f5d197723..59bfbada21 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -2562,6 +2562,7 @@ public class ShowExecutor {
LocalDateTime.ofInstant(Instant.ofEpochMilli(analysisInfo.lastExecTimeInMs),
ZoneId.systemDefault())));
row.add(analysisInfo.state.toString());
+
row.add(Env.getCurrentEnv().getAnalysisManager().getJobProgress(analysisInfo.jobId));
row.add(analysisInfo.scheduleType.toString());
resultRows.add(row);
}
@@ -2762,6 +2763,7 @@ public class ShowExecutor {
row.add(TimeUtils.DATETIME_FORMAT.format(
LocalDateTime.ofInstant(Instant.ofEpochMilli(analysisInfo.lastExecTimeInMs),
ZoneId.systemDefault())));
+ row.add(String.valueOf(analysisInfo.timeCostInMs));
row.add(analysisInfo.state.toString());
rows.add(row);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index b84ebfa007..2c7039ac82 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -20,6 +20,7 @@ package org.apache.doris.qe;
import org.apache.doris.analysis.AddPartitionLikeClause;
import org.apache.doris.analysis.AlterClause;
import org.apache.doris.analysis.AlterTableStmt;
+import org.apache.doris.analysis.AnalyzeDBStmt;
import org.apache.doris.analysis.AnalyzeStmt;
import org.apache.doris.analysis.AnalyzeTblStmt;
import org.apache.doris.analysis.Analyzer;
@@ -1148,7 +1149,7 @@ public class StmtExecutor {
if (mysqlLoadId != null) {
Env.getCurrentEnv().getLoadManager().getMysqlLoadManager().cancelMySqlLoad(mysqlLoadId);
}
- if (parsedStmt instanceof AnalyzeTblStmt) {
+ if (parsedStmt instanceof AnalyzeTblStmt || parsedStmt instanceof
AnalyzeDBStmt) {
Env.getCurrentEnv().getAnalysisManager().cancelSyncTask(context);
}
}
@@ -2484,7 +2485,7 @@ public class StmtExecutor {
analyze(context.getSessionVariable().toThrift());
}
} catch (Exception e) {
- throw new RuntimeException("Failed to execute internal SQL",
e);
+ throw new RuntimeException("Failed to execute internal SQL. "
+ Util.getRootCauseMessage(e), e);
}
planner.getFragments();
RowBatch batch;
@@ -2494,7 +2495,7 @@ public class StmtExecutor {
QeProcessorImpl.INSTANCE.registerQuery(context.queryId(),
new QeProcessorImpl.QueryInfo(context,
originStmt.originStmt, coord));
} catch (UserException e) {
- throw new RuntimeException("Failed to execute internal SQL",
e);
+ throw new RuntimeException("Failed to execute internal SQL. "
+ Util.getRootCauseMessage(e), e);
}
Span queryScheduleSpan = context.getTracer()
@@ -2503,7 +2504,7 @@ public class StmtExecutor {
coord.exec();
} catch (Exception e) {
queryScheduleSpan.recordException(e);
- throw new RuntimeException("Failed to execute internal SQL",
e);
+ throw new RuntimeException("Failed to execute internal SQL. "
+ Util.getRootCauseMessage(e), e);
} finally {
queryScheduleSpan.end();
}
@@ -2520,7 +2521,7 @@ public class StmtExecutor {
}
} catch (Exception e) {
fetchResultSpan.recordException(e);
- throw new RuntimeException("Failed to execute internal SQL",
e);
+ throw new RuntimeException("Failed to execute internal SQL. "
+ Util.getRootCauseMessage(e), e);
} finally {
fetchResultSpan.end();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
index 5a00e3471e..592fb4a99b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
@@ -129,6 +129,10 @@ public class AnalysisInfo implements Writable {
@SerializedName("lastExecTimeInMs")
public long lastExecTimeInMs;
+ // finished or failed
+ @SerializedName("timeCostInMs")
+ public long timeCostInMs;
+
@SerializedName("state")
public AnalysisState state;
@@ -153,8 +157,8 @@ public class AnalysisInfo implements Writable {
Map<String, Set<String>> colToPartitions, Set<String>
partitionNames, String colName, Long indexId,
JobType jobType, AnalysisMode analysisMode, AnalysisMethod
analysisMethod, AnalysisType analysisType,
int samplePercent, int sampleRows, int maxBucketNum, long
periodTimeInMs, String message,
- long lastExecTimeInMs, AnalysisState state, ScheduleType
scheduleType, boolean isExternalTableLevelTask,
- boolean partitionOnly, boolean samplingPartition) {
+ long lastExecTimeInMs, long timeCostInMs, AnalysisState state,
ScheduleType scheduleType,
+ boolean isExternalTableLevelTask, boolean partitionOnly, boolean
samplingPartition) {
this.jobId = jobId;
this.taskId = taskId;
this.catalogName = catalogName;
@@ -174,6 +178,7 @@ public class AnalysisInfo implements Writable {
this.periodTimeInMs = periodTimeInMs;
this.message = message;
this.lastExecTimeInMs = lastExecTimeInMs;
+ this.timeCostInMs = timeCostInMs;
this.state = state;
this.scheduleType = scheduleType;
this.externalTableLevelTask = isExternalTableLevelTask;
@@ -209,6 +214,9 @@ public class AnalysisInfo implements Writable {
if (lastExecTimeInMs > 0) {
sj.add("LastExecTime: " +
StatisticsUtil.getReadableTime(lastExecTimeInMs));
}
+ if (timeCostInMs > 0) {
+ sj.add("timeCost: " + timeCostInMs);
+ }
if (periodTimeInMs > 0) {
sj.add("periodTimeInMs: " +
StatisticsUtil.getReadableTime(periodTimeInMs));
}
@@ -266,6 +274,8 @@ public class AnalysisInfo implements Writable {
analysisInfoBuilder.setPeriodTimeInMs(StatisticsUtil.convertStrToInt(periodTimeInMs));
String lastExecTimeInMs =
resultRow.getColumnValue("last_exec_time_in_ms");
analysisInfoBuilder.setLastExecTimeInMs(StatisticsUtil.convertStrToLong(lastExecTimeInMs));
+ String timeCostInMs = resultRow.getColumnValue("time_cost_in_ms");
+
analysisInfoBuilder.setTimeCostInMs(StatisticsUtil.convertStrToLong(timeCostInMs));
String message = resultRow.getColumnValue("message");
analysisInfoBuilder.setMessage(message);
return analysisInfoBuilder.build();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
index a14f262edc..7d8b99502c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
@@ -45,6 +45,7 @@ public class AnalysisInfoBuilder {
private int sampleRows;
private long periodTimeInMs;
private long lastExecTimeInMs;
+ private long timeCostInMs;
private AnalysisState state;
private ScheduleType scheduleType;
private String message = "";
@@ -75,6 +76,7 @@ public class AnalysisInfoBuilder {
maxBucketNum = info.maxBucketNum;
message = info.message;
lastExecTimeInMs = info.lastExecTimeInMs;
+ timeCostInMs = info.timeCostInMs;
state = info.state;
scheduleType = info.scheduleType;
externalTableLevelTask = info.externalTableLevelTask;
@@ -177,6 +179,11 @@ public class AnalysisInfoBuilder {
return this;
}
+ public AnalysisInfoBuilder setTimeCostInMs(long timeCostInMs) {
+ this.timeCostInMs = timeCostInMs;
+ return this;
+ }
+
public AnalysisInfoBuilder setState(AnalysisState state) {
this.state = state;
return this;
@@ -205,7 +212,7 @@ public class AnalysisInfoBuilder {
public AnalysisInfo build() {
return new AnalysisInfo(jobId, taskId, catalogName, dbName, tblName,
colToPartitions, partitionNames,
colName, indexId, jobType, analysisMode, analysisMethod,
analysisType, samplePercent,
- sampleRows, maxBucketNum, periodTimeInMs, message,
lastExecTimeInMs, state, scheduleType,
+ sampleRows, maxBucketNum, periodTimeInMs, message,
lastExecTimeInMs, timeCostInMs, state, scheduleType,
externalTableLevelTask, partitionOnly, samplingPartition);
}
@@ -229,6 +236,7 @@ public class AnalysisInfoBuilder {
.setMaxBucketNum(maxBucketNum)
.setMessage(message)
.setLastExecTimeInMs(lastExecTimeInMs)
+ .setTimeCostInMs(timeCostInMs)
.setState(state)
.setScheduleType(scheduleType)
.setExternalTableLevelTask(externalTableLevelTask);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index 91052404d4..d5135ecacb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -42,6 +42,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.Daemon;
+import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.persist.AnalyzeDeletionLog;
@@ -174,9 +175,7 @@ public class AnalysisManager extends Daemon implements
Writable {
TableName tableName = new
TableName(analyzeDBStmt.getCtlIf().getName(), db.getFullName(),
table.getName());
AnalyzeTblStmt analyzeTblStmt = new
AnalyzeTblStmt(analyzeDBStmt.getAnalyzeProperties(), tableName,
- table.getBaseSchema().stream().map(
- Column::getName).collect(
- Collectors.toList()), db.getId(), table);
+ null, db.getId(), table);
try {
analyzeTblStmt.check();
} catch (AnalysisException analysisException) {
@@ -221,8 +220,9 @@ public class AnalysisManager extends Daemon implements
Writable {
Map<Long, BaseAnalysisTask> analysisTaskInfos = new HashMap<>();
createTaskForEachColumns(jobInfo, analysisTaskInfos, isSync);
createTaskForMVIdx(jobInfo, analysisTaskInfos, isSync);
- createTaskForExternalTable(jobInfo, analysisTaskInfos, isSync);
-
+ if (stmt.isAllColumns()) {
+ createTaskForExternalTable(jobInfo, analysisTaskInfos, isSync);
+ }
if (!isSync) {
persistAnalysisJob(jobInfo);
analysisJobIdToTaskMap.put(jobInfo.jobId, analysisTaskInfos);
@@ -505,12 +505,12 @@ public class AnalysisManager extends Daemon implements
Writable {
long taskId = Env.getCurrentEnv().getNextId();
AnalysisInfoBuilder indexTaskInfoBuilder = new
AnalysisInfoBuilder(jobInfo);
AnalysisInfo analysisInfo =
indexTaskInfoBuilder.setIndexId(indexId)
- .setTaskId(taskId).build();
+
.setTaskId(taskId).setLastExecTimeInMs(System.currentTimeMillis()).build();
if (isSync) {
return;
}
analysisTasks.put(taskId, createTask(analysisInfo));
- logCreateAnalysisJob(analysisInfo);
+ logCreateAnalysisTask(analysisInfo);
}
} finally {
olapTable.readUnlock();
@@ -530,7 +530,7 @@ public class AnalysisManager extends Daemon implements
Writable {
colTaskInfoBuilder.setColToPartitions(Collections.singletonMap(colName,
entry.getValue()));
}
AnalysisInfo analysisInfo =
colTaskInfoBuilder.setColName(colName).setIndexId(indexId)
- .setTaskId(taskId).build();
+
.setTaskId(taskId).setLastExecTimeInMs(System.currentTimeMillis()).build();
analysisTasks.put(taskId, createTask(analysisInfo));
if (isSync) {
continue;
@@ -545,13 +545,15 @@ public class AnalysisManager extends Daemon implements
Writable {
}
}
- private void logCreateAnalysisTask(AnalysisInfo analysisInfo) {
- analysisTaskInfoMap.put(analysisInfo.taskId, analysisInfo);
+ // Change to public for unit test.
+ public void logCreateAnalysisTask(AnalysisInfo analysisInfo) {
+ replayCreateAnalysisTask(analysisInfo);
Env.getCurrentEnv().getEditLog().logCreateAnalysisTasks(analysisInfo);
}
- private void logCreateAnalysisJob(AnalysisInfo analysisJob) {
- analysisJobInfoMap.put(analysisJob.jobId, analysisJob);
+ // Change to public for unit test.
+ public void logCreateAnalysisJob(AnalysisInfo analysisJob) {
+ replayCreateAnalysisJob(analysisJob);
Env.getCurrentEnv().getEditLog().logCreateAnalysisJob(analysisJob);
}
@@ -570,63 +572,71 @@ public class AnalysisManager extends Daemon implements
Writable {
}
AnalysisInfoBuilder colTaskInfoBuilder = new
AnalysisInfoBuilder(jobInfo);
long taskId = Env.getCurrentEnv().getNextId();
- AnalysisInfo analysisInfo = colTaskInfoBuilder.setIndexId(-1L)
- .setTaskId(taskId).setExternalTableLevelTask(true).build();
+ AnalysisInfo analysisInfo =
colTaskInfoBuilder.setIndexId(-1L).setLastExecTimeInMs(System.currentTimeMillis())
+
.setTaskId(taskId).setColName("TableRowCount").setExternalTableLevelTask(true).build();
analysisTasks.put(taskId, createTask(analysisInfo));
if (isSync) {
// For sync job, don't need to persist, return here and execute it
immediately.
return;
}
try {
- logCreateAnalysisJob(analysisInfo);
+ logCreateAnalysisTask(analysisInfo);
} catch (Exception e) {
throw new DdlException("Failed to create analysis task", e);
}
}
- public void updateTaskStatus(AnalysisInfo info, AnalysisState jobState,
String message, long time) {
+ public void updateTaskStatus(AnalysisInfo info, AnalysisState taskState,
String message, long time) {
if (analysisJobIdToTaskMap.get(info.jobId) == null) {
return;
}
- info.state = jobState;
+ info.state = taskState;
info.message = message;
+ // Update the task cost time when task finished or failed. And only
log the final state.
+ if (taskState.equals(AnalysisState.FINISHED) ||
taskState.equals(AnalysisState.FAILED)) {
+ info.timeCostInMs = time - info.lastExecTimeInMs;
+ info.lastExecTimeInMs = time;
+ logCreateAnalysisTask(info);
+ }
info.lastExecTimeInMs = time;
- logCreateAnalysisTask(info);
-
AnalysisInfo job = analysisJobInfoMap.get(info.jobId);
- job.lastExecTimeInMs = time;
- if (info.state.equals(AnalysisState.RUNNING) &&
!job.state.equals(AnalysisState.PENDING)) {
- job.state = AnalysisState.RUNNING;
- Env.getCurrentEnv().getEditLog().logCreateAnalysisTasks(job);
- }
- boolean allFinished = true;
- boolean hasFailure = false;
- for (BaseAnalysisTask task :
analysisJobIdToTaskMap.get(info.jobId).values()) {
- AnalysisInfo taskInfo = task.info;
- if (taskInfo.state.equals(AnalysisState.RUNNING) ||
taskInfo.state.equals(AnalysisState.PENDING)) {
- allFinished = false;
- break;
+ // Synchronize the job state change in job level.
+ synchronized (job) {
+ job.lastExecTimeInMs = time;
+ // Set the job state to RUNNING when its first task becomes
RUNNING.
+ if (info.state.equals(AnalysisState.RUNNING) &&
job.state.equals(AnalysisState.PENDING)) {
+ job.state = AnalysisState.RUNNING;
+ replayCreateAnalysisJob(job);
}
- if (taskInfo.state.equals(AnalysisState.FAILED)) {
- hasFailure = true;
+ boolean allFinished = true;
+ boolean hasFailure = false;
+ for (BaseAnalysisTask task :
analysisJobIdToTaskMap.get(info.jobId).values()) {
+ AnalysisInfo taskInfo = task.info;
+ if (taskInfo.state.equals(AnalysisState.RUNNING) ||
taskInfo.state.equals(AnalysisState.PENDING)) {
+ allFinished = false;
+ break;
+ }
+ if (taskInfo.state.equals(AnalysisState.FAILED)) {
+ hasFailure = true;
+ }
}
- }
- if (allFinished) {
- if (hasFailure) {
- job.state = AnalysisState.FAILED;
- logCreateAnalysisJob(job);
- } else {
- job.state = AnalysisState.FINISHED;
- if (job.jobType.equals(JobType.SYSTEM)) {
- try {
- updateTableStats(job);
- } catch (Throwable e) {
- LOG.warn("Failed to update Table statistics in job:
{}", info.toString());
+ if (allFinished) {
+ if (hasFailure) {
+ job.state = AnalysisState.FAILED;
+ logCreateAnalysisJob(job);
+ } else {
+ job.state = AnalysisState.FINISHED;
+ if (job.jobType.equals(JobType.SYSTEM)) {
+ try {
+ updateTableStats(job);
+ } catch (Throwable e) {
+ LOG.warn("Failed to update Table statistics in
job: {}", info.toString(), e);
+ }
}
+ logCreateAnalysisJob(job);
}
- logCreateAnalysisJob(job);
+ analysisJobIdToTaskMap.remove(job.jobId);
}
- analysisJobIdToTaskMap.remove(job.jobId);
}
}
@@ -692,6 +702,28 @@ public class AnalysisManager extends Daemon implements
Writable {
.collect(Collectors.toList());
}
+ public String getJobProgress(long jobId) {
+ List<AnalysisInfo> tasks = findTasks(jobId);
+ int finished = 0;
+ int failed = 0;
+ int inProgress = 0;
+ int total = tasks.size();
+ for (AnalysisInfo info : tasks) {
+ switch (info.state) {
+ case FINISHED:
+ finished++;
+ break;
+ case FAILED:
+ failed++;
+ break;
+ default:
+ inProgress++;
+ break;
+ }
+ }
+ return String.format("%d Finished/%d Failed/%d In Progress/%d Total",
finished, failed, inProgress, total);
+ }
+
private void syncExecute(Collection<BaseAnalysisTask> tasks) {
SyncTaskCollection syncTaskCollection = new SyncTaskCollection(tasks);
ConnectContext ctx = ConnectContext.get();
@@ -805,8 +837,11 @@ public class AnalysisManager extends Daemon implements
Writable {
public void execute() {
List<String> colNames = new ArrayList<>();
+ List<String> errorMessages = new ArrayList<>();
for (BaseAnalysisTask task : tasks) {
if (cancelled) {
+ colNames.add(task.info.colName);
+ errorMessages.add("Cancelled");
continue;
}
try {
@@ -814,12 +849,14 @@ public class AnalysisManager extends Daemon implements
Writable {
updateSyncTaskStatus(task, AnalysisState.FINISHED);
} catch (Throwable t) {
colNames.add(task.info.colName);
+ errorMessages.add(Util.getRootCauseMessage(t));
updateSyncTaskStatus(task, AnalysisState.FAILED);
LOG.warn("Failed to analyze, info: {}", task, t);
}
}
if (!colNames.isEmpty()) {
- throw new RuntimeException("Failed to analyze following
columns: " + String.join(",", colNames));
+ throw new RuntimeException("Failed to analyze following
columns:[" + String.join(",", colNames)
+ + "] Reasons: " + String.join(",", errorMessages));
}
}
@@ -904,4 +941,9 @@ public class AnalysisManager extends Daemon implements
Writable {
entry.getValue().write(out);
}
}
+
+ // For unit test use only.
+ public void addToJobIdTasksMap(long jobId, Map<Long, BaseAnalysisTask>
tasks) {
+ analysisJobIdToTaskMap.put(jobId, tasks);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskExecutor.java
index 11f244afd8..b5ec7aeb87 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskExecutor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskExecutor.java
@@ -17,7 +17,6 @@
package org.apache.doris.statistics;
-import org.apache.doris.catalog.Env;
import org.apache.doris.common.Config;
import org.apache.doris.common.ThreadPoolManager;
import org.apache.doris.common.ThreadPoolManager.BlockedPolicy;
@@ -101,9 +100,6 @@ public class AnalysisTaskExecutor extends Thread {
BaseAnalysisTask task = taskScheduler.getPendingTasks();
AnalysisTaskWrapper taskWrapper = new AnalysisTaskWrapper(this, task);
executors.submit(taskWrapper);
- Env.getCurrentEnv().getAnalysisManager()
- .updateTaskStatus(task.info,
- AnalysisState.RUNNING, "", System.currentTimeMillis());
}
public void putJob(AnalysisTaskWrapper wrapper) throws Exception {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskWrapper.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskWrapper.java
index 9a0d6ec2fc..7f55469f53 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskWrapper.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskWrapper.java
@@ -18,6 +18,7 @@
package org.apache.doris.statistics;
import org.apache.doris.catalog.Env;
+import org.apache.doris.common.util.Util;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -62,16 +63,17 @@ public class AnalysisTaskWrapper extends FutureTask<Void> {
} finally {
if (!task.killed) {
if (except != null) {
- LOG.warn("Failed to execute task", except);
+ LOG.warn("Analyze {} failed.", task.toString(), except);
Env.getCurrentEnv().getAnalysisManager()
.updateTaskStatus(task.info,
- AnalysisState.FAILED, except.getMessage(),
System.currentTimeMillis());
+ AnalysisState.FAILED,
Util.getRootCauseMessage(except), System.currentTimeMillis());
} else {
+ LOG.debug("Analyze {} finished, cost time:{}",
task.toString(),
+ System.currentTimeMillis() - startTime);
Env.getCurrentEnv().getAnalysisManager()
.updateTaskStatus(task.info,
AnalysisState.FINISHED, "",
System.currentTimeMillis());
}
- LOG.warn("{} finished, cost time:{}", task.toString(),
System.currentTimeMillis() - startTime);
}
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index aca18bd05f..9c15c49c38 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -171,12 +171,20 @@ public abstract class BaseAnalysisTask {
break;
} catch (Throwable t) {
LOG.warn("Failed to execute analysis task, retried times: {}",
retriedTimes++, t);
+ if (retriedTimes >
StatisticConstants.ANALYZE_TASK_RETRY_TIMES) {
+ throw new RuntimeException(t);
+ }
}
}
}
public abstract void doExecute() throws Exception;
+ protected void setTaskStateToRunning() {
+ Env.getCurrentEnv().getAnalysisManager()
+ .updateTaskStatus(info, AnalysisState.RUNNING, "",
System.currentTimeMillis());
+ }
+
public void cancel() {
killed = true;
if (stmtExecutor != null) {
@@ -184,7 +192,7 @@ public abstract class BaseAnalysisTask {
}
Env.getCurrentEnv().getAnalysisManager()
.updateTaskStatus(info, AnalysisState.FAILED,
- String.format("Job has been cancelled: %s",
info.toString()), -1);
+ String.format("Job has been cancelled: %s",
info.message), System.currentTimeMillis());
}
public long getLastExecTime() {
@@ -215,4 +223,11 @@ public abstract class BaseAnalysisTask {
return String.format("TABLESAMPLE(%d ROWS)", info.sampleRows);
}
}
+
+ @Override
+ public String toString() {
+ return String.format("Job id [%d], Task id [%d], catalog [%s], db
[%s], table [%s], column [%s]",
+ info.jobId, info.taskId, catalog.getName(), db.getFullName(),
tbl.getName(),
+ col == null ? "TableRowCount" : col.getName());
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java
index 87e7bbe1b5..03840f091d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java
@@ -22,6 +22,7 @@ import org.apache.doris.catalog.external.HMSExternalTable;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.qe.AutoCloseConnectContext;
+import org.apache.doris.qe.QueryState;
import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.statistics.util.InternalQueryResult;
import org.apache.doris.statistics.util.StatisticsUtil;
@@ -105,6 +106,7 @@ public class HMSAnalysisTask extends BaseAnalysisTask {
}
public void doExecute() throws Exception {
+ setTaskStateToRunning();
if (isTableLevelTask) {
getTableStats();
} else {
@@ -190,11 +192,7 @@ public class HMSAnalysisTask extends BaseAnalysisTask {
params.put("dataSizeFunction", getDataSizeFunction(col));
StringSubstitutor stringSubstitutor = new
StringSubstitutor(params);
String sql = stringSubstitutor.replace(sb.toString());
- try (AutoCloseConnectContext r =
StatisticsUtil.buildConnectContext()) {
-
r.connectContext.getSessionVariable().disableNereidsPlannerOnce();
- this.stmtExecutor = new StmtExecutor(r.connectContext,
sql);
- this.stmtExecutor.execute();
- }
+ executeInsertSql(sql);
}
} else {
StringBuilder sb = new StringBuilder();
@@ -233,12 +231,27 @@ public class HMSAnalysisTask extends BaseAnalysisTask {
params.put("dataSizeFunction", getDataSizeFunction(col));
StringSubstitutor stringSubstitutor = new
StringSubstitutor(params);
String sql = stringSubstitutor.replace(sb.toString());
- try (AutoCloseConnectContext r =
StatisticsUtil.buildConnectContext()) {
-
r.connectContext.getSessionVariable().disableNereidsPlannerOnce();
- this.stmtExecutor = new StmtExecutor(r.connectContext, sql);
- this.stmtExecutor.execute();
+ executeInsertSql(sql);
+ Env.getCurrentEnv().getStatisticsCache().refreshColStatsSync(
+ catalog.getId(), db.getId(), tbl.getId(), -1,
col.getName());
+ }
+ }
+
+ private void executeInsertSql(String sql) throws Exception {
+ long startTime = System.currentTimeMillis();
+ try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext())
{
+ r.connectContext.getSessionVariable().disableNereidsPlannerOnce();
+ this.stmtExecutor = new StmtExecutor(r.connectContext, sql);
+ r.connectContext.setExecutor(stmtExecutor);
+ this.stmtExecutor.execute();
+ QueryState queryState = r.connectContext.getState();
+ if
(queryState.getStateType().equals(QueryState.MysqlStateType.ERR)) {
+ LOG.warn(String.format("Failed to analyze %s.%s.%s, sql: [%s],
error: [%s]",
+ info.catalogName, info.dbName, info.colName, sql,
queryState.getErrorMessage()));
+ throw new RuntimeException(queryState.getErrorMessage());
}
-
Env.getCurrentEnv().getStatisticsCache().refreshColStatsSync(tbl.getId(), -1,
col.getName());
+ LOG.debug(String.format("Analyze %s.%s.%s done. SQL: [%s]. Cost %d
ms.",
+ info.catalogName, info.dbName, info.colName, sql,
(System.currentTimeMillis() - startTime)));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java
index a0b6e6d9a5..a1b53cb3be 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java
@@ -59,6 +59,7 @@ public class HistogramTask extends BaseAnalysisTask {
@Override
public void doExecute() throws Exception {
+ setTaskStateToRunning();
Map<String, String> params = new HashMap<>();
params.put("internalDB", FeConstants.INTERNAL_DB_NAME);
params.put("histogramStatTbl", StatisticConstants.HISTOGRAM_TBL_NAME);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java
index 31b3b76d5d..ed33252172 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/MVAnalysisTask.java
@@ -87,6 +87,7 @@ public class MVAnalysisTask extends BaseAnalysisTask {
@Override
public void doExecute() throws Exception {
+ setTaskStateToRunning();
for (Column column : meta.getSchema()) {
SelectStmt selectOne = (SelectStmt) selectStmt.clone();
TableRef tableRef = selectOne.getTableRefs().get(0);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
index f23c90f798..9fb04f9394 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -60,6 +60,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
}
public void doExecute() throws Exception {
+ setTaskStateToRunning();
Map<String, String> params = new HashMap<>();
params.put("internalDB", FeConstants.INTERNAL_DB_NAME);
params.put("columnStatTbl", StatisticConstants.STATISTIC_TBL_NAME);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
index 284083ac01..1149ecdd5a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
@@ -169,7 +169,11 @@ public class StatisticsCache {
}
public void refreshColStatsSync(long tblId, long idxId, String colName) {
- columnStatisticsCache.synchronous().refresh(new
StatisticsCacheKey(tblId, idxId, colName));
+ columnStatisticsCache.synchronous().refresh(new StatisticsCacheKey(-1,
-1, tblId, idxId, colName));
+ }
+
+ public void refreshColStatsSync(long catalogId, long dbId, long tblId,
long idxId, String colName) {
+ columnStatisticsCache.synchronous().refresh(new
StatisticsCacheKey(catalogId, dbId, tblId, idxId, colName));
}
public void refreshHistogramSync(long tblId, long idxId, String colName) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index 0797a7e10f..61551b4f20 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -178,6 +178,8 @@ public class StatisticsUtil {
sessionVariable.parallelPipelineTaskNum =
Config.statistics_sql_parallel_exec_instance_num;
sessionVariable.setEnableNereidsPlanner(false);
sessionVariable.enableProfile = false;
+ sessionVariable.queryTimeoutS = Config.analyze_task_timeout_in_hours *
60 * 60;
+ sessionVariable.insertTimeoutS = Config.analyze_task_timeout_in_hours
* 60 * 60;
connectContext.setEnv(Env.getCurrentEnv());
connectContext.setDatabase(FeConstants.INTERNAL_DB_NAME);
connectContext.setQualifiedUser(UserIdentity.ROOT.getQualifiedUser());
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
new file mode 100644
index 0000000000..253f9c9332
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics;
+
+import mockit.Mock;
+import mockit.MockUp;
+import mockit.Mocked;
+import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class AnalysisManagerTest {
+ @Test
+ public void testUpdateTaskStatus(@Mocked BaseAnalysisTask task1,
+ @Mocked BaseAnalysisTask task2) {
+
+ new MockUp<AnalysisManager>() {
+ @Mock
+ public void logCreateAnalysisTask(AnalysisInfo job) {
+ }
+
+ @Mock
+ public void logCreateAnalysisJob(AnalysisInfo job) {
+ }
+
+ };
+
+ AnalysisInfo job = new AnalysisInfoBuilder().setJobId(1)
+
.setState(AnalysisState.PENDING).setJobType(AnalysisInfo.JobType.MANUAL).build();
+ AnalysisInfo taskInfo1 = new AnalysisInfoBuilder().setJobId(1)
+ .setTaskId(2).setState(AnalysisState.PENDING).build();
+ AnalysisInfo taskInfo2 = new AnalysisInfoBuilder().setJobId(1)
+ .setTaskId(3).setState(AnalysisState.PENDING).build();
+ AnalysisManager manager = new AnalysisManager();
+ manager.replayCreateAnalysisJob(job);
+ manager.replayCreateAnalysisTask(taskInfo1);
+ manager.replayCreateAnalysisTask(taskInfo2);
+ Map<Long, BaseAnalysisTask> tasks = new HashMap<>();
+
+ task1.info = taskInfo1;
+ task2.info = taskInfo2;
+ tasks.put(2L, task1);
+ tasks.put(3L, task2);
+ manager.addToJobIdTasksMap(1, tasks);
+
+ Assertions.assertEquals(job.state, AnalysisState.PENDING);
+ manager.updateTaskStatus(taskInfo1, AnalysisState.RUNNING, "", 0);
+ Assertions.assertEquals(job.state, AnalysisState.RUNNING);
+ manager.updateTaskStatus(taskInfo2, AnalysisState.RUNNING, "", 0);
+ Assertions.assertEquals(job.state, AnalysisState.RUNNING);
+ manager.updateTaskStatus(taskInfo1, AnalysisState.FINISHED, "", 0);
+ Assertions.assertEquals(job.state, AnalysisState.RUNNING);
+ manager.updateTaskStatus(taskInfo2, AnalysisState.FINISHED, "", 0);
+ Assertions.assertEquals(job.state, AnalysisState.FINISHED);
+ }
+}
diff --git
a/regression-test/data/external_table_emr_p2/hive/test_hive_analyze_db.out
b/regression-test/data/external_table_emr_p2/hive/test_hive_analyze_db.out
deleted file mode 100644
index 0b75316476..0000000000
--- a/regression-test/data/external_table_emr_p2/hive/test_hive_analyze_db.out
+++ /dev/null
@@ -1,20 +0,0 @@
--- This file is automatically generated. You should know what you did if you
want to edit this
--- !1 --
-lo_orderpriority 100.0 5.0 0.0 888.8000000000001 8.8
'1-URGENT' '5-LOW'
-lo_custkey 100.0 26.0 0.0 404.0 4.0 67423 2735521
-lo_partkey 100.0 100.0 0.0 404.0 4.0 2250 989601
-lo_revenue 100.0 100.0 0.0 404.0 4.0 101171 8703450
-lo_commitdate 100.0 95.0 0.0 404.0 4.0 19920515 19981016
-lo_quantity 100.0 46.0 0.0 404.0 4.0 1 50
-lo_orderkey 100.0 26.0 0.0 404.0 4.0 1 98
-lo_suppkey 100.0 100.0 0.0 404.0 4.0 4167 195845
-lo_supplycost 100.0 100.0 0.0 404.0 4.0 58023 121374
-lo_shipmode 100.0 7.0 0.0 425.21 4.21 'AIR' 'TRUCK'
-lo_orderdate 100.0 26.0 0.0 404.0 4.0 19920221 19980721
-lo_linenumber 100.0 7.0 0.0 404.0 4.0 1 7
-lo_shippriority 100.0 1.0 0.0 404.0 4.0 0 0
-lo_ordtotalprice 100.0 26.0 0.0 404.0 4.0 3428256 36771805
-lo_extendedprice 100.0 100.0 0.0 404.0 4.0 104300 9066094
-lo_tax 100.0 9.0 0.0 404.0 4.0 0 8
-lo_discount 100.0 11.0 0.0 404.0 4.0 0 10
-
diff --git
a/regression-test/data/external_table_emr_p2/hive/test_hive_statistic.out
b/regression-test/data/external_table_emr_p2/hive/test_hive_statistic.out
deleted file mode 100644
index ad8b494c00..0000000000
--- a/regression-test/data/external_table_emr_p2/hive/test_hive_statistic.out
+++ /dev/null
@@ -1,57 +0,0 @@
--- This file is automatically generated. You should know what you did if you
want to edit this
--- !1 --
-lo_quantity 100.0 46.0 0.0 404.0 4.0 1 50
-
--- !2 --
-lo_orderkey 100.0 26.0 0.0 404.0 4.0 1 98
-
--- !3 --
-lo_linenumber 100.0 7.0 0.0 404.0 4.0 1 7
-
--- !4 --
-lo_custkey 100.0 26.0 0.0 404.0 4.0 67423 2735521
-
--- !5 --
-lo_partkey 100.0 100.0 0.0 404.0 4.0 2250 989601
-
--- !6 --
-lo_suppkey 100.0 100.0 0.0 404.0 4.0 4167 195845
-
--- !7 --
-lo_orderdate 100.0 26.0 0.0 404.0 4.0 19920221 19980721
-
--- !8 --
-lo_orderpriority 100.0 5.0 0.0 888.8000000000001 8.8
'1-URGENT' '5-LOW'
-
--- !9 --
-lo_shippriority 100.0 1.0 0.0 404.0 4.0 0 0
-
--- !10 --
-lo_extendedprice 100.0 100.0 0.0 404.0 4.0 104300 9066094
-
--- !11 --
-lo_ordtotalprice 100.0 26.0 0.0 404.0 4.0 3428256 36771805
-
--- !12 --
-lo_discount 100.0 11.0 0.0 404.0 4.0 0 10
-
--- !13 --
-lo_revenue 100.0 100.0 0.0 404.0 4.0 101171 8703450
-
--- !14 --
-lo_supplycost 100.0 100.0 0.0 404.0 4.0 58023 121374
-
--- !15 --
-lo_tax 100.0 9.0 0.0 404.0 4.0 0 8
-
--- !16 --
-lo_commitdate 100.0 95.0 0.0 404.0 4.0 19920515 19981016
-
--- !17 --
-lo_shipmode 100.0 7.0 0.0 425.21 4.21 'AIR' 'TRUCK'
-
--- !18 --
-lo_shipmode 6001215.0 0.0 0.0 0.0 0.0 'NULL' 'NULL'
-
--- !19 --
-
diff --git
a/regression-test/data/external_table_emr_p2/hive/test_hive_statistic_cache.out
b/regression-test/data/external_table_emr_p2/hive/test_hive_statistic_cache.out
deleted file mode 100644
index dc5ded3837..0000000000
---
a/regression-test/data/external_table_emr_p2/hive/test_hive_statistic_cache.out
+++ /dev/null
@@ -1,58 +0,0 @@
--- This file is automatically generated. You should know what you did if you
want to edit this
--- !1 --
-100
-
--- !2 --
-lo_orderkey 100.0 26.0 0.0 404.0 4.0 1 98
-
--- !3 --
-lo_linenumber 100.0 6.0 0.0 404.0 4.0 1 7
-
--- !4 --
-lo_custkey 100.0 30.0 0.0 404.0 4.0 67423 2735521
-
--- !5 --
-lo_partkey 100.0 107.0 0.0 404.0 4.0 2250 989601
-
--- !6 --
-lo_suppkey 100.0 90.0 0.0 404.0 4.0 4167 195845
-
--- !7 --
-lo_orderdate 100.0 22.0 0.0 404.0 4.0 19920221 19980721
-
--- !8 --
-lo_orderpriority 100.0 5.0 0.0 888.8000000000001 8.8
N/A N/A
-
--- !9 --
-lo_shippriority 100.0 2.0 0.0 404.0 4.0 0 0
-
--- !10 --
-lo_extendedprice 100.0 112.0 0.0 404.0 4.0 104300 9066094
-
--- !11 --
-lo_ordtotalprice 100.0 31.0 0.0 404.0 4.0 3428256 36771805
-
--- !12 --
-lo_discount 100.0 11.0 0.0 404.0 4.0 0 10
-
--- !13 --
-lo_revenue 100.0 127.0 0.0 404.0 4.0 101171 8703450
-
--- !14 --
-lo_supplycost 100.0 112.0 0.0 404.0 4.0 58023 121374
-
--- !15 --
-lo_tax 100.0 9.0 0.0 404.0 4.0 0 8
-
--- !16 --
-lo_commitdate 100.0 122.0 0.0 404.0 4.0 19920515 19981016
-
--- !17 --
-lo_shipmode 100.0 7.0 0.0 425.21 4.21 N/A N/A
-
--- !18 --
-lo_orderkey 100.0 26.0 0.0 404.0 4.0 1 98
-
--- !19 --
-lo_quantity 100.0 34.0 0.0 404.0 4.0 1 50
-
diff --git
a/regression-test/suites/external_table_emr_p2/hive/test_hive_analyze_db.groovy
b/regression-test/suites/external_table_emr_p2/hive/test_hive_analyze_db.groovy
index a30c87275b..fd724a67f8 100644
---
a/regression-test/suites/external_table_emr_p2/hive/test_hive_analyze_db.groovy
+++
b/regression-test/suites/external_table_emr_p2/hive/test_hive_analyze_db.groovy
@@ -33,8 +33,163 @@ suite("test_hive_analyze_db", "p2") {
sql """switch ${catalog_name};"""
logger.info("switched to catalog " + catalog_name)
sql """use statistics;"""
+ sql """set query_timeout=300"""
sql """analyze database statistics with sync"""
- qt_1 "show column stats statistics"
+ def result = sql """show column stats statistics"""
+ assertTrue(result.size() == 17)
+
+ assertTrue(result[0][0] == "lo_orderpriority")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "5.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "888.8000000000001")
+ assertTrue(result[0][5] == "8.8")
+ assertTrue(result[0][6] == "'1-URGENT'")
+ assertTrue(result[0][7] == "'5-LOW'")
+
+ assertTrue(result[1][0] == "lo_custkey")
+ assertTrue(result[1][1] == "100.0")
+ assertTrue(result[1][2] == "26.0")
+ assertTrue(result[1][3] == "0.0")
+ assertTrue(result[1][4] == "404.0")
+ assertTrue(result[1][5] == "4.0")
+ assertTrue(result[1][6] == "67423")
+ assertTrue(result[1][7] == "2735521")
+
+ assertTrue(result[2][0] == "lo_partkey")
+ assertTrue(result[2][1] == "100.0")
+ assertTrue(result[2][2] == "100.0")
+ assertTrue(result[2][3] == "0.0")
+ assertTrue(result[2][4] == "404.0")
+ assertTrue(result[2][5] == "4.0")
+ assertTrue(result[2][6] == "2250")
+ assertTrue(result[2][7] == "989601")
+
+ assertTrue(result[3][0] == "lo_revenue")
+ assertTrue(result[3][1] == "100.0")
+ assertTrue(result[3][2] == "100.0")
+ assertTrue(result[3][3] == "0.0")
+ assertTrue(result[3][4] == "404.0")
+ assertTrue(result[3][5] == "4.0")
+ assertTrue(result[3][6] == "101171")
+ assertTrue(result[3][7] == "8703450")
+
+ assertTrue(result[4][0] == "lo_commitdate")
+ assertTrue(result[4][1] == "100.0")
+ assertTrue(result[4][2] == "95.0")
+ assertTrue(result[4][3] == "0.0")
+ assertTrue(result[4][4] == "404.0")
+ assertTrue(result[4][5] == "4.0")
+ assertTrue(result[4][6] == "19920515")
+ assertTrue(result[4][7] == "19981016")
+
+ assertTrue(result[5][0] == "lo_quantity")
+ assertTrue(result[5][1] == "100.0")
+ assertTrue(result[5][2] == "46.0")
+ assertTrue(result[5][3] == "0.0")
+ assertTrue(result[5][4] == "404.0")
+ assertTrue(result[5][5] == "4.0")
+ assertTrue(result[5][6] == "1")
+ assertTrue(result[5][7] == "50")
+
+ assertTrue(result[6][0] == "lo_orderkey")
+ assertTrue(result[6][1] == "100.0")
+ assertTrue(result[6][2] == "26.0")
+ assertTrue(result[6][3] == "0.0")
+ assertTrue(result[6][4] == "404.0")
+ assertTrue(result[6][5] == "4.0")
+ assertTrue(result[6][6] == "1")
+ assertTrue(result[6][7] == "98")
+
+ assertTrue(result[7][0] == "lo_suppkey")
+ assertTrue(result[7][1] == "100.0")
+ assertTrue(result[7][2] == "100.0")
+ assertTrue(result[7][3] == "0.0")
+ assertTrue(result[7][4] == "404.0")
+ assertTrue(result[7][5] == "4.0")
+ assertTrue(result[7][6] == "4167")
+ assertTrue(result[7][7] == "195845")
+
+ assertTrue(result[8][0] == "lo_supplycost")
+ assertTrue(result[8][1] == "100.0")
+ assertTrue(result[8][2] == "100.0")
+ assertTrue(result[8][3] == "0.0")
+ assertTrue(result[8][4] == "404.0")
+ assertTrue(result[8][5] == "4.0")
+ assertTrue(result[8][6] == "58023")
+ assertTrue(result[8][7] == "121374")
+
+ assertTrue(result[9][0] == "lo_shipmode")
+ assertTrue(result[9][1] == "100.0")
+ assertTrue(result[9][2] == "7.0")
+ assertTrue(result[9][3] == "0.0")
+ assertTrue(result[9][4] == "425.21")
+ assertTrue(result[9][5] == "4.21")
+ assertTrue(result[9][6] == "'AIR'")
+ assertTrue(result[9][7] == "'TRUCK'")
+
+ assertTrue(result[10][0] == "lo_orderdate")
+ assertTrue(result[10][1] == "100.0")
+ assertTrue(result[10][2] == "26.0")
+ assertTrue(result[10][3] == "0.0")
+ assertTrue(result[10][4] == "404.0")
+ assertTrue(result[10][5] == "4.0")
+ assertTrue(result[10][6] == "19920221")
+ assertTrue(result[10][7] == "19980721")
+
+ assertTrue(result[11][0] == "lo_linenumber")
+ assertTrue(result[11][1] == "100.0")
+ assertTrue(result[11][2] == "7.0")
+ assertTrue(result[11][3] == "0.0")
+ assertTrue(result[11][4] == "404.0")
+ assertTrue(result[11][5] == "4.0")
+ assertTrue(result[11][6] == "1")
+ assertTrue(result[11][7] == "7")
+
+ assertTrue(result[12][0] == "lo_shippriority")
+ assertTrue(result[12][1] == "100.0")
+ assertTrue(result[12][2] == "1.0")
+ assertTrue(result[12][3] == "0.0")
+ assertTrue(result[12][4] == "404.0")
+ assertTrue(result[12][5] == "4.0")
+ assertTrue(result[12][6] == "0")
+ assertTrue(result[12][7] == "0")
+
+ assertTrue(result[13][0] == "lo_ordtotalprice")
+ assertTrue(result[13][1] == "100.0")
+ assertTrue(result[13][2] == "26.0")
+ assertTrue(result[13][3] == "0.0")
+ assertTrue(result[13][4] == "404.0")
+ assertTrue(result[13][5] == "4.0")
+ assertTrue(result[13][6] == "3428256")
+ assertTrue(result[13][7] == "36771805")
+
+ assertTrue(result[14][0] == "lo_extendedprice")
+ assertTrue(result[14][1] == "100.0")
+ assertTrue(result[14][2] == "100.0")
+ assertTrue(result[14][3] == "0.0")
+ assertTrue(result[14][4] == "404.0")
+ assertTrue(result[14][5] == "4.0")
+ assertTrue(result[14][6] == "104300")
+ assertTrue(result[14][7] == "9066094")
+
+ assertTrue(result[15][0] == "lo_tax")
+ assertTrue(result[15][1] == "100.0")
+ assertTrue(result[15][2] == "9.0")
+ assertTrue(result[15][3] == "0.0")
+ assertTrue(result[15][4] == "404.0")
+ assertTrue(result[15][5] == "4.0")
+ assertTrue(result[15][6] == "0")
+ assertTrue(result[15][7] == "8")
+
+ assertTrue(result[16][0] == "lo_discount")
+ assertTrue(result[16][1] == "100.0")
+ assertTrue(result[16][2] == "11.0")
+ assertTrue(result[16][3] == "0.0")
+ assertTrue(result[16][4] == "404.0")
+ assertTrue(result[16][5] == "4.0")
+ assertTrue(result[16][6] == "0")
+ assertTrue(result[16][7] == "10")
}
}
diff --git
a/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic.groovy
b/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic.groovy
index ff184e4506..c6df0b0eaa 100644
---
a/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic.groovy
+++
b/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic.groovy
@@ -34,29 +34,202 @@ suite("test_hive_statistic", "p2") {
logger.info("switched to catalog " + catalog_name)
sql """use statistics;"""
sql """analyze table `statistics` with sync"""
- qt_1 "show column stats `statistics` (lo_quantity)"
- qt_2 "show column stats `statistics` (lo_orderkey)"
- qt_3 "show column stats `statistics` (lo_linenumber)"
- qt_4 "show column stats `statistics` (lo_custkey)"
- qt_5 "show column stats `statistics` (lo_partkey)"
- qt_6 "show column stats `statistics` (lo_suppkey)"
- qt_7 "show column stats `statistics` (lo_orderdate)"
- qt_8 "show column stats `statistics` (lo_orderpriority)"
- qt_9 "show column stats `statistics` (lo_shippriority)"
- qt_10 "show column stats `statistics` (lo_extendedprice)"
- qt_11 "show column stats `statistics` (lo_ordtotalprice)"
- qt_12 "show column stats `statistics` (lo_discount)"
- qt_13 "show column stats `statistics` (lo_revenue)"
- qt_14 "show column stats `statistics` (lo_supplycost)"
- qt_15 "show column stats `statistics` (lo_tax)"
- qt_16 "show column stats `statistics` (lo_commitdate)"
- qt_17 "show column stats `statistics` (lo_shipmode)"
-
- sql """ALTER TABLE statistics MODIFY COLUMN lo_shipmode SET STATS
('row_count'='6001215')"""
- qt_18 "show column stats `statistics` (lo_shipmode)"
+ def result = sql """show column stats `statistics` (lo_quantity)"""
+ assertTrue(result.size() == 1)
+ assertTrue(result[0][0] == "lo_quantity")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "46.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "1")
+ assertTrue(result[0][7] == "50")
+
+ result = sql """show column stats `statistics` (lo_orderkey)"""
+ assertTrue(result.size() == 1)
+ assertTrue(result[0][0] == "lo_orderkey")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "26.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "1")
+ assertTrue(result[0][7] == "98")
+
+ result = sql """show column stats `statistics` (lo_linenumber)"""
+ assertTrue(result.size() == 1)
+ assertTrue(result[0][0] == "lo_linenumber")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "7.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "1")
+ assertTrue(result[0][7] == "7")
+
+ result = sql """show column stats `statistics` (lo_custkey)"""
+ assertTrue(result.size() == 1)
+ assertTrue(result[0][0] == "lo_custkey")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "26.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "67423")
+ assertTrue(result[0][7] == "2735521")
+
+ result = sql """show column stats `statistics` (lo_partkey)"""
+ assertTrue(result.size() == 1)
+ assertTrue(result[0][0] == "lo_partkey")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "100.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "2250")
+ assertTrue(result[0][7] == "989601")
+
+ result = sql """show column stats `statistics` (lo_suppkey)"""
+ assertTrue(result.size() == 1)
+ assertTrue(result[0][0] == "lo_suppkey")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "100.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "4167")
+ assertTrue(result[0][7] == "195845")
+
+ result = sql """show column stats `statistics` (lo_orderdate)"""
+ assertTrue(result.size() == 1)
+ assertTrue(result[0][0] == "lo_orderdate")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "26.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "19920221")
+ assertTrue(result[0][7] == "19980721")
+
+ result = sql """show column stats `statistics` (lo_orderpriority)"""
+ assertTrue(result.size() == 1)
+ assertTrue(result[0][0] == "lo_orderpriority")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "5.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "888.8000000000001")
+ assertTrue(result[0][5] == "8.8")
+ assertTrue(result[0][6] == "'1-URGENT'")
+ assertTrue(result[0][7] == "'5-LOW'")
+
+ result = sql """show column stats `statistics` (lo_shippriority)"""
+ assertTrue(result.size() == 1)
+ assertTrue(result[0][0] == "lo_shippriority")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "1.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "0")
+ assertTrue(result[0][7] == "0")
+
+ result = sql """show column stats `statistics` (lo_extendedprice)"""
+ assertTrue(result.size() == 1)
+ assertTrue(result[0][0] == "lo_extendedprice")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "100.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "104300")
+ assertTrue(result[0][7] == "9066094")
+
+ result = sql """show column stats `statistics` (lo_ordtotalprice)"""
+ assertTrue(result.size() == 1)
+ assertTrue(result[0][0] == "lo_ordtotalprice")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "26.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "3428256")
+ assertTrue(result[0][7] == "36771805")
+
+ result = sql """show column stats `statistics` (lo_discount)"""
+ assertTrue(result.size() == 1)
+ assertTrue(result[0][0] == "lo_discount")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "11.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "0")
+ assertTrue(result[0][7] == "10")
+
+ result = sql """show column stats `statistics` (lo_revenue)"""
+ assertTrue(result.size() == 1)
+ assertTrue(result[0][0] == "lo_revenue")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "100.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "101171")
+ assertTrue(result[0][7] == "8703450")
+
+ result = sql """show column stats `statistics` (lo_supplycost)"""
+ assertTrue(result.size() == 1)
+ assertTrue(result[0][0] == "lo_supplycost")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "100.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "58023")
+ assertTrue(result[0][7] == "121374")
+
+ result = sql """show column stats `statistics` (lo_tax)"""
+ assertTrue(result.size() == 1)
+ assertTrue(result[0][0] == "lo_tax")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "9.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "0")
+ assertTrue(result[0][7] == "8")
+
+ result = sql """show column stats `statistics` (lo_commitdate)"""
+ assertTrue(result.size() == 1)
+ assertTrue(result[0][0] == "lo_commitdate")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "95.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "19920515")
+ assertTrue(result[0][7] == "19981016")
+
+ result = sql """show column stats `statistics` (lo_shipmode)"""
+ assertTrue(result.size() == 1)
+ assertTrue(result[0][0] == "lo_shipmode")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "7.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "425.21")
+ assertTrue(result[0][5] == "4.21")
+ assertTrue(result[0][6] == "'AIR'")
+ assertTrue(result[0][7] == "'TRUCK'")
+
+ // sql """ALTER TABLE statistics MODIFY COLUMN lo_shipmode SET STATS
('row_count'='6001215')"""
+ // result = sql "show column stats `statistics` (lo_shipmode)"
+ // assertTrue(result.size() == 1)
+ // assertTrue(result[0][0] == "lo_shipmode")
+ // assertTrue(result[0][1] == "6001215.0")
sql """drop stats statistics"""
- qt_19 "show column stats statistics"
+ result = sql """show column stats statistics"""
+ assertTrue(result.size() == 0)
}
}
diff --git
a/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic_cache.groovy
b/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic_cache.groovy
index 2fb574a0da..d1399ef49b 100644
---
a/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic_cache.groovy
+++
b/regression-test/suites/external_table_emr_p2/hive/test_hive_statistic_cache.groovy
@@ -33,26 +33,179 @@ suite("test_hive_statistic_cache", "p2") {
sql """switch ${catalog_name};"""
logger.info("switched to catalog " + catalog_name)
sql """use statistics;"""
- qt_1 "select count(*) from stats"
+ sql """set query_timeout=300;"""
+ sql """analyze table `stats` with sync;"""
+ sql """select count(*) from stats"""
Thread.sleep(5000);
- qt_2 "show column cached stats `stats` (lo_orderkey)"
- qt_3 "show column cached stats `stats` (lo_linenumber)"
- qt_4 "show column cached stats `stats` (lo_custkey)"
- qt_5 "show column cached stats `stats` (lo_partkey)"
- qt_6 "show column cached stats `stats` (lo_suppkey)"
- qt_7 "show column cached stats `stats` (lo_orderdate)"
- qt_8 "show column cached stats `stats` (lo_orderpriority)"
- qt_9 "show column cached stats `stats` (lo_shippriority)"
- qt_10 "show column cached stats `stats` (lo_extendedprice)"
- qt_11 "show column cached stats `stats` (lo_ordtotalprice)"
- qt_12 "show column cached stats `stats` (lo_discount)"
- qt_13 "show column cached stats `stats` (lo_revenue)"
- qt_14 "show column cached stats `stats` (lo_supplycost)"
- qt_15 "show column cached stats `stats` (lo_tax)"
- qt_16 "show column cached stats `stats` (lo_commitdate)"
- qt_17 "show column cached stats `stats` (lo_shipmode)"
- qt_18 "show column cached stats `stats` (lo_orderkey)"
- qt_19 "show column cached stats `stats` (lo_quantity)"
+ def result = sql """show column cached stats `stats` (lo_orderkey)"""
+ assertTrue(result[0][0] == "lo_orderkey")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "26.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "1")
+ assertTrue(result[0][7] == "98")
+
+ result = sql """show column cached stats `stats` (lo_linenumber)"""
+ assertTrue(result[0][0] == "lo_linenumber")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "7.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "1")
+ assertTrue(result[0][7] == "7")
+
+ result = sql """show column cached stats `stats` (lo_custkey)"""
+ assertTrue(result[0][0] == "lo_custkey")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "26.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "67423")
+ assertTrue(result[0][7] == "2735521")
+
+ result = sql """show column cached stats `stats` (lo_partkey)"""
+ assertTrue(result[0][0] == "lo_partkey")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "100.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "2250")
+ assertTrue(result[0][7] == "989601")
+
+ result = sql """show column cached stats `stats` (lo_suppkey)"""
+ assertTrue(result[0][0] == "lo_suppkey")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "100.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "4167")
+ assertTrue(result[0][7] == "195845")
+
+ result = sql """show column cached stats `stats` (lo_orderdate)"""
+ assertTrue(result[0][0] == "lo_orderdate")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "26.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "19920221")
+ assertTrue(result[0][7] == "19980721")
+
+ result = sql """show column cached stats `stats` (lo_orderpriority)"""
+ assertTrue(result[0][0] == "lo_orderpriority")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "5.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "888.8000000000001")
+ assertTrue(result[0][5] == "8.8")
+ assertTrue(result[0][6] == "'1-URGENT'")
+ assertTrue(result[0][7] == "'5-LOW'")
+
+ result = sql """show column cached stats `stats` (lo_shippriority)"""
+ assertTrue(result[0][0] == "lo_shippriority")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "1.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "0")
+ assertTrue(result[0][7] == "0")
+
+ result = sql """show column cached stats `stats` (lo_extendedprice)"""
+ assertTrue(result[0][0] == "lo_extendedprice")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "100.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "104300")
+ assertTrue(result[0][7] == "9066094")
+
+ result = sql """show column cached stats `stats` (lo_ordtotalprice)"""
+ assertTrue(result[0][0] == "lo_ordtotalprice")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "26.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "3428256")
+ assertTrue(result[0][7] == "36771805")
+
+ result = sql """show column cached stats `stats` (lo_discount)"""
+ assertTrue(result[0][0] == "lo_discount")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "11.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "0")
+ assertTrue(result[0][7] == "10")
+
+ result = sql """show column cached stats `stats` (lo_revenue)"""
+ assertTrue(result[0][0] == "lo_revenue")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "100.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "101171")
+ assertTrue(result[0][7] == "8703450")
+
+ result = sql """show column cached stats `stats` (lo_supplycost)"""
+ assertTrue(result[0][0] == "lo_supplycost")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "100.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "58023")
+ assertTrue(result[0][7] == "121374")
+
+ result = sql """show column cached stats `stats` (lo_tax)"""
+ assertTrue(result[0][0] == "lo_tax")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "9.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "0")
+ assertTrue(result[0][7] == "8")
+
+ result = sql """show column cached stats `stats` (lo_commitdate)"""
+ assertTrue(result[0][0] == "lo_commitdate")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "95.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "19920515")
+ assertTrue(result[0][7] == "19981016")
+
+ result = sql """show column cached stats `stats` (lo_shipmode)"""
+ assertTrue(result[0][0] == "lo_shipmode")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "7.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "425.21")
+ assertTrue(result[0][5] == "4.21")
+ assertTrue(result[0][6] == "'AIR'")
+ assertTrue(result[0][7] == "'TRUCK'")
+
+ result = sql """show column cached stats `stats` (lo_quantity)"""
+ assertTrue(result[0][0] == "lo_quantity")
+ assertTrue(result[0][1] == "100.0")
+ assertTrue(result[0][2] == "46.0")
+ assertTrue(result[0][3] == "0.0")
+ assertTrue(result[0][4] == "404.0")
+ assertTrue(result[0][5] == "4.0")
+ assertTrue(result[0][6] == "1")
+ assertTrue(result[0][7] == "50")
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
