This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d0a8a30998023137b73a89119cde95a6a1e8a7f5
Author: wuwenchi <wuwenchi...@hotmail.com>
AuthorDate: Wed Feb 28 11:44:23 2024 +0800
[improvement](iceberg/paimon)add show table stats (#31473)
---
.../datasource/iceberg/IcebergExternalTable.java | 12 ----
.../java/org/apache/doris/qe/ShowExecutor.java | 4 +-
.../iceberg/test_iceberg_table_stats.groovy | 65 ++++++++++++++++++++++
.../paimon/test_paimon_table_stats.groovy | 59 ++++++++++++++++++++
4 files changed, 125 insertions(+), 15 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java
index eef679c6f13..5266d8745de 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java
@@ -19,12 +19,9 @@ package org.apache.doris.datasource.iceberg;
import org.apache.doris.catalog.Column;
import org.apache.doris.datasource.ExternalTable;
-import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper;
import org.apache.doris.statistics.AnalysisInfo;
import org.apache.doris.statistics.BaseAnalysisTask;
-import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.ExternalAnalysisTask;
-import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.doris.thrift.THiveTable;
import org.apache.doris.thrift.TIcebergTable;
import org.apache.doris.thrift.TTableDescriptor;
@@ -32,7 +29,6 @@ import org.apache.doris.thrift.TTableType;
import java.util.HashMap;
import java.util.List;
-import java.util.Optional;
public class IcebergExternalTable extends ExternalTable {
@@ -74,14 +70,6 @@ public class IcebergExternalTable extends ExternalTable {
}
}
- @Override
- public Optional<ColumnStatistic> getColumnStatistic(String colName) {
- makeSureInitialized();
- return HiveMetaStoreClientHelper.ugiDoAs(catalog.getConfiguration(),
- () -> StatisticsUtil.getIcebergColumnStats(colName,
- IcebergUtils.getIcebergTable(catalog, dbName, name)));
- }
-
@Override
public BaseAnalysisTask createAnalysisTask(AnalysisInfo info) {
makeSureInitialized();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 25b39ce5fa3..e9fcb5a0663 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -2549,11 +2549,9 @@ public class ShowExecutor {
TableIf tableIf = showTableStatsStmt.getTable();
TableStatsMeta tableStats =
Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(tableIf.getId());
/*
- HMSExternalTable table will fetch row count from HMS
- or estimate with file size and schema if it's not analyzed.
tableStats == null means it's not analyzed, in this case show the
estimated row count.
*/
- if (tableStats == null && tableIf instanceof HMSExternalTable) {
+ if (tableStats == null) {
resultSet =
showTableStatsStmt.constructResultSet(tableIf.getRowCount());
} else {
resultSet = showTableStatsStmt.constructResultSet(tableStats);
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_stats.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_stats.groovy
new file mode 100644
index 00000000000..b8eacf6d9e2
--- /dev/null
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_stats.groovy
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_iceberg_table_stats",
"p0,external,doris,external_docker,external_docker_doris") {
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ try {
+ String rest_port =
context.config.otherConfigs.get("iceberg_rest_uri_port")
+ String minio_port =
context.config.otherConfigs.get("iceberg_minio_port")
+ String externalEnvIp =
context.config.otherConfigs.get("externalEnvIp")
+ String catalog_name = "test_iceberg_rest_catalog"
+ String db_name = "format_v2"
+
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """CREATE CATALOG ${catalog_name} PROPERTIES (
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='rest',
+ 'uri' = 'http://${externalEnvIp}:${rest_port}',
+ "s3.access_key" = "admin",
+ "s3.secret_key" = "password",
+ "s3.endpoint" = "http://${externalEnvIp}:${minio_port}";,
+ "s3.region" = "us-east-1"
+ );"""
+
+ def assert_stats = { table_name, cnt ->
+ def retry = 0
+ def act = ""
+ while (retry < 10) {
+ def result = sql """ show table stats ${table_name} """
+ act = result[0][2]
+ if (act != "0") {
+ break;
+ }
+ Thread.sleep(2000)
+ retry++
+ }
+ assertEquals(act, cnt)
+ }
+
+ sql """ switch ${catalog_name} """
+ sql """ use format_v2 """
+ assert_stats("sample_cow_orc", "1000")
+ assert_stats("sample_cow_parquet", "1000")
+ assert_stats("sample_mor_orc", "1000")
+ assert_stats("sample_mor_parquet", "1000")
+
+ } finally {
+ }
+ }
+}
+
diff --git
a/regression-test/suites/external_table_p0/paimon/test_paimon_table_stats.groovy
b/regression-test/suites/external_table_p0/paimon/test_paimon_table_stats.groovy
new file mode 100644
index 00000000000..006969bba45
--- /dev/null
+++
b/regression-test/suites/external_table_p0/paimon/test_paimon_table_stats.groovy
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_paimon_table_stats",
"p0,external,doris,external_docker,external_docker_doris") {
+ String enabled = context.config.otherConfigs.get("enablePaimonTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ try {
+ String hdfs_port = context.config.otherConfigs.get("hdfs_port")
+ String catalog_name = "paimon1"
+ String externalEnvIp =
context.config.otherConfigs.get("externalEnvIp")
+
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """create catalog if not exists ${catalog_name} properties (
+ "type" = "paimon",
+ "paimon.catalog.type"="filesystem",
+ "warehouse" =
"hdfs://${externalEnvIp}:${hdfs_port}/user/doris/paimon1"
+ );"""
+
+ def assert_stats = { table_name, cnt ->
+ def retry = 0
+ def act = ""
+ while (retry < 10) {
+ def result = sql """ show table stats ${table_name} """
+ act = result[0][2]
+ if (act != "0") {
+ break;
+ }
+ Thread.sleep(2000)
+ retry++
+ }
+ assertEquals(act, cnt)
+ }
+
+ // select
+ sql """ switch ${catalog_name} """
+ sql """ use db1 """
+ assert_stats("all_table", "2")
+ assert_stats("auto_bucket", "2")
+ assert_stats("complex_all", "3")
+ assert_stats("complex_tab", "3")
+ } finally {
+ }
+ }
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
