This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new eafdab0cfd [Enhancement](tvf) Add frontends_disks
table-valued-function (#22568)
eafdab0cfd is described below
commit eafdab0cfd104060f8fa13b2faae1958be5c16df
Author: herry2038 <5303...@qq.com>
AuthorDate: Thu Aug 10 10:40:24 2023 +0800
[Enhancement](tvf) Add frontends_disks table-valued-function (#22568)
---------
Co-authored-by: yuxianbing <yuxianb...@yy.com>
Co-authored-by: yuxianbing <iloveqaz123>
---
be/src/vec/exec/scan/vmeta_scanner.cpp | 22 +++++
be/src/vec/exec/scan/vmeta_scanner.h | 2 +
.../table-functions/frontends_disks.md | 93 ++++++++++++++++++++
docs/sidebars.json | 3 +-
.../table-functions/frontends_disks.md | 92 ++++++++++++++++++++
.../FrontendsDisksTableValuedFunction.java | 98 ++++++++++++++++++++++
.../doris/tablefunction/MetadataGenerator.java | 26 ++++++
.../tablefunction/MetadataTableValuedFunction.java | 2 +
.../doris/tablefunction/TableValuedFunctionIf.java | 2 +
gensrc/thrift/Types.thrift | 3 +-
.../test_frontends_disks_tvf.groovy | 49 +++++++++++
11 files changed, 390 insertions(+), 2 deletions(-)
diff --git a/be/src/vec/exec/scan/vmeta_scanner.cpp
b/be/src/vec/exec/scan/vmeta_scanner.cpp
index f0adefef90..aea604e8f5 100644
--- a/be/src/vec/exec/scan/vmeta_scanner.cpp
+++ b/be/src/vec/exec/scan/vmeta_scanner.cpp
@@ -213,6 +213,9 @@ Status VMetaScanner::_fetch_metadata(const TMetaScanRange&
meta_scan_range) {
case TMetadataType::FRONTENDS:
RETURN_IF_ERROR(_build_frontends_metadata_request(meta_scan_range,
&request));
break;
+ case TMetadataType::FRONTENDS_DISKS:
+
RETURN_IF_ERROR(_build_frontends_disks_metadata_request(meta_scan_range,
&request));
+ break;
case TMetadataType::WORKLOAD_GROUPS:
RETURN_IF_ERROR(_build_workload_groups_metadata_request(meta_scan_range,
&request));
break;
@@ -309,6 +312,25 @@ Status
VMetaScanner::_build_frontends_metadata_request(const TMetaScanRange& met
return Status::OK();
}
+Status VMetaScanner::_build_frontends_disks_metadata_request(
+ const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest*
request) {
+ VLOG_CRITICAL << "VMetaScanner::_build_frontends_metadata_request";
+ if (!meta_scan_range.__isset.frontends_params) {
+ return Status::InternalError("Can not find TFrontendsMetadataParams
from meta_scan_range.");
+ }
+ // create request
+ request->__set_cluster_name("");
+ request->__set_schema_table_name(TSchemaTableName::METADATA_TABLE);
+
+ // create TMetadataTableRequestParams
+ TMetadataTableRequestParams metadata_table_params;
+ metadata_table_params.__set_metadata_type(TMetadataType::FRONTENDS_DISKS);
+
metadata_table_params.__set_frontends_metadata_params(meta_scan_range.frontends_params);
+
+ request->__set_metada_table_params(metadata_table_params);
+ return Status::OK();
+}
+
Status VMetaScanner::_build_workload_groups_metadata_request(
const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest*
request) {
VLOG_CRITICAL << "VMetaScanner::_build_workload_groups_metadata_request";
diff --git a/be/src/vec/exec/scan/vmeta_scanner.h
b/be/src/vec/exec/scan/vmeta_scanner.h
index 4c2dd00584..5eb2296d27 100644
--- a/be/src/vec/exec/scan/vmeta_scanner.h
+++ b/be/src/vec/exec/scan/vmeta_scanner.h
@@ -71,6 +71,8 @@ private:
TFetchSchemaTableDataRequest*
request);
Status _build_frontends_metadata_request(const TMetaScanRange&
meta_scan_range,
TFetchSchemaTableDataRequest*
request);
+ Status _build_frontends_disks_metadata_request(const TMetaScanRange&
meta_scan_range,
+
TFetchSchemaTableDataRequest* request);
Status _build_workload_groups_metadata_request(const TMetaScanRange&
meta_scan_range,
TFetchSchemaTableDataRequest* request);
Status _build_catalogs_metadata_request(const TMetaScanRange&
meta_scan_range,
diff --git
a/docs/en/docs/sql-manual/sql-functions/table-functions/frontends_disks.md
b/docs/en/docs/sql-manual/sql-functions/table-functions/frontends_disks.md
new file mode 100644
index 0000000000..c92a076f5f
--- /dev/null
+++ b/docs/en/docs/sql-manual/sql-functions/table-functions/frontends_disks.md
@@ -0,0 +1,93 @@
+---
+{
+ "title": "frontends_disks",
+ "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## `frontends`
+
+### Name
+
+<version since="dev">
+
+frontends
+
+</version>
+
+### description
+
+Table-Value-Function, generate a temporary table named `frontends_disks`. This
tvf is used to view the information of FE nodes 's disks in the doris cluster.
+
+This function is used in `FROM` clauses.
+
+#### syntax
+
+`frontends_disks()`
+
+The table schema of `frontends_disks()` tvf：
+```
+mysql> desc function frontends_disks();
++-------------+------+------+-------+---------+-------+
+| Field | Type | Null | Key | Default | Extra |
++-------------+------+------+-------+---------+-------+
+| Name | TEXT | No | false | NULL | NONE |
+| Host | TEXT | No | false | NULL | NONE |
+| EditLogPort | TEXT | No | false | NULL | NONE |
+| DirType | TEXT | No | false | NULL | NONE |
+| Dir | TEXT | No | false | NULL | NONE |
+| Filesystem | TEXT | No | false | NULL | NONE |
+| Capacity | TEXT | No | false | NULL | NONE |
+| Used | TEXT | No | false | NULL | NONE |
+| Available | TEXT | No | false | NULL | NONE |
+| UseRate | TEXT | No | false | NULL | NONE |
+| MountOn | TEXT | No | false | NULL | NONE |
++-------------+------+------+-------+---------+-------+
+11 rows in set (0.14 sec)
+```
+
+The information displayed by the `frontends_disks` tvf is basically consistent
with the information displayed by the `show frontends disks` statement.
However, the types of each field in the `frontends_disks` tvf are more
specific, and you can use the `frontends_disks` tvf to perform operations such
as filtering and joining.
+
+The information displayed by the `frontends_disks` tvf is authenticated, which
is consistent with the behavior of `show frontends disks`, user must have
ADMIN/OPERATOR privelege.
+
+### example
+```
+mysql> select * from frontends_disk()\G
+*************************** 1. row ***************************
+ Name: fe_fe1d5bd9_d1e5_4ccc_9b03_ca79b95c9941
+ Host: 172.XX.XX.1
+EditLogPort: 9010
+ DirType: log
+ Dir: /data/doris/fe-github/log
+ Filesystem: /dev/sdc5
+ Capacity: 366G
+ Used: 119G
+ Available: 228G
+ UseRate: 35%
+ MountOn: /data
+......
+12 row in set (0.03 sec)
+```
+
+### keywords
+
+ frontends_disks
\ No newline at end of file
diff --git a/docs/sidebars.json b/docs/sidebars.json
index c43e6d1e86..160d665490 100644
--- a/docs/sidebars.json
+++ b/docs/sidebars.json
@@ -726,7 +726,8 @@
"sql-manual/sql-functions/table-functions/backends",
"sql-manual/sql-functions/table-functions/frontends",
"sql-manual/sql-functions/table-functions/workload-group",
-
"sql-manual/sql-functions/table-functions/catalogs"
+
"sql-manual/sql-functions/table-functions/catalogs",
+
"sql-manual/sql-functions/table-functions/frontends_disks"
]
},
{
diff --git
a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/frontends_disks.md
b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/frontends_disks.md
new file mode 100644
index 0000000000..fcba673a0a
--- /dev/null
+++
b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/frontends_disks.md
@@ -0,0 +1,92 @@
+---
+{
+ "title": "frontends_disks",
+ "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## `frontends_disks`
+
+### Name
+
+<version since="dev">
+
+frontends_disks
+
+</version>
+
+### description
+
+表函数，生成frontends_disks临时表，可以查看当前doris集群中的 FE 节点的磁盘信息。
+
+该函数用于from子句中。
+
+#### syntax
+`frontends_disks()`
+
+frontends_disks()表结构：
+```
+mysql> desc function frontends_disks();
++-------------+------+------+-------+---------+-------+
+| Field | Type | Null | Key | Default | Extra |
++-------------+------+------+-------+---------+-------+
+| Name | TEXT | No | false | NULL | NONE |
+| Host | TEXT | No | false | NULL | NONE |
+| EditLogPort | TEXT | No | false | NULL | NONE |
+| DirType | TEXT | No | false | NULL | NONE |
+| Dir | TEXT | No | false | NULL | NONE |
+| Filesystem | TEXT | No | false | NULL | NONE |
+| Capacity | TEXT | No | false | NULL | NONE |
+| Used | TEXT | No | false | NULL | NONE |
+| Available | TEXT | No | false | NULL | NONE |
+| UseRate | TEXT | No | false | NULL | NONE |
+| MountOn | TEXT | No | false | NULL | NONE |
++-------------+------+------+-------+---------+-------+
+11 rows in set (0.14 sec)
+```
+
+`frontends_disks()` tvf展示出来的信息基本与 `show frontends disks` 语句展示出的信息一致,但是
`frontends_disks()` tvf的各个字段类型更加明确，且可以利用tvf生成的表去做过滤、join等操作。
+
+对 `frontends_disks()` tvf信息展示进行了鉴权，与 `show frontends disks` 行为保持一致，要求用户具有
ADMIN/OPERATOR 权限。
+
+### example
+```
+mysql> select * from frontends_disk()\G
+*************************** 1. row ***************************
+ Name: fe_fe1d5bd9_d1e5_4ccc_9b03_ca79b95c9941
+ Host: 172.XX.XX.1
+EditLogPort: 9010
+ DirType: log
+ Dir: /data/doris/fe-github/log
+ Filesystem: /dev/sdc5
+ Capacity: 366G
+ Used: 119G
+ Available: 228G
+ UseRate: 35%
+ MountOn: /data
+......
+12 row in set (0.03 sec)
+```
+
+### keywords
+
+ frontends_disks
\ No newline at end of file
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsDisksTableValuedFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsDisksTableValuedFunction.java
new file mode 100644
index 0000000000..f7f9f9e7a7
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsDisksTableValuedFunction.java
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.tablefunction;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.thrift.TFrontendsMetadataParams;
+import org.apache.doris.thrift.TMetaScanRange;
+import org.apache.doris.thrift.TMetadataType;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The Implement of table valued function
+ * frontends_disks().
+ */
+public class FrontendsDisksTableValuedFunction extends
MetadataTableValuedFunction {
+ public static final String NAME = "frontends_disks";
+
+ private static final ImmutableList<Column> SCHEMA = ImmutableList.of(
+ new Column("Name", ScalarType.createStringType()),
+ new Column("Host", ScalarType.createStringType()),
+ new Column("EditLogPort", ScalarType.createStringType()),
+ new Column("DirType", ScalarType.createStringType()),
+ new Column("Dir", ScalarType.createStringType()),
+ new Column("Filesystem", ScalarType.createStringType()),
+ new Column("Capacity", ScalarType.createStringType()),
+ new Column("Used", ScalarType.createStringType()),
+ new Column("Available", ScalarType.createStringType()),
+ new Column("UseRate", ScalarType.createStringType()),
+ new Column("MountOn", ScalarType.createStringType()));
+
+ private static final ImmutableMap<String, Integer> COLUMN_TO_INDEX;
+
+ static {
+ ImmutableMap.Builder<String, Integer> builder = new
ImmutableMap.Builder();
+ for (int i = 0; i < SCHEMA.size(); i++) {
+ builder.put(SCHEMA.get(i).getName().toLowerCase(), i);
+ }
+ COLUMN_TO_INDEX = builder.build();
+ }
+
+ public static Integer getColumnIndexFromColumnName(String columnName) {
+ return COLUMN_TO_INDEX.get(columnName.toLowerCase());
+ }
+
+ public FrontendsDisksTableValuedFunction(Map<String, String> params)
throws AnalysisException {
+ if (params.size() != 0) {
+ throw new AnalysisException("frontends_disks table-valued-function
does not support any params");
+ }
+ }
+
+ @Override
+ public TMetadataType getMetadataType() {
+ return TMetadataType.FRONTENDS_DISKS;
+ }
+
+ @Override
+ public TMetaScanRange getMetaScanRange() {
+ TMetaScanRange metaScanRange = new TMetaScanRange();
+ metaScanRange.setMetadataType(TMetadataType.FRONTENDS_DISKS);
+ TFrontendsMetadataParams frontendsMetadataParams = new
TFrontendsMetadataParams();
+ frontendsMetadataParams.setClusterName("");
+ metaScanRange.setFrontendsParams(frontendsMetadataParams);
+ return metaScanRange;
+ }
+
+ @Override
+ public String getTableName() {
+ return "FrontendsDisksTableValuedFunction";
+ }
+
+ @Override
+ public List<Column> getTableColumns() throws AnalysisException {
+ return SCHEMA;
+ }
+}
+
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
index 00bcf168cd..dfab4bb8fb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
@@ -83,6 +83,9 @@ public class MetadataGenerator {
case FRONTENDS:
result = frontendsMetadataResult(params);
break;
+ case FRONTENDS_DISKS:
+ result = frontendsDisksMetadataResult(params);
+ break;
case WORKLOAD_GROUPS:
result = workloadGroupsMetadataResult(params);
break;
@@ -266,6 +269,29 @@ public class MetadataGenerator {
return result;
}
+ private static TFetchSchemaTableDataResult
frontendsDisksMetadataResult(TMetadataTableRequestParams params) {
+ if (!params.isSetFrontendsMetadataParams()) {
+ return errorResult("frontends metadata param is not set.");
+ }
+
+ TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult();
+
+ List<TRow> dataBatch = Lists.newArrayList();
+ List<List<String>> infos = Lists.newArrayList();
+ FrontendsProcNode.getFrontendsDiskInfo(Env.getCurrentEnv(), infos);
+ for (List<String> info : infos) {
+ TRow trow = new TRow();
+ for (String item : info) {
+ trow.addToColumnValue(new TCell().setStringVal(item));
+ }
+ dataBatch.add(trow);
+ }
+
+ result.setDataBatch(dataBatch);
+ result.setStatus(new TStatus(TStatusCode.OK));
+ return result;
+ }
+
private static TFetchSchemaTableDataResult
catalogsMetadataResult(TMetadataTableRequestParams params) {
TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult();
List<CatalogIf> info =
Env.getCurrentEnv().getCatalogMgr().listCatalogs();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java
index c5740c739b..c09687c9b4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java
@@ -33,6 +33,8 @@ public abstract class MetadataTableValuedFunction extends
TableValuedFunctionIf
return
BackendsTableValuedFunction.getColumnIndexFromColumnName(columnName);
case FRONTENDS:
return
FrontendsTableValuedFunction.getColumnIndexFromColumnName(columnName);
+ case FRONTENDS_DISKS:
+ return
FrontendsDisksTableValuedFunction.getColumnIndexFromColumnName(columnName);
case ICEBERG:
return
IcebergTableValuedFunction.getColumnIndexFromColumnName(columnName);
case WORKLOAD_GROUPS:
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
index 6deb85cf51..5d0b3b00bd 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java
@@ -57,6 +57,8 @@ public abstract class TableValuedFunctionIf {
return new BackendsTableValuedFunction(params);
case FrontendsTableValuedFunction.NAME:
return new FrontendsTableValuedFunction(params);
+ case FrontendsDisksTableValuedFunction.NAME:
+ return new FrontendsDisksTableValuedFunction(params);
case WorkloadGroupsTableValuedFunction.NAME:
return new WorkloadGroupsTableValuedFunction(params);
case CatalogsTableValuedFunction.NAME:
diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift
index 522b80c0d8..baca98b228 100644
--- a/gensrc/thrift/Types.thrift
+++ b/gensrc/thrift/Types.thrift
@@ -690,7 +690,8 @@ enum TMetadataType {
BACKENDS,
WORKLOAD_GROUPS,
FRONTENDS,
- CATALOGS
+ CATALOGS,
+ FRONTENDS_DISKS,
}
enum TIcebergQueryType {
diff --git
a/regression-test/suites/correctness_p0/table_valued_function/test_frontends_disks_tvf.groovy
b/regression-test/suites/correctness_p0/table_valued_function/test_frontends_disks_tvf.groovy
new file mode 100644
index 0000000000..93bd2490e8
--- /dev/null
+++
b/regression-test/suites/correctness_p0/table_valued_function/test_frontends_disks_tvf.groovy
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// This suit test the `frontends_disks` tvf
+suite("test_frontends_disks_tvf") {
+ List<List<Object>> table = sql """ select * from `frontends_disks`(); """
+ assertTrue(table.size() > 0)
+ assertTrue(table[0].size == 11)
+
+ // filter columns
+ table = sql """ select Name from `frontends_disks`();"""
+ assertTrue(table.size() > 0)
+ assertTrue(table[0].size == 1)
+
+ // case insensitive
+ table = sql """ select name, host, editlogport, dirtype, dir from
frontends_disks() order by dirtype;"""
+ assertTrue(table.size() > 0)
+ assertTrue(table[0].size == 5)
+ assertEquals("audit-log", table[0][3])
+
+ // test aliase columns
+ table = sql """ select name as n, host as h, dirtype as a, editlogport as
e from frontends_disks() order by dirtype; """
+ assertTrue(table.size() > 0)
+ assertTrue(table[0].size == 4)
+ assertEquals("audit-log", table[0][2])
+
+ // test changing position of columns
+ def res = sql """ select count(*) from frontends_disks() where dirtype =
'audit-log'; """
+ assertTrue(res[0][0] > 0)
+
+ sql """ select Name, Host, EditLogPort,
+ DirType, Dir, Filesystem, Capacity, Used
+ Available, UseRate, MountOn from frontends_disks();
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
