This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push: new ca6e33ec0c [feature](table-value-functions)add catalogs table-value-function (#21790) ca6e33ec0c is described below commit ca6e33ec0c5ab151596215963e49d082744a5c31 Author: daidai <2017501...@qq.com> AuthorDate: Fri Jul 14 10:25:16 2023 +0800 [feature](table-value-functions)add catalogs table-value-function (#21790) mysql> select * from catalogs() order by CatalogId; --- be/src/vec/exec/scan/vmeta_scanner.cpp | 18 +++++ be/src/vec/exec/scan/vmeta_scanner.h | 3 +- .../sql-functions/table-functions/catalogs.md | 91 +++++++++++++++++++++ docs/sidebars.json | 3 +- .../sql-functions/table-functions/catalogs.md | 92 ++++++++++++++++++++++ .../tablefunction/CatalogsTableValuedFunction.java | 88 +++++++++++++++++++++ .../doris/tablefunction/MetadataGenerator.java | 37 +++++++++ .../tablefunction/MetadataTableValuedFunction.java | 2 + .../doris/tablefunction/TableValuedFunctionIf.java | 2 + gensrc/thrift/Types.thrift | 3 +- .../table_valued_function/test_catalogs_tvf.out | 10 +++ .../table_valued_function/test_catalogs_tvf.groovy | 71 +++++++++++++++++ 12 files changed, 417 insertions(+), 3 deletions(-) diff --git a/be/src/vec/exec/scan/vmeta_scanner.cpp b/be/src/vec/exec/scan/vmeta_scanner.cpp index 49cd0f1043..f0adefef90 100644 --- a/be/src/vec/exec/scan/vmeta_scanner.cpp +++ b/be/src/vec/exec/scan/vmeta_scanner.cpp @@ -216,6 +216,9 @@ Status VMetaScanner::_fetch_metadata(const TMetaScanRange& meta_scan_range) { case TMetadataType::WORKLOAD_GROUPS: RETURN_IF_ERROR(_build_workload_groups_metadata_request(meta_scan_range, &request)); break; + case TMetadataType::CATALOGS: + RETURN_IF_ERROR(_build_catalogs_metadata_request(meta_scan_range, &request)); + break; default: _meta_eos = true; return Status::OK(); @@ -323,6 +326,21 @@ Status VMetaScanner::_build_workload_groups_metadata_request( return Status::OK(); } +Status VMetaScanner::_build_catalogs_metadata_request(const TMetaScanRange& meta_scan_range, + TFetchSchemaTableDataRequest* request) { + VLOG_CRITICAL << "VMetaScanner::_build_catalogs_metadata_request"; + + // create request + request->__set_schema_table_name(TSchemaTableName::METADATA_TABLE); + + // create TMetadataTableRequestParams + TMetadataTableRequestParams metadata_table_params; + metadata_table_params.__set_metadata_type(TMetadataType::CATALOGS); + + request->__set_metada_table_params(metadata_table_params); + return Status::OK(); +} + Status VMetaScanner::close(RuntimeState* state) { VLOG_CRITICAL << "VMetaScanner::close"; RETURN_IF_ERROR(VScanner::close(state)); diff --git a/be/src/vec/exec/scan/vmeta_scanner.h b/be/src/vec/exec/scan/vmeta_scanner.h index 22f3dfe681..4c2dd00584 100644 --- a/be/src/vec/exec/scan/vmeta_scanner.h +++ b/be/src/vec/exec/scan/vmeta_scanner.h @@ -73,7 +73,8 @@ private: TFetchSchemaTableDataRequest* request); Status _build_workload_groups_metadata_request(const TMetaScanRange& meta_scan_range, TFetchSchemaTableDataRequest* request); - + Status _build_catalogs_metadata_request(const TMetaScanRange& meta_scan_range, + TFetchSchemaTableDataRequest* request); bool _meta_eos; TupleId _tuple_id; TUserIdentity _user_identity; diff --git a/docs/en/docs/sql-manual/sql-functions/table-functions/catalogs.md b/docs/en/docs/sql-manual/sql-functions/table-functions/catalogs.md new file mode 100644 index 0000000000..d0c821f7bb --- /dev/null +++ b/docs/en/docs/sql-manual/sql-functions/table-functions/catalogs.md @@ -0,0 +1,91 @@ +--- +{ + "title": "catalogs", + "language": "en" +} +--- + +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +## `catalogs` + +### Name + + +catalogs + + +### description + +The table function generates a temporary table of catalogs to view the information of the catalogs created in the current Doris. + +This function is used in the from clause. + +#### syntax + +`catalogs()` + +Catalogs () table structure: +``` +mysql> desc function catalogs(); ++-------------+--------+------+-------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++-------------+--------+------+-------+---------+-------+ +| CatalogId | BIGINT | No | false | NULL | NONE | +| CatalogName | TEXT | No | false | NULL | NONE | +| CatalogType | TEXT | No | false | NULL | NONE | +| Property | TEXT | No | false | NULL | NONE | +| Value | TEXT | No | false | NULL | NONE | ++-------------+--------+------+-------+---------+-------+ +5 rows in set (0.04 sec) +``` + +The information presented by `catalogs()` tvf is the result of synthesizing `show catalogs` and `show catalog xxx` statements. + +The table generated by tvf can be used for filtering, join and other operations. + + +### example + +``` +mysql> select * from catalogs(); ++-----------+-------------+-------------+--------------------------------------------+---------------------------------------------------------------------------+ +| CatalogId | CatalogName | CatalogType | Property | Value | ++-----------+-------------+-------------+--------------------------------------------+---------------------------------------------------------------------------+ +| 16725 | hive | hms | dfs.client.failover.proxy.provider.HANN | org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider | +| 16725 | hive | hms | dfs.ha.namenodes.HANN | nn1,nn2 | +| 16725 | hive | hms | create_time | 2023-07-13 16:24:38.968 | +| 16725 | hive | hms | ipc.client.fallback-to-simple-auth-allowed | true | +| 16725 | hive | hms | dfs.namenode.rpc-address.HANN.nn1 | nn1_host:rpc_port | +| 16725 | hive | hms | hive.metastore.uris | thrift://127.0.0.1:7004 | +| 16725 | hive | hms | dfs.namenode.rpc-address.HANN.nn2 | nn2_host:rpc_port | +| 16725 | hive | hms | type | hms | +| 16725 | hive | hms | dfs.nameservices | HANN | +| 0 | internal | internal | NULL | NULL | +| 16726 | es | es | create_time | 2023-07-13 16:24:44.922 | +| 16726 | es | es | type | es | +| 16726 | es | es | hosts | http://127.0.0.1:9200 | ++-----------+-------------+-------------+--------------------------------------------+---------------------------------------------------------------------------+ +13 rows in set (0.01 sec) +``` + +### keywords + + catalogs diff --git a/docs/sidebars.json b/docs/sidebars.json index 715681bd43..b1b2b94d18 100644 --- a/docs/sidebars.json +++ b/docs/sidebars.json @@ -711,7 +711,8 @@ "sql-manual/sql-functions/table-functions/iceberg_meta", "sql-manual/sql-functions/table-functions/backends", "sql-manual/sql-functions/table-functions/frontends", - "sql-manual/sql-functions/table-functions/workload-group" + "sql-manual/sql-functions/table-functions/workload-group", + "sql-manual/sql-functions/table-functions/catalogs" ] }, { diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/catalogs.md b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/catalogs.md new file mode 100644 index 0000000000..b95795e8e6 --- /dev/null +++ b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/catalogs.md @@ -0,0 +1,92 @@ +--- +{ + "title": "catalogs", + "language": "zh-CN" +} +--- + +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +## `catalogs` + +### Name + + +catalogs + + +### description + +表函数,生成 catalogs 临时表,可以查看当前doris中的创建的 catalogs 信息。 + +该函数用于 from 子句中。 + +#### syntax + +`catalogs()` + +catalogs()表结构: +``` +mysql> desc function catalogs(); ++-------------+--------+------+-------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++-------------+--------+------+-------+---------+-------+ +| CatalogId | BIGINT | No | false | NULL | NONE | +| CatalogName | TEXT | No | false | NULL | NONE | +| CatalogType | TEXT | No | false | NULL | NONE | +| Property | TEXT | No | false | NULL | NONE | +| Value | TEXT | No | false | NULL | NONE | ++-------------+--------+------+-------+---------+-------+ +5 rows in set (0.04 sec) +``` + +`catalogs()` tvf展示的信息是综合了 `show catalogs` 与 `show catalog xxx` 语句的结果。 + +可以利用tvf生成的表去做过滤、join等操作。 + + + +### example + +``` +mysql> select * from catalogs(); ++-----------+-------------+-------------+--------------------------------------------+---------------------------------------------------------------------------+ +| CatalogId | CatalogName | CatalogType | Property | Value | ++-----------+-------------+-------------+--------------------------------------------+---------------------------------------------------------------------------+ +| 16725 | hive | hms | dfs.client.failover.proxy.provider.HANN | org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider | +| 16725 | hive | hms | dfs.ha.namenodes.HANN | nn1,nn2 | +| 16725 | hive | hms | create_time | 2023-07-13 16:24:38.968 | +| 16725 | hive | hms | ipc.client.fallback-to-simple-auth-allowed | true | +| 16725 | hive | hms | dfs.namenode.rpc-address.HANN.nn1 | nn1_host:rpc_port | +| 16725 | hive | hms | hive.metastore.uris | thrift://127.0.0.1:7004 | +| 16725 | hive | hms | dfs.namenode.rpc-address.HANN.nn2 | nn2_host:rpc_port | +| 16725 | hive | hms | type | hms | +| 16725 | hive | hms | dfs.nameservices | HANN | +| 0 | internal | internal | NULL | NULL | +| 16726 | es | es | create_time | 2023-07-13 16:24:44.922 | +| 16726 | es | es | type | es | +| 16726 | es | es | hosts | http://127.0.0.1:9200 | ++-----------+-------------+-------------+--------------------------------------------+---------------------------------------------------------------------------+ +13 rows in set (0.01 sec) +``` + +### keywords + + catalogs diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/CatalogsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/CatalogsTableValuedFunction.java new file mode 100644 index 0000000000..062cb29b19 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/CatalogsTableValuedFunction.java @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + + + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.thrift.TMetaScanRange; +import org.apache.doris.thrift.TMetadataType; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.util.List; +import java.util.Map; + +public class CatalogsTableValuedFunction extends MetadataTableValuedFunction { + public static final String NAME = "catalogs"; + + private static final ImmutableList<Column> SCHEMA = ImmutableList.of( + new Column("CatalogId", ScalarType.createType(PrimitiveType.BIGINT)), + new Column("CatalogName", ScalarType.createStringType()), + new Column("CatalogType", ScalarType.createStringType()), + new Column("Property", ScalarType.createStringType()), + new Column("Value", ScalarType.createStringType())); + + public CatalogsTableValuedFunction(Map<String, String> params) + throws org.apache.doris.nereids.exceptions.AnalysisException { + if (params.size() != 0) { + throw new org.apache.doris.nereids.exceptions + .AnalysisException("catalogs table-valued-function does not support any params"); + } + } + + private static final ImmutableMap<String, Integer> COLUMN_TO_INDEX; + + static { + ImmutableMap.Builder<String, Integer> builder = new ImmutableMap.Builder(); + for (int i = 0; i < SCHEMA.size(); i++) { + builder.put(SCHEMA.get(i).getName().toLowerCase(), i); + } + COLUMN_TO_INDEX = builder.build(); + } + + public static Integer getColumnIndexFromColumnName(String columnName) { + return COLUMN_TO_INDEX.get(columnName.toLowerCase()); + } + + @Override + public String getTableName() { + return "CatalogsTableValuedFunction"; + } + + @Override + public List<Column> getTableColumns() throws AnalysisException { + return SCHEMA; + } + + @Override + public TMetadataType getMetadataType() { + return TMetadataType.CATALOGS; + } + + @Override + public TMetaScanRange getMetaScanRange() { + TMetaScanRange metaScanRange = new TMetaScanRange(); + metaScanRange.setMetadataType(TMetadataType.CATALOGS); + return metaScanRange; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index ff2d9ce55c..1dcc8418d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -22,6 +22,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.proc.FrontendsProcNode; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.HMSExternalCatalog; import org.apache.doris.datasource.property.constants.HMSProperties; import org.apache.doris.system.Backend; @@ -59,6 +60,8 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; + + public class MetadataGenerator { private static final Logger LOG = LogManager.getLogger(MetadataGenerator.class); @@ -81,6 +84,9 @@ public class MetadataGenerator { case WORKLOAD_GROUPS: result = workloadGroupsMetadataResult(params); break; + case CATALOGS: + result = catalogsMetadataResult(params); + break; default: return errorResult("Metadata table params is not set."); } @@ -257,6 +263,37 @@ public class MetadataGenerator { return result; } + private static TFetchSchemaTableDataResult catalogsMetadataResult(TMetadataTableRequestParams params) { + TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult(); + List<CatalogIf> info = Env.getCurrentEnv().getCatalogMgr().listCatalogs(); + List<TRow> dataBatch = Lists.newArrayList(); + + for (CatalogIf catalog : info) { + TRow trow = new TRow(); + trow.addToColumnValue(new TCell().setLongVal(catalog.getId())); + trow.addToColumnValue(new TCell().setStringVal(catalog.getName())); + trow.addToColumnValue(new TCell().setStringVal(catalog.getType())); + + Map<String, String> properties = catalog.getProperties(); + + for (Map.Entry<String, String> entry : properties.entrySet()) { + TRow subTrow = new TRow(trow); + subTrow.addToColumnValue(new TCell().setStringVal(entry.getKey())); + subTrow.addToColumnValue(new TCell().setStringVal(entry.getValue())); + dataBatch.add(subTrow); + } + if (properties.isEmpty()) { + trow.addToColumnValue(new TCell().setStringVal("NULL")); + trow.addToColumnValue(new TCell().setStringVal("NULL")); + dataBatch.add(trow); + } + } + + result.setDataBatch(dataBatch); + result.setStatus(new TStatus(TStatusCode.OK)); + return result; + } + private static TFetchSchemaTableDataResult workloadGroupsMetadataResult(TMetadataTableRequestParams params) { if (!params.isSetCurrentUserIdent()) { return errorResult("current user ident is not set."); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java index ff01524e76..c5740c739b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataTableValuedFunction.java @@ -37,6 +37,8 @@ public abstract class MetadataTableValuedFunction extends TableValuedFunctionIf return IcebergTableValuedFunction.getColumnIndexFromColumnName(columnName); case WORKLOAD_GROUPS: return WorkloadGroupsTableValuedFunction.getColumnIndexFromColumnName(columnName); + case CATALOGS: + return CatalogsTableValuedFunction.getColumnIndexFromColumnName(columnName); default: throw new AnalysisException("Unknown Metadata TableValuedFunction type"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java index 0cc0a61067..ea135b8b1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TableValuedFunctionIf.java @@ -57,6 +57,8 @@ public abstract class TableValuedFunctionIf { return new FrontendsTableValuedFunction(params); case WorkloadGroupsTableValuedFunction.NAME: return new WorkloadGroupsTableValuedFunction(params); + case CatalogsTableValuedFunction.NAME: + return new CatalogsTableValuedFunction(params); default: throw new AnalysisException("Could not find table function " + funcName); } diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index 9dec29db22..522b80c0d8 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -689,7 +689,8 @@ enum TMetadataType { ICEBERG, BACKENDS, WORKLOAD_GROUPS, - FRONTENDS + FRONTENDS, + CATALOGS } enum TIcebergQueryType { diff --git a/regression-test/data/correctness_p0/table_valued_function/test_catalogs_tvf.out b/regression-test/data/correctness_p0/table_valued_function/test_catalogs_tvf.out new file mode 100644 index 0000000000..310da69d76 --- /dev/null +++ b/regression-test/data/correctness_p0/table_valued_function/test_catalogs_tvf.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !create -- +catalog_test_es00 es type es +catalog_test_hive00 hms type hms + +-- !delete -- + +-- !create -- +catalog_test_es00 es type es + diff --git a/regression-test/suites/correctness_p0/table_valued_function/test_catalogs_tvf.groovy b/regression-test/suites/correctness_p0/table_valued_function/test_catalogs_tvf.groovy new file mode 100644 index 0000000000..e1a98b0037 --- /dev/null +++ b/regression-test/suites/correctness_p0/table_valued_function/test_catalogs_tvf.groovy @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_catalogs_tvf") { + List<List<Object>> table = sql """ select * from catalogs(); """ + assertTrue(table.size() > 0) + assertEquals(5, table[0].size) + + + table = sql """ select CatalogId,CatalogName from catalogs();""" + assertTrue(table.size() > 0) + assertTrue(table[0].size == 2) + + + table = sql """ select * from catalogs() where CatalogId=0;""" + assertTrue(table.size() > 0) + assertEquals("NULL", table[0][3]) + assertEquals("NULL", table[0][4]) + + + def res = sql """ select count(*) from catalogs(); """ + assertTrue(res[0][0] > 0) + + res = sql """ select * from catalogs() order by CatalogId; """ + assertTrue(res[0][0] == 0) + assertEquals(res[0][1],"internal") + assertEquals(res[0][2],"internal") + + sql """ drop catalog if exists catalog_test_hive00 """ + sql """ drop catalog if exists catalog_test_es00 """ + + sql """ CREATE CATALOG catalog_test_hive00 PROPERTIES ( + 'type'='hms', + 'hive.metastore.uris' = 'thrift://127.0.0.1:7004', + 'dfs.nameservices'='HANN', + 'dfs.ha.namenodes.HANN'='nn1,nn2', + 'dfs.namenode.rpc-address.HANN.nn1'='nn1_host:rpc_port', + 'dfs.namenode.rpc-address.HANN.nn2'='nn2_host:rpc_port', + 'dfs.client.failover.proxy.provider.HANN'='org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider' + ) """ + + sql """ CREATE CATALOG catalog_test_es00 PROPERTIES ( + "type"="es", + "hosts"="http://127.0.0.1:9200" + ) """ + + + qt_create """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName in ("catalog_test_es00","catalog_test_hive00") and Property="type" order by Value""" + + sql """ drop catalog catalog_test_hive00 """ + + qt_delete """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName="catalog_test_hive00" """ + + qt_create """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName in ("catalog_test_es00","catalog_test_hive00") and Property="type" order by Value""" + + sql """ drop catalog catalog_test_es00 """ +} --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org