This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push: new 078cb3b4db [feature-wip](multi-catalog) end to end to support multi-catalog (#10521) 078cb3b4db is described below commit 078cb3b4dbc69736521749e779c4947d83591bd7 Author: Ashin Gau <ashin...@users.noreply.github.com> AuthorDate: Sat Jul 2 20:43:10 2022 +0800 [feature-wip](multi-catalog) end to end to support multi-catalog (#10521) Get through the previous pull requests that support multi-catalog, and end to end to achieve multi-catalog. --- .../doris/analysis/AdminCancelRepairTableStmt.java | 16 +----- .../doris/analysis/AdminCompactTableStmt.java | 16 +----- .../doris/analysis/AdminRepairTableStmt.java | 16 +----- .../analysis/AdminShowReplicaDistributionStmt.java | 16 +----- .../doris/analysis/AdminShowReplicaStatusStmt.java | 16 +----- .../java/org/apache/doris/analysis/Analyzer.java | 19 ++----- .../org/apache/doris/analysis/DescribeStmt.java | 64 +++++++++++++++++----- .../java/org/apache/doris/analysis/FromClause.java | 32 ----------- .../apache/doris/analysis/FunctionCallExpr.java | 3 +- .../org/apache/doris/analysis/SlotDescriptor.java | 4 +- .../java/org/apache/doris/analysis/SlotRef.java | 8 +-- .../java/org/apache/doris/catalog/Catalog.java | 10 ++-- .../org/apache/doris/datasource/DataSourceMgr.java | 14 +++++ .../doris/datasource/ExternalDataSource.java | 8 +++ .../doris/datasource/HMSExternalDataSource.java | 5 +- .../apache/doris/httpv2/rest/MetaInfoAction.java | 13 ++++- .../org/apache/doris/planner/HashJoinNode.java | 10 ++-- .../main/java/org/apache/doris/policy/Policy.java | 9 ++- .../java/org/apache/doris/qe/ConnectContext.java | 11 +++- .../java/org/apache/doris/qe/ConnectProcessor.java | 2 +- .../java/org/apache/doris/qe/ShowExecutor.java | 37 +++++++------ .../rewrite/mvrewrite/CountDistinctToBitmap.java | 4 +- .../doris/rewrite/mvrewrite/CountFieldToSum.java | 4 +- .../rewrite/mvrewrite/HLLHashToSlotRefRule.java | 4 +- .../apache/doris/rewrite/mvrewrite/NDVToHll.java | 4 +- .../rewrite/mvrewrite/ToBitmapToSlotRefRule.java | 4 +- .../apache/doris/service/FrontendServiceImpl.java | 4 +- .../org/apache/doris/analysis/AccessTestUtil.java | 44 +++++++++++++++ .../apache/doris/analysis/VirtualSlotRefTest.java | 4 +- .../org/apache/doris/http/DorisHttpTestCase.java | 23 ++++++++ .../org/apache/doris/qe/ConnectProcessorTest.java | 7 +++ .../org/apache/doris/qe/PartitionCacheTest.java | 39 ++++++++++--- .../java/org/apache/doris/qe/ShowExecutorTest.java | 23 ++++++++ 33 files changed, 303 insertions(+), 190 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminCancelRepairTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminCancelRepairTableStmt.java index 1c6c442615..d1643b72cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminCancelRepairTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminCancelRepairTableStmt.java @@ -18,15 +18,14 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.Catalog; -import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.Util; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; -import com.google.common.base.Strings; import com.google.common.collect.Lists; import java.util.List; @@ -48,17 +47,8 @@ public class AdminCancelRepairTableStmt extends DdlStmt { ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); } - String dbName = null; - if (Strings.isNullOrEmpty(tblRef.getName().getDb())) { - dbName = analyzer.getDefaultDb(); - if (Strings.isNullOrEmpty(dbName)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR); - } - } else { - dbName = ClusterNamespace.getFullName(getClusterName(), tblRef.getName().getDb()); - } - - tblRef.getName().setDb(dbName); + tblRef.getName().analyze(analyzer); + Util.prohibitExternalCatalog(tblRef.getName().getCtl(), this.getClass().getSimpleName()); PartitionNames partitionNames = tblRef.getPartitionNames(); if (partitionNames != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminCompactTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminCompactTableStmt.java index d65ad0acff..143aa4daaf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminCompactTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminCompactTableStmt.java @@ -18,15 +18,14 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.Catalog; -import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.Util; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; -import com.google.common.base.Strings; import com.google.common.collect.Lists; import java.util.List; @@ -59,17 +58,8 @@ public class AdminCompactTableStmt extends DdlStmt { ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); } - String dbName = null; - if (Strings.isNullOrEmpty(tblRef.getName().getDb())) { - dbName = analyzer.getDefaultDb(); - if (Strings.isNullOrEmpty(dbName)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR); - } - } else { - dbName = ClusterNamespace.getFullName(getClusterName(), tblRef.getName().getDb()); - } - - tblRef.getName().setDb(dbName); + tblRef.getName().analyze(analyzer); + Util.prohibitExternalCatalog(tblRef.getName().getCtl(), this.getClass().getSimpleName()); PartitionNames partitionNames = tblRef.getPartitionNames(); if (partitionNames != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminRepairTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminRepairTableStmt.java index 6cb1f2deb6..c8d9ad18df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminRepairTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminRepairTableStmt.java @@ -18,15 +18,14 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.Catalog; -import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.Util; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; -import com.google.common.base.Strings; import com.google.common.collect.Lists; import java.util.List; @@ -51,17 +50,8 @@ public class AdminRepairTableStmt extends DdlStmt { ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); } - String dbName = null; - if (Strings.isNullOrEmpty(tblRef.getName().getDb())) { - dbName = analyzer.getDefaultDb(); - if (Strings.isNullOrEmpty(dbName)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR); - } - } else { - dbName = ClusterNamespace.getFullName(getClusterName(), tblRef.getName().getDb()); - } - - tblRef.getName().setDb(dbName); + tblRef.getName().analyze(analyzer); + Util.prohibitExternalCatalog(tblRef.getName().getCtl(), this.getClass().getSimpleName()); PartitionNames partitionNames = tblRef.getPartitionNames(); if (partitionNames != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminShowReplicaDistributionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminShowReplicaDistributionStmt.java index 759bf6b588..6c444d554f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminShowReplicaDistributionStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminShowReplicaDistributionStmt.java @@ -20,16 +20,15 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.ScalarType; -import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.Util; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSetMetaData; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; // admin show replica distribution from tbl [partition(p1, p2, ...)] @@ -55,17 +54,8 @@ public class AdminShowReplicaDistributionStmt extends ShowStmt { ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); } - String dbName = null; - if (Strings.isNullOrEmpty(tblRef.getName().getDb())) { - dbName = analyzer.getDefaultDb(); - if (Strings.isNullOrEmpty(dbName)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR); - } - } else { - dbName = ClusterNamespace.getFullName(getClusterName(), tblRef.getName().getDb()); - } - - tblRef.getName().setDb(dbName); + tblRef.getName().analyze(analyzer); + Util.prohibitExternalCatalog(tblRef.getName().getCtl(), this.getClass().getSimpleName()); } public String getDbName() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminShowReplicaStatusStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminShowReplicaStatusStmt.java index b095a25a6b..634f7e1864 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminShowReplicaStatusStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminShowReplicaStatusStmt.java @@ -22,16 +22,15 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Replica.ReplicaStatus; import org.apache.doris.catalog.ScalarType; -import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.Util; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSetMetaData; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -65,17 +64,8 @@ public class AdminShowReplicaStatusStmt extends ShowStmt { ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); } - String dbName = null; - if (Strings.isNullOrEmpty(tblRef.getName().getDb())) { - dbName = analyzer.getDefaultDb(); - if (Strings.isNullOrEmpty(dbName)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR); - } - } else { - dbName = ClusterNamespace.getFullName(getClusterName(), tblRef.getName().getDb()); - } - - tblRef.getName().setDb(dbName); + tblRef.getName().analyze(analyzer); + Util.prohibitExternalCatalog(tblRef.getName().getCtl(), this.getClass().getSimpleName()); PartitionNames partitionNames = tblRef.getPartitionNames(); if (partitionNames != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java index 957b396a64..ac0ed79bcb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java @@ -31,7 +31,6 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.catalog.View; -import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; @@ -649,17 +648,10 @@ public class Analyzer { // Resolve the table ref's path and determine what resolved table ref // to replace it with. - String dbName = tableName.getDb(); - if (Strings.isNullOrEmpty(dbName)) { - dbName = getDefaultDb(); - } else { - dbName = ClusterNamespace.getFullName(getClusterName(), tableName.getDb()); - } - if (Strings.isNullOrEmpty(dbName)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR); - } + tableName.analyze(this); - DatabaseIf database = globalState.catalog.getCurrentDataSource().getDbOrAnalysisException(dbName); + DatabaseIf database = globalState.catalog.getDataSourceMgr().getCatalogOrAnalysisException(tableName.getCtl()) + .getDbOrAnalysisException(tableName.getDb()); TableIf table = database.getTableOrAnalysisException(tableName.getTbl()); if (table.getType() == TableType.OLAP && (((OlapTable) table).getState() == OlapTableState.RESTORE @@ -688,7 +680,7 @@ public class Analyzer { // which may not be the same as the user's reference to the table name, causing the table name not to be found // in registerColumnRef(). So here the tblName is constructed using tableName.getTbl() // instead of table.getName(). - TableName tblName = new TableName(dbName, tableName.getTbl()); + TableName tblName = new TableName(tableName.getDb(), tableName.getTbl()); if (table instanceof View) { return new InlineViewRef((View) table, tableRef); } else { @@ -698,7 +690,8 @@ public class Analyzer { } public TableIf getTableOrAnalysisException(TableName tblName) throws AnalysisException { - DatabaseIf db = globalState.catalog.getCurrentDataSource().getDbOrAnalysisException(tblName.getDb()); + DatabaseIf db = globalState.catalog.getDataSourceMgr().getCatalogOrAnalysisException(tblName.getCtl()) + .getDbOrAnalysisException(tblName.getDb()); return db.getTableOrAnalysisException(tblName.getTbl()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java index 911fa5fb3b..48b4e4d664 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java @@ -19,13 +19,13 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.catalog.MysqlTable; import org.apache.doris.catalog.OdbcTable; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.ScalarType; -import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; @@ -36,11 +36,13 @@ import org.apache.doris.common.proc.ProcNodeInterface; import org.apache.doris.common.proc.ProcResult; import org.apache.doris.common.proc.ProcService; import org.apache.doris.common.proc.TableProcDir; +import org.apache.doris.common.util.Util; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSetMetaData; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import org.apache.commons.lang.StringUtils; @@ -48,8 +50,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; public class DescribeStmt extends ShowStmt { private static final ShowResultSetMetaData DESC_OLAP_TABLE_ALL_META_DATA = @@ -75,6 +79,17 @@ public class DescribeStmt extends ShowStmt { .addColumn(new Column("Table", ScalarType.createVarchar(30))) .build(); + // The same columns in IndexSchemaProcNode.TITLE_NAMES + private static final ShowResultSetMetaData HMS_EXTERNAL_TABLE_META_DATA = + ShowResultSetMetaData.builder() + .addColumn(new Column("Field", ScalarType.createVarchar(20))) + .addColumn(new Column("Type", ScalarType.createVarchar(20))) + .addColumn(new Column("Null", ScalarType.createVarchar(20))) + .addColumn(new Column("Key", ScalarType.createVarchar(20))) + .addColumn(new Column("Default", ScalarType.createVarchar(20))) + .addColumn(new Column("Extra", ScalarType.createVarchar(20))) + .build(); + // empty col num equals to DESC_OLAP_TABLE_ALL_META_DATA.size() private static final List<String> EMPTY_ROW = initEmptyRow(); @@ -86,6 +101,8 @@ public class DescribeStmt extends ShowStmt { private boolean isAllTables; private boolean isOlapTable; + private List<List<String>> hmsSchema = null; + public DescribeStmt(TableName dbTableName, boolean isAllTables) { this.dbTableName = dbTableName; this.totalRows = new LinkedList<List<String>>(); @@ -108,26 +125,39 @@ public class DescribeStmt extends ShowStmt { dbTableName.toString()); } - Database db = Catalog.getCurrentInternalCatalog().getDbOrAnalysisException(dbTableName.getDb()); - Table table = db.getTableOrAnalysisException(dbTableName.getTbl()); + DatabaseIf db = Catalog.getCurrentCatalog().getDataSourceMgr() + .getCatalogOrAnalysisException(dbTableName.getCtl()).getDbOrAnalysisException(dbTableName.getDb()); + TableIf table = db.getTableOrAnalysisException(dbTableName.getTbl()); table.readLock(); try { if (!isAllTables) { - // show base table schema only - String procString = "/dbs/" + db.getId() + "/" + table.getId() + "/" + TableProcDir.INDEX_SCHEMA - + "/"; - if (table.getType() == TableType.OLAP) { - procString += ((OlapTable) table).getBaseIndexId(); + if (table.getType() == TableType.HMS_EXTERNAL_TABLE) { + hmsSchema = table.getFullSchema().stream().map(col -> Arrays.asList( + col.getName(), + col.getType().toSql().toUpperCase(Locale.ROOT), + Boolean.toString(col.isAllowNull()), + Boolean.toString(col.isKey()), + Strings.nullToEmpty(col.getDefaultValue()), + "" /* no extra field */)) + .collect(Collectors.toList()); } else { - procString += table.getId(); - } + // show base table schema only + String procString = "/dbs/" + db.getId() + "/" + table.getId() + "/" + TableProcDir.INDEX_SCHEMA + + "/"; + if (table.getType() == TableType.OLAP) { + procString += ((OlapTable) table).getBaseIndexId(); + } else { + procString += table.getId(); + } - node = ProcService.getInstance().open(procString); - if (node == null) { - throw new AnalysisException("Describe table[" + dbTableName.getTbl() + "] failed"); + node = ProcService.getInstance().open(procString); + if (node == null) { + throw new AnalysisException("Describe table[" + dbTableName.getTbl() + "] failed"); + } } } else { + Util.prohibitExternalCatalog(dbTableName.getCtl(), this.getClass().getSimpleName() + " ALL"); if (table.getType() == TableType.OLAP) { isOlapTable = true; OlapTable olapTable = (OlapTable) table; @@ -231,6 +261,9 @@ public class DescribeStmt extends ShowStmt { if (isAllTables) { return totalRows; } else { + if (hmsSchema != null) { + return hmsSchema; + } Preconditions.checkNotNull(node); return node.fetchResult().getRows(); } @@ -239,6 +272,9 @@ public class DescribeStmt extends ShowStmt { @Override public ShowResultSetMetaData getMetaData() { if (!isAllTables) { + if (hmsSchema != null) { + return HMS_EXTERNAL_TABLE_META_DATA; + } ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); ProcResult result = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java index dfd40a1718..7dd7b9698d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java @@ -20,16 +20,10 @@ package org.apache.doris.analysis; - -import org.apache.doris.catalog.Database; -import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.ErrorCode; -import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.collect.Lists; import java.util.ArrayList; @@ -70,29 +64,6 @@ public class FromClause implements ParseNode, Iterable<TableRef> { this.needToSql = needToSql; } - private void checkFromHiveTable(Analyzer analyzer) throws AnalysisException { - for (TableRef tblRef : tablerefs) { - if (!(tblRef instanceof BaseTableRef)) { - continue; - } - - TableName tableName = tblRef.getName(); - String dbName = tableName.getDb(); - if (Strings.isNullOrEmpty(dbName)) { - dbName = analyzer.getDefaultDb(); - } else { - dbName = ClusterNamespace.getFullName(analyzer.getClusterName(), tblRef.getName().getDb()); - } - if (Strings.isNullOrEmpty(dbName)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_NO_DB_ERROR); - } - - Database db = analyzer.getCatalog().getInternalDataSource().getDbOrAnalysisException(dbName); - String tblName = tableName.getTbl(); - db.getTableOrAnalysisException(tblName); - } - } - /** * In some cases, the reorder method of select stmt will incorrectly sort the tableRef with on clause. * The meaning of this function is to reset those tableRefs with on clauses. @@ -155,9 +126,6 @@ public class FromClause implements ParseNode, Iterable<TableRef> { leftTblRef = tblRef; } - // TODO: remove when query from hive table is supported - checkFromHiveTable(analyzer); - analyzed = true; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 6b4ee81dc7..5f7af9643f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -936,7 +936,8 @@ public class FunctionCallExpr extends Expr { .checkDbPriv(ConnectContext.get(), dbName, PrivPredicate.SELECT)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "SELECT"); } - DatabaseIf db = Catalog.getCurrentCatalog().getCurrentDataSource().getDbNullable(dbName); + // TODO(gaoxin): ExternalDatabase not implement udf yet. + DatabaseIf db = Catalog.getCurrentCatalog().getInternalDataSource().getDbNullable(dbName); if (db != null && (db instanceof Database)) { Function searchDesc = new Function(fnName, Arrays.asList(collectChildReturnTypes()), Type.INVALID, false); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java index 1eb41ae6cc..4f041dfe01 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java @@ -23,7 +23,6 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.ColumnStats; import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Type; import org.apache.doris.thrift.TSlotDescriptor; @@ -329,7 +328,6 @@ public class SlotDescriptor { } public boolean isScanSlot() { - Table table = (Table) parent.getTable(); - return table instanceof OlapTable; + return parent.getTable() instanceof OlapTable; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java index 706d82e79c..e8c1c82b12 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java @@ -21,7 +21,7 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.io.Text; @@ -373,7 +373,7 @@ public class SlotRef extends Expr { expr.getTableIdToColumnNames(tableIdToColumnNames); } } else { - Table table = (Table) desc.getParent().getTable(); + TableIf table = desc.getParent().getTable(); if (table == null) { // Maybe this column comes from inline view. return; @@ -388,9 +388,9 @@ public class SlotRef extends Expr { } } - public Table getTable() { + public TableIf getTable() { Preconditions.checkState(desc != null); - return (Table) desc.getParent().getTable(); + return desc.getParent().getTable(); } public void setLabel(String label) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java index b1818105ff..7b79774fef 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java @@ -479,9 +479,11 @@ public class Catalog { } public DataSourceIf getCurrentDataSource() { - // TODO: this should be got from connect context. - // Will be fixed later. - return dataSourceMgr.getInternalDataSource(); + ConnectContext ctx = ConnectContext.get(); + if (ctx == null) { + return dataSourceMgr.getInternalDataSource(); + } + return ctx.getCurrentDataSource(); } public InternalDataSource getInternalDataSource() { @@ -4163,7 +4165,7 @@ public class Catalog { ErrorReport.reportDdlException(ErrorCode.ERR_DBACCESS_DENIED_ERROR, ctx.getQualifiedUser(), qualifiedDb); } - getInternalDataSource().getDbOrDdlException(qualifiedDb); + ctx.getCurrentDataSource().getDbOrDdlException(qualifiedDb); ctx.setDatabase(qualifiedDb); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceMgr.java index 116efe65e7..f61b0c42a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceMgr.java @@ -48,6 +48,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -84,6 +85,19 @@ public class DataSourceMgr implements Writable { return nameToCatalogs.get(name); } + public <E extends Exception> DataSourceIf getCatalogOrException(String name, Function<String, E> e) throws E { + DataSourceIf ds = nameToCatalogs.get(name); + if (ds == null) { + throw e.apply(name); + } + return ds; + } + + public DataSourceIf getCatalogOrAnalysisException(String name) throws AnalysisException { + return getCatalogOrException(name, ds -> new AnalysisException( + ErrorCode.ERR_UNKNOWN_CATALOG.formatErrorMsg(ds), ErrorCode.ERR_UNKNOWN_CATALOG)); + } + public DatabaseIf getDbNullable(long dbId) { DatabaseIf db = internalDataSource.getDbNullable(dbId); if (db != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDataSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDataSource.java index 3ae70e5c2c..ca08ff2e67 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDataSource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDataSource.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource; import org.apache.doris.catalog.external.ExternalDatabase; +import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; @@ -176,6 +177,13 @@ public abstract class ExternalDataSource implements DataSourceIf<ExternalDatabas Text.writeString(out, GsonUtils.GSON.toJson(this)); } + /** + * External catalog has no cluster semantics. + */ + protected static String getRealTableName(String tableName) { + return ClusterNamespace.getNameFromFullName(tableName); + } + public static ExternalDataSource read(DataInput in) throws IOException { String json = Text.readString(in); return GsonUtils.GSON.fromJson(json, ExternalDataSource.class); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalDataSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalDataSource.java index 4d1a84c4b3..0bb90a8db1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalDataSource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalDataSource.java @@ -121,7 +121,7 @@ public class HMSExternalDataSource extends ExternalDataSource { public List<String> listTableNames(SessionContext ctx, String dbName) { makeSureInitialized(); try { - return client.getAllTables(dbName); + return client.getAllTables(getRealTableName(dbName)); } catch (MetaException e) { LOG.warn("List Table Names failed. {}", e.getMessage()); } @@ -132,7 +132,7 @@ public class HMSExternalDataSource extends ExternalDataSource { public boolean tableExist(SessionContext ctx, String dbName, String tblName) { makeSureInitialized(); try { - return client.tableExists(dbName, tblName); + return client.tableExists(getRealTableName(dbName), tblName); } catch (TException e) { LOG.warn("Check table exist failed. {}", e.getMessage()); } @@ -143,6 +143,7 @@ public class HMSExternalDataSource extends ExternalDataSource { @Override public ExternalDatabase getDbNullable(String dbName) { makeSureInitialized(); + dbName = getRealTableName(dbName); try { client.getDatabase(dbName); } catch (TException e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetaInfoAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetaInfoAction.java index 6f3bbbce5c..1196359713 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetaInfoAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/MetaInfoAction.java @@ -30,6 +30,8 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.proc.ProcNodeInterface; import org.apache.doris.common.proc.ProcResult; import org.apache.doris.common.proc.ProcService; +import org.apache.doris.datasource.DataSourceIf; +import org.apache.doris.datasource.InternalDataSource; import org.apache.doris.httpv2.entity.ResponseEntityBuilder; import org.apache.doris.httpv2.exception.BadRequestException; import org.apache.doris.mysql.privilege.PrivPredicate; @@ -86,12 +88,17 @@ public class MetaInfoAction extends RestBaseController { HttpServletRequest request, HttpServletResponse response) { checkWithCookie(request, response, false); - if (!ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { - return ResponseEntityBuilder.badRequest("Only support 'default_cluster' now"); + // use NS_KEY as catalog, but NS_KEY's default value is 'default_cluster'. + if (ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + ns = InternalDataSource.INTERNAL_DS_NAME; } // 1. get all database with privilege - List<String> dbNames = Catalog.getCurrentCatalog().getCurrentDataSource().getDbNames(); + DataSourceIf ds = Catalog.getCurrentCatalog().getDataSourceMgr().getCatalog(ns); + if (ds == null) { + return ResponseEntityBuilder.badRequest("Unknown catalog " + ns); + } + List<String> dbNames = ds.getDbNames(); List<String> dbNameSet = Lists.newArrayList(); for (String fullName : dbNames) { final String db = ClusterNamespace.getNameFromFullName(fullName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java index 9192c0981f..ddfbe47da7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java @@ -32,7 +32,7 @@ import org.apache.doris.analysis.TableRef; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.ColumnStats; import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.CheckedMath; import org.apache.doris.common.NotImplementedException; import org.apache.doris.common.Pair; @@ -363,15 +363,15 @@ public class HashJoinNode extends PlanNode { } public double lhsNumRows() { - Table table = (Table) lhs.getParent().getTable(); + TableIf table = lhs.getParent().getTable(); Preconditions.checkState(table instanceof OlapTable); - return ((OlapTable) (table)).getRowCount(); + return table.getRowCount(); } public double rhsNumRows() { - Table table = (Table) rhs.getParent().getTable(); + TableIf table = rhs.getParent().getTable(); Preconditions.checkState(table instanceof OlapTable); - return ((OlapTable) (table)).getRowCount(); + return table.getRowCount(); } public TupleId lhsTid() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java b/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java index 47f8dcb149..3fddc3cf59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/policy/Policy.java @@ -84,11 +84,10 @@ public abstract class Policy implements Writable, GsonPostProcessable { return storagePolicy; case ROW: default: - String curDb = stmt.getTableName().getDb(); - if (curDb == null) { - curDb = ConnectContext.get().getDatabase(); - } - DatabaseIf db = Catalog.getCurrentCatalog().getCurrentDataSource().getDbOrAnalysisException(curDb); + // stmt must be analyzed. + DatabaseIf db = Catalog.getCurrentCatalog().getDataSourceMgr() + .getCatalogOrAnalysisException(stmt.getTableName().getCtl()) + .getDbOrAnalysisException(stmt.getTableName().getDb()); UserIdentity userIdent = stmt.getUser(); userIdent.analyze(ConnectContext.get().getClusterName()); TableIf table = db.getTableOrAnalysisException(stmt.getTableName().getTbl()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java index 33f15e0f1f..1b53407967 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.UserException; import org.apache.doris.common.util.DebugUtil; +import org.apache.doris.datasource.DataSourceIf; import org.apache.doris.datasource.InternalDataSource; import org.apache.doris.datasource.SessionContext; import org.apache.doris.mysql.MysqlCapability; @@ -418,6 +419,14 @@ public class ConnectContext { return defaultCatalog; } + public DataSourceIf getCurrentDataSource() { + // defaultCatalog is switched by SwitchStmt, so we don't need to check to exist of catalog. + if (catalog == null) { + return Catalog.getCurrentCatalog().getDataSourceMgr().getCatalog(defaultCatalog); + } + return catalog.getDataSourceMgr().getCatalog(defaultCatalog); + } + public void changeDefaultCatalog(String catalogName) { defaultCatalog = catalogName; } @@ -428,7 +437,7 @@ public class ConnectContext { public void setDatabase(String db) { currentDb = db; - Optional<DatabaseIf> dbInstance = Catalog.getCurrentCatalog().getCurrentDataSource().getDb(db); + Optional<DatabaseIf> dbInstance = getCurrentDataSource().getDb(db); currentDbId = dbInstance.isPresent() ? dbInstance.get().getId() : -1; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 48400a820f..fcf5f8787a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -311,7 +311,7 @@ public class ConnectProcessor { ctx.getState().setError(ErrorCode.ERR_UNKNOWN_TABLE, "Empty tableName"); return; } - DatabaseIf db = ctx.getCatalog().getCurrentDataSource().getDbNullable(ctx.getDatabase()); + DatabaseIf db = ctx.getCurrentDataSource().getDbNullable(ctx.getDatabase()); if (db == null) { ctx.getState().setError(ErrorCode.ERR_BAD_DB_ERROR, "Unknown database(" + ctx.getDatabase() + ")"); return; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index ac1a79d608..f032932afe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -151,6 +151,7 @@ import org.apache.doris.common.util.PrintableMap; import org.apache.doris.common.util.ProfileManager; import org.apache.doris.common.util.RuntimeProfile; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.common.util.Util; import org.apache.doris.external.iceberg.IcebergTableCreationRecord; import org.apache.doris.load.DeleteHandler; import org.apache.doris.load.ExportJob; @@ -413,7 +414,8 @@ public class ShowExecutor { // Handle show functions private void handleShowFunctions() throws AnalysisException { ShowFunctionsStmt showStmt = (ShowFunctionsStmt) stmt; - DatabaseIf db = ctx.getCatalog().getCurrentDataSource().getDbOrAnalysisException(showStmt.getDbName()); + Util.prohibitExternalCatalog(ctx.getDefaultCatalog(), stmt.getClass().getSimpleName()); + DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(showStmt.getDbName()); List<List<String>> resultRowSet = Lists.newArrayList(); if (db instanceof Database) { @@ -460,8 +462,8 @@ public class ShowExecutor { // Handle show create function private void handleShowCreateFunction() throws AnalysisException { ShowCreateFunctionStmt showCreateFunctionStmt = (ShowCreateFunctionStmt) stmt; - DatabaseIf db = - ctx.getCatalog().getCurrentDataSource().getDbOrAnalysisException(showCreateFunctionStmt.getDbName()); + Util.prohibitExternalCatalog(ctx.getDefaultCatalog(), stmt.getClass().getSimpleName()); + DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(showCreateFunctionStmt.getDbName()); List<List<String>> resultRowSet = Lists.newArrayList(); if (db instanceof Database) { Function function = ((Database) db).getFunction(showCreateFunctionStmt.getFunction()); @@ -476,7 +478,8 @@ public class ShowExecutor { // Handle show encryptkeys private void handleShowEncryptKeys() throws AnalysisException { ShowEncryptKeysStmt showStmt = (ShowEncryptKeysStmt) stmt; - DatabaseIf db = ctx.getCatalog().getCurrentDataSource().getDbOrAnalysisException(showStmt.getDbName()); + Util.prohibitExternalCatalog(ctx.getDefaultCatalog(), stmt.getClass().getSimpleName()); + DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(showStmt.getDbName()); List<List<String>> resultRowSet = Lists.newArrayList(); if (db instanceof Database) { List<EncryptKey> encryptKeys = ((Database) db).getEncryptKeys(); @@ -571,8 +574,7 @@ public class ShowExecutor { ShowDbIdStmt showStmt = (ShowDbIdStmt) stmt; long dbId = showStmt.getDbId(); List<List<String>> rows = Lists.newArrayList(); - Catalog catalog = ctx.getCatalog(); - DatabaseIf database = catalog.getCurrentDataSource().getDbNullable(dbId); + DatabaseIf database = ctx.getCurrentDataSource().getDbNullable(dbId); if (database != null) { List<String> row = new ArrayList<>(); row.add(database.getFullName()); @@ -647,7 +649,8 @@ public class ShowExecutor { private void handleShowDb() throws AnalysisException { ShowDbStmt showDbStmt = (ShowDbStmt) stmt; List<List<String>> rows = Lists.newArrayList(); - List<String> dbNames = ctx.getCatalog().getInternalDataSource().getClusterDbNames(ctx.getClusterName()); + // cluster feature is deprecated. + List<String> dbNames = ctx.getCurrentDataSource().getDbNames(); PatternMatcher matcher = null; if (showDbStmt.getPattern() != null) { matcher = PatternMatcher.createMysqlPattern(showDbStmt.getPattern(), @@ -682,8 +685,7 @@ public class ShowExecutor { List<List<String>> rows = Lists.newArrayList(); // TODO(gaoxin): Whether to support "show tables from `ctl.db`" syntax in show statement? String catalogName = ctx.getDefaultCatalog(); - DatabaseIf<TableIf> db = - ctx.getCatalog().getCurrentDataSource().getDbOrAnalysisException(showTableStmt.getDb()); + DatabaseIf<TableIf> db = ctx.getCurrentDataSource().getDbOrAnalysisException(showTableStmt.getDb()); PatternMatcher matcher = null; if (showTableStmt.getPattern() != null) { matcher = PatternMatcher.createMysqlPattern(showTableStmt.getPattern(), @@ -715,7 +717,7 @@ public class ShowExecutor { private void handleShowTableStatus() throws AnalysisException { ShowTableStatusStmt showStmt = (ShowTableStatusStmt) stmt; List<List<String>> rows = Lists.newArrayList(); - DatabaseIf<TableIf> db = ctx.getCatalog().getCurrentDataSource().getDbNullable(showStmt.getDb()); + DatabaseIf<TableIf> db = ctx.getCurrentDataSource().getDbNullable(showStmt.getDb()); if (db != null) { PatternMatcher matcher = null; if (showStmt.getPattern() != null) { @@ -799,7 +801,7 @@ public class ShowExecutor { private void handleShowCreateDb() throws AnalysisException { ShowCreateDbStmt showStmt = (ShowCreateDbStmt) stmt; List<List<String>> rows = Lists.newArrayList(); - DatabaseIf db = ctx.getCatalog().getCurrentDataSource().getDbOrAnalysisException(showStmt.getDb()); + DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(showStmt.getDb()); StringBuilder sb = new StringBuilder(); sb.append("CREATE DATABASE `").append(ClusterNamespace.getNameFromFullName(showStmt.getDb())).append("`"); if (db.getDbProperties().getProperties().size() > 0) { @@ -815,7 +817,7 @@ public class ShowExecutor { // Show create table private void handleShowCreateTable() throws AnalysisException { ShowCreateTableStmt showStmt = (ShowCreateTableStmt) stmt; - DatabaseIf db = ctx.getCatalog().getCurrentDataSource().getDbOrAnalysisException(showStmt.getDb()); + DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(showStmt.getDb()); TableIf table = db.getTableOrAnalysisException(showStmt.getTable()); List<List<String>> rows = Lists.newArrayList(); @@ -854,7 +856,7 @@ public class ShowExecutor { private void handleShowColumn() throws AnalysisException { ShowColumnStmt showStmt = (ShowColumnStmt) stmt; List<List<String>> rows = Lists.newArrayList(); - DatabaseIf db = ctx.getCatalog().getCurrentDataSource().getDbOrAnalysisException(showStmt.getDb()); + DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(showStmt.getDb()); TableIf table = db.getTableOrAnalysisException(showStmt.getTable()); PatternMatcher matcher = null; if (showStmt.getPattern() != null) { @@ -906,7 +908,7 @@ public class ShowExecutor { private void handleShowIndex() throws AnalysisException { ShowIndexStmt showStmt = (ShowIndexStmt) stmt; List<List<String>> rows = Lists.newArrayList(); - DatabaseIf db = ctx.getCatalog().getCurrentDataSource().getDbOrAnalysisException(showStmt.getDbName()); + DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(showStmt.getDbName()); OlapTable table = db.getOlapTableOrAnalysisException(showStmt.getTableName().getTbl()); table.readLock(); @@ -1005,8 +1007,9 @@ public class ShowExecutor { private void handleShowLoad() throws AnalysisException { ShowLoadStmt showStmt = (ShowLoadStmt) stmt; - Catalog catalog = Catalog.getCurrentCatalog(); - DatabaseIf db = catalog.getCurrentDataSource().getDbOrAnalysisException(showStmt.getDbName()); + Util.prohibitExternalCatalog(ctx.getDefaultCatalog(), stmt.getClass().getSimpleName()); + Catalog catalog = ctx.getCatalog(); + DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(showStmt.getDbName()); long dbId = db.getId(); // combine the List<LoadInfo> of load(v1) and loadManager(v2) @@ -2125,7 +2128,7 @@ public class ShowExecutor { private void handleShowTableCreation() throws AnalysisException { ShowTableCreationStmt showStmt = (ShowTableCreationStmt) stmt; String dbName = showStmt.getDbName(); - DatabaseIf db = ctx.getCatalog().getCurrentDataSource().getDbOrAnalysisException(dbName); + DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(dbName); List<IcebergTableCreationRecord> records = ctx.getCatalog().getIcebergTableCreationRecordMgr().getTableCreationRecordByDbId(db.getId()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmap.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmap.java index 8a02b8ae7f..284fbf62ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmap.java @@ -27,7 +27,7 @@ import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import org.apache.doris.rewrite.ExprRewriteRule; import org.apache.doris.rewrite.ExprRewriter; @@ -69,7 +69,7 @@ public class CountDistinctToBitmap implements ExprRewriteRule { } SlotRef fnChild0 = (SlotRef) fnExpr.getChild(0); Column column = fnChild0.getColumn(); - Table table = fnChild0.getTable(); + TableIf table = fnChild0.getTable(); if (column == null || table == null || !(table instanceof OlapTable)) { return expr; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/CountFieldToSum.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/CountFieldToSum.java index 11e9981121..a636c39f94 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/CountFieldToSum.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/CountFieldToSum.java @@ -25,7 +25,7 @@ import org.apache.doris.analysis.SlotRef; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import org.apache.doris.rewrite.ExprRewriteRule; import org.apache.doris.rewrite.ExprRewriter; @@ -65,7 +65,7 @@ public class CountFieldToSum implements ExprRewriteRule { } SlotRef fnChild0 = (SlotRef) fnExpr.getChild(0); Column column = fnChild0.getColumn(); - Table table = fnChild0.getTable(); + TableIf table = fnChild0.getTable(); if (column == null || table == null || !(table instanceof OlapTable)) { return expr; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java index e1540a4c91..22a311d0f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java @@ -27,7 +27,7 @@ import org.apache.doris.analysis.TableName; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import org.apache.doris.rewrite.ExprRewriteRule; import org.apache.doris.rewrite.ExprRewriter; @@ -81,7 +81,7 @@ public class HLLHashToSlotRefRule implements ExprRewriteRule { return expr; } Column column = queryColumnSlotRef.getColumn(); - Table table = queryColumnSlotRef.getTable(); + TableIf table = queryColumnSlotRef.getTable(); if (column == null || table == null || !(table instanceof OlapTable)) { return expr; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java index 652a8e2d2d..9f20f0a70d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java @@ -26,7 +26,7 @@ import org.apache.doris.analysis.TableName; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import org.apache.doris.rewrite.ExprRewriteRule; import org.apache.doris.rewrite.ExprRewriter; @@ -66,7 +66,7 @@ public class NDVToHll implements ExprRewriteRule { } SlotRef fnChild0 = (SlotRef) fnExpr.getChild(0); Column column = fnChild0.getColumn(); - Table table = fnChild0.getTable(); + TableIf table = fnChild0.getTable(); if (column == null || table == null || !(table instanceof OlapTable)) { return expr; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/ToBitmapToSlotRefRule.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/ToBitmapToSlotRefRule.java index 95495cd6d4..c97f069cf8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/ToBitmapToSlotRefRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/mvrewrite/ToBitmapToSlotRefRule.java @@ -28,7 +28,7 @@ import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import org.apache.doris.rewrite.ExprRewriteRule; import org.apache.doris.rewrite.ExprRewriter; @@ -80,7 +80,7 @@ public class ToBitmapToSlotRefRule implements ExprRewriteRule { return expr; } Column column = queryColumnSlotRef.getColumn(); - Table table = queryColumnSlotRef.getTable(); + TableIf table = queryColumnSlotRef.getTable(); if (column == null || table == null || !(table instanceof OlapTable)) { return expr; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 31e44cd711..b8041b03a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -204,7 +204,9 @@ public class FrontendServiceImpl implements FrontendService.Iface { } else { currentUser = UserIdentity.createAnalyzedUserIdentWithIp(params.user, params.user_ip); } - DatabaseIf<TableIf> db = Catalog.getCurrentCatalog().getCurrentDataSource().getDbNullable(params.db); + String catalog = Strings.isNullOrEmpty(params.catalog) ? InternalDataSource.INTERNAL_DS_NAME : params.catalog; + DatabaseIf<TableIf> db = Catalog.getCurrentCatalog().getDataSourceMgr() + .getCatalogOrException(catalog, ds -> new TException("Unknown catalog " + ds)).getDbNullable(params.db); if (db != null) { for (String tableName : db.getTableNamesWithLock()) { LOG.debug("get table: {}, wait to check", tableName); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/AccessTestUtil.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/AccessTestUtil.java index 879d771299..0a2cf58052 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/AccessTestUtil.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/AccessTestUtil.java @@ -33,6 +33,7 @@ import org.apache.doris.catalog.SinglePartitionInfo; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.DataSourceMgr; import org.apache.doris.datasource.InternalDataSource; import org.apache.doris.load.Load; import org.apache.doris.mysql.privilege.PaloAuth; @@ -48,6 +49,7 @@ import mockit.Expectations; import java.util.LinkedList; import java.util.List; +import java.util.function.Function; public class AccessTestUtil { private static FakeEditLog fakeEditLog; @@ -142,6 +144,23 @@ public class AccessTestUtil { } }; + DataSourceMgr dsMgr = new DataSourceMgr(); + new Expectations(dsMgr) { + { + dsMgr.getCatalog((String) any); + minTimes = 0; + result = ds; + + dsMgr.getCatalogOrException((String) any, (Function) any); + minTimes = 0; + result = ds; + + dsMgr.getCatalogOrAnalysisException((String) any); + minTimes = 0; + result = ds; + } + }; + new Expectations(catalog, ds) { { catalog.getAuth(); @@ -178,6 +197,10 @@ public class AccessTestUtil { catalog.getBrokerMgr(); minTimes = 0; result = new BrokerMgr(); + + catalog.getDataSourceMgr(); + minTimes = 0; + result = dsMgr; } }; return catalog; @@ -353,6 +376,23 @@ public class AccessTestUtil { } }; + DataSourceMgr dsMgr = new DataSourceMgr(); + new Expectations(dsMgr) { + { + dsMgr.getCatalog((String) any); + minTimes = 0; + result = ds; + + dsMgr.getCatalogOrException((String) any, (Function) any); + minTimes = 0; + result = ds; + + dsMgr.getCatalogOrAnalysisException((String) any); + minTimes = 0; + result = ds; + } + }; + new Expectations(catalog) { { catalog.getAuth(); @@ -370,6 +410,10 @@ public class AccessTestUtil { catalog.getCurrentDataSource(); minTimes = 0; result = ds; + + catalog.getDataSourceMgr(); + minTimes = 0; + result = dsMgr; } }; return catalog; diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/VirtualSlotRefTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/VirtualSlotRefTest.java index 843473d9a3..f7db83e944 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/VirtualSlotRefTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/VirtualSlotRefTest.java @@ -67,7 +67,9 @@ public class VirtualSlotRefTest { f.setAccessible(true); Multimap<String, TupleDescriptor> tupleByAlias = ArrayListMultimap.create(); TupleDescriptor td = new TupleDescriptor(new TupleId(0)); - td.setTable(analyzerBase.getTableOrAnalysisException(new TableName("testdb", "t"))); + TableName tableName = new TableName("testdb", "t"); + tableName.analyze(analyzerBase); + td.setTable(analyzerBase.getTableOrAnalysisException(tableName)); tupleByAlias.put("testdb.t", td); f.set(analyzer, tupleByAlias); } catch (NoSuchFieldException | IllegalAccessException e) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java b/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java index e2e9c5a906..49099ccf04 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java +++ b/fe/fe-core/src/test/java/org/apache/doris/http/DorisHttpTestCase.java @@ -42,6 +42,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.ExceptionChecker.ThrowingRunnable; import org.apache.doris.common.FeConstants; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.DataSourceMgr; import org.apache.doris.datasource.InternalDataSource; import org.apache.doris.httpv2.HttpServer; import org.apache.doris.httpv2.IllegalArgException; @@ -78,6 +79,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.function.Function; public abstract class DorisHttpTestCase { @@ -245,6 +247,23 @@ public abstract class DorisHttpTestCase { } }; + DataSourceMgr dsMgr = new DataSourceMgr(); + new Expectations(dsMgr) { + { + dsMgr.getCatalog((String) any); + minTimes = 0; + result = internalDataSource; + + dsMgr.getCatalogOrException((String) any, (Function) any); + minTimes = 0; + result = internalDataSource; + + dsMgr.getCatalogOrAnalysisException((String) any); + minTimes = 0; + result = internalDataSource; + } + }; + new Expectations(catalog) { { catalog.getAuth(); @@ -279,6 +298,10 @@ public abstract class DorisHttpTestCase { catalog.initDefaultCluster(); minTimes = 0; + + catalog.getDataSourceMgr(); + minTimes = 0; + result = dsMgr; } }; return catalog; diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectProcessorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectProcessorTest.java index ef7b588790..a82df5c2b8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectProcessorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectProcessorTest.java @@ -20,6 +20,7 @@ package org.apache.doris.qe; import org.apache.doris.analysis.AccessTestUtil; import org.apache.doris.catalog.Catalog; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.DataSourceIf; import org.apache.doris.metric.MetricRepo; import org.apache.doris.mysql.MysqlChannel; import org.apache.doris.mysql.MysqlCommand; @@ -191,6 +192,8 @@ public class ConnectProcessorTest { } }; + DataSourceIf ds = catalog.getCurrentDataSource(); + new Expectations(context) { { context.getMysqlChannel(); @@ -236,6 +239,10 @@ public class ConnectProcessorTest { context.queryId(); minTimes = 0; result = new TUniqueId(); + + context.getCurrentDataSource(); + minTimes = 0; + result = ds; } }; diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java index 8daaff95b9..24189fdc5e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/PartitionCacheTest.java @@ -47,6 +47,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.common.util.SqlParserUtils; import org.apache.doris.common.util.Util; +import org.apache.doris.datasource.DataSourceMgr; import org.apache.doris.datasource.InternalDataSource; import org.apache.doris.metric.MetricRepo; import org.apache.doris.mysql.MysqlChannel; @@ -90,6 +91,7 @@ import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.function.Function; public class PartitionCacheTest { private static final Logger LOG = LogManager.getLogger(PartitionCacheTest.class); @@ -191,6 +193,23 @@ public class PartitionCacheTest { } }; + DataSourceMgr dsMgr = new DataSourceMgr(); + new Expectations(dsMgr) { + { + dsMgr.getCatalog((String) any); + minTimes = 0; + result = ds; + + dsMgr.getCatalogOrException((String) any, (Function) any); + minTimes = 0; + result = ds; + + dsMgr.getCatalogOrAnalysisException((String) any); + minTimes = 0; + result = ds; + } + }; + new Expectations(catalog) { { catalog.getAuth(); @@ -204,6 +223,10 @@ public class PartitionCacheTest { catalog.getInternalDataSource(); minTimes = 0; result = ds; + + catalog.getDataSourceMgr(); + minTimes = 0; + result = dsMgr; } }; FunctionSet fs = new FunctionSet(); @@ -1067,8 +1090,8 @@ public class PartitionCacheTest { SqlCache sqlCache = (SqlCache) ca.getCache(); String cacheKey = sqlCache.getSqlWithViewStmt(); - Assert.assertEquals(cacheKey, "SELECT `testDb`.`view1`.`eventdate` AS `eventdate`, `testDb`.`view1`." - + "`count(`userid`)` AS `count(``userid``)` FROM `testDb`.`view1`|select eventdate, COUNT(userid) " + Assert.assertEquals(cacheKey, "SELECT `testCluster:testDb`.`view1`.`eventdate` AS `eventdate`, `testCluster:testDb`.`view1`." + + "`count(`userid`)` AS `count(``userid``)` FROM `testCluster:testDb`.`view1`|select eventdate, COUNT(userid) " + "FROM appevent WHERE eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate"); } @@ -1094,7 +1117,7 @@ public class PartitionCacheTest { String cacheKey = sqlCache.getSqlWithViewStmt(); Assert.assertEquals(cacheKey, "SELECT `origin`.`eventdate` AS `eventdate`, `origin`.`userid` AS " + "`userid` FROM (SELECT `view2`.`eventdate` AS `eventdate`, `view2`.`userid` AS `userid` FROM " - + "`testDb`.`view2` view2 WHERE `view2`.`eventdate` >= '2020-01-12 00:00:00' AND `view2`.`eventdate`" + + "`testCluster:testDb`.`view2` view2 WHERE `view2`.`eventdate` >= '2020-01-12 00:00:00' AND `view2`.`eventdate`" + " <= '2020-01-14 00:00:00') origin|select eventdate, userid FROM appevent"); } @@ -1114,9 +1137,9 @@ public class PartitionCacheTest { cache.rewriteSelectStmt(null); Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); - Assert.assertEquals(cache.getSqlWithViewStmt(), "SELECT `testDb`.`view3`.`eventdate` AS " - + "`eventdate`, `testDb`.`view3`.`count(`userid`)` AS `count(``userid``)` FROM " - + "`testDb`.`view3`|select eventdate, COUNT(userid) FROM appevent WHERE eventdate>=" + Assert.assertEquals(cache.getSqlWithViewStmt(), "SELECT `testCluster:testDb`.`view3`.`eventdate` AS " + + "`eventdate`, `testCluster:testDb`.`view3`.`count(`userid`)` AS `count(``userid``)` FROM " + + "`testCluster:testDb`.`view3`|select eventdate, COUNT(userid) FROM appevent WHERE eventdate>=" + "\"2020-01-12\" and eventdate<=\"2020-01-15\" GROUP BY eventdate"); } catch (Exception e) { LOG.warn("ex={}", e); @@ -1170,8 +1193,8 @@ public class PartitionCacheTest { SqlCache sqlCache = (SqlCache) ca.getCache(); String cacheKey = sqlCache.getSqlWithViewStmt(); - Assert.assertEquals(cacheKey, "SELECT `testDb`.`view4`.`eventdate` AS `eventdate`, " - + "`testDb`.`view4`.`count(`userid`)` AS `count(``userid``)` FROM `testDb`.`view4`|select " + Assert.assertEquals(cacheKey, "SELECT `testCluster:testDb`.`view4`.`eventdate` AS `eventdate`, " + + "`testCluster:testDb`.`view4`.`count(`userid`)` AS `count(``userid``)` FROM `testCluster:testDb`.`view4`|select " + "eventdate, COUNT(userid) FROM view2 WHERE eventdate>=\"2020-01-12\" and " + "eventdate<=\"2020-01-14\" GROUP BY eventdate|select eventdate, userid FROM appevent"); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java index a554d61bc1..00cc083a91 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java @@ -50,6 +50,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.PatternMatcher; import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.datasource.DataSourceMgr; import org.apache.doris.datasource.InternalDataSource; import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.mysql.privilege.PaloAuth; @@ -70,6 +71,7 @@ import org.junit.rules.ExpectedException; import java.io.IOException; import java.net.URL; import java.util.List; +import java.util.function.Function; public class ShowExecutorTest { private ConnectContext ctx; @@ -189,6 +191,23 @@ public class ShowExecutorTest { } }; + DataSourceMgr dsMgr = new DataSourceMgr(); + new Expectations(dsMgr) { + { + dsMgr.getCatalog((String) any); + minTimes = 0; + result = ds; + + dsMgr.getCatalogOrException((String) any, (Function) any); + minTimes = 0; + result = ds; + + dsMgr.getCatalogOrAnalysisException((String) any); + minTimes = 0; + result = ds; + } + }; + // mock catalog. catalog = Deencapsulation.newInstance(Catalog.class); new Expectations(catalog) { @@ -218,6 +237,10 @@ public class ShowExecutorTest { Catalog.getDdlStmt((Table) any, (List) any, null, null, anyBoolean, anyBoolean); minTimes = 0; + + catalog.getDataSourceMgr(); + minTimes = 0; + result = dsMgr; } }; --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org