morningman commented on code in PR #14527: URL: https://github.com/apache/doris/pull/14527#discussion_r1031572303
########## fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java: ########## @@ -0,0 +1,422 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.external.jdbc; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.util.Util; + +import com.google.common.collect.Lists; +import lombok.Data; +import lombok.Getter; +import org.apache.commons.codec.binary.Hex; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLClassLoader; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; + +@Getter +public class JdbcClient { + private static final Logger LOG = LogManager.getLogger(JdbcClient.class); + private static final String MYSQL = "MYSQL"; + private static final String ORACLE = "ORACLE"; + private static final String HANA = "HANA"; + private static final String SQLSERVER = "SQLSERVER"; + private static final String H2 = "H2"; + private static final String POSTGRESQL = "POSTGRESQL"; + private static final String ZENITH = "ZENITH"; + private static final int HTTP_TIMEOUT_MS = 10000; + + private String dbType; + private String jdbcUser; + private String jdbcPasswd; + private String jdbcUrl; + private String driverUrl; + private String driverClass; + private String checkSum; + + + private URLClassLoader classLoader; + + + public JdbcClient(String user, String password, String jdbcUrl, String driverUrl, String driverClass) { + this.jdbcUser = user; + this.jdbcPasswd = password; + this.jdbcUrl = jdbcUrl; + this.driverUrl = driverUrl; + this.driverClass = driverClass; + this.dbType = parseDbType(jdbcUrl); + this.checkSum = computeObjectChecksum(); + + try { + URL[] urls = {new URL(driverUrl)}; + // set parent ClassLoader to null, we can achieve class loading isolation. + classLoader = URLClassLoader.newInstance(urls, null); + } catch (Exception e) { + throw new JdbcClientException("failed to load jar"); Review Comment: add detail error msg ########## fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java: ########## @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.external.ExternalDatabase; +import org.apache.doris.catalog.external.JdbcExternalDatabase; +import org.apache.doris.external.jdbc.JdbcClient; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import lombok.Getter; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +@Getter +public class JdbcExternalCatalog extends ExternalCatalog { + private static final Logger LOG = LogManager.getLogger(JdbcExternalCatalog.class); + + public static final String PROP_USER = "jdbc.user"; + public static final String PROP_PASSWORD = "jdbc.password"; + public static final String PROP_JDBC_URL = "jdbc.jdbc_url"; + public static final String PROP_DRIVER_URL = "jdbc.driver_url"; + public static final String PROP_DRIVER_CLASS = "jdbc.driver_class"; + + private JdbcClient jdbcClient; + private String databaseTypeName; + private String jdbcUser; + private String jdbcPasswd; + private String jdbcUrl; + private String driverUrl; + private String driverClass; + private String checkSum; + + public JdbcExternalCatalog(long catalogId, String name, Map<String, String> props) { + this.id = catalogId; + this.name = name; + this.type = "jdbc"; + setProperties(props); + this.catalogProperty = new CatalogProperty(); + this.catalogProperty.setProperties(props); + } + + private void setProperties(Map<String, String> props) { + jdbcUser = props.getOrDefault(PROP_USER, ""); + jdbcPasswd = props.getOrDefault(PROP_PASSWORD, ""); + jdbcUrl = props.getOrDefault(PROP_JDBC_URL, ""); + driverUrl = props.getOrDefault(PROP_DRIVER_URL, ""); + driverClass = props.getOrDefault(PROP_DRIVER_CLASS, ""); + } + + @Override + protected void initLocalObjectsImpl() { + jdbcClient = new JdbcClient(jdbcUser, jdbcPasswd, jdbcUrl, driverUrl, driverClass); + databaseTypeName = jdbcClient.getDbType(); + checkSum = jdbcClient.getCheckSum(); + } + + @Override + protected void init() { + Map<String, Long> tmpDbNameToId = Maps.newConcurrentMap(); + Map<Long, ExternalDatabase> tmpIdToDb = Maps.newConcurrentMap(); + InitCatalogLog initCatalogLog = new InitCatalogLog(); + initCatalogLog.setCatalogId(id); + initCatalogLog.setType(InitCatalogLog.Type.JDBC); + List<String> allDatabaseNames = jdbcClient.getDatabaseNameList(); + for (String dbName : allDatabaseNames) { + long dbId; + if (dbNameToId != null && dbNameToId.containsKey(dbName)) { + dbId = dbNameToId.get(dbName); + tmpDbNameToId.put(dbName, dbId); + ExternalDatabase db = idToDb.get(dbId); + db.setUnInitialized(); + tmpIdToDb.put(dbId, db); + initCatalogLog.addRefreshDb(dbId); + } else { + dbId = Env.getCurrentEnv().getNextId(); + tmpDbNameToId.put(dbName, dbId); + JdbcExternalDatabase db = new JdbcExternalDatabase(this, dbId, dbName); + tmpIdToDb.put(dbId, db); + initCatalogLog.addCreateDb(dbId, dbName); + } + } + dbNameToId = tmpDbNameToId; + idToDb = tmpIdToDb; + Env.getCurrentEnv().getEditLog().logInitCatalog(initCatalogLog); + } + + @Override + public List<String> listDatabaseNames(SessionContext ctx) { + makeSureInitialized(); + return Lists.newArrayList(dbNameToId.keySet()); + } + + @Override + public List<String> listTableNames(SessionContext ctx, String dbName) { + makeSureInitialized(); + JdbcExternalDatabase db = (JdbcExternalDatabase) idToDb.get(dbNameToId.get(dbName)); + if (db != null && db.isInitialized()) { + List<String> names = Lists.newArrayList(); + db.getTables().stream().forEach(table -> names.add(table.getName())); + return names; + } else { + return jdbcClient.getTablesNameList(dbName); + } + } + + @Override + public boolean tableExist(SessionContext ctx, String dbName, String tblName) { + makeSureInitialized(); + return jdbcClient.isTableExist(dbName, tblName); + } + + @Override + public void gsonPostProcess() throws IOException { + super.gsonPostProcess(); + setProperties(this.catalogProperty.getProperties()); + } + + // TODO(ftw): 每次调用getSchema都会建立jdbc conn，会不会性能差？ Review Comment: This is only called when schema cache is missing, so I think it is ok -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
