morningman commented on code in PR #13746:
URL: https://github.com/apache/doris/pull/13746#discussion_r1010181566
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java:
##########
@@ -406,6 +413,105 @@ public void replayAlterCatalogProps(CatalogLog log) {
}
}
+ public void replayInitCatalog(InitCatalogLog log) {
+ writeLock();
+ try {
+ ExternalCatalog catalog = (ExternalCatalog)
idToCatalog.get(log.getCatalogId());
+ for (int i = 0; i < log.getRefreshCount(); i++) {
+ ExternalDatabase db =
catalog.getDbNullable(log.getRefreshDbIds().get(i));
+ db.setUnInitialized();
+ }
+ switch (log.getType()) {
+ case HMS:
+ for (int i = 0; i < log.getCreateCount(); i++) {
+ catalog.addDb(new HMSExternalDatabase(
+ catalog, log.getCreateDbIds().get(i),
log.getCreateDbNames().get(i)));
+ }
+ break;
+ case ES:
+ for (int i = 0; i < log.getCreateCount(); i++) {
+ catalog.addDb(new EsExternalDatabase(
+ catalog, log.getCreateDbIds().get(i),
log.getCreateDbNames().get(i)));
+ }
+ break;
+ default:
+ break;
+ }
+ catalog.setInitialized(true);
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ public void replayInitExternalDb(InitDatabaseLog log) {
+ writeLock();
+ try {
+ ExternalCatalog catalog = (ExternalCatalog)
idToCatalog.get(log.getCatalogId());
+ ExternalDatabase db = catalog.getDbNullable(log.getDbId());
+ for (int i = 0; i < log.getRefreshCount(); i++) {
+ ExternalTable table =
db.getTableNullable(log.getRefreshTableIds().get(i));
+ table.setUnInitialized();
+ }
+ switch (log.getType()) {
+ case HMS:
+ for (int i = 0; i < log.getCreateCount(); i++) {
+ db.addTable(new
HMSExternalTable(log.getCreateTableIds().get(i),
+ log.getCreateTableNames().get(i),
db.getFullName(), (HMSExternalCatalog) catalog));
+ }
+ break;
+ case ES:
+ for (int i = 0; i < log.getCreateCount(); i++) {
+ db.addTable(new
EsExternalTable(log.getCreateTableIds().get(i),
+ log.getCreateTableNames().get(i),
db.getFullName(), (EsExternalCatalog) catalog));
+ }
+ break;
+ default:
+ break;
+ }
+ db.setInitialized();
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ public void replayInitExternalTable(InitTableLog log) {
+ writeLock();
Review Comment:
Sink to the `Database` class
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalDatabase.java:
##########
@@ -53,15 +60,46 @@ public EsExternalDatabase(ExternalCatalog extCatalog, long
id, String name) {
super(extCatalog, id, name);
}
- private synchronized void makeSureInitialized() {
+ public void addTable(ExternalTable table) {
+ if (!(table instanceof HMSExternalTable)) {
+ LOG.warn("Table {} is not HMSExternalTable.", table.getName());
+ return;
+ }
+ tableNameToId.put(table.getName(), table.getId());
+ idToTbl.put(table.getId(), (EsExternalTable) table);
+ }
+
+ public void setTableExtCatalog(ExternalCatalog extCatalog) {
+ for (EsExternalTable table : idToTbl.values()) {
+ table.setCatalog(extCatalog);
+ }
+ }
+
+ public synchronized void makeSureInitialized() {
if (!initialized) {
- init();
+ if (!Env.getCurrentEnv().isMaster()) {
+ tableNameToId = null;
+ idToTbl = null;
+ // Forward to master and wait the journal to replay.
+ MasterCatalogExecutor remoteExecutor = new
MasterCatalogExecutor();
+ try {
+ remoteExecutor.forward(extCatalog.getName(), name, null);
+ } catch (Exception e) {
+ LOG.warn("Failed to forward init db {} operation to
master. {}", name, e.getMessage());
+ }
+ return;
+ }
+ InitDatabaseLog initDatabaseLog = new InitDatabaseLog();
+ init(initDatabaseLog);
initialized = true;
+
Env.getCurrentEnv().getEditLog().logInitExternalDb(initDatabaseLog);
Review Comment:
I think we can put the line 92~95 into the `init()` method
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalDatabase.java:
##########
@@ -53,10 +55,35 @@ public EsExternalDatabase(ExternalCatalog extCatalog, long
id, String name) {
super(extCatalog, id, name);
}
- private synchronized void makeSureInitialized() {
+ public void addTable(ExternalTable table) {
+ if (!(table instanceof HMSExternalTable)) {
Review Comment:
should be `EsTable`?
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java:
##########
@@ -156,5 +161,10 @@ private void refreshExternalCtlTable(String dbName, String
tableName, CatalogIf
throw new DdlException("Table " + tableName + " does not exist in
db " + dbName);
}
((ExternalTable) table).setUnInitialized();
+ ExternalObjectLog log = new ExternalObjectLog();
+ log.setCatalogName(catalog.getName());
Review Comment:
Better use `id` instead of `name` in edit log
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java:
##########
@@ -138,28 +136,67 @@ private void validate(Map<String, String> properties)
throws DdlException {
* Datasource can't be init when creating because the external datasource
may depend on third system.
* So you have to make sure the client of third system is initialized
before any method was called.
*/
- private synchronized void makeSureInitialized() {
+ @Override
+ public synchronized void makeSureInitialized() {
+ if (!objectCreated) {
+ try {
+ validate(catalogProperty.getProperties());
+ } catch (DdlException e) {
+ LOG.warn("validate error", e);
+ }
+ esRestClient = new EsRestClient(this.nodes, this.username,
this.password, this.enableSsl);
+ objectCreated = true;
+ }
if (!initialized) {
- init();
+ if (!Env.getCurrentEnv().isMaster()) {
+ dbNameToId = null;
+ idToDb = null;
+ // Forward to master and wait the journal to replay.
+ MasterCatalogExecutor remoteExecutor = new
MasterCatalogExecutor();
+ try {
+ remoteExecutor.forward(name, null, null);
+ } catch (Exception e) {
+ LOG.warn("Failed to forward init catalog {} operation to
master. {}", name, e.getMessage());
+ }
+ return;
+ }
+ InitCatalogLog initCatalogLog = new InitCatalogLog();
+ init(initCatalogLog);
initialized = true;
+ Env.getCurrentEnv().getEditLog().logInitCatalog(initCatalogLog);
Review Comment:
Move line 163~166 into the `init()` method.
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java:
##########
@@ -138,28 +136,67 @@ private void validate(Map<String, String> properties)
throws DdlException {
* Datasource can't be init when creating because the external datasource
may depend on third system.
* So you have to make sure the client of third system is initialized
before any method was called.
*/
- private synchronized void makeSureInitialized() {
+ @Override
+ public synchronized void makeSureInitialized() {
+ if (!objectCreated) {
+ try {
+ validate(catalogProperty.getProperties());
+ } catch (DdlException e) {
+ LOG.warn("validate error", e);
+ }
+ esRestClient = new EsRestClient(this.nodes, this.username,
this.password, this.enableSsl);
+ objectCreated = true;
+ }
if (!initialized) {
- init();
+ if (!Env.getCurrentEnv().isMaster()) {
+ dbNameToId = null;
+ idToDb = null;
+ // Forward to master and wait the journal to replay.
+ MasterCatalogExecutor remoteExecutor = new
MasterCatalogExecutor();
+ try {
+ remoteExecutor.forward(name, null, null);
+ } catch (Exception e) {
+ LOG.warn("Failed to forward init catalog {} operation to
master. {}", name, e.getMessage());
+ }
+ return;
+ }
+ InitCatalogLog initCatalogLog = new InitCatalogLog();
+ init(initCatalogLog);
initialized = true;
+ Env.getCurrentEnv().getEditLog().logInitCatalog(initCatalogLog);
+ }
+ }
+
+ @Override
+ public void addDb(ExternalDatabase db) {
+ if (!(db instanceof HMSExternalDatabase)) {
Review Comment:
```suggestion
if (!(db instanceof EsExternalDatabase)) {
```
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java:
##########
@@ -406,6 +413,105 @@ public void replayAlterCatalogProps(CatalogLog log) {
}
}
+ public void replayInitCatalog(InitCatalogLog log) {
+ writeLock();
+ try {
+ ExternalCatalog catalog = (ExternalCatalog)
idToCatalog.get(log.getCatalogId());
+ for (int i = 0; i < log.getRefreshCount(); i++) {
+ ExternalDatabase db =
catalog.getDbNullable(log.getRefreshDbIds().get(i));
+ db.setUnInitialized();
+ }
+ switch (log.getType()) {
+ case HMS:
+ for (int i = 0; i < log.getCreateCount(); i++) {
+ catalog.addDb(new HMSExternalDatabase(
+ catalog, log.getCreateDbIds().get(i),
log.getCreateDbNames().get(i)));
+ }
+ break;
+ case ES:
+ for (int i = 0; i < log.getCreateCount(); i++) {
+ catalog.addDb(new EsExternalDatabase(
+ catalog, log.getCreateDbIds().get(i),
log.getCreateDbNames().get(i)));
+ }
+ break;
+ default:
+ break;
+ }
+ catalog.setInitialized(true);
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ public void replayInitExternalDb(InitDatabaseLog log) {
+ writeLock();
Review Comment:
The lock scope is too large for initializing a database.
I think we can sink the `replayInitExternalDb()` to the Database class.
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalDatabase.java:
##########
@@ -53,15 +60,46 @@ public EsExternalDatabase(ExternalCatalog extCatalog, long
id, String name) {
super(extCatalog, id, name);
}
- private synchronized void makeSureInitialized() {
+ public void addTable(ExternalTable table) {
+ if (!(table instanceof HMSExternalTable)) {
+ LOG.warn("Table {} is not HMSExternalTable.", table.getName());
+ return;
+ }
+ tableNameToId.put(table.getName(), table.getId());
+ idToTbl.put(table.getId(), (EsExternalTable) table);
+ }
+
+ public void setTableExtCatalog(ExternalCatalog extCatalog) {
+ for (EsExternalTable table : idToTbl.values()) {
+ table.setCatalog(extCatalog);
+ }
+ }
+
+ public synchronized void makeSureInitialized() {
if (!initialized) {
- init();
+ if (!Env.getCurrentEnv().isMaster()) {
+ tableNameToId = null;
+ idToTbl = null;
+ // Forward to master and wait the journal to replay.
+ MasterCatalogExecutor remoteExecutor = new
MasterCatalogExecutor();
+ try {
+ remoteExecutor.forward(extCatalog.getName(), name, null);
Review Comment:
Better to use ID instead of NAME
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
