This is an automated email from the ASF dual-hosted git repository.

diwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-kafka-connector.git


The following commit(s) were added to refs/heads/master by this push:
     new 67aa079  [Improve]add cache for schema change fetch doris table (#19)
67aa079 is described below

commit 67aa07991352307b136958331997468ab4fcf6dc
Author: wudongliang <[email protected]>
AuthorDate: Fri May 17 16:36:42 2024 +0800

    [Improve]add cache for schema change fetch doris table (#19)
---
 .../kafka/connector/converter/RecordService.java   | 29 +++++++++++++++++++---
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git 
a/src/main/java/org/apache/doris/kafka/connector/converter/RecordService.java 
b/src/main/java/org/apache/doris/kafka/connector/converter/RecordService.java
index 8bc2480..9487307 100644
--- 
a/src/main/java/org/apache/doris/kafka/connector/converter/RecordService.java
+++ 
b/src/main/java/org/apache/doris/kafka/connector/converter/RecordService.java
@@ -66,6 +66,7 @@ public class RecordService {
     private DorisOptions dorisOptions;
     private RecordTypeRegister recordTypeRegister;
     private Set<RecordDescriptor.FieldDescriptor> missingFields;
+    private Map<String, TableDescriptor> dorisTableDescriptorCache;
 
     public RecordService() {
         this.converter = new JsonConverter();
@@ -80,6 +81,7 @@ public class RecordService {
         this.recordTypeRegister = new RecordTypeRegister(dorisOptions);
         this.dorisSystemService = new DorisSystemService(dorisOptions);
         this.schemaChangeManager = new SchemaChangeManager(dorisOptions);
+        this.dorisTableDescriptorCache = new HashMap<>();
     }
 
     /**
@@ -140,6 +142,9 @@ public class RecordService {
             String tableName, RecordDescriptor recordDescriptor) {
         if (!hasTable(tableName)) {
             // TODO Table does not exist, lets attempt to create it.
+            LOG.warn("The {} table does not exist, please create it 
manually.", tableName);
+            throw new DorisException(
+                    "The " + tableName + " table does not exist, please create 
it manually.");
         } else {
             // Table exists, lets attempt to alter it if necessary.
             alterTableIfNeeded(tableName, recordDescriptor);
@@ -147,12 +152,19 @@ public class RecordService {
     }
 
     private boolean hasTable(String tableName) {
-        return dorisSystemService.tableExists(dorisOptions.getDatabase(), 
tableName);
+        if (!dorisTableDescriptorCache.containsKey(tableName)) {
+            boolean exist = 
dorisSystemService.tableExists(dorisOptions.getDatabase(), tableName);
+            if (exist) {
+                dorisTableDescriptorCache.put(tableName, null);
+            }
+            return exist;
+        }
+        return true;
     }
 
     private void alterTableIfNeeded(String tableName, RecordDescriptor record) 
{
         // Resolve table metadata from the database
-        final TableDescriptor table = obtainTableSchema(tableName);
+        final TableDescriptor table = fetchDorisTableDescriptor(tableName);
 
         missingFields = resolveMissingFields(record, table);
         if (missingFields.isEmpty()) {
@@ -177,6 +189,8 @@ public class RecordService {
         for (RecordDescriptor.FieldDescriptor missingField : missingFields) {
             schemaChangeManager.addColumnDDL(tableName, missingField);
         }
+        TableDescriptor newTableDescriptor = obtainTableSchema(tableName);
+        dorisTableDescriptorCache.put(tableName, newTableDescriptor);
     }
 
     private Set<RecordDescriptor.FieldDescriptor> resolveMissingFields(
@@ -192,9 +206,16 @@ public class RecordService {
         return missingFields;
     }
 
+    private TableDescriptor fetchDorisTableDescriptor(String tableName) {
+        if (!dorisTableDescriptorCache.containsKey(tableName)
+                || Objects.isNull(dorisTableDescriptorCache.get(tableName))) {
+            TableDescriptor tableDescriptor = obtainTableSchema(tableName);
+            dorisTableDescriptorCache.put(tableName, tableDescriptor);
+        }
+        return dorisTableDescriptorCache.get(tableName);
+    }
+
     private TableDescriptor obtainTableSchema(String tableName) {
-        // TODO when the table structure is obtained from doris for first 
time, it should be
-        // obtained in the cache later.
         Schema schema =
                 RestService.getSchema(dorisOptions, 
dorisOptions.getDatabase(), tableName, LOG);
         List<ColumnDescriptor> columnDescriptors = new ArrayList<>();


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to