This is an automated email from the ASF dual-hosted git repository.
diwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris-kafka-connector.git
The following commit(s) were added to refs/heads/master by this push:
new 67aa079 [Improve]add cache for schema change fetch doris table (#19)
67aa079 is described below
commit 67aa07991352307b136958331997468ab4fcf6dc
Author: wudongliang <[email protected]>
AuthorDate: Fri May 17 16:36:42 2024 +0800
[Improve]add cache for schema change fetch doris table (#19)
---
.../kafka/connector/converter/RecordService.java | 29 +++++++++++++++++++---
1 file changed, 25 insertions(+), 4 deletions(-)
diff --git
a/src/main/java/org/apache/doris/kafka/connector/converter/RecordService.java
b/src/main/java/org/apache/doris/kafka/connector/converter/RecordService.java
index 8bc2480..9487307 100644
---
a/src/main/java/org/apache/doris/kafka/connector/converter/RecordService.java
+++
b/src/main/java/org/apache/doris/kafka/connector/converter/RecordService.java
@@ -66,6 +66,7 @@ public class RecordService {
private DorisOptions dorisOptions;
private RecordTypeRegister recordTypeRegister;
private Set<RecordDescriptor.FieldDescriptor> missingFields;
+ private Map<String, TableDescriptor> dorisTableDescriptorCache;
public RecordService() {
this.converter = new JsonConverter();
@@ -80,6 +81,7 @@ public class RecordService {
this.recordTypeRegister = new RecordTypeRegister(dorisOptions);
this.dorisSystemService = new DorisSystemService(dorisOptions);
this.schemaChangeManager = new SchemaChangeManager(dorisOptions);
+ this.dorisTableDescriptorCache = new HashMap<>();
}
/**
@@ -140,6 +142,9 @@ public class RecordService {
String tableName, RecordDescriptor recordDescriptor) {
if (!hasTable(tableName)) {
// TODO Table does not exist, lets attempt to create it.
+ LOG.warn("The {} table does not exist, please create it
manually.", tableName);
+ throw new DorisException(
+ "The " + tableName + " table does not exist, please create
it manually.");
} else {
// Table exists, lets attempt to alter it if necessary.
alterTableIfNeeded(tableName, recordDescriptor);
@@ -147,12 +152,19 @@ public class RecordService {
}
private boolean hasTable(String tableName) {
- return dorisSystemService.tableExists(dorisOptions.getDatabase(),
tableName);
+ if (!dorisTableDescriptorCache.containsKey(tableName)) {
+ boolean exist =
dorisSystemService.tableExists(dorisOptions.getDatabase(), tableName);
+ if (exist) {
+ dorisTableDescriptorCache.put(tableName, null);
+ }
+ return exist;
+ }
+ return true;
}
private void alterTableIfNeeded(String tableName, RecordDescriptor record)
{
// Resolve table metadata from the database
- final TableDescriptor table = obtainTableSchema(tableName);
+ final TableDescriptor table = fetchDorisTableDescriptor(tableName);
missingFields = resolveMissingFields(record, table);
if (missingFields.isEmpty()) {
@@ -177,6 +189,8 @@ public class RecordService {
for (RecordDescriptor.FieldDescriptor missingField : missingFields) {
schemaChangeManager.addColumnDDL(tableName, missingField);
}
+ TableDescriptor newTableDescriptor = obtainTableSchema(tableName);
+ dorisTableDescriptorCache.put(tableName, newTableDescriptor);
}
private Set<RecordDescriptor.FieldDescriptor> resolveMissingFields(
@@ -192,9 +206,16 @@ public class RecordService {
return missingFields;
}
+ private TableDescriptor fetchDorisTableDescriptor(String tableName) {
+ if (!dorisTableDescriptorCache.containsKey(tableName)
+ || Objects.isNull(dorisTableDescriptorCache.get(tableName))) {
+ TableDescriptor tableDescriptor = obtainTableSchema(tableName);
+ dorisTableDescriptorCache.put(tableName, tableDescriptor);
+ }
+ return dorisTableDescriptorCache.get(tableName);
+ }
+
private TableDescriptor obtainTableSchema(String tableName) {
- // TODO when the table structure is obtained from doris for first
time, it should be
- // obtained in the cache later.
Schema schema =
RestService.getSchema(dorisOptions,
dorisOptions.getDatabase(), tableName, LOG);
List<ColumnDescriptor> columnDescriptors = new ArrayList<>();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]