This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new f3a8be6f3d7 [feature](binlog) Wrap rename table/column binlog in
BarrierLog (#43033)
f3a8be6f3d7 is described below
commit f3a8be6f3d7e42a74714f646e975aa0938b48d34
Author: walter <w41te...@gmail.com>
AuthorDate: Fri Nov 1 16:18:49 2024 +0800
[feature](binlog) Wrap rename table/column binlog in BarrierLog (#43033)
This PR is part of #37031, #39782
---
.../java/org/apache/doris/binlog/BinlogManager.java | 20 ++++++++++++++++++++
.../main/java/org/apache/doris/persist/EditLog.java | 10 ++++++++--
.../java/org/apache/doris/persist/TableInfo.java | 18 ++++++++++++++++++
.../apache/doris/persist/TableRenameColumnInfo.java | 4 ++++
gensrc/thrift/FrontendService.thrift | 2 ++
5 files changed, 52 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
index 350a6c70917..47b0bb3c767 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java
@@ -32,6 +32,8 @@ import org.apache.doris.persist.DropPartitionInfo;
import org.apache.doris.persist.ModifyTablePropertyOperationLog;
import org.apache.doris.persist.ReplacePartitionOperationLog;
import org.apache.doris.persist.TableAddOrDropColumnsInfo;
+import org.apache.doris.persist.TableInfo;
+import org.apache.doris.persist.TableRenameColumnInfo;
import org.apache.doris.persist.TruncateTableInfo;
import org.apache.doris.thrift.TBinlog;
import org.apache.doris.thrift.TBinlogType;
@@ -355,6 +357,24 @@ public class BinlogManager {
}
}
+ public void addTableRename(TableInfo info, long commitSeq) {
+ long dbId = info.getDbId();
+ long tableId = info.getTableId();
+ TBinlogType type = TBinlogType.RENAME_TABLE;
+ String data = info.toJson();
+ BarrierLog log = new BarrierLog(dbId, tableId, type, data);
+ addBarrierLog(log, commitSeq);
+ }
+
+ public void addColumnRename(TableRenameColumnInfo info, long commitSeq) {
+ long dbId = info.getDbId();
+ long tableId = info.getTableId();
+ TBinlogType type = TBinlogType.RENAME_COLUMN;
+ String data = info.toJson();
+ BarrierLog log = new BarrierLog(dbId, tableId, type, data);
+ addBarrierLog(log, commitSeq);
+ }
+
// get the dropped partitions of the db.
public List<Long> getDroppedPartitions(long dbId) {
lock.readLock().lock();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
index a1e033d6d1f..87b2b746d5e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
@@ -296,6 +296,7 @@ public class EditLog {
case OperationType.OP_RENAME_TABLE: {
TableInfo info = (TableInfo) journal.getData();
env.replayRenameTable(info);
+ env.getBinlogManager().addTableRename(info, logId);
break;
}
case OperationType.OP_MODIFY_VIEW_DEF: {
@@ -311,6 +312,7 @@ public class EditLog {
case OperationType.OP_RENAME_COLUMN: {
TableRenameColumnInfo info = (TableRenameColumnInfo)
journal.getData();
env.replayRenameColumn(info);
+
Env.getCurrentEnv().getBinlogManager().addColumnRename(info, logId);
break;
}
case OperationType.OP_BACKUP_JOB: {
@@ -1435,7 +1437,9 @@ public class EditLog {
}
public void logTableRename(TableInfo tableInfo) {
- logEdit(OperationType.OP_RENAME_TABLE, tableInfo);
+ long logId = logEdit(OperationType.OP_RENAME_TABLE, tableInfo);
+ LOG.info("log table rename, logId : {}, infos: {}", logId, tableInfo);
+ Env.getCurrentEnv().getBinlogManager().addTableRename(tableInfo,
logId);
}
public void logModifyViewDef(AlterViewInfo alterViewInfo) {
@@ -1451,7 +1455,9 @@ public class EditLog {
}
public void logColumnRename(TableRenameColumnInfo info) {
- logEdit(OperationType.OP_RENAME_COLUMN, info);
+ long logId = logEdit(OperationType.OP_RENAME_COLUMN, info);
+ LOG.info("log column rename, logId : {}, infos: {}", logId, info);
+ Env.getCurrentEnv().getBinlogManager().addColumnRename(info, logId);
}
public void logAddBroker(BrokerMgr.ModifyBrokerInfo info) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/TableInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/TableInfo.java
index 2d08a8a43fa..69360a0e08b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/TableInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/TableInfo.java
@@ -19,6 +19,9 @@ package org.apache.doris.persist;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.gson.annotations.SerializedName;
import java.io.DataInput;
import java.io.DataOutput;
@@ -26,13 +29,20 @@ import java.io.IOException;
public class TableInfo implements Writable {
+ @SerializedName("db")
private long dbId;
+ @SerializedName("tb")
private long tableId;
+ @SerializedName("ind")
private long indexId;
+ @SerializedName("p")
private long partitionId;
+ @SerializedName("nT")
private String newTableName;
+ @SerializedName("nR")
private String newRollupName;
+ @SerializedName("nP")
private String newPartitionName;
public TableInfo() {
@@ -124,4 +134,12 @@ public class TableInfo implements Writable {
info.readFields(in);
return info;
}
+
+ public String toJson() {
+ return GsonUtils.GSON.toJson(this);
+ }
+
+ public static TableInfo fromJson(String json) {
+ return GsonUtils.GSON.fromJson(json, TableInfo.class);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/persist/TableRenameColumnInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/persist/TableRenameColumnInfo.java
index eafdb943e11..9434f7302c0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/persist/TableRenameColumnInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/persist/TableRenameColumnInfo.java
@@ -72,6 +72,10 @@ public class TableRenameColumnInfo implements Writable {
return indexIdToSchemaVersion;
}
+ public String toJson() {
+ return GsonUtils.GSON.toJson(this);
+ }
+
@Override
public void write(DataOutput out) throws IOException {
Text.writeString(out, GsonUtils.GSON.toJson(this));
diff --git a/gensrc/thrift/FrontendService.thrift
b/gensrc/thrift/FrontendService.thrift
index a3c25c17616..0140eeff5e1 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -1027,6 +1027,8 @@ enum TBinlogType {
MODIFY_PARTITIONS = 11,
REPLACE_PARTITIONS = 12,
TRUNCATE_TABLE = 13,
+ RENAME_TABLE = 14,
+ RENAME_COLUMN = 15,
}
struct TBinlog {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
