This is an automated email from the ASF dual-hosted git repository.
w41ter pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 9c74a96d42d [feat](backup) GetSnapshot returns snapshot expiration
time #43731 (#43863)
9c74a96d42d is described below
commit 9c74a96d42d5e23fc4e877695d384efe1f75d2ac
Author: walter <maoch...@selectdb.com>
AuthorDate: Wed Nov 13 18:31:42 2024 +0800
[feat](backup) GetSnapshot returns snapshot expiration time #43731 (#43863)
cherry pick from #43731
---
.../main/java/org/apache/doris/backup/BackupJob.java | 8 +++++++-
.../src/main/java/org/apache/doris/backup/Snapshot.java | 17 ++++++++++++++---
.../org/apache/doris/service/FrontendServiceImpl.java | 9 +++++++--
gensrc/thrift/FrontendService.thrift | 1 +
gensrc/thrift/Status.thrift | 7 +++++++
.../org/apache/doris/regression/suite/Syncer.groovy | 2 ++
6 files changed, 38 insertions(+), 6 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index dc92e9a07c3..8a5043ebe7f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -978,12 +978,18 @@ public class BackupJob extends AbstractJob {
return null;
}
+ // Avoid loading expired meta.
+ long expiredAt = createTime + timeoutMs;
+ if (System.currentTimeMillis() >= expiredAt) {
+ return new Snapshot(label, new byte[0], new byte[0], expiredAt);
+ }
+
try {
File metaInfoFile = new File(localMetaInfoFilePath);
File jobInfoFile = new File(localJobInfoFilePath);
byte[] metaInfoBytes = Files.readAllBytes(metaInfoFile.toPath());
byte[] jobInfoBytes = Files.readAllBytes(jobInfoFile.toPath());
- return new Snapshot(label, metaInfoBytes, jobInfoBytes);
+ return new Snapshot(label, metaInfoBytes, jobInfoBytes, expiredAt);
} catch (IOException e) {
LOG.warn("failed to load meta info and job info file, meta info
file {}, job info file {}: ",
localMetaInfoFilePath, localJobInfoFilePath, e);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java
index e31309b19a5..c4c93548177 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java
@@ -31,16 +31,19 @@ public class Snapshot {
@SerializedName(value = "jobInfo")
private byte[] jobInfo = null;
+ @SerializedName(value = "expired_at")
+ private long expiredAt = 0;
+
public Snapshot() {
}
- public Snapshot(String label, byte[] meta, byte[] jobInfo) {
+ public Snapshot(String label, byte[] meta, byte[] jobInfo, long expiredAt)
{
this.label = label;
this.meta = meta;
this.jobInfo = jobInfo;
+ this.expiredAt = expiredAt;
}
-
public byte[] getMeta() {
return meta;
}
@@ -49,17 +52,25 @@ public class Snapshot {
return jobInfo;
}
+ public long getExpiredAt() {
+ return expiredAt;
+ }
+
+ public boolean isExpired() {
+ return System.currentTimeMillis() > expiredAt;
+ }
+
public String toJson() {
return GsonUtils.GSON.toJson(this);
}
@Override
public String toString() {
- // return toJson();
return "Snapshot{"
+ "label='" + label + '\''
+ ", meta=" + meta
+ ", jobInfo=" + jobInfo
+ + ", expiredAt=" + expiredAt
+ '}';
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index d83ff7e0815..8c521cc73b0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -2813,12 +2813,16 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
if (snapshot == null) {
result.getStatus().setStatusCode(TStatusCode.SNAPSHOT_NOT_EXIST);
result.getStatus().addToErrorMsgs(String.format("snapshot %s not
exist", label));
+ } else if (snapshot.isExpired()) {
+ result.getStatus().setStatusCode(TStatusCode.SNAPSHOT_EXPIRED);
+ result.getStatus().addToErrorMsgs(String.format("snapshot %s is
expired", label));
} else {
byte[] meta = snapshot.getMeta();
byte[] jobInfo = snapshot.getJobInfo();
+ long expiredAt = snapshot.getExpiredAt();
- LOG.info("get snapshot info, snapshot: {}, meta size: {}, job info
size: {}",
- label, meta.length, jobInfo.length);
+ LOG.info("get snapshot info, snapshot: {}, meta size: {}, job info
size: {}, expired at: {}",
+ label, meta.length, jobInfo.length, expiredAt);
if (request.isEnableCompress()) {
meta = GZIPUtils.compress(meta);
jobInfo = GZIPUtils.compress(jobInfo);
@@ -2830,6 +2834,7 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
}
result.setMeta(meta);
result.setJobInfo(jobInfo);
+ result.setExpiredAt(expiredAt);
}
return result;
diff --git a/gensrc/thrift/FrontendService.thrift
b/gensrc/thrift/FrontendService.thrift
index 2739b1c2096..6f9242cdd0e 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -1087,6 +1087,7 @@ struct TGetSnapshotResult {
3: optional binary job_info
4: optional Types.TNetworkAddress master_address
5: optional bool compressed;
+ 6: optional i64 expiredAt; // in millis
}
struct TTableRef {
diff --git a/gensrc/thrift/Status.thrift b/gensrc/thrift/Status.thrift
index 2cc9adb344a..bb3450dd120 100644
--- a/gensrc/thrift/Status.thrift
+++ b/gensrc/thrift/Status.thrift
@@ -103,6 +103,13 @@ enum TStatusCode {
TABLET_MISSING = 72,
NOT_MASTER = 73,
+
+ OBTAIN_LOCK_FAILED = 74,
+
+ SNAPSHOT_EXPIRED = 75,
+
+ // Not be larger than 200, see status.h
+ // And all error code defined here, should also be defined in status.h
}
struct TStatus {
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy
index 20a4cf1ffd3..398f1ec2624 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy
@@ -459,6 +459,8 @@ class Syncer {
logger.error("TGetSnapshotResult meta is unset.")
} else if (!result.isSetJobInfo()) {
logger.error("TGetSnapshotResult job info is
unset.")
+ } else if (!result.isSetExpiredAt()) {
+ logger.error("TGetSnapshotResult expiredAt is
unset.")
} else {
isCheckedOK = true
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
