This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-8342
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-8342 by this push:
new 3b5cb65c82a HDDS-12784. Add bucket object ID to lifecycle
configuration proto definition (#9255)
3b5cb65c82a is described below
commit 3b5cb65c82a1b1c1307660b01765bf876c3199c0
Author: Sammi Chen <[email protected]>
AuthorDate: Mon Nov 17 12:04:06 2025 +0800
HDDS-12784. Add bucket object ID to lifecycle configuration proto
definition (#9255)
---
.../ozone/om/helpers/OmLifecycleConfiguration.java | 38 ++++++++++++++++++++--
.../src/main/proto/OmClientProtocol.proto | 1 +
.../OMLifecycleConfigurationSetRequest.java | 21 +++++++-----
.../om/response/bucket/OMBucketDeleteResponse.java | 3 +-
.../OMLifecycleConfigurationSetResponse.java | 4 +++
.../ozone/om/service/KeyLifecycleService.java | 6 ++++
.../TestOMLifecycleConfigurationSetRequest.java | 5 ++-
.../ozone/om/service/TestKeyLifecycleService.java | 6 +++-
.../s3/endpoint/S3LifecycleConfiguration.java | 1 +
9 files changed, 70 insertions(+), 15 deletions(-)
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleConfiguration.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleConfiguration.java
index ad6d31d7cf1..b19a0c5e713 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleConfiguration.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleConfiguration.java
@@ -56,6 +56,7 @@ public static Codec<OmLifecycleConfiguration> getCodec() {
public static final int LC_MAX_RULES = 1000;
private final String volume;
private final String bucket;
+ private final Long bucketObjectID;
private final BucketLayout bucketLayout;
private final long creationTime;
private final List<OmLCRule> rules;
@@ -68,6 +69,7 @@ private OmLifecycleConfiguration() {
super(builder);
this.volume = builder.volume;
this.bucket = builder.bucket;
+ this.bucketObjectID = builder.bucketObjectID;
this.rules = Collections.unmodifiableList(builder.rules);
this.creationTime = builder.creationTime;
this.bucketLayout = builder.bucketLayout;
@@ -85,6 +87,10 @@ public String getVolume() {
return volume;
}
+ public Long getBucketObjectID() {
+ return bucketObjectID;
+ }
+
public long getCreationTime() {
return creationTime;
}
@@ -142,12 +148,16 @@ private boolean hasNoDuplicateID() {
}
public Builder toBuilder() {
- return new Builder(this)
- .setVolume(this.volume)
+ Builder builder = new Builder(this);
+ builder.setVolume(this.volume)
.setBucket(this.bucket)
.setBucketLayout(bucketLayout)
.setCreationTime(this.creationTime)
.setRules(this.rules);
+ if (bucketObjectID != null) {
+ builder.setBucketObjectID(bucketObjectID);
+ }
+ return builder;
}
@Override
@@ -155,6 +165,7 @@ public String toString() {
return "OmLifecycleConfiguration{" +
"volume='" + volume + '\'' +
", bucket='" + bucket + '\'' +
+ ", bucketObjectID='" + bucketObjectID + '\'' +
", creationTime=" + creationTime +
", rulesCount=" + rules.size() +
", objectID=" + getObjectID() +
@@ -167,6 +178,9 @@ public Map<String, String> toAuditMap() {
Map<String, String> auditMap = new LinkedHashMap<>();
auditMap.put(OzoneConsts.VOLUME, this.volume);
auditMap.put(OzoneConsts.BUCKET, this.bucket);
+ if (this.bucketObjectID != null) {
+ auditMap.put(OzoneConsts.OBJECT_ID, String.valueOf(this.bucketObjectID));
+ }
auditMap.put(OzoneConsts.CREATION_TIME, String.valueOf(this.creationTime));
return auditMap;
@@ -195,11 +209,20 @@ public LifecycleConfiguration getProtobuf() {
.setObjectID(getObjectID())
.setUpdateID(getUpdateID());
+ if (bucketObjectID != null) {
+ b.setBucketObjectID(bucketObjectID);
+ }
+
return b.build();
}
public static OmLifecycleConfiguration getFromProtobuf(
LifecycleConfiguration lifecycleConfiguration) throws OMException {
+ return getBuilderFromProtobuf(lifecycleConfiguration).build();
+ }
+
+ public static OmLifecycleConfiguration.Builder getBuilderFromProtobuf(
+ LifecycleConfiguration lifecycleConfiguration) throws OMException {
List<OmLCRule> rulesList = new ArrayList<>();
BucketLayout layout =
BucketLayout.fromProto(lifecycleConfiguration.getBucketLayout());
for (LifecycleRule lifecycleRule : lifecycleConfiguration.getRulesList()) {
@@ -220,8 +243,11 @@ public static OmLifecycleConfiguration getFromProtobuf(
if (lifecycleConfiguration.hasUpdateID()) {
builder.setUpdateID(lifecycleConfiguration.getUpdateID());
}
+ if (lifecycleConfiguration.hasBucketObjectID()) {
+ builder.setBucketObjectID(lifecycleConfiguration.getBucketObjectID());
+ }
- return builder.build();
+ return builder;
}
/**
@@ -230,6 +256,7 @@ public static OmLifecycleConfiguration getFromProtobuf(
public static class Builder extends WithObjectID.Builder {
private String volume = "";
private String bucket = "";
+ private Long bucketObjectID;
private BucketLayout bucketLayout;
private long creationTime;
private List<OmLCRule> rules = new ArrayList<>();
@@ -251,6 +278,11 @@ public Builder setBucket(String bucketName) {
return this;
}
+ public Builder setBucketObjectID(long bucketID) {
+ this.bucketObjectID = bucketID;
+ return this;
+ }
+
public Builder setBucketLayout(BucketLayout layout) {
this.bucketLayout = layout;
return this;
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 027f03a4ad7..ba35dab8470 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -2395,6 +2395,7 @@ message LifecycleConfiguration {
repeated LifecycleRule rules = 5;
optional uint64 objectID = 6;
optional uint64 updateID = 7;
+ optional uint64 bucketObjectID = 8;
}
message SetLifecycleConfigurationRequest {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleConfigurationSetRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleConfigurationSetRequest.java
index da395fa30c1..27f797eb1e8 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleConfigurationSetRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleConfigurationSetRequest.java
@@ -34,6 +34,7 @@
import org.apache.hadoop.ozone.om.ResolvedBucket;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmLifecycleConfiguration;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.request.util.OmResponseUtil;
@@ -127,29 +128,31 @@ public OMClientResponse
validateAndUpdateCache(OzoneManager ozoneManager, Execut
AuditLogger auditLogger = ozoneManager.getAuditLogger();
UserInfo userInfo = getOmRequest().getUserInfo();
- String bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
IOException exception = null;
boolean acquiredBucketLock = false;
OMClientResponse omClientResponse = null;
Map<String, String> auditMap = new HashMap<>();
-
try {
- OmLifecycleConfiguration omLifecycleConfiguration =
- OmLifecycleConfiguration.getFromProtobuf(lifecycleConfiguration);
- auditMap = omLifecycleConfiguration.toAuditMap();
-
mergeOmLockDetails(metadataManager.getLock()
.acquireWriteLock(BUCKET_LOCK, volumeName, bucketName));
acquiredBucketLock = getOmLockDetails().isLockAcquired();
- omLifecycleConfiguration.valid();
- if (!metadataManager.getBucketTable().isExist(bucketKey)) {
+ String bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
+ OmBucketInfo bucketInfo =
metadataManager.getBucketTable().get(bucketKey);
+ if (bucketInfo == null) {
LOG.debug("bucket: {} in volume: {} doesn't exist", bucketName,
volumeName);
throw new OMException("Bucket doesn't exist", BUCKET_NOT_FOUND);
}
- omLifecycleConfiguration.setUpdateID(transactionLogIndex);
+ OmLifecycleConfiguration.Builder lcBuilder =
+
OmLifecycleConfiguration.getBuilderFromProtobuf(lifecycleConfiguration);
+ OmLifecycleConfiguration omLifecycleConfiguration =
+ lcBuilder.setBucketObjectID(bucketInfo.getObjectID()).build();
+ omLifecycleConfiguration.valid();
+ auditMap = omLifecycleConfiguration.toAuditMap();
+
+ omLifecycleConfiguration.setUpdateID(transactionLogIndex);
metadataManager.getLifecycleConfigurationTable().addCacheEntry(
new CacheKey<>(bucketKey),
CacheValue.get(transactionLogIndex, omLifecycleConfiguration));
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/bucket/OMBucketDeleteResponse.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/bucket/OMBucketDeleteResponse.java
index b6bd1cf0373..fd7c27defee 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/bucket/OMBucketDeleteResponse.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/bucket/OMBucketDeleteResponse.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.om.response.bucket;
import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.BUCKET_TABLE;
+import static
org.apache.hadoop.ozone.om.OmMetadataManagerImpl.LIFECYCLE_CONFIGURATION_TABLE;
import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.VOLUME_TABLE;
import jakarta.annotation.Nonnull;
@@ -32,7 +33,7 @@
/**
* Response for DeleteBucket request.
*/
-@CleanupTableInfo(cleanupTables = {BUCKET_TABLE, VOLUME_TABLE})
+@CleanupTableInfo(cleanupTables = {BUCKET_TABLE, VOLUME_TABLE,
LIFECYCLE_CONFIGURATION_TABLE})
public final class OMBucketDeleteResponse extends OMClientResponse {
private String volumeName;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleConfigurationSetResponse.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleConfigurationSetResponse.java
index fcfbaa761ea..bed7cf21789 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleConfigurationSetResponse.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleConfigurationSetResponse.java
@@ -61,4 +61,8 @@ protected void addToDBBatch(OMMetadataManager
omMetadataManager,
omMetadataManager.getLifecycleConfigurationTable().putWithBatch(
batchOperation, dbLifecycleKey, omLifecycleConfiguration);
}
+
+ public OmLifecycleConfiguration getOmLifecycleConfiguration() {
+ return omLifecycleConfiguration;
+ }
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
index 9e7112f1b31..680ba2bdbb0 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
@@ -270,6 +270,12 @@ public BackgroundTaskResult call() {
onFailure(bucketKey);
return result;
}
+ if (bucket.getObjectID() != policy.getBucketObjectID()) {
+ LOG.warn("Bucket object ID doesn't match. ID in bucket is {}, ID
in LifecycleConfiguration is {}.",
+ bucket.getObjectID(), policy.getBucketObjectID());
+ onFailure(bucketKey);
+ return result;
+ }
} catch (IOException e) {
LOG.warn("Failed to get Bucket {}", bucketKey, e);
onFailure(bucketKey);
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationSetRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationSetRequest.java
index 75de1eee84d..3aac4a8f573 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationSetRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationSetRequest.java
@@ -35,6 +35,7 @@
import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
+import
org.apache.hadoop.ozone.om.response.lifecycle.OMLifecycleConfigurationSetResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleConfiguration;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
@@ -108,7 +109,8 @@ public void testValidateAndUpdateCacheSuccess() throws
Exception {
OMClientResponse omClientResponse =
request.validateAndUpdateCache(ozoneManager, txLogIndex);
OMResponse omResponse = omClientResponse.getOMResponse();
-
+ OMLifecycleConfigurationSetResponse response =
(OMLifecycleConfigurationSetResponse) omClientResponse;
+ assertNotNull(response.getOmLifecycleConfiguration().getBucketObjectID());
assertNotNull(omResponse.getSetLifecycleConfigurationResponse());
assertEquals(OzoneManagerProtocolProtos.Status.OK,
omResponse.getStatus());
@@ -177,6 +179,7 @@ public void testValidateAndUpdateInvalidLCC() throws
Exception {
String ownerName = "ownerName";
OMRequestTestUtils.addVolumeToDB(volumeName, ownerName, omMetadataManager);
+ OMRequestTestUtils.addBucketToDB(volumeName, bucketName,
omMetadataManager);
OMRequest originalRequest = setLifecycleConfigurationRequest(volumeName,
bucketName, ownerName, false);
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
index cb54173266b..8d9d457ef61 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
@@ -158,6 +158,7 @@ class TestKeyLifecycleService extends OzoneTestBase {
private DirectoryDeletingService directoryDeletingService;
private ScmBlockLocationTestingClient scmBlockTestingClient;
private KeyLifecycleServiceMetrics metrics;
+ private long bucketObjectID;
@BeforeAll
void setup() {
@@ -1779,6 +1780,7 @@ private void createLifecyclePolicy(String volume, String
bucket, BucketLayout la
.setVolume(volume)
.setBucket(bucket)
.setBucketLayout(layout)
+ .setBucketObjectID(bucketObjectID)
.setRules(Collections.singletonList(new OmLCRule.Builder()
.setId(String.valueOf(OBJECT_ID_COUNTER.getAndIncrement()))
.setEnabled(enabled)
@@ -1803,6 +1805,7 @@ private void createLifecyclePolicy(String volume, String
bucket, BucketLayout la
OmLifecycleConfiguration lcc = new OmLifecycleConfiguration.Builder()
.setVolume(volume)
.setBucket(bucket)
+ .setBucketObjectID(bucketObjectID)
.setBucketLayout(layout)
.setRules(ruleList)
.build();
@@ -1835,12 +1838,13 @@ private void createVolumeAndBucket(String volumeName,
.setObjectID(OBJECT_ID_COUNTER.incrementAndGet())
.build());
+ bucketObjectID = OBJECT_ID_COUNTER.incrementAndGet();
OMRequestTestUtils.addBucketToOM(keyManager.getMetadataManager(),
OmBucketInfo.newBuilder().setVolumeName(volumeName)
.setBucketName(bucketName)
.setBucketLayout(bucketLayout)
.setOwner(owner)
- .setObjectID(OBJECT_ID_COUNTER.incrementAndGet())
+ .setObjectID(bucketObjectID)
.build());
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3LifecycleConfiguration.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3LifecycleConfiguration.java
index 8aa3e52f433..8e459281965 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3LifecycleConfiguration.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3LifecycleConfiguration.java
@@ -251,6 +251,7 @@ public void setAndOperator(AndOperator andOperator) {
}
}
+ // OzoneBucket doesn't have objectID info.
public OmLifecycleConfiguration toOmLifecycleConfiguration(OzoneBucket
ozoneBucket)
throws OS3Exception, OMException {
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
