This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-8342
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-8342 by this push:
new 8ee502aa31b HDDS-12794. Improve Lifecycle rule validity checking
(#9593)
8ee502aa31b is described below
commit 8ee502aa31baf8a7eb50201fbff59cd4b4aca4df
Author: XiChen <[email protected]>
AuthorDate: Fri Jan 16 13:12:49 2026 +0800
HDDS-12794. Improve Lifecycle rule validity checking (#9593)
---
.../apache/hadoop/ozone/om/helpers/OmLCFilter.java | 40 ++++---
.../apache/hadoop/ozone/om/helpers/OmLCRule.java | 38 +++---
.../om/helpers/OmLifecycleRuleAndOperator.java | 33 ++++--
.../hadoop/ozone/om/helpers/OmLifecycleUtils.java | 131 +++++++++++++++++++++
.../hadoop/ozone/om/helpers/TestOmLCFilter.java | 28 +++++
.../hadoop/ozone/om/helpers/TestOmLCRule.java | 86 ++++++++++++++
.../om/helpers/TestOmLifecycleRuleAndOperator.java | 32 +++++
.../ozone/om/service/KeyLifecycleService.java | 16 ---
.../ozone/om/service/TestKeyLifecycleService.java | 26 ++--
9 files changed, 350 insertions(+), 80 deletions(-)
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCFilter.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCFilter.java
index 0ef39db6f28..387c4b24df1 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCFilter.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCFilter.java
@@ -17,10 +17,13 @@
package org.apache.hadoop.ozone.om.helpers;
-import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.isValidKeyPath;
-import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.normalizePrefix;
+import static
org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateAndNormalizePrefix;
+import static
org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validatePrefixLength;
+import static
org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTagUniqAndLength;
+import static
org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTrashPrefix;
import jakarta.annotation.Nullable;
+import java.util.Collections;
import net.jcip.annotations.Immutable;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -59,12 +62,17 @@ private OmLCFilter(Builder builder) {
/**
* Validates the OmLCFilter.
- * Ensures that only one of prefix, tag, or andOperator is set.
- * You can specify an empty filter, in which case the rule applies to all
objects in the bucket.
- * Prefix can be "", in which case the rule applies to all objects in the
bucket.
* Ref: <a
href="https://docs.aws.amazon.com/AmazonS3/latest/userguide/intro-lifecycle-filters.html#filter-examples";>...</a>
- * If the validation fails, an OMException is thrown.
+ * - Only one of prefix, tag, or andOperator is set.
+ * - You can specify an empty filter, in which case the rule applies to all
objects in the bucket.
+ * - Prefix can be "", in which case the rule applies to all objects in the
bucket.
+ * - Prefix length must be a length between 0 and 1024.
+ * - Tag's key must be a length between 1 and 128.
+ * - Tag's value must be a length between 0 and 256.
+ * - For FSO bucket, the prefix must be normalized and valid path.
+ * - Prefix cannot be the Trash directory or any of its subdirectories.
*
+ * @param layout The bucket layout for validation
* @throws OMException if the filter is invalid.
*/
public void valid(BucketLayout layout) throws OMException {
@@ -78,17 +86,17 @@ public void valid(BucketLayout layout) throws OMException {
OMException.ResultCodes.INVALID_REQUEST);
}
+ if (hasPrefix) {
+ validatePrefixLength(prefix);
+ validateTrashPrefix(prefix);
+ }
+
+ if (hasTag()) {
+ validateTagUniqAndLength(Collections.singletonMap(tagKey, tagValue));
+ }
+
if (hasPrefix && layout == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
- String normalizedPrefix = normalizePrefix(prefix);
- if (!normalizedPrefix.equals(prefix)) {
- throw new OMException("Prefix format is not supported. Please use " +
normalizedPrefix +
- " instead of " + prefix + ".",
OMException.ResultCodes.INVALID_REQUEST);
- }
- try {
- isValidKeyPath(normalizedPrefix);
- } catch (OMException e) {
- throw new OMException("Prefix is not a valid key path: " + prefix,
OMException.ResultCodes.INVALID_REQUEST);
- }
+ validateAndNormalizePrefix(prefix);
}
if (andOperator != null) {
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java
index b539830006a..00875842655 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java
@@ -17,8 +17,9 @@
package org.apache.hadoop.ozone.om.helpers;
-import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.isValidKeyPath;
-import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.normalizePrefix;
+import static
org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateAndNormalizePrefix;
+import static
org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validatePrefixLength;
+import static
org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTrashPrefix;
import jakarta.annotation.Nullable;
import java.util.ArrayList;
@@ -142,14 +143,17 @@ public boolean isTagEnable() {
/**
* Validates the lifecycle rule.
- * - ID length should not exceed the allowed limit
- * - At least one action must be specified
- * - Filter and Prefix cannot be used together
- * - Filter and prefix cannot both be null
+ * - ID length should not exceed the allowed limit.
+ * - At least one action must be specified, and the expiration type Action
can have at most one.
+ * - Filter and Prefix cannot be used together.
+ * - Filter and prefix cannot both be null.
* - Prefix can be "", in which case the rule applies to all objects in the
bucket.
- * - Actions must be valid
- * - Filter must be valid
- * - There must be at most one Expiration action per rule
+ * - Prefix length must be a length between 0 and 1024.
+ * - For FSO bucket, the prefix must be normalized and valid path.
+ * - Prefix cannot be the Trash directory or any of its subdirectories.
+ * - Actions must be valid.
+ * - Filter must be valid.
+ * - There must be at most one Expiration action per rule.
*
* @param bucketLayout The bucket layout for validation
* @param creationTime The creation time of the lifecycle configuration in
milliseconds since epoch
@@ -189,17 +193,13 @@ public void valid(BucketLayout bucketLayout, Long
creationTime) throws OMExcepti
OMException.ResultCodes.INVALID_REQUEST);
}
+ if (prefix != null) {
+ validatePrefixLength(prefix);
+ validateTrashPrefix(prefix);
+ }
+
if (prefix != null && bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
- String normalizedPrefix = normalizePrefix(prefix);
- if (!normalizedPrefix.equals(prefix)) {
- throw new OMException("Prefix format is not supported. Please use " +
normalizedPrefix +
- " instead of " + prefix + ".",
OMException.ResultCodes.INVALID_REQUEST);
- }
- try {
- isValidKeyPath(normalizedPrefix);
- } catch (OMException e) {
- throw new OMException("Prefix is not a valid key path: " + prefix,
OMException.ResultCodes.INVALID_REQUEST);
- }
+ validateAndNormalizePrefix(prefix);
}
if (filter != null) {
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleRuleAndOperator.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleRuleAndOperator.java
index 2abef21c855..70228846c22 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleRuleAndOperator.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleRuleAndOperator.java
@@ -17,8 +17,10 @@
package org.apache.hadoop.ozone.om.helpers;
-import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.isValidKeyPath;
-import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.normalizePrefix;
+import static
org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateAndNormalizePrefix;
+import static
org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validatePrefixLength;
+import static
org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTagUniqAndLength;
+import static
org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTrashPrefix;
import jakarta.annotation.Nonnull;
import jakarta.annotation.Nullable;
@@ -77,7 +79,14 @@ public boolean isDirectoryStylePrefix() {
* - If there are tags and no prefix, the tags should be more than one.
* - Prefix can be "".
* - Prefix alone is not allowed.
+ * - Prefix length must be a length between 0 and 1024.
+ * - The key of a tag must be unique.
+ * - Tag's key must be a length between 1 and 128.
+ * - Tag's value must be a length between 0 and 256.
+ * - Prefix cannot be the Trash directory or any of its subdirectories.
+ * - For FSO bucket, the prefix must be normalized and valid path
*
+ * @param layout The bucket layout for validation
* @throws OMException if the validation fails.
*/
public void valid(BucketLayout layout) throws OMException {
@@ -102,17 +111,17 @@ public void valid(BucketLayout layout) throws OMException
{
OMException.ResultCodes.INVALID_REQUEST);
}
+ if (hasTags) {
+ validateTagUniqAndLength(tags);
+ }
+
+ if (hasPrefix) {
+ validatePrefixLength(prefix);
+ validateTrashPrefix(prefix);
+ }
+
if (hasPrefix && layout == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
- String normalizedPrefix = normalizePrefix(prefix);
- if (!normalizedPrefix.equals(prefix)) {
- throw new OMException("Prefix format is not supported. Please use " +
normalizedPrefix +
- " instead of " + prefix + ".",
OMException.ResultCodes.INVALID_REQUEST);
- }
- try {
- isValidKeyPath(normalizedPrefix);
- } catch (OMException e) {
- throw new OMException("Prefix is not a valid key path: " + prefix,
OMException.ResultCodes.INVALID_REQUEST);
- }
+ validateAndNormalizePrefix(prefix);
}
}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleUtils.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleUtils.java
new file mode 100644
index 00000000000..32781280919
--- /dev/null
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleUtils.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.isValidKeyPath;
+import static org.apache.hadoop.ozone.om.helpers.OzoneFSUtils.normalizePrefix;
+
+import java.nio.charset.StandardCharsets;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+
+/**
+ * Utility class for Ozone Lifecycle.
+ */
+public final class OmLifecycleUtils {
+
+ // Ref:
https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html
+ public static final int MAX_PREFIX_LENGTH = 1024;
+
+ // Ref:
https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html
+ public static final int MAX_TAG_KEY_LENGTH = 128;
+ public static final int MAX_TAG_VALUE_LENGTH = 256;
+
+ private OmLifecycleUtils() {
+ }
+
+ /**
+ * Check if the prefix is a Trash path.
+ *
+ * @param prefix the prefix to check
+ * @throws OMException if the prefix is a trash path
+ */
+ public static void validateTrashPrefix(String prefix) throws OMException {
+ if (StringUtils.isEmpty(prefix)) {
+ return;
+ }
+ // Remove leading slash if present for validation
+ String p = prefix.startsWith(OZONE_URI_DELIMITER) ? prefix.substring(1) :
prefix;
+
+ if (p.startsWith(FileSystem.TRASH_PREFIX + OZONE_URI_DELIMITER) ||
+ p.equals(FileSystem.TRASH_PREFIX)) {
+ throw new OMException("Lifecycle rule prefix cannot be trash root " +
+ FileSystem.TRASH_PREFIX + OZONE_URI_DELIMITER,
OMException.ResultCodes.INVALID_REQUEST);
+ }
+ }
+
+ /**
+ * Normalize and validate the prefix for FILE_SYSTEM_OPTIMIZED layout.
+ *
+ * @param prefix the prefix to validate
+ * @throws OMException if the prefix is invalid
+ */
+ public static void validateAndNormalizePrefix(String prefix) throws
OMException {
+ String normalizedPrefix = normalizePrefix(prefix);
+ if (!normalizedPrefix.equals(prefix)) {
+ throw new OMException("Prefix format is not supported. Please use " +
normalizedPrefix +
+ " instead of " + prefix + ".",
OMException.ResultCodes.INVALID_REQUEST);
+ }
+ try {
+ isValidKeyPath(normalizedPrefix);
+ } catch (OMException e) {
+ throw new OMException("Prefix is not a valid key path: " + prefix,
OMException.ResultCodes.INVALID_REQUEST);
+ }
+ }
+
+ /**
+ * Validate prefix length.
+ *
+ * @param prefix the prefix to validate
+ * @throws OMException if the prefix length exceeds 1024
+ */
+ public static void validatePrefixLength(String prefix) throws OMException {
+ if (prefix != null && prefix.getBytes(StandardCharsets.UTF_8).length >
MAX_PREFIX_LENGTH) {
+ throw new OMException("The maximum size of a prefix is " +
MAX_PREFIX_LENGTH,
+ OMException.ResultCodes.INVALID_REQUEST);
+ }
+ }
+
+ /**
+ * Validate tag key, value length and the uniqueness of the key.
+ *
+ * @param tags the tags to validate
+ * @throws OMException if the tag key or value is invalid
+ */
+ public static void validateTagUniqAndLength(Map<String, String> tags) throws
OMException {
+ if (tags == null) {
+ return;
+ }
+ Set<String> keys = new HashSet<>();
+ for (Map.Entry<String, String> entry : tags.entrySet()) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+
+ if (StringUtils.isEmpty(key) ||
key.getBytes(StandardCharsets.UTF_8).length > MAX_TAG_KEY_LENGTH) {
+ throw new OMException("A Tag's Key must be a length between 1 and " +
+ MAX_TAG_KEY_LENGTH, OMException.ResultCodes.INVALID_REQUEST);
+ }
+
+ if (!StringUtils.isEmpty(value) &&
value.getBytes(StandardCharsets.UTF_8).length > MAX_TAG_VALUE_LENGTH) {
+ throw new OMException("A Tag's Value must be a length between 0 and " +
+ MAX_TAG_VALUE_LENGTH, OMException.ResultCodes.INVALID_REQUEST);
+ }
+
+ if (!keys.add(key)) {
+ throw new OMException("Duplicate Tag Keys are not allowed",
+ OMException.ResultCodes.INVALID_REQUEST);
+ }
+ }
+ }
+}
+
diff --git
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCFilter.java
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCFilter.java
index bd3809ba10a..75f693d8461 100644
---
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCFilter.java
+++
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCFilter.java
@@ -30,7 +30,9 @@
import static org.junit.jupiter.api.Assertions.assertNull;
import java.util.Collections;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleFilter;
import org.junit.jupiter.api.Test;
@@ -90,6 +92,32 @@ public void testInValidFilter() {
}
+ @Test
+ public void testFilterValidation() {
+ // 1. Prefix is Trash path
+ OmLCFilter.Builder trashPrefixFilter =
getOmLCFilterBuilder(FileSystem.TRASH_PREFIX, null, null);
+ assertOMException(() ->
trashPrefixFilter.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST,
+ "Lifecycle rule prefix cannot be trash root");
+
+ // 2. Prefix too long
+ String longPrefix = RandomStringUtils.randomAlphanumeric(1025);
+ OmLCFilter.Builder longPrefixFilter = getOmLCFilterBuilder(longPrefix,
null, null);
+ assertOMException(() ->
longPrefixFilter.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST,
+ "The maximum size of a prefix is 1024");
+
+ // 3. Tag key too long
+ String longKey = RandomStringUtils.randomAlphanumeric(129);
+ OmLCFilter.Builder longKeyFilter = getOmLCFilterBuilder(null,
Pair.of(longKey, "value"), null);
+ assertOMException(() -> longKeyFilter.build().valid(BucketLayout.DEFAULT),
INVALID_REQUEST,
+ "A Tag's Key must be a length between 1 and 128");
+
+ // 4. Tag value too long
+ String longValue = RandomStringUtils.randomAlphanumeric(257);
+ OmLCFilter.Builder longValueFilter = getOmLCFilterBuilder(null,
Pair.of("key", longValue), null);
+ assertOMException(() ->
longValueFilter.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST,
+ "A Tag's Value must be a length between 0 and 256");
+ }
+
@Test
public void testProtobufConversion() throws OMException {
// Only prefix
diff --git
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java
index b9993bce114..4524e4adfa8 100644
---
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java
+++
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java
@@ -34,7 +34,9 @@
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleRule;
import org.junit.jupiter.api.Test;
@@ -186,6 +188,90 @@ public void testDuplicateRuleIDs() throws OMException {
assertOMException(() -> config.buildAndValid(), INVALID_REQUEST,
"Duplicate rule IDs found");
}
+ @Test
+ public void testTrashPrefixValidation() throws OMException {
+ long currentTime = System.currentTimeMillis();
+ OmLCExpiration exp = new OmLCExpiration.Builder()
+ .setDays(30)
+ .build();
+
+ // Case 1: Prefix is .Trash
+ OmLCRule.Builder r1 = new OmLCRule.Builder()
+ .setId("trash-rule-1")
+ .setEnabled(true)
+ .setPrefix(FileSystem.TRASH_PREFIX)
+ .setAction(exp);
+ assertOMException(() -> r1.build().valid(BucketLayout.DEFAULT,
currentTime), INVALID_REQUEST,
+ "Lifecycle rule prefix cannot be trash root");
+
+ // Case 4: Prefix is .Trash/subdir
+ OmLCRule.Builder r2 = new OmLCRule.Builder()
+ .setId("trash-rule-2")
+ .setEnabled(true)
+ .setPrefix(FileSystem.TRASH_PREFIX + "/user")
+ .setAction(exp);
+ assertOMException(() -> r2.build().valid(BucketLayout.DEFAULT,
currentTime), INVALID_REQUEST,
+ "Lifecycle rule prefix cannot be trash root");
+ }
+
+ @Test
+ public void testTagValidation() throws OMException {
+ long currentTime = System.currentTimeMillis();
+ OmLCExpiration exp = new OmLCExpiration.Builder()
+ .setDays(30)
+ .build();
+
+ // Case 1: Tag key too long
+ String longKey =
RandomStringUtils.randomAlphanumeric(OmLifecycleUtils.MAX_TAG_KEY_LENGTH + 1);
+ OmLCFilter filterKeyTooLong = getOmLCFilterBuilder(null, Pair.of(longKey,
"value"), null).build();
+ OmLCRule.Builder r1 = new OmLCRule.Builder()
+ .setId("long-tag-key")
+ .setEnabled(true)
+ .setFilter(filterKeyTooLong)
+ .setAction(exp);
+ assertOMException(() -> r1.build().valid(BucketLayout.DEFAULT,
currentTime), INVALID_REQUEST,
+ "A Tag's Key must be a length between 1 and 128");
+
+ // Case 2: Tag value too long
+ String longValue =
RandomStringUtils.randomAlphanumeric(OmLifecycleUtils.MAX_TAG_VALUE_LENGTH + 1);
+ OmLCFilter filterValueTooLong = getOmLCFilterBuilder(null, Pair.of("key",
longValue), null).build();
+ OmLCRule.Builder r2 = new OmLCRule.Builder()
+ .setId("long-tag-value")
+ .setEnabled(true)
+ .setFilter(filterValueTooLong)
+ .setAction(exp);
+ assertOMException(() -> r2.build().valid(BucketLayout.DEFAULT,
currentTime), INVALID_REQUEST,
+ "A Tag's Value must be a length between 0 and 256");
+ }
+
+ @Test
+ public void testPrefixLengthValidation() throws OMException {
+ long currentTime = System.currentTimeMillis();
+ OmLCExpiration exp = new OmLCExpiration.Builder()
+ .setDays(30)
+ .build();
+
+ // Case 1: Prefix too long
+ String longPrefix =
RandomStringUtils.randomAlphanumeric(OmLifecycleUtils.MAX_PREFIX_LENGTH + 1);
+ OmLCRule.Builder r1 = new OmLCRule.Builder()
+ .setId("long-prefix")
+ .setEnabled(true)
+ .setPrefix(longPrefix)
+ .setAction(exp);
+ assertOMException(() -> r1.build().valid(BucketLayout.DEFAULT,
currentTime), INVALID_REQUEST,
+ "The maximum size of a prefix is 1024");
+
+ // Case 2: Filter Prefix too long
+ OmLCFilter filterPrefixTooLong = getOmLCFilterBuilder(longPrefix, null,
null).build();
+ OmLCRule.Builder r2 = new OmLCRule.Builder()
+ .setId("filter-long-prefix")
+ .setEnabled(true)
+ .setFilter(filterPrefixTooLong)
+ .setAction(exp);
+ assertOMException(() -> r2.build().valid(BucketLayout.DEFAULT,
currentTime), INVALID_REQUEST,
+ "The maximum size of a prefix is 1024");
+ }
+
@Test
public void testProtobufConversion() throws OMException {
// Only Filter
diff --git
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifecycleRuleAndOperator.java
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifecycleRuleAndOperator.java
index 43c7ffbc41b..164846bba7e 100644
---
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifecycleRuleAndOperator.java
+++
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifecycleRuleAndOperator.java
@@ -28,6 +28,8 @@
import com.google.common.collect.ImmutableMap;
import java.util.Collections;
import java.util.Map;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleRuleAndOperator;
import org.junit.jupiter.api.Test;
@@ -72,6 +74,36 @@ public void testInValidAndOperator() {
"Either 'Tags' or 'Prefix' must be specified.");
}
+ @Test
+ public void testValidation() {
+ // 1. Prefix is Trash path
+ OmLifecycleRuleAndOperator.Builder trashPrefixAndOp =
getOmLCAndOperatorBuilder(
+ FileSystem.TRASH_PREFIX, Collections.singletonMap("tag1", "value1"));
+ assertOMException(() ->
trashPrefixAndOp.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST,
+ "Lifecycle rule prefix cannot be trash root");
+
+ // 2. Prefix too long
+ String longPrefix = RandomStringUtils.randomAlphanumeric(1025);
+ OmLifecycleRuleAndOperator.Builder longPrefixAndOp =
getOmLCAndOperatorBuilder(
+ longPrefix, Collections.singletonMap("tag1", "value1"));
+ assertOMException(() ->
longPrefixAndOp.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST,
+ "The maximum size of a prefix is 1024");
+
+ // 3. Tag key too long
+ String longKey = RandomStringUtils.randomAlphanumeric(129);
+ OmLifecycleRuleAndOperator.Builder longKeyAndOp =
getOmLCAndOperatorBuilder(
+ "prefix", Collections.singletonMap(longKey, "value"));
+ assertOMException(() -> longKeyAndOp.build().valid(BucketLayout.DEFAULT),
INVALID_REQUEST,
+ "A Tag's Key must be a length between 1 and 128");
+
+ // 4. Tag value too long
+ String longValue = RandomStringUtils.randomAlphanumeric(257);
+ OmLifecycleRuleAndOperator.Builder longValueAndOp =
getOmLCAndOperatorBuilder(
+ "prefix", Collections.singletonMap("key", longValue));
+ assertOMException(() ->
longValueAndOp.build().valid(BucketLayout.DEFAULT), INVALID_REQUEST,
+ "A Tag's Value must be a length between 0 and 256");
+ }
+
@Test
public void testProtobufConversion() throws OMException {
// Prefix and tags
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
index d009b166a1c..3666401da16 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
@@ -377,19 +377,12 @@ public BackgroundTaskResult call() {
private void evaluateFSOBucket(OmVolumeArgs volume, OmBucketInfo bucket,
String bucketKey,
Table<String, OmKeyInfo> keyTable, List<OmLCRule> ruleList,
LimitedExpiredObjectList expiredKeyList, LimitedExpiredObjectList
expiredDirList) {
- List<OmLCRule> prefixStartsWithTrashRuleList =
- ruleList.stream().filter(r -> r.isPrefixEnable() &&
r.getEffectivePrefix().startsWith(
- TRASH_PREFIX +
OzoneConsts.OM_KEY_PREFIX)).collect(Collectors.toList());
List<OmLCRule> prefixRuleList =
ruleList.stream().filter(r ->
r.isPrefixEnable()).collect(Collectors.toList());
// r.isPrefixEnable() == false means empty filter
List<OmLCRule> noPrefixRuleList =
ruleList.stream().filter(r ->
!r.isPrefixEnable()).collect(Collectors.toList());
- prefixRuleList.removeAll(prefixStartsWithTrashRuleList);
- prefixStartsWithTrashRuleList.stream().forEach(
- r -> LOG.info("Skip rule {} as its prefix starts with {}", r,
TRASH_PREFIX + OzoneConsts.OM_KEY_PREFIX));
-
for (OmLCRule rule : prefixRuleList) {
// find KeyInfo of each directory for prefix
List<OmDirectoryInfo> dirList;
@@ -715,15 +708,6 @@ private void evaluateBucket(OmBucketInfo bucketInfo,
String volumeName = bucketInfo.getVolumeName();
String bucketName = bucketInfo.getBucketName();
- if (bucketInfo.getBucketLayout() == BucketLayout.LEGACY) {
- List<OmLCRule> prefixStartsWithTrashRuleList =
- ruleList.stream().filter(r -> r.isPrefixEnable() &&
r.getEffectivePrefix().startsWith(
- TRASH_PREFIX +
OzoneConsts.OM_KEY_PREFIX)).collect(Collectors.toList());
- ruleList.removeAll(prefixStartsWithTrashRuleList);
- prefixStartsWithTrashRuleList.stream().forEach(
- r -> LOG.info("Skip rule {} as its prefix starts with {}", r,
TRASH_PREFIX + OzoneConsts.OM_KEY_PREFIX));
- }
-
// use bucket name as key iterator prefix
try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
keyTblItr =
keyTable.iterator(omMetadataManager.getBucketKey(volumeName,
bucketName))) {
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
index 8d53ab8f8a9..3769ba41a8c 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
@@ -38,6 +38,7 @@
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertThrowsExactly;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
@@ -1405,24 +1406,14 @@ void testMoveToTrash(BucketLayout bucketLayout, String
prefix) throws IOExceptio
}
// create new policy to test rule with prefix ".Trash/" is ignored
during lifecycle evaluation
now = ZonedDateTime.now(ZoneOffset.UTC);
- date = now.plusSeconds(EXPIRE_SECONDS);
- createLifecyclePolicy(volumeName, bucketName, bucketLayout, TRASH_PREFIX
+ OM_KEY_PREFIX,
- null, date.toString(), true);
-
- GenericTestUtils.waitFor(
- () -> log.getOutput().contains("Skip rule") &&
- log.getOutput().contains("as its prefix starts with " +
TRASH_PREFIX + OM_KEY_PREFIX),
- WAIT_CHECK_INTERVAL, 5000);
- deleteLifecyclePolicy(volumeName, bucketName);
+ final String expiredDate = now.plusSeconds(EXPIRE_SECONDS).toString();
+ assertThrowsExactly(OMException.class, () -> createLifecyclePolicy(
+ volumeName, bucketName, bucketLayout, TRASH_PREFIX + OM_KEY_PREFIX,
null, expiredDate, true));
// create new policy to test rule with prefix ".Trash" is ignored during
lifecycle evaluation
- now = ZonedDateTime.now(ZoneOffset.UTC);
- date = now.plusSeconds(EXPIRE_SECONDS);
- createLifecyclePolicy(volumeName, bucketName, bucketLayout,
TRASH_PREFIX, null, date.toString(), true);
-
- GenericTestUtils.waitFor(
- () -> log.getOutput().contains("Skip evaluate trash directory " +
TRASH_PREFIX), WAIT_CHECK_INTERVAL, 5000);
- deleteLifecyclePolicy(volumeName, bucketName);
+ assertThrowsExactly(OMException.class, () -> createLifecyclePolicy(
+ volumeName, bucketName, bucketLayout, TRASH_PREFIX,
+ null, expiredDate, true));
// create new policy to test rule with prefix ".Tras" is ignored during
lifecycle evaluation
now = ZonedDateTime.now(ZoneOffset.UTC);
@@ -1430,8 +1421,9 @@ void testMoveToTrash(BucketLayout bucketLayout, String
prefix) throws IOExceptio
createLifecyclePolicy(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED,
".Tras", null, date.toString(), true);
GenericTestUtils.waitFor(
- () -> log.getOutput().contains("Skip evaluate trash directory " +
TRASH_PREFIX), WAIT_CHECK_INTERVAL, 5000);
+ () -> log.getOutput().contains("No expired keys/dirs found/remained
for bucket"), WAIT_CHECK_INTERVAL, 5000);
deleteLifecyclePolicy(volumeName, bucketName);
+ log.clearOutput();
// create new policy to test trash directory is skipped during lifecycle
evaluation
now = ZonedDateTime.now(ZoneOffset.UTC);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
