This is an automated email from the ASF dual-hosted git repository.
peterxcli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 66ae5d34402 HDDS-13041. Add basic snapshot diff report test for object
tag, stream key, rewriteKey (#8923)
66ae5d34402 is described below
commit 66ae5d34402761bc34461ba2a01f1f41fe5bba20
Author: Bolin Lin <[email protected]>
AuthorDate: Mon Aug 18 10:13:38 2025 -0400
HDDS-13041. Add basic snapshot diff report test for object tag, stream key,
rewriteKey (#8923)
---
.../hadoop/ozone/om/snapshot/TestOmSnapshot.java | 1213 ++++++++++++++++++++
1 file changed, 1213 insertions(+)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java
index 038fc491fe7..aac18d5d36d 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java
@@ -93,6 +93,7 @@
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdds.StringUtils;
import org.apache.hadoop.hdds.client.OzoneQuota;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@@ -118,9 +119,12 @@
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.client.OzoneMultipartUploadList;
+import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts;
import org.apache.hadoop.ozone.client.OzoneSnapshot;
import org.apache.hadoop.ozone.client.OzoneSnapshotDiff;
import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.om.KeyManagerImpl;
@@ -134,6 +138,7 @@
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
@@ -175,6 +180,7 @@ public abstract class TestOmSnapshot {
Pattern.compile(SNAPSHOT_KEY_PATTERN_STRING);
private static final int POLL_INTERVAL_MILLIS = 500;
private static final int POLL_MAX_WAIT_MILLIS = 120_000;
+ private static final int MIN_PART_SIZE = 5 * 1024 * 1024;
private MiniOzoneCluster cluster;
private OzoneClient client;
@@ -2676,4 +2682,1211 @@ public void testSnapdiffWithNoOpAPI() throws Exception
{
assertEquals(0, diff.getDiffList().size());
}
}
+
+ @Test
+ public void testSnapshotdiffWithObjectTagModification() throws Exception {
+ String testVolumeName = "vol" + counter.incrementAndGet();
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String snap1 = "snap1";
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ Map<String, String> tags = new HashMap<>();
+ tags.put("environment", "test");
+ tags.put("owner", "testuser");
+ bucket.putObjectTagging(key1, tags);
+
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ SnapshotDiffReport diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap1, snap2);
+
+ List<DiffReportEntry> diffEntries = Lists.newArrayList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.MODIFY,
+ key1));
+ assertEquals(diffEntries, diff.getDiffList());
+ }
+
+ @Test
+ public void testSnapdiffWithMultipleTagOperations() throws Exception {
+ String testVolumeName = "vol" + counter.incrementAndGet();
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+
+ Map<String, String> initialTags = new HashMap<>();
+ initialTags.put("version", "1.0");
+ initialTags.put("team", "dev");
+
+ bucket.putObjectTagging(key1, initialTags);
+
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ Map<String, String> updatedTags = new HashMap<>();
+ updatedTags.put("version", "2.0");
+ updatedTags.put("team", "dev");
+ updatedTags.put("priority", "high");
+
+ bucket.putObjectTagging(key1, updatedTags);
+
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ SnapshotDiffReport diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap1, snap2);
+
+ List<DiffReportEntry> diffEntries = Lists.newArrayList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.MODIFY,
+ key1));
+ assertEquals(diffEntries, diff.getDiffList());
+ }
+
+ @Test
+ public void testSnapdiffWithTagRemoval() throws Exception {
+ String testVolumeName = "vol" + counter.incrementAndGet();
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+
+ Map<String, String> initialTags = new HashMap<>();
+ initialTags.put("env", "staging");
+ initialTags.put("cost-center", "engineering");
+ initialTags.put("backup", "daily");
+
+ bucket.putObjectTagging(key1, initialTags);
+
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ bucket.deleteObjectTagging(key1);
+
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ SnapshotDiffReport diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap1, snap2);
+
+ List<DiffReportEntry> diffEntries = Lists.newArrayList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.MODIFY,
+ key1));
+ assertEquals(diffEntries, diff.getDiffList());
+ }
+
+ private String createStreamKeyWithPrefix(OzoneBucket bucket, String
keyPrefix)
+ throws Exception {
+ String key = keyPrefix + counter.incrementAndGet();
+ return createStreamKey(bucket, key);
+ }
+
+ private String createStreamKey(OzoneBucket bucket, String key)
+ throws Exception {
+ byte[] value = RandomStringUtils.secure().nextAscii(10240).getBytes(UTF_8);
+ OzoneDataStreamOutput streamKey = bucket.createStreamKey(key,
value.length);
+ streamKey.write(value);
+ streamKey.close();
+ GenericTestUtils.waitFor(() -> {
+ try {
+ getOmKeyInfo(bucket.getVolumeName(), bucket.getName(), key);
+ } catch (IOException e) {
+ return false;
+ }
+ return true;
+ }, 1000, 30000);
+ return key;
+ }
+
+ @Test
+ public void testSnapDiffWithStreamKeyModification() throws Exception {
+ String testVolumeName = "vol" + counter.incrementAndGet();
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String key1 = "stream-key-1";
+ key1 = createStreamKeyWithPrefix(bucket, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ String key2 = "stream-key-2";
+ key2 = createStreamKeyWithPrefix(bucket, key2);
+ bucket.deleteKey(key1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
testBucketName,
+ snap1, snap2);
+ assertEquals(2, diff.getDiffList().size());
+ assertEquals(Arrays.asList(SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, key1),
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.CREATE, key2)), diff.getDiffList());
+ }
+
+ @Test
+ public void testSnapDiffWithStreamKeyRewrite() throws Exception {
+ String testVolumeName = "vol" + counter.incrementAndGet();
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String key1 = "stream-key-1";
+ key1 = createStreamKeyWithPrefix(bucket, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ createStreamKey(bucket, key1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
testBucketName, snap1, snap2);
+ assertEquals(1, diff.getDiffList().size());
+ assertEquals(Collections.singletonList(
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.MODIFY,
key1)),
+ diff.getDiffList());
+ }
+
+ @Test
+ public void testSnapDiffWithStreamKeyRename() throws Exception {
+ String testVolumeName = "vol" + counter.incrementAndGet();
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String key1 = "stream-key-1";
+ key1 = createStreamKeyWithPrefix(bucket, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ String renamedKey = key1 + "_renamed";
+ bucket.renameKey(key1, renamedKey);
+ GenericTestUtils.waitFor(() -> {
+ try {
+ getOmKeyInfo(testVolumeName, testBucketName, renamedKey);
+ } catch (IOException e) {
+ return false;
+ }
+ return true;
+ }, 1000, 10000);
+
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap2);
+ assertEquals(1, diff.getDiffList().size());
+ assertEquals(Collections.singletonList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.RENAME, key1, renamedKey)),
+ diff.getDiffList());
+ }
+
+ @Test
+ public void testSnapDiffWithStreamKeyRecreation() throws Exception {
+ String testVolumeName = "vol" + counter.incrementAndGet();
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String key1 = "stream-key-1";
+ key1 = createStreamKeyWithPrefix(bucket, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ getOmKeyInfo(testVolumeName, testBucketName, key1);
+ bucket.deleteKey(key1);
+ key1 = createStreamKey(bucket, key1);
+ getOmKeyInfo(testVolumeName, testBucketName, key1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap2);
+ assertEquals(2, diff.getDiffList().size());
+ assertEquals(Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, key1),
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.CREATE, key1)),
+ diff.getDiffList());
+ }
+
+ @Test
+ public void testSnapDiffWithMixedStreamAndRegularKeys() throws Exception {
+ String testVolumeName = "vol" + counter.incrementAndGet();
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String regularKey1 = "regular-key-1";
+ String streamKey1 = "stream-key-1";
+ regularKey1 = createFileKeyWithPrefix(bucket, regularKey1);
+ streamKey1 = createStreamKeyWithPrefix(bucket, streamKey1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ String regularKey2 = "regular-key-2";
+ String streamKey2 = "stream-key-2";
+ regularKey2 = createFileKeyWithPrefix(bucket, regularKey2);
+ streamKey2 = createStreamKeyWithPrefix(bucket, streamKey2);
+ bucket.deleteKey(regularKey1);
+ bucket.deleteKey(streamKey1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap2);
+ assertEquals(4, diff.getDiffList().size());
+ List<SnapshotDiffReport.DiffReportEntry> expectedDiffs = Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, regularKey1),
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, streamKey1),
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.CREATE, regularKey2),
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.CREATE, streamKey2));
+ assertEquals(expectedDiffs, diff.getDiffList());
+ }
+
+ private String createStreamFileWithPrefix(OzoneBucket bucket, String
filePrefix)
+ throws Exception {
+ String file = filePrefix + counter.incrementAndGet();
+ return createStreamFile(bucket, file, false);
+ }
+
+ private String createStreamFile(OzoneBucket bucket, String fileName, boolean
overWrite)
+ throws Exception {
+ byte[] value = RandomStringUtils.secure().nextAscii(10240).getBytes(UTF_8);
+ OzoneDataStreamOutput streamFile = bucket.createStreamFile(
+ fileName, value.length, bucket.getReplicationConfig(), overWrite,
true);
+ streamFile.write(value);
+ streamFile.close();
+ GenericTestUtils.waitFor(() -> {
+ try {
+ getOmKeyInfo(bucket.getVolumeName(), bucket.getName(), fileName);
+ } catch (IOException e) {
+ return false;
+ }
+ return true;
+ }, 300, 10000);
+ return fileName;
+ }
+
+ @Test
+ public void testSnapDiffWithStreamFileModification() throws Exception {
+ assumeTrue(!bucketLayout.isObjectStore(enabledFileSystemPaths));
+
+ String testVolumeName = "vol" + counter.incrementAndGet();
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String file1 = "stream-file-1";
+ file1 = createStreamFileWithPrefix(bucket, file1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ String file2 = "stream-file-2";
+ file2 = createStreamFileWithPrefix(bucket, file2);
+ bucket.deleteKey(file1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap2);
+ assertEquals(2, diff.getDiffList().size());
+ assertEquals(Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, file1),
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.CREATE, file2)),
+ diff.getDiffList());
+ }
+
+ @Test
+ public void testSnapDiffWithStreamFileRewrite() throws Exception {
+ assumeTrue(!bucketLayout.isObjectStore(enabledFileSystemPaths));
+
+ String testVolumeName = "vol" + counter.incrementAndGet();
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String file1 = "stream-file-1";
+ file1 = createStreamFileWithPrefix(bucket, file1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ createStreamFile(bucket, file1, true);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap2);
+ assertEquals(1, diff.getDiffList().size());
+ assertEquals(Collections.singletonList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.MODIFY, file1)),
+ diff.getDiffList());
+ }
+
+ @Test
+ public void testSnapDiffWithStreamFileRename() throws Exception {
+ assumeTrue(!bucketLayout.isObjectStore(enabledFileSystemPaths));
+
+ String testVolumeName = "vol" + counter.incrementAndGet();
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String file1 = "stream-file-1";
+ file1 = createStreamFileWithPrefix(bucket, file1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ String renamedFile = file1 + "_renamed";
+ bucket.renameKey(file1, renamedFile);
+ GenericTestUtils.waitFor(() -> {
+ try {
+ getOmKeyInfo(testVolumeName, testBucketName, renamedFile);
+ } catch (IOException e) {
+ return false;
+ }
+ return true;
+ }, 1000, 30000);
+
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap2);
+ assertEquals(1, diff.getDiffList().size());
+ assertEquals(Collections.singletonList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.RENAME, file1, renamedFile)),
+ diff.getDiffList());
+ }
+
+ @Test
+ public void testSnapDiffWithStreamFileRecreation() throws Exception {
+ assumeTrue(!bucketLayout.isObjectStore(enabledFileSystemPaths));
+
+ String testVolumeName = "vol" + counter.incrementAndGet();
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String file1 = "stream-file-1";
+ file1 = createStreamFileWithPrefix(bucket, file1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ getOmKeyInfo(testVolumeName, testBucketName, file1);
+ bucket.deleteKey(file1);
+ file1 = createStreamFile(bucket, file1, true);
+ getOmKeyInfo(testVolumeName, testBucketName, file1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap2);
+ assertEquals(2, diff.getDiffList().size());
+ assertEquals(Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, file1),
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.CREATE, file1)),
+ diff.getDiffList());
+ }
+
+ @Test
+ public void testSnapDiffWithStreamFileAndDirectoryOperations() throws
Exception {
+ assumeTrue(bucketLayout.isFileSystemOptimized());
+ String testVolumeName = "vol" + counter.incrementAndGet();
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ bucket.createDirectory("dir1/dir2");
+ String file1 = "dir1/dir2/stream-file-1";
+ createStreamFile(bucket, file1, false);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ String file2 = "dir1/dir2/stream-file-2";
+ createStreamFile(bucket, file2, false);
+ bucket.renameKey("dir1/dir2", "dir1/dir3");
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap2);
+ assertEquals(3, diff.getDiffList().size());
+
+ List<SnapshotDiffReport.DiffReportEntry> expectedDiffs = Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.RENAME, "dir1/dir2", "dir1/dir3"),
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.CREATE, "dir1/dir3/stream-file-2"),
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.MODIFY, "dir1"));
+ assertEquals(expectedDiffs, diff.getDiffList());
+ }
+
+ private ReplicationConfig getDefaultReplication() {
+ return
StandaloneReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE);
+ }
+
+ @Test
+ public void testSnapshotDiffWithInitiateMultipartUpload() throws Exception {
+ String testVolumeName = "vol-initiate" + counter.incrementAndGet();
+ String testBucketName = "bucket-initiate-" + counter.incrementAndGet();
+
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String snap1 = "snap-before-initiate-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ String mpuKey1 = "mpu-key-1" + counter.incrementAndGet();
+ String mpuKey2 = "mpu-key-2" + counter.incrementAndGet();
+
+ Map<String, String> metadata = new HashMap<>();
+ metadata.put("test-metadata", "initiate-test");
+
+ Map<String, String> tags = new HashMap<>();
+ tags.put("environment", "test");
+ tags.put("operation", "initiate");
+
+ OmMultipartInfo mpuInfo1 = bucket.initiateMultipartUpload(mpuKey1);
+ OmMultipartInfo mpuInfo2 = bucket.initiateMultipartUpload(mpuKey2,
getDefaultReplication(), metadata, tags);
+
+ String snap2 = "snap-after-initiate-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
testBucketName, snap1, snap2);
+ assertEquals(0, diff.getDiffList().size());
+
+ OzoneMultipartUploadList mpuList = bucket.
+ listMultipartUploads("", null, null, 100);
+ assertEquals(2, mpuList.getUploads().size());
+
+ bucket.abortMultipartUpload(mpuKey1, mpuInfo1.getUploadID());
+ bucket.abortMultipartUpload(mpuKey2, mpuInfo2.getUploadID());
+ }
+
+ @Test
+ public void testSnapshotDiffWithCreateMultipartKeys() throws Exception {
+ String testVolumeName = "vol-create-part-" + counter.incrementAndGet();
+ String testBucketName = "bucket-create-part-" + counter.incrementAndGet();
+
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String regularPartsKey = "regular-parts-key" + counter.incrementAndGet();
+ String streamPartsKey = "stream-parts-key" + counter.incrementAndGet();
+ String mixedPartsKey = "mixed-parts-key" + counter.incrementAndGet();
+
+ OmMultipartInfo regularMpuInfo =
bucket.initiateMultipartUpload(regularPartsKey, getDefaultReplication());
+ OmMultipartInfo streamMpuInfo =
bucket.initiateMultipartUpload(streamPartsKey, getDefaultReplication());
+ OmMultipartInfo mixedMpuInfo =
bucket.initiateMultipartUpload(mixedPartsKey, getDefaultReplication());
+
+ String snap1 = "snap-before-parts-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ byte[] regularPart = "regular multipart key".getBytes(UTF_8);
+ try (OzoneOutputStream stream = bucket.createMultipartKey(
+ regularPartsKey, regularPart.length, 1, regularMpuInfo.getUploadID()))
{
+ stream.write(regularPart);
+ }
+
+ byte[] streamPart = "stream data".getBytes(UTF_8);
+ try (OzoneDataStreamOutput streamOut = bucket.createMultipartStreamKey(
+ streamPartsKey, streamPart.length, 1, streamMpuInfo.getUploadID())) {
+ streamOut.write(streamPart);
+ }
+
+ byte[] mixedPart = "mixed data".getBytes(UTF_8);
+ try (OzoneOutputStream mixedStream = bucket.createMultipartKey(
+ mixedPartsKey, mixedPart.length, 1, mixedMpuInfo.getUploadID())) {
+ mixedStream.write(mixedPart);
+ }
+
+ assertEquals(1,
+ bucket.listParts(regularPartsKey, regularMpuInfo.getUploadID(), 0,
100).getPartInfoList().size());
+ assertEquals(1,
+ bucket.listParts(streamPartsKey, streamMpuInfo.getUploadID(), 0,
100).getPartInfoList().size());
+ assertEquals(1,
+ bucket.listParts(mixedPartsKey, mixedMpuInfo.getUploadID(), 0,
100).getPartInfoList().size());
+
+ String snap2 = "snap-after-parts-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
testBucketName, snap1, snap2);
+ assertEquals(Collections.emptyList(), diff.getDiffList());
+
+ bucket.abortMultipartUpload(regularPartsKey, regularMpuInfo.getUploadID());
+ bucket.abortMultipartUpload(streamPartsKey, streamMpuInfo.getUploadID());
+ bucket.abortMultipartUpload(mixedPartsKey, mixedMpuInfo.getUploadID());
+ }
+
+ @Test
+ public void testSnapshotDiffWithAbortMultipartUpload() throws Exception {
+ String testVolumeName = "vol-abort-" + counter.incrementAndGet();
+ String testBucketName = "bucket-abort-" + counter.incrementAndGet();
+
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String snap1 = "snap-before-abort-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ // Abort immediately after initiate
+ String immediateAbortKey = "immediate-abort-" + counter.incrementAndGet();
+ OmMultipartInfo immediateInfo =
bucket.initiateMultipartUpload(immediateAbortKey, getDefaultReplication());
+ bucket.abortMultipartUpload(immediateAbortKey,
immediateInfo.getUploadID());
+
+ // Abort after uploading some parts
+ String partialAbortKey = "partial-abort-" + counter.incrementAndGet();
+ OmMultipartInfo partialInfo =
bucket.initiateMultipartUpload(partialAbortKey, getDefaultReplication());
+
+ byte[] part1Data = "Part 1 - will be aborted".getBytes(UTF_8);
+ byte[] part2Data = "Part 2 - will be aborted".getBytes(UTF_8);
+ byte[] part3Data = "Part 3 - stream part, will be aborted".getBytes(UTF_8);
+
+ try (OzoneOutputStream part1Stream = bucket.createMultipartKey(
+ partialAbortKey, part1Data.length, 1, partialInfo.getUploadID())) {
+ part1Stream.write(part1Data);
+ }
+ try (OzoneOutputStream part2Stream = bucket.createMultipartKey(
+ partialAbortKey, part2Data.length, 2, partialInfo.getUploadID())) {
+ part2Stream.write(part2Data);
+ }
+ try (OzoneDataStreamOutput part3Stream = bucket.createMultipartStreamKey(
+ partialAbortKey, part3Data.length, 3, partialInfo.getUploadID())) {
+ part3Stream.write(part3Data);
+ }
+
+ OzoneMultipartUploadPartListParts partsList = bucket.listParts(
+ partialAbortKey, partialInfo.getUploadID(), 0, 100);
+ assertEquals(3, partsList.getPartInfoList().size());
+
+ bucket.abortMultipartUpload(partialAbortKey, partialInfo.getUploadID());
+
+ // Multiple aborts in same snapshot window
+ String multiAbortKey1 = "multi-abort-1-" + counter.incrementAndGet();
+ String multiAbortKey2 = "multi-abort-2-" + counter.incrementAndGet();
+
+ OmMultipartInfo multiInfo1 =
bucket.initiateMultipartUpload(multiAbortKey1, getDefaultReplication());
+ OmMultipartInfo multiInfo2 =
bucket.initiateMultipartUpload(multiAbortKey2, getDefaultReplication());
+
+ try (OzoneOutputStream stream = bucket.createMultipartKey(
+ multiAbortKey1, part1Data.length, 1, multiInfo1.getUploadID())) {
+ stream.write(part1Data);
+ }
+ try (OzoneDataStreamOutput stream = bucket.createMultipartStreamKey(
+ multiAbortKey2, part2Data.length, 1, multiInfo2.getUploadID())) {
+ stream.write(part2Data);
+ }
+
+ bucket.abortMultipartUpload(multiAbortKey1, multiInfo1.getUploadID());
+ bucket.abortMultipartUpload(multiAbortKey2, multiInfo2.getUploadID());
+
+ String snap2 = "snap-after-aborts-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
testBucketName, snap1, snap2);
+ assertEquals(Collections.emptyList(), diff.getDiffList());
+
+ OzoneMultipartUploadList finalMpuList = bucket.listMultipartUploads("",
null, null, 100);
+ assertEquals(0, finalMpuList.getUploads().size());
+ }
+
+ @Test
+ public void testSnapshotDiffWithCompleteInvisibleMPULifecycle() throws
Exception {
+ String testVolumeName = "vol-invisible-" + counter.incrementAndGet();
+ String testBucketName = "bucket-invisible-" + counter.incrementAndGet();
+
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String regularKey1 = "regular-before-" + counter.incrementAndGet();
+ String regularKey2 = "regular-after-" + counter.incrementAndGet();
+ createFileKey(bucket, regularKey1);
+
+ String snap1 = "snap-invisible-baseline-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ String mpuKey1 = "invisible-mpu-1-" + counter.incrementAndGet();
+ String mpuKey2 = "invisible-mpu-2-" + counter.incrementAndGet();
+ String mpuKey3 = "invisible-mpu-3-" + counter.incrementAndGet();
+
+ OmMultipartInfo mpuInfo1 = bucket.initiateMultipartUpload(mpuKey1,
getDefaultReplication());
+ OmMultipartInfo mpuInfo2 = bucket.initiateMultipartUpload(mpuKey2,
getDefaultReplication());
+ OmMultipartInfo mpuInfo3 = bucket.initiateMultipartUpload(mpuKey3,
getDefaultReplication());
+
+ byte[] regularData1 = "Regular multipart data 1".getBytes(UTF_8);
+ byte[] regularData2 = "Regular multipart data 2".getBytes(UTF_8);
+
+ try (OzoneOutputStream stream = bucket.createMultipartKey(
+ mpuKey1, regularData1.length, 1, mpuInfo1.getUploadID())) {
+ stream.write(regularData1);
+ }
+ try (OzoneOutputStream stream = bucket.createMultipartKey(
+ mpuKey1, regularData2.length, 2, mpuInfo1.getUploadID())) {
+ stream.write(regularData2);
+ }
+
+ byte[] streamData1 = "Stream multipart data 1".getBytes(UTF_8);
+ byte[] streamData2 = "Stream multipart data 2".getBytes(UTF_8);
+
+ try (OzoneDataStreamOutput stream = bucket.createMultipartStreamKey(
+ mpuKey2, streamData1.length, 1, mpuInfo2.getUploadID())) {
+ stream.write(streamData1);
+ }
+ try (OzoneDataStreamOutput stream = bucket.createMultipartStreamKey(
+ mpuKey2, streamData2.length, 2, mpuInfo2.getUploadID())) {
+ stream.write(streamData2);
+ }
+
+
+ byte[] mixedRegular = "Mixed - regular part".getBytes(UTF_8);
+ byte[] mixedStream = "Mixed - stream part".getBytes(UTF_8);
+
+ try (OzoneOutputStream stream = bucket.createMultipartKey(
+ mpuKey3, mixedRegular.length, 1, mpuInfo3.getUploadID())) {
+ stream.write(mixedRegular);
+ }
+ try (OzoneDataStreamOutput stream = bucket.createMultipartStreamKey(
+ mpuKey3, mixedStream.length, 2, mpuInfo3.getUploadID())) {
+ stream.write(mixedStream);
+ }
+
+ assertEquals(2,
+ bucket.listParts(mpuKey1, mpuInfo1.getUploadID(), 0,
100).getPartInfoList().size());
+ assertEquals(2,
+ bucket.listParts(mpuKey2, mpuInfo2.getUploadID(), 0,
100).getPartInfoList().size());
+ assertEquals(2,
+ bucket.listParts(mpuKey3, mpuInfo3.getUploadID(), 0,
100).getPartInfoList().size());
+
+ createFileKey(bucket, regularKey2);
+
+ bucket.abortMultipartUpload(mpuKey1, mpuInfo1.getUploadID());
+ bucket.abortMultipartUpload(mpuKey2, mpuInfo2.getUploadID());
+ bucket.abortMultipartUpload(mpuKey3, mpuInfo3.getUploadID());
+
+ String snap2 = "snap-after-invisible-lifecycle-" +
counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
testBucketName, snap1, snap2);
+
+ List<SnapshotDiffReport.DiffReportEntry> expectedDiffs = Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.CREATE, regularKey2));
+ assertEquals(expectedDiffs, diff.getDiffList());
+
+ OzoneMultipartUploadList finalMpuList = bucket.listMultipartUploads("",
null, null, 100);
+ assertEquals(0, finalMpuList.getUploads().size());
+
+ assertDoesNotThrow(() -> bucket.getKey(regularKey1));
+ assertDoesNotThrow(() -> bucket.getKey(regularKey2));
+
+ assertThrows(OMException.class, () -> bucket.getKey(mpuKey1));
+ assertThrows(OMException.class, () -> bucket.getKey(mpuKey2));
+ assertThrows(OMException.class, () -> bucket.getKey(mpuKey3));
+ }
+
+ private void completeSinglePartMPU(OzoneBucket bucket, String keyName,
String data) throws IOException {
+ OmMultipartInfo mpuInfo = bucket.initiateMultipartUpload(keyName,
getDefaultReplication());
+ String uploadId = mpuInfo.getUploadID();
+
+ byte[] partData = createLargePartData(data, MIN_PART_SIZE);
+ OzoneOutputStream partStream = bucket.createMultipartKey(keyName,
partData.length, 1, uploadId);
+ partStream.write(partData);
+ partStream.close();
+
+ OzoneMultipartUploadPartListParts partsList = bucket.listParts(keyName,
uploadId, 0, 100);
+ String realETag = partsList.getPartInfoList().get(0).getPartName();
+
+ Map<Integer, String> partsMap = Collections.singletonMap(1, realETag);
+ bucket.completeMultipartUpload(keyName, uploadId, partsMap);
+ }
+
+ private void completeMultiplePartMPU(
+ OzoneBucket bucket, String keyName, List<String> partDataList) throws
IOException {
+ OmMultipartInfo mpuInfo = bucket.initiateMultipartUpload(keyName,
getDefaultReplication());
+ String uploadId = mpuInfo.getUploadID();
+
+ for (int i = 0; i < partDataList.size(); i++) {
+ int partNum = i + 1;
+ byte[] partData = createLargePartData(partDataList.get(i),
MIN_PART_SIZE);
+
+ try (OzoneOutputStream partStream = bucket.createMultipartKey(
+ keyName, partData.length, partNum, uploadId)) {
+ partStream.write(partData);
+ }
+ }
+
+ OzoneMultipartUploadPartListParts partsList = bucket.
+ listParts(keyName, uploadId, 0, partDataList.size());
+ Map<Integer, String> partsMap = new HashMap<>();
+
+ for (OzoneMultipartUploadPartListParts.PartInfo partInfo :
partsList.getPartInfoList()) {
+ partsMap.put(partInfo.getPartNumber(), partInfo.getPartName());
+ }
+
+ bucket.completeMultipartUpload(keyName, uploadId, partsMap);
+ }
+
+ private void completeMixedPartMPU(
+ OzoneBucket bucket, String keyName, String regularData, String
streamData) throws IOException {
+ OmMultipartInfo mpuInfo = bucket.initiateMultipartUpload(keyName,
getDefaultReplication());
+ String uploadId = mpuInfo.getUploadID();
+
+ byte[] part1Data = createLargePartData(regularData, MIN_PART_SIZE);
+ try (OzoneOutputStream partStream = bucket.createMultipartKey(
+ keyName, part1Data.length, 1, uploadId)) {
+ partStream.write(part1Data);
+ }
+
+ byte[] part2Data = createLargePartData(streamData, MIN_PART_SIZE);
+ try (OzoneDataStreamOutput partStream = bucket.createMultipartStreamKey(
+ keyName, part2Data.length, 2, uploadId)) {
+ partStream.write(part2Data);
+ }
+
+ OzoneMultipartUploadPartListParts partsList = bucket.listParts(keyName,
uploadId, 0, 2);
+ Map<Integer, String> partsMap = new HashMap<>();
+
+ for (OzoneMultipartUploadPartListParts.PartInfo partInfo :
partsList.getPartInfoList()) {
+ partsMap.put(partInfo.getPartNumber(), partInfo.getPartName());
+ }
+
+ bucket.completeMultipartUpload(keyName, uploadId, partsMap);
+ }
+
+ private void completeMPUWithReplication(
+ OzoneBucket bucket, String keyName, ReplicationConfig replicationConfig)
throws IOException {
+ OmMultipartInfo mpuInfo;
+ if (replicationConfig != null) {
+ mpuInfo = bucket.initiateMultipartUpload(keyName, replicationConfig);
+ } else {
+ mpuInfo = bucket.initiateMultipartUpload(keyName);
+ }
+
+ String uploadId = mpuInfo.getUploadID();
+
+ byte[] partData = createLargePartData("Replication test data for " +
keyName, MIN_PART_SIZE);
+ try (OzoneOutputStream partStream = bucket.createMultipartKey(
+ keyName, partData.length, 1, uploadId)) {
+ partStream.write(partData);
+ }
+
+ OzoneMultipartUploadPartListParts partsList = bucket.listParts(keyName,
uploadId, 0, 1);
+ String realETag = partsList.getPartInfoList().get(0).getPartName();
+
+ Map<Integer, String> partsMap = Collections.singletonMap(1, realETag);
+ bucket.completeMultipartUpload(keyName, uploadId, partsMap);
+ }
+
+ private void completeMPUWithMetadata(OzoneBucket bucket, String keyName,
+ Map<String, String> metadata,
Map<String, String> tags) throws IOException {
+
+ OmMultipartInfo mpuInfo = bucket.initiateMultipartUpload(
+ keyName, getDefaultReplication(), metadata, tags);
+ String uploadId = mpuInfo.getUploadID();
+
+ byte[] partData = createLargePartData("MPU with metadata and tags",
MIN_PART_SIZE);
+ OzoneOutputStream partStream = bucket.createMultipartKey(keyName,
partData.length, 1, uploadId);
+ partStream.write(partData);
+ partStream.close();
+
+ OzoneMultipartUploadPartListParts partsList = bucket.listParts(keyName,
uploadId, 0, 100);
+ String realETag = partsList.getPartInfoList().get(0).getPartName();
+
+ Map<Integer, String> partsMap = Collections.singletonMap(1, realETag);
+ bucket.completeMultipartUpload(keyName, uploadId, partsMap);
+ }
+
+ private byte[] createLargePartData(String baseContent, int targetSize) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(baseContent);
+
+ String padding = " - padding data";
+ while (sb.length() < targetSize) {
+ sb.append(padding);
+ if (sb.length() + padding.length() > targetSize) {
+ int remaining = targetSize - sb.length();
+ sb.append(padding.substring(0, Math.min(remaining, padding.length())));
+ }
+ }
+
+ String result = sb.toString();
+ if (result.length() > targetSize) {
+ result = result.substring(0, targetSize);
+ }
+
+ return result.getBytes(UTF_8);
+ }
+
+ @Test
+ public void testSnapshotDiffMPUCreateNewKey() throws Exception {
+ String testVolumeName = "vol-create-new-" + counter.incrementAndGet();
+ String testBucketName = "bucket-create-new-" + counter.incrementAndGet();
+
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String existingKey = "existing-baseline-" + counter.incrementAndGet();
+ createFileKey(bucket, existingKey);
+
+ String snap1 = "snap-before-create-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ String mpuKey1 = "multi-mpu-1-" + counter.incrementAndGet();
+ String mpuKey2 = "multi-mpu-2-" + counter.incrementAndGet();
+ String mpuKey3 = "multi-mpu-3-" + counter.incrementAndGet();
+
+ completeSinglePartMPU(bucket, mpuKey1, "Single part MPU data");
+
+ completeMultiplePartMPU(bucket, mpuKey2,
+ Arrays.asList("Multi part 1", "Multi part 2", "Multi part 3"));
+
+ completeMixedPartMPU(bucket, mpuKey3,
+ "Mixed regular part", "Mixed stream part");
+
+ String regularKey = "regular-compare-" + counter.incrementAndGet();
+ createFileKey(bucket, regularKey);
+
+ String snap2 = "snap-multiple-final-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
testBucketName, snap1, snap2);
+
+ List<SnapshotDiffReport.DiffReportEntry> expectedDiffs = Arrays.asList(
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE,
mpuKey1),
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE,
mpuKey2),
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE,
mpuKey3),
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE,
regularKey));
+ assertEquals(expectedDiffs, diff.getDiffList());
+ }
+
+ @Test
+ public void testSnapshotDiffMPUCreateMultipleKeys() throws Exception {
+ String testVolumeName = "vol-create-multiple-" + counter.incrementAndGet();
+ String testBucketName = "bucket-create-multiple-" +
counter.incrementAndGet();
+
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String snap1 = "snap-multiple-baseline-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ String mpuKey1 = "multi-mpu-1-" + counter.incrementAndGet();
+ String mpuKey2 = "multi-mpu-2-" + counter.incrementAndGet();
+ String mpuKey3 = "multi-mpu-3-" + counter.incrementAndGet();
+
+ completeSinglePartMPU(bucket, mpuKey1, "Single part MPU data");
+ completeMultiplePartMPU(bucket, mpuKey2,
+ Arrays.asList("Multi part 1", "Multi part 2", "Multi part 3"));
+ completeMixedPartMPU(bucket, mpuKey3,
+ "Mixed regular part", "Mixed stream part");
+
+ String regularKey = "regular-compare-" + counter.incrementAndGet();
+ createFileKey(bucket, regularKey);
+
+ String snap2 = "snap-multiple-final-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
testBucketName, snap1, snap2);
+
+ List<SnapshotDiffReport.DiffReportEntry> expectedDiffs = Arrays.asList(
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE,
mpuKey1),
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE,
mpuKey2),
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE,
mpuKey3),
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE,
regularKey));
+ assertEquals(expectedDiffs, diff.getDiffList());
+ }
+
+ @Test
+ public void testSnapshotDiffMPUCreateWithMetadataAndTags() throws Exception {
+ String testVolumeName = "vol-create-meta-" + counter.incrementAndGet();
+ String testBucketName = "bucket-create-meta-" + counter.incrementAndGet();
+
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String snap1 = "snap-meta-baseline-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ String mpuKeyWithMeta = "mpu-with-metadata-" + counter.incrementAndGet();
+
+ Map<String, String> richMetadata = new HashMap<>();
+ richMetadata.put("content-type", "application/octet-stream");
+ richMetadata.put("content-encoding", "gzip");
+ richMetadata.put("cache-control", "no-cache");
+ richMetadata.put("custom-header", "test-value");
+
+ Map<String, String> richTags = new HashMap<>();
+ richTags.put("project", "ozone-testing");
+ richTags.put("environment", "integration");
+ richTags.put("cost-center", "engineering");
+ richTags.put("backup-policy", "daily");
+
+ completeMPUWithMetadata(bucket, mpuKeyWithMeta, richMetadata, richTags);
+
+ String snap2 = "snap-meta-final-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
testBucketName, snap1, snap2);
+
+ List<SnapshotDiffReport.DiffReportEntry> expectedDiffs = Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.CREATE, mpuKeyWithMeta));
+ assertEquals(expectedDiffs, diff.getDiffList());
+ }
+
+ @Test
+ public void testSnapshotDiffMPUCreateWithDifferentReplication() throws
Exception {
+ String testVolumeName = "vol-create-repl-" + counter.incrementAndGet();
+ String testBucketName = "bucket-create-repl-" + counter.incrementAndGet();
+
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String snap1 = "snap-repl-baseline-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ String standaloneKey = "standalone-mpu-" + counter.incrementAndGet();
+ String defaultKey = "default-mpu-" + counter.incrementAndGet();
+
+ ReplicationConfig standaloneConfig =
StandaloneReplicationConfig.getInstance(
+ HddsProtos.ReplicationFactor.ONE);
+ completeMPUWithReplication(bucket, standaloneKey, standaloneConfig);
+ completeMPUWithReplication(bucket, defaultKey, null);
+
+ String snap2 = "snap-repl-final-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
testBucketName, snap1, snap2);
+
+ List<SnapshotDiffReport.DiffReportEntry> expectedDiffs = Arrays.asList(
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE,
standaloneKey),
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE,
defaultKey));
+ assertEquals(expectedDiffs, diff.getDiffList());
+ }
+
+ @Test
+ public void testSnapshotDiffMPUModifyExistingKey() throws Exception {
+ String testVolumeName = "vol-modify-existing-" + counter.incrementAndGet();
+ String testBucketName = "bucket-modify-existing-" +
counter.incrementAndGet();
+
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String existingKey = "existing-key-" + counter.incrementAndGet();
+ createFileKey(bucket, existingKey);
+
+ String snap1 = "snap-before-modify-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ completeSinglePartMPU(bucket, existingKey, "MPU overwritten content");
+
+ String snap2 = "snap-after-modify-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
testBucketName, snap1, snap2);
+
+ List<SnapshotDiffReport.DiffReportEntry> expectedDiffs =
Collections.singletonList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.MODIFY, existingKey));
+ assertEquals(expectedDiffs, diff.getDiffList());
+ }
+
+ @Test
+ public void testSnapshotDiffMPUModifyMultipleKeys() throws Exception {
+ String testVolumeName = "vol-modify-multiple-" + counter.incrementAndGet();
+ String testBucketName = "bucket-modify-multiple-" +
counter.incrementAndGet();
+
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String existingKey1 = "existing-1-" + counter.incrementAndGet();
+ String existingKey2 = "existing-2-" + counter.incrementAndGet();
+ String existingKey3 = "existing-3-" + counter.incrementAndGet();
+ String unchangedKey = "unchanged-" + counter.incrementAndGet();
+
+ createFileKey(bucket, existingKey1);
+ createFileKey(bucket, existingKey2);
+ createFileKey(bucket, existingKey3);
+ createFileKey(bucket, unchangedKey);
+
+ String snap1 = "snap-modify-multiple-baseline-" +
counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ completeSinglePartMPU(bucket, existingKey1, "Single part overwrite");
+ completeMultiplePartMPU(bucket, existingKey2,
+ Arrays.asList("Multi part overwrite 1", "Multi part overwrite 2"));
+ completeMixedPartMPU(bucket, existingKey3,
+ "Mixed regular overwrite", "Mixed stream overwrite");
+
+ String snap2 = "snap-modify-multiple-final-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
testBucketName, snap1, snap2);
+
+ List<SnapshotDiffReport.DiffReportEntry> expectedDiffs = Arrays.asList(
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.MODIFY,
existingKey1),
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.MODIFY,
existingKey2),
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.MODIFY,
existingKey3));
+ assertEquals(expectedDiffs, diff.getDiffList());
+ }
+
+ @Test
+ public void testSnapshotDiffMPUModifyWithMetadataChange() throws Exception {
+ String testVolumeName = "vol-modify-meta-" + counter.incrementAndGet();
+ String testBucketName = "bucket-modify-meta-" + counter.incrementAndGet();
+
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String existingKey = "existing-meta-key-" + counter.incrementAndGet();
+ createFileKey(bucket, existingKey);
+
+ Map<String, String> originalTags = new HashMap<>();
+ originalTags.put("version", "1.0");
+ originalTags.put("environment", "test");
+ bucket.putObjectTagging(existingKey, originalTags);
+
+ String snap1 = "snap-modify-meta-baseline-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ Map<String, String> newMetadata = new HashMap<>();
+ newMetadata.put("content-type", "application/json");
+ newMetadata.put("updated-by", "mpu-test");
+
+ Map<String, String> newTags = new HashMap<>();
+ newTags.put("version", "2.0");
+ newTags.put("environment", "updated");
+ newTags.put("method", "mpu-overwrite");
+
+ completeMPUWithMetadata(bucket, existingKey, newMetadata, newTags);
+
+ String snap2 = "snap-modify-meta-final-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
testBucketName, snap1, snap2);
+
+ List<SnapshotDiffReport.DiffReportEntry> expectedDiffs = Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.MODIFY, existingKey));
+ assertEquals(expectedDiffs, diff.getDiffList());
+ }
+
+ @Test
+ public void testSnapshotDiffMPUMixedCreateAndModify() throws Exception {
+ String testVolumeName = "vol-mixed-ops-" + counter.incrementAndGet();
+ String testBucketName = "bucket-mixed-ops-" + counter.incrementAndGet();
+
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ createBucket(volume, testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+
+ String existingKey1 = "existing-1-" + counter.incrementAndGet();
+ String existingKey2 = "existing-2-" + counter.incrementAndGet();
+ String unchangedKey = "unchanged-" + counter.incrementAndGet();
+
+ createFileKey(bucket, existingKey1);
+ createFileKey(bucket, existingKey2);
+ createFileKey(bucket, unchangedKey);
+
+ String snap1 = "snap-mixed-baseline-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap1);
+
+ String newKey1 = "new-mpu-1-" + counter.incrementAndGet();
+ String newKey2 = "new-mpu-2-" + counter.incrementAndGet();
+
+ completeSinglePartMPU(bucket, newKey1, "New key via single part MPU");
+ completeMixedPartMPU(bucket, newKey2, "New key regular part", "New key
stream part");
+
+ completeSinglePartMPU(bucket, existingKey1, "Modified via single part
MPU");
+ completeMultiplePartMPU(bucket, existingKey2,
+ Arrays.asList("Modified part 1", "Modified part 2"));
+
+ String regularNewKey = "regular-new-" + counter.incrementAndGet();
+ createFileKey(bucket, regularNewKey);
+
+ String snap2 = "snap-mixed-final-" + counter.incrementAndGet();
+ createSnapshot(testVolumeName, testBucketName, snap2);
+
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
testBucketName, snap1, snap2);
+
+ List<SnapshotDiffReport.DiffReportEntry> expectedDiffs = Arrays.asList(
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE,
newKey1),
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE,
newKey2),
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.CREATE,
regularNewKey),
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.MODIFY,
existingKey1),
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.MODIFY,
existingKey2));
+ assertEquals(expectedDiffs, diff.getDiffList());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
