This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 6df3077fe19 HDDS-9223. Use protobuf for SnapshotDiffJobCodec (#8503)
6df3077fe19 is described below
commit 6df3077fe19d2280a6bba34d9c10c6f00eb92ebc
Author: Yu-Syuan Jheng <[email protected]>
AuthorDate: Tue Jun 17 17:16:26 2025 -0400
HDDS-9223. Use protobuf for SnapshotDiffJobCodec (#8503)
---
.../hadoop/ozone/om/helpers/SnapshotDiffJob.java | 13 +++-
.../helpers/OldSnapshotDiffJobCodecForTesting.java | 56 +++++++++++++++++
.../om/helpers/TestOmSnapshotDiffJobCodec.java | 73 ++++++++++++++++++++++
3 files changed, 139 insertions(+), 3 deletions(-)
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotDiffJob.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotDiffJob.java
index ce8a62b1620..b09ea1a4278 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotDiffJob.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotDiffJob.java
@@ -20,6 +20,7 @@
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.protobuf.InvalidProtocolBufferException;
import java.io.IOException;
import java.util.Objects;
import org.apache.commons.lang3.StringUtils;
@@ -315,13 +316,19 @@ public Class<SnapshotDiffJob> getTypeClass() {
}
@Override
- public byte[] toPersistedFormatImpl(SnapshotDiffJob object) throws
IOException {
- return MAPPER.writeValueAsBytes(object);
+ public byte[] toPersistedFormat(SnapshotDiffJob object) {
+ return object.toProtoBuf().toByteArray();
}
@Override
public SnapshotDiffJob fromPersistedFormatImpl(byte[] rawData) throws
IOException {
- return MAPPER.readValue(rawData, SnapshotDiffJob.class);
+ try {
+ SnapshotDiffJobProto proto = SnapshotDiffJobProto.parseFrom(rawData);
+ return SnapshotDiffJob.getFromProtoBuf(proto);
+ } catch (InvalidProtocolBufferException e) {
+ // the rawData was in old format, fallback to the old implementation
+ return MAPPER.readValue(rawData, SnapshotDiffJob.class);
+ }
}
@Override
diff --git
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/OldSnapshotDiffJobCodecForTesting.java
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/OldSnapshotDiffJobCodecForTesting.java
new file mode 100644
index 00000000000..dde5842e141
--- /dev/null
+++
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/OldSnapshotDiffJobCodecForTesting.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import org.apache.hadoop.hdds.utils.db.Codec;
+
+/**
+ * Codec to serialize / deserialize SnapshotDiffJob.
+ */
+public class OldSnapshotDiffJobCodecForTesting
+ implements Codec<SnapshotDiffJob> {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper()
+ .setSerializationInclusion(JsonInclude.Include.NON_NULL)
+ .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+ @Override
+ public Class<SnapshotDiffJob> getTypeClass() {
+ return SnapshotDiffJob.class;
+ }
+
+ @Override
+ public byte[] toPersistedFormatImpl(SnapshotDiffJob object) throws
IOException {
+ return MAPPER.writeValueAsBytes(object);
+ }
+
+ @Override
+ public SnapshotDiffJob fromPersistedFormatImpl(byte[] rawData) throws
IOException {
+ return MAPPER.readValue(rawData, SnapshotDiffJob.class);
+ }
+
+ @Override
+ public SnapshotDiffJob copyObject(SnapshotDiffJob object) {
+ // Note: Not really a "copy". from OmDBDiffReportEntryCodec
+ return object;
+ }
+}
diff --git
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotDiffJobCodec.java
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotDiffJobCodec.java
new file mode 100644
index 00000000000..6e86a73d472
--- /dev/null
+++
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotDiffJobCodec.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import org.apache.hadoop.hdds.utils.db.Codec;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.SubStatus;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Testing serialization of SnapshotDiffJobCodec objects to/from RocksDB.
+ */
+public class TestOmSnapshotDiffJobCodec {
+ private final OldSnapshotDiffJobCodecForTesting oldCodec
+ = new OldSnapshotDiffJobCodecForTesting();
+ private final Codec<SnapshotDiffJob> newCodec = SnapshotDiffJob.getCodec();
+
+ @Test
+ public void testOldJsonSerializedDataCanBeReadByNewCodec() throws Exception {
+ // Step 1: Construct a SnapshotDiffJob instance
+ SnapshotDiffJob original = new SnapshotDiffJob(
+ 123456789L,
+ "job-001",
+ JobStatus.IN_PROGRESS,
+ "volA",
+ "buckB",
+ "snap1",
+ "snap2",
+ true,
+ false,
+ 100L,
+ SubStatus.SST_FILE_DELTA_DAG_WALK,
+ 0.0
+ );
+
+ // Step 2: Serialize using the old Jackson-based codec
+ byte[] oldFormatData = oldCodec.toPersistedFormatImpl(original);
+
+ // Step 3: Deserialize using the new default codec (with Protobuf + JSON
fallback)
+ SnapshotDiffJob parsed = newCodec.fromPersistedFormatImpl(oldFormatData);
+
+ // Step 4: Verify critical fields remain consistent after round-trip
+ assertEquals(original.getJobId(), parsed.getJobId());
+ assertEquals(original.getStatus(), parsed.getStatus());
+ assertEquals(original.getVolume(), parsed.getVolume());
+ assertEquals(original.getBucket(), parsed.getBucket());
+ assertEquals(original.getFromSnapshot(), parsed.getFromSnapshot());
+ assertEquals(original.getToSnapshot(), parsed.getToSnapshot());
+ assertEquals(original.isForceFullDiff(), parsed.isForceFullDiff());
+ assertEquals(original.isNativeDiffDisabled(),
parsed.isNativeDiffDisabled());
+ assertEquals(original.getSubStatus(), parsed.getSubStatus());
+ assertEquals(original.getTotalDiffEntries(), parsed.getTotalDiffEntries());
+
+ assertEquals(0.0, parsed.getKeysProcessedPct());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
