This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e08b001044 HDDS-13006. Use yaml files to host Ozone snapshot local
properties (#8555)
e08b001044 is described below
commit e08b0010449342abca417510714a08e49f7ad132
Author: Siyao Meng <[email protected]>
AuthorDate: Wed Jul 9 19:31:35 2025 -0700
HDDS-13006. Use yaml files to host Ozone snapshot local properties (#8555)
Co-authored-by: Wei-Chiu Chuang <[email protected]>
Generated-by: Claude 3.7 Sonnet
---
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 9 +
hadoop-ozone/ozone-manager/pom.xml | 4 +
.../hadoop/ozone/om/OmSnapshotLocalData.java | 288 +++++++++++++++++++++
.../hadoop/ozone/om/OmSnapshotLocalDataYaml.java | 259 ++++++++++++++++++
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 5 +
.../ozone/om/TestOmSnapshotLocalDataYaml.java | 209 +++++++++++++++
6 files changed, 774 insertions(+)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index 6bee926336..c87fcc4bf0 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -202,6 +202,15 @@ public final class OzoneConsts {
public static final String SCM_CONTEXT_ATTRIBUTE = "ozone.scm";
+ // YAML field constants for OmSnapshotLocalData (thus the OM_SLD_ prefix)
YAML files
+ public static final String OM_SLD_VERSION = "version";
+ public static final String OM_SLD_CHECKSUM = "checksum";
+ public static final String OM_SLD_IS_SST_FILTERED = "isSSTFiltered";
+ public static final String OM_SLD_UNCOMPACTED_SST_FILE_LIST =
"uncompactedSSTFileList";
+ public static final String OM_SLD_LAST_COMPACTION_TIME =
"lastCompactionTime";
+ public static final String OM_SLD_NEEDS_COMPACTION = "needsCompaction";
+ public static final String OM_SLD_COMPACTED_SST_FILE_LIST =
"compactedSSTFileList";
+
// YAML fields for .container files
public static final String CONTAINER_ID = "containerID";
public static final String CONTAINER_TYPE = "containerType";
diff --git a/hadoop-ozone/ozone-manager/pom.xml
b/hadoop-ozone/ozone-manager/pom.xml
index ba32592990..6347ee2722 100644
--- a/hadoop-ozone/ozone-manager/pom.xml
+++ b/hadoop-ozone/ozone-manager/pom.xml
@@ -298,6 +298,10 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-docs</artifactId>
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java
new file mode 100644
index 0000000000..ce127fb3a6
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.yaml.snakeyaml.Yaml;
+
+/**
+ * OmSnapshotLocalData is the in-memory representation of snapshot local
metadata.
+ * Inspired by org.apache.hadoop.ozone.container.common.impl.ContainerData
+ */
+public abstract class OmSnapshotLocalData {
+
+ // Version of the snapshot local data. A valid version shall be greater than
0.
+ private int version;
+
+ // Checksum of the YAML representation
+ private String checksum;
+
+ // Whether SST is filtered
+ private boolean isSSTFiltered;
+
+ // Map of Table to uncompacted SST file list on snapshot create
+ private Map<String, List<String>> uncompactedSSTFileList;
+
+ // Time of last compaction, in epoch milliseconds
+ private long lastCompactionTime;
+
+ // Whether the snapshot needs compaction
+ private boolean needsCompaction;
+
+ // Map of version to compacted SST file list
+ // Map<version, Map<Table, sstFileList>>
+ private Map<Integer, Map<String, List<String>>> compactedSSTFileList;
+
+ public static final Charset CHARSET_ENCODING = StandardCharsets.UTF_8;
+ private static final String DUMMY_CHECKSUM = new String(new byte[64],
CHARSET_ENCODING);
+
+ /**
+ * Creates a OmSnapshotLocalData object with default values.
+ */
+ public OmSnapshotLocalData() {
+ this.isSSTFiltered = false;
+ this.uncompactedSSTFileList = new HashMap<>();
+ this.lastCompactionTime = 0L;
+ this.needsCompaction = false;
+ this.compactedSSTFileList = new HashMap<>();
+ this.version = 0;
+ setChecksumTo0ByteArray();
+ }
+
+ /**
+ * Copy constructor to create a deep copy of OmSnapshotLocalData object.
+ * @param source The source OmSnapshotLocalData to copy from
+ */
+ public OmSnapshotLocalData(OmSnapshotLocalData source) {
+ // Copy primitive fields directly
+ this.isSSTFiltered = source.isSSTFiltered;
+ this.lastCompactionTime = source.lastCompactionTime;
+ this.needsCompaction = source.needsCompaction;
+ this.checksum = source.checksum;
+ this.version = source.version;
+
+ // Deep copy for uncompactedSSTFileList
+ this.uncompactedSSTFileList = new HashMap<>();
+ for (Map.Entry<String, List<String>> entry :
+ source.uncompactedSSTFileList.entrySet()) {
+ this.uncompactedSSTFileList.put(
+ entry.getKey(),
+ Lists.newArrayList(entry.getValue()));
+ }
+
+ // Deep copy for compactedSSTFileList
+ this.compactedSSTFileList = new HashMap<>();
+ for (Map.Entry<Integer, Map<String, List<String>>> versionEntry :
+ source.compactedSSTFileList.entrySet()) {
+ Map<String, List<String>> tableMap = new HashMap<>();
+
+ for (Map.Entry<String, List<String>> tableEntry :
+ versionEntry.getValue().entrySet()) {
+ tableMap.put(
+ tableEntry.getKey(),
+ Lists.newArrayList(tableEntry.getValue()));
+ }
+
+ this.compactedSSTFileList.put(versionEntry.getKey(), tableMap);
+ }
+ }
+
+ /**
+ * Returns whether SST is filtered for this snapshot.
+ * @return true if SST is filtered, false otherwise
+ */
+ public boolean getSstFiltered() {
+ return isSSTFiltered;
+ }
+
+ /**
+ * Sets whether SST is filtered for this snapshot.
+ * @param sstFiltered
+ */
+ public void setSstFiltered(boolean sstFiltered) {
+ this.isSSTFiltered = sstFiltered;
+ }
+
+ /**
+ * Returns the uncompacted SST file list.
+ * @return Map of Table to uncompacted SST file list
+ */
+ public Map<String, List<String>> getUncompactedSSTFileList() {
+ return Collections.unmodifiableMap(this.uncompactedSSTFileList);
+ }
+
+ /**
+ * Sets the uncompacted SST file list.
+ * @param uncompactedSSTFileList Map of Table to uncompacted SST file list
+ */
+ public void setUncompactedSSTFileList(
+ Map<String, List<String>> uncompactedSSTFileList) {
+ this.uncompactedSSTFileList.clear();
+ this.uncompactedSSTFileList.putAll(uncompactedSSTFileList);
+ }
+
+ /**
+ * Adds an entry to the uncompacted SST file list.
+ * @param table Table name
+ * @param sstFile SST file name
+ */
+ public void addUncompactedSSTFile(String table, String sstFile) {
+ this.uncompactedSSTFileList.computeIfAbsent(table, k ->
Lists.newArrayList())
+ .add(sstFile);
+ }
+
+ /**
+ * Returns the last compaction time, in epoch milliseconds.
+ * @return Timestamp of the last compaction
+ */
+ public long getLastCompactionTime() {
+ return lastCompactionTime;
+ }
+
+ /**
+ * Sets the last compaction time, in epoch milliseconds.
+ * @param lastCompactionTime Timestamp of the last compaction
+ */
+ public void setLastCompactionTime(Long lastCompactionTime) {
+ this.lastCompactionTime = lastCompactionTime;
+ }
+
+ /**
+ * Returns whether the snapshot needs compaction.
+ * @return true if the snapshot needs compaction, false otherwise
+ */
+ public boolean getNeedsCompaction() {
+ return needsCompaction;
+ }
+
+ /**
+ * Sets whether the snapshot needs compaction.
+ * @param needsCompaction true if the snapshot needs compaction, false
otherwise
+ */
+ public void setNeedsCompaction(boolean needsCompaction) {
+ this.needsCompaction = needsCompaction;
+ }
+
+ /**
+ * Returns the compacted SST file list.
+ * @return Map of version to compacted SST file list
+ */
+ public Map<Integer, Map<String, List<String>>> getCompactedSSTFileList() {
+ return Collections.unmodifiableMap(this.compactedSSTFileList);
+ }
+
+ /**
+ * Sets the compacted SST file list.
+ * @param compactedSSTFileList Map of version to compacted SST file list
+ */
+ public void setCompactedSSTFileList(
+ Map<Integer, Map<String, List<String>>> compactedSSTFileList) {
+ this.compactedSSTFileList.clear();
+ this.compactedSSTFileList.putAll(compactedSSTFileList);
+ }
+
+ /**
+ * Adds an entry to the compacted SST file list.
+ * @param ver Version number (TODO: to be clarified)
+ * @param table Table name
+ * @param sstFile SST file name
+ */
+ public void addCompactedSSTFile(Integer ver, String table, String sstFile) {
+ this.compactedSSTFileList.computeIfAbsent(ver, k -> Maps.newHashMap())
+ .computeIfAbsent(table, k -> Lists.newArrayList())
+ .add(sstFile);
+ }
+
+ /**
+ * Returns the checksum of the YAML representation.
+ * @return checksum
+ */
+ public String getChecksum() {
+ return checksum;
+ }
+
+ /**
+ * Sets the checksum of the YAML representation.
+ * @param checksum checksum
+ */
+ public void setChecksum(String checksum) {
+ this.checksum = checksum;
+ }
+
+ /**
+ * Sets the checksum to a 0 byte array.
+ */
+ public void setChecksumTo0ByteArray() {
+ this.checksum = DUMMY_CHECKSUM;
+ }
+
+ /**
+ * Compute and set checksum for the snapshot data.
+ * @param yaml Yaml instance for serialization
+ * @throws IOException if checksum computation fails
+ */
+ public void computeAndSetChecksum(Yaml yaml) throws IOException {
+ // Set checksum to dummy value - 0 byte array, to calculate the checksum
+ // of rest of the data.
+ setChecksumTo0ByteArray();
+
+ // Dump yaml data into a string to compute its checksum
+ String snapshotDataYamlStr = yaml.dump(this);
+
+ this.checksum = getChecksum(snapshotDataYamlStr);
+ }
+
+ /**
+ * Computes SHA-256 hash for a given string.
+ * @param data String data for which checksum needs to be calculated
+ * @return SHA-256 checksum as hex string
+ * @throws IOException If checksum calculation fails
+ */
+ private static String getChecksum(String data) throws IOException {
+ try {
+ return DigestUtils.sha256Hex(data.getBytes(StandardCharsets.UTF_8));
+ } catch (Exception ex) {
+ throw new IOException("Unable to calculate checksum", ex);
+ }
+ }
+
+ /**
+ * Returns the version of the snapshot local data.
+ * @return version
+ */
+ public int getVersion() {
+ return version;
+ }
+
+ /**
+ * Sets the version of the snapshot local data. A valid version shall be
greater than 0.
+ * @param version version
+ */
+ public void setVersion(int version) {
+ this.version = version;
+ }
+}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java
new file mode 100644
index 0000000000..97b401d8cb
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import com.google.common.base.Preconditions;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.hdds.server.YamlUtils;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.LoaderOptions;
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.constructor.AbstractConstruct;
+import org.yaml.snakeyaml.constructor.SafeConstructor;
+import org.yaml.snakeyaml.error.YAMLException;
+import org.yaml.snakeyaml.introspector.BeanAccess;
+import org.yaml.snakeyaml.introspector.PropertyUtils;
+import org.yaml.snakeyaml.nodes.MappingNode;
+import org.yaml.snakeyaml.nodes.Node;
+import org.yaml.snakeyaml.nodes.Tag;
+import org.yaml.snakeyaml.representer.Representer;
+
+/**
+ * Class for creating and reading snapshot local properties / data YAML files.
+ * Checksum of the YAML fields are computed and stored in the YAML file
transparently to callers.
+ * Inspired by org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml
+ */
+public final class OmSnapshotLocalDataYaml extends OmSnapshotLocalData {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(OmSnapshotLocalDataYaml.class);
+
+ public static final Tag SNAPSHOT_YAML_TAG = new Tag("OmSnapshotLocalData");
+
+ /**
+ * Creates a new OmSnapshotLocalDataYaml with default values.
+ */
+ public OmSnapshotLocalDataYaml() {
+ super();
+ }
+
+ /**
+ * Copy constructor to create a deep copy.
+ * @param source The source OmSnapshotLocalData to copy from
+ */
+ public OmSnapshotLocalDataYaml(OmSnapshotLocalData source) {
+ super(source);
+ }
+
+ /**
+ * Verifies the checksum of the snapshot data.
+ * @param snapshotData The snapshot data to verify
+ * @return true if the checksum is valid, false otherwise
+ * @throws IOException if there's an error computing the checksum
+ */
+ public static boolean verifyChecksum(OmSnapshotLocalData snapshotData)
+ throws IOException {
+ Preconditions.checkNotNull(snapshotData, "snapshotData cannot be null");
+
+ // Get the stored checksum
+ String storedChecksum = snapshotData.getChecksum();
+ if (storedChecksum == null) {
+ LOG.warn("No checksum found in snapshot data for verification");
+ return false;
+ }
+
+ // Create a copy of the snapshot data for computing checksum
+ OmSnapshotLocalDataYaml snapshotDataCopy = new
OmSnapshotLocalDataYaml(snapshotData);
+
+ // Clear the existing checksum in the copy
+ snapshotDataCopy.setChecksum(null);
+
+ // Get the YAML representation
+ final Yaml yaml = getYamlForSnapshotLocalData();
+
+ // Compute new checksum
+ snapshotDataCopy.computeAndSetChecksum(yaml);
+
+ // Compare the stored and computed checksums
+ String computedChecksum = snapshotDataCopy.getChecksum();
+ boolean isValid = storedChecksum.equals(computedChecksum);
+
+ if (!isValid) {
+ LOG.warn("Checksum verification failed for snapshot local data. " +
+ "Stored: {}, Computed: {}", storedChecksum, computedChecksum);
+ }
+
+ return isValid;
+ }
+
+ /**
+ * Constructor class for OmSnapshotLocalData.
+ * This is used when parsing YAML files into OmSnapshotLocalDataYaml objects.
+ */
+ private static class SnapshotLocalDataConstructor extends SafeConstructor {
+ SnapshotLocalDataConstructor() {
+ super(new LoaderOptions());
+ //Adding our own specific constructors for tags.
+ this.yamlConstructors.put(SNAPSHOT_YAML_TAG, new
ConstructSnapshotLocalData());
+ }
+
+ private final class ConstructSnapshotLocalData extends AbstractConstruct {
+ @SuppressWarnings("unchecked")
+ @Override
+ public Object construct(Node node) {
+ MappingNode mnode = (MappingNode) node;
+ Map<Object, Object> nodes = constructMapping(mnode);
+
+ OmSnapshotLocalDataYaml snapshotLocalData = new
OmSnapshotLocalDataYaml();
+
+ // Set version from YAML
+ Integer version = (Integer) nodes.get(OzoneConsts.OM_SLD_VERSION);
+ // Validate version.
+ if (version <= 0) {
+ // If version is not set or invalid, log a warning, but do not throw.
+ LOG.warn("Invalid version ({}) detected in snapshot local data YAML.
Proceed with caution.", version);
+ }
+ snapshotLocalData.setVersion(version);
+
+ // Set other fields from parsed YAML
+ snapshotLocalData.setSstFiltered((Boolean)
nodes.getOrDefault(OzoneConsts.OM_SLD_IS_SST_FILTERED, false));
+
+ Map<String, List<String>> uncompactedSSTFileList =
+ (Map<String, List<String>>)
nodes.get(OzoneConsts.OM_SLD_UNCOMPACTED_SST_FILE_LIST);
+ if (uncompactedSSTFileList != null) {
+ snapshotLocalData.setUncompactedSSTFileList(uncompactedSSTFileList);
+ }
+
+ snapshotLocalData.setLastCompactionTime(
+ (Long) nodes.getOrDefault(OzoneConsts.OM_SLD_LAST_COMPACTION_TIME,
-1L));
+ snapshotLocalData.setNeedsCompaction((Boolean)
nodes.getOrDefault(OzoneConsts.OM_SLD_NEEDS_COMPACTION, false));
+
+ Map<Integer, Map<String, List<String>>> compactedSSTFileList =
+ (Map<Integer, Map<String, List<String>>>)
nodes.get(OzoneConsts.OM_SLD_COMPACTED_SST_FILE_LIST);
+ if (compactedSSTFileList != null) {
+ snapshotLocalData.setCompactedSSTFileList(compactedSSTFileList);
+ }
+
+ String checksum = (String) nodes.get(OzoneConsts.OM_SLD_CHECKSUM);
+ if (checksum != null) {
+ snapshotLocalData.setChecksum(checksum);
+ }
+
+ return snapshotLocalData;
+ }
+ }
+ }
+
+ /**
+ * Returns the YAML representation of this object as a String
+ * (without triggering checksum computation or persistence).
+ * @return YAML string representation
+ */
+ public String getYaml() {
+ final Yaml yaml = getYamlForSnapshotLocalData();
+ return yaml.dump(this);
+ }
+
+ /**
+ * Computes checksum (stored in this object), and writes this object to a
YAML file.
+ * @param yamlFile The file to write to
+ * @throws IOException If there's an error writing to the file
+ */
+ public void writeToYaml(File yamlFile) throws IOException {
+ // Create Yaml
+ final Yaml yaml = getYamlForSnapshotLocalData();
+ // Compute Checksum and update SnapshotData
+ computeAndSetChecksum(yaml);
+ // Write the SnapshotData with checksum to Yaml file.
+ YamlUtils.dump(yaml, this, yamlFile, LOG);
+ }
+
+ /**
+ * Creates a OmSnapshotLocalDataYaml instance from a YAML file.
+ * @param yamlFile The YAML file to read from
+ * @return A new OmSnapshotLocalDataYaml instance
+ * @throws IOException If there's an error reading the file
+ */
+ public static OmSnapshotLocalDataYaml getFromYamlFile(File yamlFile) throws
IOException {
+ Preconditions.checkNotNull(yamlFile, "yamlFile cannot be null");
+ try (InputStream inputFileStream =
Files.newInputStream(yamlFile.toPath())) {
+ return getFromYamlStream(inputFileStream);
+ }
+ }
+
+ /**
+ * Returns a Yaml representation of the snapshot properties.
+ * @return Yaml representation of snapshot properties
+ */
+ public static Yaml getYamlForSnapshotLocalData() {
+ PropertyUtils propertyUtils = new PropertyUtils();
+ propertyUtils.setBeanAccess(BeanAccess.FIELD);
+ propertyUtils.setAllowReadOnlyProperties(true);
+
+ DumperOptions options = new DumperOptions();
+ Representer representer = new Representer(options);
+ representer.setPropertyUtils(propertyUtils);
+ representer.addClassTag(OmSnapshotLocalDataYaml.class, SNAPSHOT_YAML_TAG);
+
+ SafeConstructor snapshotDataConstructor = new
SnapshotLocalDataConstructor();
+ return new Yaml(snapshotDataConstructor, representer);
+ }
+
+ /**
+ * Read the YAML content InputStream, and return OmSnapshotLocalDataYaml
instance.
+ * @throws IOException
+ */
+ public static OmSnapshotLocalDataYaml getFromYamlStream(InputStream input)
throws IOException {
+ OmSnapshotLocalDataYaml dataYaml;
+
+ PropertyUtils propertyUtils = new PropertyUtils();
+ propertyUtils.setBeanAccess(BeanAccess.FIELD);
+ propertyUtils.setAllowReadOnlyProperties(true);
+
+ DumperOptions options = new DumperOptions();
+ Representer representer = new Representer(options);
+ representer.setPropertyUtils(propertyUtils);
+
+ SafeConstructor snapshotDataConstructor = new
SnapshotLocalDataConstructor();
+
+ Yaml yaml = new Yaml(snapshotDataConstructor, representer);
+
+ try {
+ dataYaml = yaml.load(input);
+ } catch (YAMLException ex) {
+ // Unchecked exception. Convert to IOException
+ throw new IOException(ex);
+ }
+
+ if (dataYaml == null) {
+ // If Yaml#load returned null, then the file is empty. This is valid yaml
+ // but considered an error in this case since we have lost data about
+ // the snapshot.
+ throw new IOException("Failed to load snapshot file. File is empty.");
+ }
+
+ return dataYaml;
+ }
+}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
index 24ae4e03ce..7457fdfb4f 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
@@ -769,6 +769,11 @@ public static String getSnapshotPath(OzoneConfiguration
conf,
OM_DB_NAME + checkpointDirName;
}
+ public static String getSnapshotLocalPropertyYamlPath(OzoneConfiguration
conf,
+ SnapshotInfo snapshotInfo) {
+ return getSnapshotPath(conf, snapshotInfo) + ".yaml";
+ }
+
public static boolean isSnapshotKey(String[] keyParts) {
return (keyParts.length > 1) &&
(keyParts[0].compareTo(OM_SNAPSHOT_INDICATOR) == 0);
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java
new file mode 100644
index 0000000000..00fea3e360
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.fs.FileSystemTestHelper;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * This class tests creating and reading snapshot data YAML files.
+ */
+public class TestOmSnapshotLocalDataYaml {
+
+ private static String testRoot = new FileSystemTestHelper().getTestRootDir();
+
+ private static final Instant NOW = Instant.now();
+
+ @BeforeEach
+ public void setUp() {
+ assertTrue(new File(testRoot).mkdirs());
+ }
+
+ @AfterEach
+ public void cleanup() {
+ FileUtil.fullyDelete(new File(testRoot));
+ }
+
+ /**
+ * Creates a snapshot local data YAML file.
+ */
+ private File writeToYaml(String snapshotName) throws IOException {
+ String yamlFilePath = snapshotName + ".yaml";
+
+ OmSnapshotLocalDataYaml dataYaml = new OmSnapshotLocalDataYaml();
+
+ // Set version
+ dataYaml.setVersion(42);
+ // Set SST filtered flag
+ dataYaml.setSstFiltered(true);
+
+ // Add some uncompacted SST files
+ dataYaml.addUncompactedSSTFile("table1", "sst1");
+ dataYaml.addUncompactedSSTFile("table1", "sst2");
+ dataYaml.addUncompactedSSTFile("table2", "sst3");
+
+ // Set last compaction time
+ dataYaml.setLastCompactionTime(NOW.toEpochMilli());
+
+ // Set needs compaction flag
+ dataYaml.setNeedsCompaction(true);
+
+ // Add some compacted SST files
+ dataYaml.addCompactedSSTFile(1, "table1", "compacted-sst1");
+ dataYaml.addCompactedSSTFile(1, "table2", "compacted-sst2");
+ dataYaml.addCompactedSSTFile(2, "table1", "compacted-sst3");
+
+ File yamlFile = new File(testRoot, yamlFilePath);
+
+ // Create YAML file with SnapshotData
+ dataYaml.writeToYaml(yamlFile);
+
+ // Check YAML file exists
+ assertTrue(yamlFile.exists());
+
+ return yamlFile;
+ }
+
+ @Test
+ public void testWriteToYaml() throws IOException {
+ File yamlFile = writeToYaml("snapshot1");
+
+ // Read from YAML file
+ OmSnapshotLocalDataYaml snapshotData =
OmSnapshotLocalDataYaml.getFromYamlFile(yamlFile);
+
+ // Verify fields
+ assertEquals(42, snapshotData.getVersion());
+ assertTrue(snapshotData.getSstFiltered());
+
+ Map<String, List<String>> uncompactedFiles =
snapshotData.getUncompactedSSTFileList();
+ assertEquals(2, uncompactedFiles.size());
+ assertEquals(2, uncompactedFiles.get("table1").size());
+ assertEquals(1, uncompactedFiles.get("table2").size());
+ assertTrue(uncompactedFiles.get("table1").contains("sst1"));
+ assertTrue(uncompactedFiles.get("table1").contains("sst2"));
+ assertTrue(uncompactedFiles.get("table2").contains("sst3"));
+
+ assertEquals(NOW.toEpochMilli(), snapshotData.getLastCompactionTime());
+ assertTrue(snapshotData.getNeedsCompaction());
+
+ Map<Integer, Map<String, List<String>>> compactedFiles =
snapshotData.getCompactedSSTFileList();
+ assertEquals(2, compactedFiles.size());
+ assertTrue(compactedFiles.containsKey(1));
+ assertTrue(compactedFiles.containsKey(2));
+ assertEquals(2, compactedFiles.get(1).size());
+ assertEquals(1, compactedFiles.get(2).size());
+ assertTrue(compactedFiles.get(1).get("table1").contains("compacted-sst1"));
+ assertTrue(compactedFiles.get(1).get("table2").contains("compacted-sst2"));
+ assertTrue(compactedFiles.get(2).get("table1").contains("compacted-sst3"));
+ }
+
+ @Test
+ public void testUpdateSnapshotDataFile() throws IOException {
+ File yamlFile = writeToYaml("snapshot2");
+
+ // Read from YAML file
+ OmSnapshotLocalDataYaml dataYaml =
+ OmSnapshotLocalDataYaml.getFromYamlFile(yamlFile);
+
+ // Update snapshot data
+ dataYaml.setSstFiltered(false);
+ dataYaml.setNeedsCompaction(false);
+ dataYaml.addUncompactedSSTFile("table3", "sst4");
+ dataYaml.addCompactedSSTFile(3, "table3", "compacted-sst4");
+
+ // Write updated data back to file
+ dataYaml.writeToYaml(yamlFile);
+
+ // Read back the updated data
+ dataYaml = OmSnapshotLocalDataYaml.getFromYamlFile(yamlFile);
+
+ // Verify updated data
+ assertThat(dataYaml.getSstFiltered()).isFalse();
+ assertThat(dataYaml.getNeedsCompaction()).isFalse();
+
+ Map<String, List<String>> uncompactedFiles =
dataYaml.getUncompactedSSTFileList();
+ assertEquals(3, uncompactedFiles.size());
+ assertTrue(uncompactedFiles.containsKey("table3"));
+ assertTrue(uncompactedFiles.get("table3").contains("sst4"));
+
+ Map<Integer, Map<String, List<String>>> compactedFiles =
dataYaml.getCompactedSSTFileList();
+ assertEquals(3, compactedFiles.size());
+ assertTrue(compactedFiles.containsKey(3));
+ assertTrue(compactedFiles.get(3).containsKey("table3"));
+ assertTrue(compactedFiles.get(3).get("table3").contains("compacted-sst4"));
+ }
+
+ @Test
+ public void testEmptyFile() throws IOException {
+ File emptyFile = new File(testRoot, "empty.yaml");
+ assertTrue(emptyFile.createNewFile());
+
+ IOException ex = assertThrows(IOException.class, () ->
+ OmSnapshotLocalDataYaml.getFromYamlFile(emptyFile));
+
+ assertThat(ex).hasMessageContaining("Failed to load snapshot file. File is
empty.");
+ }
+
+ @Test
+ public void testChecksum() throws IOException {
+ File yamlFile = writeToYaml("snapshot3");
+
+ // Read from YAML file
+ OmSnapshotLocalDataYaml snapshotData =
OmSnapshotLocalDataYaml.getFromYamlFile(yamlFile);
+
+ // Get the original checksum
+ String originalChecksum = snapshotData.getChecksum();
+
+ // Verify the checksum is not null or empty
+ assertThat(originalChecksum).isNotNull().isNotEmpty();
+
+ assertTrue(OmSnapshotLocalDataYaml.verifyChecksum(snapshotData));
+ }
+
+ @Test
+ public void testYamlContainsAllFields() throws IOException {
+ File yamlFile = writeToYaml("snapshot4");
+
+ String content = FileUtils.readFileToString(yamlFile,
Charset.defaultCharset());
+
+ // Verify the YAML content contains all expected fields
+ assertThat(content).contains(OzoneConsts.OM_SLD_VERSION);
+ assertThat(content).contains(OzoneConsts.OM_SLD_CHECKSUM);
+ assertThat(content).contains(OzoneConsts.OM_SLD_IS_SST_FILTERED);
+ assertThat(content).contains(OzoneConsts.OM_SLD_UNCOMPACTED_SST_FILE_LIST);
+ assertThat(content).contains(OzoneConsts.OM_SLD_LAST_COMPACTION_TIME);
+ assertThat(content).contains(OzoneConsts.OM_SLD_NEEDS_COMPACTION);
+ assertThat(content).contains(OzoneConsts.OM_SLD_COMPACTED_SST_FILE_LIST);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
