This is an automated email from the ASF dual-hosted git repository.
swamirishi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 74d4bd220f7 HDDS-13627. In-memory Manager for Snapshot Local Data
(#9141)
74d4bd220f7 is described below
commit 74d4bd220f757f191089527de2e2b5afeb471b81
Author: Swaminathan Balachandran <[email protected]>
AuthorDate: Tue Oct 14 22:59:28 2025 -0400
HDDS-13627. In-memory Manager for Snapshot Local Data (#9141)
---
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 9 +-
.../response/snapshot/OMSnapshotPurgeResponse.java | 9 +-
.../om/snapshot/OmSnapshotLocalDataManager.java | 190 ++++++++++-
.../TestOMSnapshotPurgeRequestAndResponse.java | 2 -
.../snapshot/TestOmSnapshotLocalDataManager.java | 374 +++++++++++++++++++++
.../filter/AbstractReclaimableFilterTest.java | 11 +-
6 files changed, 578 insertions(+), 17 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
index 19fe367bb92..7b9beb80cf6 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
@@ -195,7 +195,7 @@ public final class OmSnapshotManager implements
AutoCloseable {
private int fsSnapshotMaxLimit;
private final AtomicInteger inFlightSnapshotCount = new AtomicInteger(0);
- public OmSnapshotManager(OzoneManager ozoneManager) {
+ public OmSnapshotManager(OzoneManager ozoneManager) throws IOException {
this.snapshotLocalDataManager = new
OmSnapshotLocalDataManager(ozoneManager.getMetadataManager());
boolean isFilesystemSnapshotEnabled =
ozoneManager.isFilesystemSnapshotEnabled();
@@ -803,6 +803,13 @@ public static Path getSnapshotPath(OMMetadataManager
omMetadataManager, Snapshot
checkpointPrefix + snapshotInfo.getCheckpointDir());
}
+ public static Path getSnapshotPath(OMMetadataManager omMetadataManager, UUID
snapshotId) {
+ RDBStore store = (RDBStore) omMetadataManager.getStore();
+ String checkpointPrefix = store.getDbLocation().getName();
+ return Paths.get(store.getSnapshotsParentDir(),
+ checkpointPrefix + SnapshotInfo.getCheckpointDirName(snapshotId));
+ }
+
public static String getSnapshotPath(OzoneConfiguration conf,
SnapshotInfo snapshotInfo) {
return getSnapshotPath(conf, snapshotInfo.getCheckpointDirName());
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeResponse.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeResponse.java
index 75ba2a8f950..267547bc1e5 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeResponse.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeResponse.java
@@ -23,9 +23,7 @@
import com.google.common.annotations.VisibleForTesting;
import jakarta.annotation.Nonnull;
import java.io.IOException;
-import java.nio.file.Files;
import java.nio.file.Path;
-import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import org.apache.commons.io.FileUtils;
@@ -130,14 +128,11 @@ private void
deleteCheckpointDirectory(OmSnapshotLocalDataManager snapshotLocalD
boolean acquiredSnapshotLock = omLockDetails.isLockAcquired();
if (acquiredSnapshotLock) {
Path snapshotDirPath =
OmSnapshotManager.getSnapshotPath(omMetadataManager, snapshotInfo);
- // TODO: Do not delete on snapshot purge. OmSnapshotLocalDataManager
should delete orphan local data files.
- Path snapshotLocalDataPath =
Paths.get(snapshotLocalDataManager.getSnapshotLocalPropertyYamlPath(snapshotInfo));
try {
FileUtils.deleteDirectory(snapshotDirPath.toFile());
- Files.deleteIfExists(snapshotLocalDataPath);
} catch (IOException ex) {
- LOG.error("Failed to delete snapshot directory {} and/or local data
file {} for snapshot {}",
- snapshotDirPath, snapshotLocalDataPath,
snapshotInfo.getTableKey(), ex);
+ LOG.error("Failed to delete snapshot directory {} for snapshot {}",
+ snapshotDirPath, snapshotInfo.getTableKey(), ex);
} finally {
omMetadataManager.getLock().releaseWriteLock(SNAPSHOT_DB_LOCK,
snapshotInfo.getSnapshotId().toString());
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java
index 98536444a61..3c529abaf3c 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java
@@ -19,14 +19,28 @@
import static
org.apache.hadoop.ozone.om.OmSnapshotLocalDataYaml.YAML_FILE_EXTENSION;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.graph.GraphBuilder;
+import com.google.common.graph.MutableGraph;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.Stack;
+import java.util.UUID;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.utils.db.RDBStore;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OmSnapshotLocalData;
+import org.apache.hadoop.ozone.om.OmSnapshotLocalData.VersionMeta;
import org.apache.hadoop.ozone.om.OmSnapshotLocalDataYaml;
import org.apache.hadoop.ozone.om.OmSnapshotManager;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
@@ -45,9 +59,12 @@ public class OmSnapshotLocalDataManager implements
AutoCloseable {
private static final Logger LOG =
LoggerFactory.getLogger(OmSnapshotLocalDataManager.class);
private final ObjectSerializer<OmSnapshotLocalData>
snapshotLocalDataSerializer;
+ private final MutableGraph<LocalDataVersionNode> localDataGraph;
+ private final Map<UUID, SnapshotVersionsMeta> versionNodeMap;
private final OMMetadataManager omMetadataManager;
- public OmSnapshotLocalDataManager(OMMetadataManager omMetadataManager) {
+ public OmSnapshotLocalDataManager(OMMetadataManager omMetadataManager)
throws IOException {
+ this.localDataGraph = GraphBuilder.directed().build();
this.omMetadataManager = omMetadataManager;
this.snapshotLocalDataSerializer = new YamlSerializer<OmSnapshotLocalData>(
new OmSnapshotLocalDataYaml.YamlFactory()) {
@@ -57,6 +74,13 @@ public void computeAndSetChecksum(Yaml yaml,
OmSnapshotLocalData data) throws IO
data.computeAndSetChecksum(yaml);
}
};
+ this.versionNodeMap = new HashMap<>();
+ init();
+ }
+
+ @VisibleForTesting
+ Map<UUID, SnapshotVersionsMeta> getVersionNodeMap() {
+ return versionNodeMap;
}
/**
@@ -76,7 +100,11 @@ public static String getSnapshotLocalPropertyYamlPath(Path
snapshotPath) {
* @return the path to the snapshot's local property YAML file
*/
public String getSnapshotLocalPropertyYamlPath(SnapshotInfo snapshotInfo) {
- Path snapshotPath = OmSnapshotManager.getSnapshotPath(omMetadataManager,
snapshotInfo);
+ return getSnapshotLocalPropertyYamlPath(snapshotInfo.getSnapshotId());
+ }
+
+ public String getSnapshotLocalPropertyYamlPath(UUID snapshotId) {
+ Path snapshotPath = OmSnapshotManager.getSnapshotPath(omMetadataManager,
snapshotId);
return getSnapshotLocalPropertyYamlPath(snapshotPath);
}
@@ -95,14 +123,100 @@ public void createNewOmSnapshotLocalDataFile(RDBStore
snapshotStore, SnapshotInf
}
public OmSnapshotLocalData getOmSnapshotLocalData(SnapshotInfo snapshotInfo)
throws IOException {
- Path snapshotLocalDataPath =
Paths.get(getSnapshotLocalPropertyYamlPath(snapshotInfo));
- return snapshotLocalDataSerializer.load(snapshotLocalDataPath.toFile());
+ return getOmSnapshotLocalData(snapshotInfo.getSnapshotId());
+ }
+
+ public OmSnapshotLocalData getOmSnapshotLocalData(UUID snapshotId) throws
IOException {
+ Path snapshotLocalDataPath =
Paths.get(getSnapshotLocalPropertyYamlPath(snapshotId));
+ OmSnapshotLocalData snapshotLocalData =
snapshotLocalDataSerializer.load(snapshotLocalDataPath.toFile());
+ if (!Objects.equals(snapshotLocalData.getSnapshotId(), snapshotId)) {
+ throw new IOException("SnapshotId in path : " + snapshotLocalDataPath +
" contains snapshotLocalData " +
+ "corresponding to snapshotId " + snapshotLocalData.getSnapshotId() +
". Expected snapshotId " + snapshotId);
+ }
+ return snapshotLocalData;
}
public OmSnapshotLocalData getOmSnapshotLocalData(File snapshotDataPath)
throws IOException {
return snapshotLocalDataSerializer.load(snapshotDataPath);
}
+ private LocalDataVersionNode getVersionNode(UUID snapshotId, int version) {
+ if (!versionNodeMap.containsKey(snapshotId)) {
+ return null;
+ }
+ return versionNodeMap.get(snapshotId).getVersionNode(version);
+ }
+
+ private void addSnapshotVersionMeta(UUID snapshotId, SnapshotVersionsMeta
snapshotVersionsMeta)
+ throws IOException {
+ if (!versionNodeMap.containsKey(snapshotId)) {
+ for (LocalDataVersionNode versionNode :
snapshotVersionsMeta.getSnapshotVersions().values()) {
+ if (getVersionNode(versionNode.snapshotId, versionNode.version) !=
null) {
+ throw new IOException("Unable to add " + versionNode + " since it
already exists");
+ }
+ LocalDataVersionNode previousVersionNode =
versionNode.previousSnapshotId == null ? null :
+ getVersionNode(versionNode.previousSnapshotId,
versionNode.previousSnapshotVersion);
+ if (versionNode.previousSnapshotId != null && previousVersionNode ==
null) {
+ throw new IOException("Unable to add " + versionNode + " since
previous snapshot with version hasn't been " +
+ "loaded");
+ }
+ localDataGraph.addNode(versionNode);
+ if (previousVersionNode != null) {
+ localDataGraph.putEdge(versionNode, previousVersionNode);
+ }
+ }
+ versionNodeMap.put(snapshotId, snapshotVersionsMeta);
+ }
+ }
+
+ void addVersionNodeWithDependents(OmSnapshotLocalData snapshotLocalData)
throws IOException {
+ if (versionNodeMap.containsKey(snapshotLocalData.getSnapshotId())) {
+ return;
+ }
+ Set<UUID> visitedSnapshotIds = new HashSet<>();
+ Stack<Pair<UUID, SnapshotVersionsMeta>> stack = new Stack<>();
+ stack.push(Pair.of(snapshotLocalData.getSnapshotId(), new
SnapshotVersionsMeta(snapshotLocalData)));
+ while (!stack.isEmpty()) {
+ Pair<UUID, SnapshotVersionsMeta> versionNodeToProcess = stack.peek();
+ UUID snapId = versionNodeToProcess.getLeft();
+ SnapshotVersionsMeta snapshotVersionsMeta =
versionNodeToProcess.getRight();
+ if (visitedSnapshotIds.contains(snapId)) {
+ addSnapshotVersionMeta(snapId, snapshotVersionsMeta);
+ stack.pop();
+ } else {
+ UUID prevSnapId = snapshotVersionsMeta.getPreviousSnapshotId();
+ if (prevSnapId != null && !versionNodeMap.containsKey(prevSnapId)) {
+ OmSnapshotLocalData prevSnapshotLocalData =
getOmSnapshotLocalData(prevSnapId);
+ stack.push(Pair.of(prevSnapshotLocalData.getSnapshotId(), new
SnapshotVersionsMeta(prevSnapshotLocalData)));
+ }
+ visitedSnapshotIds.add(snapId);
+ }
+ }
+ }
+
+ private void init() throws IOException {
+ RDBStore store = (RDBStore) omMetadataManager.getStore();
+ String checkpointPrefix = store.getDbLocation().getName();
+ File snapshotDir = new File(store.getSnapshotsParentDir());
+ File[] localDataFiles = snapshotDir.listFiles(
+ (dir, name) -> name.startsWith(checkpointPrefix) &&
name.endsWith(YAML_FILE_EXTENSION));
+ if (localDataFiles == null) {
+ throw new IOException("Error while listing yaml files inside directory:
" + snapshotDir.getAbsolutePath());
+ }
+ Arrays.sort(localDataFiles, Comparator.comparing(File::getName));
+ for (File localDataFile : localDataFiles) {
+ OmSnapshotLocalData snapshotLocalData =
snapshotLocalDataSerializer.load(localDataFile);
+ File file = new
File(getSnapshotLocalPropertyYamlPath(snapshotLocalData.getSnapshotId()));
+ String expectedPath = file.getAbsolutePath();
+ String actualPath = localDataFile.getAbsolutePath();
+ if (!expectedPath.equals(actualPath)) {
+ throw new IOException("Unexpected path for local data file with
snapshotId:" + snapshotLocalData.getSnapshotId()
+ + " : " + actualPath + ". " + "Expected: " + expectedPath);
+ }
+ addVersionNodeWithDependents(snapshotLocalData);
+ }
+ }
+
@Override
public void close() {
if (snapshotLocalDataSerializer != null) {
@@ -113,4 +227,72 @@ public void close() {
}
}
}
+
+ static final class LocalDataVersionNode {
+ private final UUID snapshotId;
+ private final int version;
+ private final UUID previousSnapshotId;
+ private final int previousSnapshotVersion;
+
+ private LocalDataVersionNode(UUID snapshotId, int version, UUID
previousSnapshotId, int previousSnapshotVersion) {
+ this.previousSnapshotId = previousSnapshotId;
+ this.previousSnapshotVersion = previousSnapshotVersion;
+ this.snapshotId = snapshotId;
+ this.version = version;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof LocalDataVersionNode)) {
+ return false;
+ }
+ LocalDataVersionNode that = (LocalDataVersionNode) o;
+ return version == that.version && previousSnapshotVersion ==
that.previousSnapshotVersion &&
+ snapshotId.equals(that.snapshotId) &&
Objects.equals(previousSnapshotId, that.previousSnapshotId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(snapshotId, version, previousSnapshotId,
previousSnapshotVersion);
+ }
+ }
+
+ static final class SnapshotVersionsMeta {
+ private final UUID previousSnapshotId;
+ private final Map<Integer, LocalDataVersionNode> snapshotVersions;
+ private int version;
+
+ private SnapshotVersionsMeta(OmSnapshotLocalData snapshotLocalData) {
+ this.previousSnapshotId = snapshotLocalData.getPreviousSnapshotId();
+ this.snapshotVersions = getVersionNodes(snapshotLocalData);
+ this.version = snapshotLocalData.getVersion();
+ }
+
+ private Map<Integer, LocalDataVersionNode>
getVersionNodes(OmSnapshotLocalData snapshotLocalData) {
+ UUID snapshotId = snapshotLocalData.getSnapshotId();
+ UUID prevSnapshotId = snapshotLocalData.getPreviousSnapshotId();
+ Map<Integer, LocalDataVersionNode> versionNodes = new HashMap<>();
+ for (Map.Entry<Integer, VersionMeta> entry :
snapshotLocalData.getVersionSstFileInfos().entrySet()) {
+ versionNodes.put(entry.getKey(), new LocalDataVersionNode(snapshotId,
entry.getKey(),
+ prevSnapshotId, entry.getValue().getPreviousSnapshotVersion()));
+ }
+ return versionNodes;
+ }
+
+ UUID getPreviousSnapshotId() {
+ return previousSnapshotId;
+ }
+
+ int getVersion() {
+ return version;
+ }
+
+ Map<Integer, LocalDataVersionNode> getSnapshotVersions() {
+ return snapshotVersions;
+ }
+
+ LocalDataVersionNode getVersionNode(int snapshotVersion) {
+ return snapshotVersions.get(snapshotVersion);
+ }
+ }
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java
index 0fb26a4cd99..35053882eed 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java
@@ -190,8 +190,6 @@ public void testValidateAndUpdateCache() throws Exception {
// Check if all the checkpoints are cleared.
for (Path checkpoint : checkpointPaths) {
assertFalse(Files.exists(checkpoint));
- assertFalse(Files.exists(Paths.get(
-
OmSnapshotLocalDataManager.getSnapshotLocalPropertyYamlPath(checkpoint))));
}
assertEquals(initialSnapshotPurgeCount + 1,
getOmSnapshotIntMetrics().getNumSnapshotPurges());
assertEquals(initialSnapshotPurgeFailCount,
getOmSnapshotIntMetrics().getNumSnapshotPurgeFails());
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLocalDataManager.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLocalDataManager.java
new file mode 100644
index 00000000000..34bde4814a6
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLocalDataManager.java
@@ -0,0 +1,374 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_SEPARATOR;
+import static
org.apache.hadoop.ozone.om.OmSnapshotLocalDataYaml.YAML_FILE_EXTENSION;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DIRECTORY_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
+import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hdds.StringUtils;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDatabase;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmSnapshotLocalData;
+import org.apache.hadoop.ozone.om.OmSnapshotLocalDataYaml;
+import org.apache.hadoop.ozone.om.OmSnapshotManager;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.util.YamlSerializer;
+import org.apache.ozone.compaction.log.SstFileInfo;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.rocksdb.LiveFileMetaData;
+import org.yaml.snakeyaml.Yaml;
+
+/**
+ * Test class for OmSnapshotLocalDataManager.
+ */
+public class TestOmSnapshotLocalDataManager {
+
+ private static YamlSerializer<OmSnapshotLocalData>
snapshotLocalDataYamlSerializer;
+
+ @Mock
+ private OMMetadataManager omMetadataManager;
+
+ @Mock
+ private RDBStore rdbStore;
+
+ @Mock
+ private RDBStore snapshotStore;
+
+ @TempDir
+ private Path tempDir;
+
+ private OmSnapshotLocalDataManager localDataManager;
+ private AutoCloseable mocks;
+
+ private File snapshotsDir;
+
+ @BeforeAll
+ public static void setupClass() {
+ snapshotLocalDataYamlSerializer = new YamlSerializer<OmSnapshotLocalData>(
+ new OmSnapshotLocalDataYaml.YamlFactory()) {
+
+ @Override
+ public void computeAndSetChecksum(Yaml yaml, OmSnapshotLocalData data)
throws IOException {
+ data.computeAndSetChecksum(yaml);
+ }
+ };
+ }
+
+ @AfterAll
+ public static void teardownClass() throws IOException {
+ snapshotLocalDataYamlSerializer.close();
+ snapshotLocalDataYamlSerializer = null;
+ }
+
+ @BeforeEach
+ public void setUp() throws IOException {
+ mocks = MockitoAnnotations.openMocks(this);
+
+ // Setup mock behavior
+ when(omMetadataManager.getStore()).thenReturn(rdbStore);
+
+ this.snapshotsDir = tempDir.resolve("snapshots").toFile();
+ FileUtils.deleteDirectory(snapshotsDir);
+ assertTrue(snapshotsDir.exists() || snapshotsDir.mkdirs());
+ File dbLocation = tempDir.resolve("db").toFile();
+ FileUtils.deleteDirectory(dbLocation);
+ assertTrue(dbLocation.exists() || dbLocation.mkdirs());
+
+
+
when(rdbStore.getSnapshotsParentDir()).thenReturn(snapshotsDir.getAbsolutePath());
+ when(rdbStore.getDbLocation()).thenReturn(dbLocation);
+ }
+
+ @AfterEach
+ public void tearDown() throws Exception {
+ if (localDataManager != null) {
+ localDataManager.close();
+ }
+ if (mocks != null) {
+ mocks.close();
+ }
+ }
+
+ @Test
+ public void testConstructor() throws IOException {
+ localDataManager = new OmSnapshotLocalDataManager(omMetadataManager);
+ assertNotNull(localDataManager);
+ }
+
+ @Test
+ public void testGetSnapshotLocalPropertyYamlPathWithSnapshotInfo() throws
IOException {
+ UUID snapshotId = UUID.randomUUID();
+ SnapshotInfo snapshotInfo = createMockSnapshotInfo(snapshotId, null);
+
+ localDataManager = new OmSnapshotLocalDataManager(omMetadataManager);
+
+ File yamlPath = new
File(localDataManager.getSnapshotLocalPropertyYamlPath(snapshotInfo));
+ assertNotNull(yamlPath);
+ Path expectedYamlPath = Paths.get(snapshotsDir.getAbsolutePath(), "db" +
OM_SNAPSHOT_SEPARATOR + snapshotId
+ + YAML_FILE_EXTENSION);
+ assertEquals(expectedYamlPath.toAbsolutePath().toString(),
yamlPath.getAbsolutePath());
+ }
+
+ @Test
+ public void testCreateNewOmSnapshotLocalDataFile() throws IOException {
+ UUID snapshotId = UUID.randomUUID();
+ SnapshotInfo snapshotInfo = createMockSnapshotInfo(snapshotId, null);
+
+ // Setup snapshot store mock
+ File snapshotDbLocation =
OmSnapshotManager.getSnapshotPath(omMetadataManager, snapshotId).toFile();
+ assertTrue(snapshotDbLocation.exists() || snapshotDbLocation.mkdirs());
+
+ List<LiveFileMetaData> sstFiles = new ArrayList<>();
+ sstFiles.add(createMockLiveFileMetaData("file1.sst", KEY_TABLE, "key1",
"key7"));
+ sstFiles.add(createMockLiveFileMetaData("file2.sst", KEY_TABLE, "key3",
"key9"));
+ sstFiles.add(createMockLiveFileMetaData("file3.sst", FILE_TABLE, "key1",
"key7"));
+ sstFiles.add(createMockLiveFileMetaData("file4.sst", FILE_TABLE, "key1",
"key7"));
+ sstFiles.add(createMockLiveFileMetaData("file5.sst", DIRECTORY_TABLE,
"key1", "key7"));
+ sstFiles.add(createMockLiveFileMetaData("file6.sst", "colFamily1", "key1",
"key7"));
+ List<SstFileInfo> sstFileInfos = IntStream.range(0, sstFiles.size() - 1)
+
.mapToObj(sstFiles::get).map(SstFileInfo::new).collect(Collectors.toList());
+ when(snapshotStore.getDbLocation()).thenReturn(snapshotDbLocation);
+ RocksDatabase rocksDatabase = mock(RocksDatabase.class);
+ when(snapshotStore.getDb()).thenReturn(rocksDatabase);
+ when(rocksDatabase.getLiveFilesMetaData()).thenReturn(sstFiles);
+ localDataManager = new OmSnapshotLocalDataManager(omMetadataManager);
+
+ localDataManager.createNewOmSnapshotLocalDataFile(snapshotStore,
snapshotInfo);
+
+ // Verify file was created
+ OmSnapshotLocalData snapshotLocalData =
localDataManager.getOmSnapshotLocalData(snapshotId);
+ assertEquals(1, snapshotLocalData.getVersionSstFileInfos().size());
+ OmSnapshotLocalData.VersionMeta versionMeta =
snapshotLocalData.getVersionSstFileInfos().get(0);
+ OmSnapshotLocalData.VersionMeta expectedVersionMeta = new
OmSnapshotLocalData.VersionMeta(0, sstFileInfos);
+ assertEquals(expectedVersionMeta, versionMeta);
+ }
+
+ @Test
+ public void testGetOmSnapshotLocalDataWithSnapshotInfo() throws IOException {
+ UUID snapshotId = UUID.randomUUID();
+ SnapshotInfo snapshotInfo = createMockSnapshotInfo(snapshotId, null);
+
+ // Create and write snapshot local data file
+ OmSnapshotLocalData localData = createMockLocalData(snapshotId, null);
+
+ localDataManager = new OmSnapshotLocalDataManager(omMetadataManager);
+
+ // Write the file manually for testing
+ Path yamlPath =
Paths.get(localDataManager.getSnapshotLocalPropertyYamlPath(snapshotInfo.getSnapshotId()));
+ writeLocalDataToFile(localData, yamlPath);
+
+ // Test retrieval
+ OmSnapshotLocalData retrieved =
localDataManager.getOmSnapshotLocalData(snapshotInfo);
+
+ assertNotNull(retrieved);
+ assertEquals(snapshotId, retrieved.getSnapshotId());
+ }
+
+ @Test
+ public void testGetOmSnapshotLocalDataWithMismatchedSnapshotId() throws
IOException {
+ UUID snapshotId = UUID.randomUUID();
+ UUID wrongSnapshotId = UUID.randomUUID();
+
+ // Create local data with wrong snapshot ID
+ OmSnapshotLocalData localData = createMockLocalData(wrongSnapshotId, null);
+
+ localDataManager = new OmSnapshotLocalDataManager(omMetadataManager);
+
+ Path yamlPath =
Paths.get(localDataManager.getSnapshotLocalPropertyYamlPath(snapshotId));
+ writeLocalDataToFile(localData, yamlPath);
+ // Should throw IOException due to mismatched IDs
+ assertThrows(IOException.class, () -> {
+ localDataManager.getOmSnapshotLocalData(snapshotId);
+ });
+ }
+
+ @Test
+ public void testGetOmSnapshotLocalDataWithFile() throws IOException {
+ UUID snapshotId = UUID.randomUUID();
+
+ OmSnapshotLocalData localData = createMockLocalData(snapshotId, null);
+
+ localDataManager = new OmSnapshotLocalDataManager(omMetadataManager);
+
+ Path yamlPath = tempDir.resolve("test-snapshot.yaml");
+ writeLocalDataToFile(localData, yamlPath);
+
+ OmSnapshotLocalData retrieved = localDataManager
+ .getOmSnapshotLocalData(yamlPath.toFile());
+
+ assertNotNull(retrieved);
+ assertEquals(snapshotId, retrieved.getSnapshotId());
+ }
+
+ @Test
+ public void testAddVersionNodeWithDependents() throws IOException {
+ List<UUID> versionIds = Stream.of(UUID.randomUUID(), UUID.randomUUID())
+
.sorted(Comparator.comparing(String::valueOf)).collect(Collectors.toList());
+ UUID snapshotId = versionIds.get(0);
+ UUID previousSnapshotId = versionIds.get(1);
+ localDataManager = new OmSnapshotLocalDataManager(omMetadataManager);
+ // Create snapshot directory structure and files
+ createSnapshotLocalDataFile(snapshotId, previousSnapshotId);
+ createSnapshotLocalDataFile(previousSnapshotId, null);
+ OmSnapshotLocalData localData = createMockLocalData(snapshotId,
previousSnapshotId);
+
+ // Should not throw exception
+ localDataManager.addVersionNodeWithDependents(localData);
+ }
+
+ @Test
+ public void testAddVersionNodeWithDependentsAlreadyExists() throws
IOException {
+ UUID snapshotId = UUID.randomUUID();
+
+ createSnapshotLocalDataFile(snapshotId, null);
+
+ localDataManager = new OmSnapshotLocalDataManager(omMetadataManager);
+
+ OmSnapshotLocalData localData = createMockLocalData(snapshotId, null);
+
+ // First addition
+ localDataManager.addVersionNodeWithDependents(localData);
+
+ // Second addition - should handle gracefully
+ localDataManager.addVersionNodeWithDependents(localData);
+ }
+
+ @Test
+ public void testInitWithExistingYamlFiles() throws IOException {
+ List<UUID> versionIds = Stream.of(UUID.randomUUID(), UUID.randomUUID())
+
.sorted(Comparator.comparing(String::valueOf)).collect(Collectors.toList());
+ UUID snapshotId = versionIds.get(0);
+ UUID previousSnapshotId = versionIds.get(1);
+
+ createSnapshotLocalDataFile(previousSnapshotId, null);
+ createSnapshotLocalDataFile(snapshotId, previousSnapshotId);
+
+ // Initialize - should load existing files
+ localDataManager = new OmSnapshotLocalDataManager(omMetadataManager);
+
+ assertNotNull(localDataManager);
+ Map<UUID, OmSnapshotLocalDataManager.SnapshotVersionsMeta> versionMap =
+ localDataManager.getVersionNodeMap();
+ assertEquals(2, versionMap.size());
+ assertEquals(versionMap.keySet(), new HashSet<>(versionIds));
+ }
+
+ @Test
+ public void testInitWithInvalidPathThrowsException() throws IOException {
+ UUID snapshotId = UUID.randomUUID();
+
+ // Create a file with wrong location
+ OmSnapshotLocalData localData = createMockLocalData(snapshotId, null);
+ Path wrongPath = Paths.get(snapshotsDir.getAbsolutePath(),
"db-wrong-name.yaml");
+ writeLocalDataToFile(localData, wrongPath);
+
+ // Should throw IOException during init
+ assertThrows(IOException.class, () -> {
+ new OmSnapshotLocalDataManager(omMetadataManager);
+ });
+ }
+
+ @Test
+ public void testClose() throws IOException {
+ localDataManager = new OmSnapshotLocalDataManager(omMetadataManager);
+
+ // Should not throw exception
+ localDataManager.close();
+ }
+
+ // Helper methods
+
+ private SnapshotInfo createMockSnapshotInfo(UUID snapshotId, UUID
previousSnapshotId) {
+ SnapshotInfo.Builder builder = SnapshotInfo.newBuilder()
+ .setSnapshotId(snapshotId)
+ .setName("snapshot-" + snapshotId);
+
+ if (previousSnapshotId != null) {
+ builder.setPathPreviousSnapshotId(previousSnapshotId);
+ }
+
+ return builder.build();
+ }
+
+ private LiveFileMetaData createMockLiveFileMetaData(String fileName, String
columnFamilyName, String smallestKey,
+ String largestKey) {
+ LiveFileMetaData liveFileMetaData = mock(LiveFileMetaData.class);
+
when(liveFileMetaData.columnFamilyName()).thenReturn(StringUtils.string2Bytes(columnFamilyName));
+ when(liveFileMetaData.fileName()).thenReturn(fileName);
+
when(liveFileMetaData.smallestKey()).thenReturn(StringUtils.string2Bytes(smallestKey));
+
when(liveFileMetaData.largestKey()).thenReturn(StringUtils.string2Bytes(largestKey));
+ return liveFileMetaData;
+ }
+
+ private OmSnapshotLocalData createMockLocalData(UUID snapshotId, UUID
previousSnapshotId) {
+ List<LiveFileMetaData> sstFiles = new ArrayList<>();
+ sstFiles.add(createMockLiveFileMetaData("file1.sst", "columnFamily1",
"key1", "key7"));
+ sstFiles.add(createMockLiveFileMetaData("file2.sst", "columnFamily1",
"key3", "key10"));
+ sstFiles.add(createMockLiveFileMetaData("file3.sst", "columnFamily2",
"key1", "key8"));
+ sstFiles.add(createMockLiveFileMetaData("file4.sst", "columnFamily2",
"key0", "key10"));
+ return new OmSnapshotLocalData(snapshotId, sstFiles, previousSnapshotId);
+ }
+
+ private void createSnapshotLocalDataFile(UUID snapshotId, UUID
previousSnapshotId)
+ throws IOException {
+ OmSnapshotLocalData localData = createMockLocalData(snapshotId,
previousSnapshotId);
+
+ String fileName = "db" + OM_SNAPSHOT_SEPARATOR + snapshotId.toString() +
YAML_FILE_EXTENSION;
+ Path yamlPath = Paths.get(snapshotsDir.getAbsolutePath(), fileName);
+
+ writeLocalDataToFile(localData, yamlPath);
+ }
+
+ private void writeLocalDataToFile(OmSnapshotLocalData localData, Path
filePath)
+ throws IOException {
+ // This is a simplified version - in real implementation,
+ // you would use the YamlSerializer
+ snapshotLocalDataYamlSerializer.save(filePath.toFile(), localData);
+ }
+}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/filter/AbstractReclaimableFilterTest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/filter/AbstractReclaimableFilterTest.java
index e8c362d9a5f..13ba79a77f8 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/filter/AbstractReclaimableFilterTest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/filter/AbstractReclaimableFilterTest.java
@@ -27,6 +27,7 @@
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockConstruction;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.when;
@@ -61,6 +62,7 @@
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock;
import org.apache.hadoop.ozone.om.lock.OMLockDetails;
+import org.apache.hadoop.ozone.om.snapshot.OmSnapshotLocalDataManager;
import org.apache.hadoop.ozone.om.snapshot.SnapshotCache;
import org.apache.hadoop.ozone.om.snapshot.SnapshotDiffManager;
import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils;
@@ -188,9 +190,9 @@ private void mockOzoneManager(BucketLayout bucketLayout)
throws IOException {
private void mockOmSnapshotManager(OzoneManager om) throws RocksDBException,
IOException {
try (MockedStatic<ManagedRocksDB> rocksdb =
Mockito.mockStatic(ManagedRocksDB.class);
MockedConstruction<SnapshotDiffManager> mockedSnapshotDiffManager =
- Mockito.mockConstruction(SnapshotDiffManager.class, (mock,
context) ->
+ mockConstruction(SnapshotDiffManager.class, (mock, context) ->
doNothing().when(mock).close());
- MockedConstruction<SnapshotCache> mockedCache =
Mockito.mockConstruction(SnapshotCache.class,
+ MockedConstruction<SnapshotCache> mockedCache =
mockConstruction(SnapshotCache.class,
(mock, context) -> {
Map<UUID, UncheckedAutoCloseableSupplier<OmSnapshot>> map = new
HashMap<>();
when(mock.get(any(UUID.class))).thenAnswer(i -> {
@@ -237,7 +239,10 @@ private void mockOmSnapshotManager(OzoneManager om) throws
RocksDBException, IOE
conf.set(OZONE_METADATA_DIRS,
testDir.toAbsolutePath().toFile().getAbsolutePath());
when(om.getConfiguration()).thenReturn(conf);
when(om.isFilesystemSnapshotEnabled()).thenReturn(true);
- this.omSnapshotManager = new OmSnapshotManager(om);
+ try (MockedConstruction<OmSnapshotLocalDataManager> ignored =
+ mockConstruction(OmSnapshotLocalDataManager.class)) {
+ this.omSnapshotManager = new OmSnapshotManager(om);
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
