This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 905d1ad567b HDDS-13538. Transfer snapshot YAML files as part of
bootstrap tarball. (#8899)
905d1ad567b is described below
commit 905d1ad567bf2c6f1fae1dec4c8164da82682923
Author: Sadanand Shenoy <[email protected]>
AuthorDate: Sat Aug 16 10:27:12 2025 +0530
HDDS-13538. Transfer snapshot YAML files as part of bootstrap tarball.
(#8899)
Co-authored-by: Wei-Chiu Chuang <[email protected]>
Generated-by: Copilot
---
.../TestOMDbCheckpointServletInodeBasedXfer.java | 24 ++++++++++++++++++++++
.../om/OMDBCheckpointServletInodeBasedXfer.java | 7 +++++++
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 14 +++++++++----
3 files changed, 41 insertions(+), 4 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java
index a43b0832c96..ec2080e9cf4 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java
@@ -291,10 +291,12 @@ public void testContentsOfTarballWithSnapshot(boolean
includeSnapshot) throws Ex
Path checkpointLocation = realCheckpoint.get().getCheckpointLocation();
populateInodesOfFilesInDirectory(dbStore, checkpointLocation,
inodesFromOmDataDir, hardLinkMapFromOmData);
+ int numSnapshots = 0;
if (includeSnapshot) {
for (String snapshotPath : snapshotPaths) {
populateInodesOfFilesInDirectory(dbStore, Paths.get(snapshotPath),
inodesFromOmDataDir, hardLinkMapFromOmData);
+ numSnapshots++;
}
}
populateInodesOfFilesInDirectory(dbStore,
Paths.get(dbStore.getRocksDBCheckpointDiffer().getSSTBackupDir()),
@@ -317,8 +319,30 @@ public void testContentsOfTarballWithSnapshot(boolean
includeSnapshot) throws Ex
assertFalse(inodesFromTarball.isEmpty());
assertTrue(inodesFromTarball.containsAll(inodesFromOmDataDir));
+ long actualYamlFiles = Files.list(newDbDir.toPath())
+ .filter(f -> f.getFileName().toString()
+ .endsWith(".yaml")).count();
+ assertEquals(numSnapshots, actualYamlFiles,
+ "Number of generated YAML files should match the number of
snapshots.");
+
// create hardlinks now
OmSnapshotUtils.createHardLinks(newDbDir.toPath(), true);
+
+ if (includeSnapshot) {
+ List<String> yamlRelativePaths = snapshotPaths.stream().map(path -> {
+ int startIndex = path.indexOf("db.snapshots");
+ if (startIndex != -1) {
+ return path.substring(startIndex) + ".yaml";
+ }
+ return path + ".yaml";
+ }).collect(Collectors.toList());
+
+ for (String yamlRelativePath : yamlRelativePaths) {
+ String yamlFileName = Paths.get(newDbDir.getPath(),
yamlRelativePath).toString();
+ assertTrue(Files.exists(Paths.get(yamlFileName)));
+ }
+ }
+
assertFalse(hardlinkFilePath.toFile().exists());
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServletInodeBasedXfer.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServletInodeBasedXfer.java
index 97e345e7677..42a28e4a781 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServletInodeBasedXfer.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServletInodeBasedXfer.java
@@ -297,6 +297,13 @@ private void transferSnapshotData(Set<String>
sstFilesToExclude, Path tmpdir, Se
om.getOmSnapshotManager().invalidateCacheEntry(UUID.fromString(snapshotId));
writeDBToArchive(sstFilesToExclude, snapshotDir, maxTotalSstSize,
archiveOutputStream, tmpdir,
hardLinkFileMap, false);
+ Path snapshotLocalPropertyYaml = Paths.get(
+ OmSnapshotManager.getSnapshotLocalPropertyYamlPath(snapshotDir));
+ if (Files.exists(snapshotLocalPropertyYaml)) {
+ File yamlFile = snapshotLocalPropertyYaml.toFile();
+ hardLinkFileMap.put(yamlFile.getAbsolutePath(), yamlFile.getName());
+ linkAndIncludeFile(yamlFile, yamlFile.getName(),
archiveOutputStream, tmpdir);
+ }
} finally {
omMetadataManager.getLock().releaseReadLock(SNAPSHOT_DB_LOCK,
snapshotId);
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
index 786a55b4bb9..91e5abb217a 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
@@ -840,12 +840,18 @@ public static String
extractSnapshotIDFromCheckpointDirName(String snapshotPath)
*/
public static String getSnapshotLocalPropertyYamlPath(OMMetadataManager
omMetadataManager,
SnapshotInfo snapshotInfo) {
- return getSnapshotPath(omMetadataManager, snapshotInfo) + ".yaml";
+ Path snapshotPath = getSnapshotPath(omMetadataManager, snapshotInfo);
+ return getSnapshotLocalPropertyYamlPath(snapshotPath);
}
- public static String getSnapshotLocalPropertyYamlPath(OzoneConfiguration
conf,
- SnapshotInfo snapshotInfo) {
- return getSnapshotPath(conf, snapshotInfo) + ".yaml";
+ /**
+ * Returns the path to the YAML file that stores local properties for the
given snapshot.
+ *
+ * @param snapshotPath path to the snapshot checkpoint dir
+ * @return the path to the snapshot's local property YAML file
+ */
+ public static String getSnapshotLocalPropertyYamlPath(Path snapshotPath) {
+ return snapshotPath.toString() + ".yaml";
}
public static boolean isSnapshotKey(String[] keyParts) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
