This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e515f3a3872 HDDS-13227. Integration test for inode based bootstrap
flow. (#8884)
e515f3a3872 is described below
commit e515f3a3872d2c120c73b7480bf31f5d1823950c
Author: Sadanand Shenoy <[email protected]>
AuthorDate: Fri Aug 1 04:00:01 2025 +0530
HDDS-13227. Integration test for inode based bootstrap flow. (#8884)
---
.../TestOMDbCheckpointServletInodeBasedXfer.java | 58 ++++++++++++++++++----
1 file changed, 47 insertions(+), 11 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java
index 724a26c4cb3..a43b0832c96 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java
@@ -27,6 +27,7 @@
import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_MAX_TOTAL_SST_SIZE_KEY;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -143,11 +144,11 @@ void shutdown() {
private void setupCluster() throws Exception {
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build();
+ conf.setBoolean(OZONE_ACL_ENABLED, false);
+ conf.set(OZONE_ADMINISTRATORS, OZONE_ADMINISTRATORS_WILDCARD);
cluster.waitForClusterToBeReady();
client = cluster.newClient();
om = cluster.getOzoneManager();
- conf.setBoolean(OZONE_ACL_ENABLED, false);
- conf.set(OZONE_ADMINISTRATORS, OZONE_ADMINISTRATORS_WILDCARD);
}
private void setupMocks() throws Exception {
@@ -224,12 +225,44 @@ public void write(int b) throws IOException {
doCallRealMethod().when(omDbCheckpointServletMock).getSstBackupDir();
}
- @Test
- void testContentsOfTarballWithSnapshot() throws Exception {
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testTarballBatching(boolean includeSnapshots) throws Exception {
String volumeName = "vol" + RandomStringUtils.secure().nextNumeric(5);
String bucketName = "buck" + RandomStringUtils.secure().nextNumeric(5);
AtomicReference<DBCheckpoint> realCheckpoint = new AtomicReference<>();
- setupClusterAndMocks(volumeName, bucketName, realCheckpoint);
+ setupClusterAndMocks(volumeName, bucketName, realCheckpoint,
includeSnapshots);
+ long maxFileSizeLimit = 4096;
+
om.getConfiguration().setLong(OZONE_OM_RATIS_SNAPSHOT_MAX_TOTAL_SST_SIZE_KEY,
maxFileSizeLimit);
+ // Get the tarball.
+ omDbCheckpointServletMock.doGet(requestMock, responseMock);
+ String testDirName = folder.resolve("testDir").toString();
+ String newDbDirName = testDirName + OM_KEY_PREFIX + OM_DB_NAME;
+ File newDbDir = new File(newDbDirName);
+ assertTrue(newDbDir.mkdirs());
+ FileUtil.unTar(tempFile, newDbDir);
+ long totalSize;
+ try (Stream<Path> list = Files.list(newDbDir.toPath())) {
+ totalSize = list.mapToLong(path -> path.toFile().length()).sum();
+ }
+ boolean obtainedFilesUnderMaxLimit = totalSize < maxFileSizeLimit;
+ if (!includeSnapshots) {
+ // If includeSnapshotData flag is set to false , it always sends all data
+ // in one batch and doesn't respect the max size config. This is how
Recon
+ // uses it today.
+ assertFalse(obtainedFilesUnderMaxLimit);
+ } else {
+ assertTrue(obtainedFilesUnderMaxLimit);
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testContentsOfTarballWithSnapshot(boolean includeSnapshot)
throws Exception {
+ String volumeName = "vol" + RandomStringUtils.secure().nextNumeric(5);
+ String bucketName = "buck" + RandomStringUtils.secure().nextNumeric(5);
+ AtomicReference<DBCheckpoint> realCheckpoint = new AtomicReference<>();
+ setupClusterAndMocks(volumeName, bucketName, realCheckpoint,
includeSnapshot);
DBStore dbStore = om.getMetadataManager().getStore();
// Get the tarball.
omDbCheckpointServletMock.doGet(requestMock, responseMock);
@@ -258,9 +291,11 @@ void testContentsOfTarballWithSnapshot() throws Exception {
Path checkpointLocation = realCheckpoint.get().getCheckpointLocation();
populateInodesOfFilesInDirectory(dbStore, checkpointLocation,
inodesFromOmDataDir, hardLinkMapFromOmData);
- for (String snapshotPath : snapshotPaths) {
- populateInodesOfFilesInDirectory(dbStore, Paths.get(snapshotPath),
- inodesFromOmDataDir, hardLinkMapFromOmData);
+ if (includeSnapshot) {
+ for (String snapshotPath : snapshotPaths) {
+ populateInodesOfFilesInDirectory(dbStore, Paths.get(snapshotPath),
+ inodesFromOmDataDir, hardLinkMapFromOmData);
+ }
}
populateInodesOfFilesInDirectory(dbStore,
Paths.get(dbStore.getRocksDBCheckpointDiffer().getSSTBackupDir()),
inodesFromOmDataDir, hardLinkMapFromOmData);
@@ -296,7 +331,7 @@ public void testSnapshotDBConsistency() throws Exception {
String volumeName = "vol" + RandomStringUtils.secure().nextNumeric(5);
String bucketName = "buck" + RandomStringUtils.secure().nextNumeric(5);
AtomicReference<DBCheckpoint> realCheckpoint = new AtomicReference<>();
- setupClusterAndMocks(volumeName, bucketName, realCheckpoint);
+ setupClusterAndMocks(volumeName, bucketName, realCheckpoint, true);
List<OzoneSnapshot> snapshots = new ArrayList<>();
client.getObjectStore().listSnapshot(volumeName, bucketName, "", null)
.forEachRemaining(snapshots::add);
@@ -420,11 +455,12 @@ private void
writeDummyKeyToDeleteTableOfSnapshotDB(OzoneSnapshot snapshotToModi
}
private void setupClusterAndMocks(String volumeName, String bucketName,
- AtomicReference<DBCheckpoint> realCheckpoint) throws Exception {
+ AtomicReference<DBCheckpoint> realCheckpoint, boolean includeSnapshots)
throws Exception {
setupCluster();
setupMocks();
om.getKeyManager().getSnapshotSstFilteringService().pause();
-
when(requestMock.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA)).thenReturn("true");
+ when(requestMock.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA))
+ .thenReturn(String.valueOf(includeSnapshots));
// Create a "spy" dbstore keep track of the checkpoint.
writeData(volumeName, bucketName, true);
DBStore dbStore = om.getMetadataManager().getStore();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
