This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 4f12199bd02 HDDS-13569. Separate OM and SCM metadata dirs in
MiniOzoneClusterImpl (#8937)
4f12199bd02 is described below
commit 4f12199bd029e7be12a3455e800d0720e33204a5
Author: Siyao Meng <[email protected]>
AuthorDate: Fri Aug 15 16:05:35 2025 -0700
HDDS-13569. Separate OM and SCM metadata dirs in MiniOzoneClusterImpl
(#8937)
---
.../hadoop/ozone/freon/TestOMSnapshotDAG.java | 3 +-
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 14 +++++
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 71 ++++++++++++++++++++--
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 2 +-
.../hadoop/ozone/UniformDatanodesFactory.java | 2 +-
5 files changed, 82 insertions(+), 10 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java
index 4a2c030526c..5429dc0f4a1 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.ozone.freon;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_S3_VOLUME_NAME_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConsts.DB_COMPACTION_LOG_DIR;
import static org.apache.hadoop.ozone.OzoneConsts.DB_COMPACTION_SST_BACKUP_DIR;
@@ -318,7 +317,7 @@ public void testSkipTrackingWithZeroSnapshot() {
assertEquals(1000L, randomKeyGenerator.getSuccessfulValidationCount());
String omMetadataDir =
- cluster.getOzoneManager().getConfiguration().get(OZONE_METADATA_DIRS);
+
cluster.getOzoneManager().getConfiguration().get(OMConfigKeys.OZONE_OM_DB_DIRS);
// Verify that no compaction log entry has been written
Path logPath = Paths.get(omMetadataDir, OM_SNAPSHOT_DIFF_DIR,
DB_COMPACTION_LOG_DIR);
diff --git
a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index a1bcadd7936..dbeeda5cdbf 100644
---
a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++
b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -26,12 +26,14 @@
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import
org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient;
import
org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OzoneManager;
import
org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.ratis.util.ExitUtils;
@@ -272,6 +274,18 @@ protected Builder(OzoneConfiguration conf) {
* between the clusters created. */
protected void prepareForNextBuild() {
conf = new OzoneConfiguration(conf);
+
+ // Remove the extra configs set in configureSCM() and configureOM() so
that MiniOzoneClusterProvider won't fail
+ conf.unset(ScmConfigKeys.OZONE_SCM_HA_RATIS_STORAGE_DIR);
+ conf.unset(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_DIR);
+ conf.unset(ScmConfigKeys.OZONE_SCM_DB_DIRS);
+ conf.unset(OzoneConfigKeys.OZONE_HTTP_BASEDIR);
+
+ conf.unset(OMConfigKeys.OZONE_OM_RATIS_STORAGE_DIR);
+ conf.unset(OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_DIR);
+ conf.unset(OMConfigKeys.OZONE_OM_DB_DIRS);
+ conf.unset(OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DB_DIR);
+
setClusterId();
}
diff --git
a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index cf18edbb5ba..a147a53bdf4 100644
---
a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++
b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -19,6 +19,8 @@
import static java.util.Collections.singletonList;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY;
+import static org.apache.hadoop.hdds.server.http.BaseHttpServer.SERVER_DIR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RATIS_SNAPSHOT_DIR;
import static org.apache.ozone.test.GenericTestUtils.PortAllocator.getFreePort;
import static
org.apache.ozone.test.GenericTestUtils.PortAllocator.localhostWithFreePort;
@@ -91,6 +93,12 @@ public class MiniOzoneClusterImpl implements
MiniOzoneCluster {
private static final String[] NO_ARGS = new String[0];
+ private static final String SCM_SUBDIR_NAME = "scm";
+ private static final String OM_SUBDIR_NAME = "om";
+ private static final String OZONE_METADATA_SUBDIR_NAME = "ozone-metadata";
+ private static final String RATIS_SUBDIR_NAME = "ratis";
+ private static final String DATA_SUBDIR_NAME = "data";
+
static {
CodecBuffer.enableLeakDetection();
}
@@ -548,7 +556,7 @@ protected void setClients(OzoneManager om) throws
IOException {
* Initializes the configuration required for starting MiniOzoneCluster.
*/
protected void initializeConfiguration() throws IOException {
- Path metaDir = Paths.get(path, "ozone-meta");
+ Path metaDir = Paths.get(path, OZONE_METADATA_SUBDIR_NAME);
Files.createDirectories(metaDir);
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.toString());
@@ -558,7 +566,7 @@ protected void initializeConfiguration() throws IOException
{
// default max retry timeout set to 30s
scmClientConfig.setMaxRetryTimeout(30 * 1000);
conf.setFromObject(scmClientConfig);
- // In this way safemode exit will happen only when atleast we have one
+ // In this way safemode exit will happen only when at least we have one
// pipeline.
conf.setInt(HddsConfigKeys.HDDS_SCM_SAFEMODE_MIN_DATANODE,
numOfDatanodes >= 3 ? 3 : 1);
@@ -583,7 +591,7 @@ protected StorageContainerManager createAndStartSingleSCM()
*/
protected StorageContainerManager createSCM()
throws IOException, AuthenticationException {
- configureSCM();
+ configureSCM(false);
SCMStorageConfig scmStore = new SCMStorageConfig(conf);
initializeScmStorage(scmStore);
@@ -642,7 +650,7 @@ protected OzoneManager createAndStartSingleOM() throws
AuthenticationException,
*/
protected OzoneManager createOM()
throws IOException, AuthenticationException {
- configureOM();
+ configureOM(false);
OMStorage omStore = new OMStorage(conf);
initializeOmStorage(omStore);
return OzoneManager.createOm(conf);
@@ -688,7 +696,7 @@ protected List<HddsDatanodeService> createHddsDatanodes()
return hddsDatanodes;
}
- protected void configureSCM() {
+ protected void configureSCM(boolean isHA) throws IOException {
conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY,
localhostWithFreePort());
conf.set(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY,
@@ -702,14 +710,65 @@ protected void configureSCM() {
conf.setInt(ScmConfigKeys.OZONE_SCM_RATIS_PORT_KEY, getFreePort());
conf.setInt(ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY, getFreePort());
conf.setIfUnset(ScmConfigKeys.OZONE_SCM_HA_RATIS_SERVER_RPC_FIRST_ELECTION_TIMEOUT,
"1s");
+
+ Path scmMetaDir = Paths.get(path, SCM_SUBDIR_NAME);
+ Files.createDirectories(scmMetaDir.resolve(DATA_SUBDIR_NAME));
+
+ // Extra configs for non-HA SCM
+ if (!isHA) {
+ // e.g. when path is /var/lib/hadoop-ozone
+ // ozone.scm.ha.ratis.storage.dir = /var/lib/hadoop-ozone/scm/ratis
+ conf.setIfUnset(ScmConfigKeys.OZONE_SCM_HA_RATIS_STORAGE_DIR,
+ scmMetaDir.resolve(RATIS_SUBDIR_NAME).toString());
+
+ // ozone.scm.ha.ratis.snapshot.dir =
/var/lib/hadoop-ozone/scm/ozone-metadata/snapshot
+ conf.setIfUnset(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_DIR,
+
scmMetaDir.resolve(OZONE_METADATA_SUBDIR_NAME).resolve(OZONE_RATIS_SNAPSHOT_DIR).toString());
+
+ // ozone.scm.db.dirs = /var/lib/hadoop-ozone/scm/data
+ conf.setIfUnset(ScmConfigKeys.OZONE_SCM_DB_DIRS,
+ scmMetaDir.resolve(DATA_SUBDIR_NAME).toString());
+
+ // ozone.http.basedir =
/var/lib/hadoop-ozone/scm/ozone-metadata/webserver
+ conf.setIfUnset(OzoneConfigKeys.OZONE_HTTP_BASEDIR,
+ scmMetaDir.resolve(OZONE_METADATA_SUBDIR_NAME) + SERVER_DIR);
+ }
}
- private void configureOM() {
+ private void configureOM(boolean isHA) throws IOException {
conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, localhostWithFreePort());
conf.set(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY,
localhostWithFreePort());
conf.set(OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY,
localhostWithFreePort());
conf.setInt(OMConfigKeys.OZONE_OM_RATIS_PORT_KEY, getFreePort());
+
+ Path omMetaDir = Paths.get(path, OM_SUBDIR_NAME);
+ Files.createDirectories(omMetaDir.resolve(DATA_SUBDIR_NAME));
+
+ // Extra configs for non-HA OM
+ if (!isHA) {
+ // e.g. when path is /var/lib/hadoop-ozone
+ // ozone.om.ratis.storage.dir = /var/lib/hadoop-ozone/om/ratis
+ conf.setIfUnset(OMConfigKeys.OZONE_OM_RATIS_STORAGE_DIR,
+ omMetaDir.resolve(RATIS_SUBDIR_NAME).toString());
+
+ // ozone.om.ratis.snapshot.dir =
/var/lib/hadoop-ozone/om/ozone-metadata/snapshot
+ conf.setIfUnset(OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_DIR,
+
omMetaDir.resolve(OZONE_METADATA_SUBDIR_NAME).resolve(OZONE_RATIS_SNAPSHOT_DIR).toString());
+
+ // ozone.om.db.dirs = /var/lib/hadoop-ozone/om/data
+ conf.setIfUnset(OMConfigKeys.OZONE_OM_DB_DIRS,
+ omMetaDir.resolve(DATA_SUBDIR_NAME).toString());
+
+ // ozone.om.snapshot.diff.db.dir =
/var/lib/hadoop-ozone/om/ozone-metadata
+ // actual dir would be
/var/lib/hadoop-ozone/om/ozone-metadata/db.snapdiff
+ conf.setIfUnset(OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DB_DIR,
+ omMetaDir.resolve(OZONE_METADATA_SUBDIR_NAME).toString());
+
+ // ozone.http.basedir =
/var/lib/hadoop-ozone/om/ozone-metadata/webserver
+ conf.setIfUnset(OzoneConfigKeys.OZONE_HTTP_BASEDIR,
+ omMetaDir.resolve(OZONE_METADATA_SUBDIR_NAME) + SERVER_DIR);
+ }
}
}
diff --git
a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
index 8612eaa1748..ee387a25885 100644
---
a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
+++
b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
@@ -565,7 +565,7 @@ protected SCMHAService createSCMService()
scmConfig.set(OZONE_METADATA_DIRS, metaDirPath);
scmConfig.set(ScmConfigKeys.OZONE_SCM_NODE_ID_KEY, nodeId);
- configureSCM();
+ configureSCM(true);
if (i == 1) {
StorageContainerManager.scmInit(scmConfig, clusterId);
} else {
diff --git
a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/UniformDatanodesFactory.java
b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/UniformDatanodesFactory.java
index 771d2b190ce..7e0a1003115 100644
---
a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/UniformDatanodesFactory.java
+++
b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/UniformDatanodesFactory.java
@@ -81,7 +81,7 @@ public OzoneConfiguration apply(OzoneConfiguration conf)
throws IOException {
Path baseDir =
Paths.get(Objects.requireNonNull(conf.get(OZONE_METADATA_DIRS)), "datanode-" +
i);
- Path metaDir = baseDir.resolve("meta");
+ Path metaDir = baseDir.resolve("ozone-metadata");
Files.createDirectories(metaDir);
dnConf.set(OZONE_METADATA_DIRS, metaDir.toString());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
