This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 7f2e0e37de HDDS-11554. OMDBDefinition should be singleton. (#7292)
7f2e0e37de is described below
commit 7f2e0e37de53baccc9764f9feac2f73ff540acd8
Author: Tsz-Wo Nicholas Sze <[email protected]>
AuthorDate: Wed Oct 9 22:57:12 2024 -0700
HDDS-11554. OMDBDefinition should be singleton. (#7292)
---
.../apache/hadoop/hdds/utils/db/DBDefinition.java | 22 ++++++++++++++++
.../hadoop/ozone/om/codec/OMDBDefinition.java | 10 ++++++--
.../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 6 +----
.../om/ratis/utils/OzoneManagerRatisUtils.java | 3 +--
.../apache/hadoop/ozone/om/TestOMDBDefinition.java | 3 +--
.../ozone/recon/tasks/OMDBUpdatesHandler.java | 6 ++---
.../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 2 +-
.../recon/tasks/TestOmUpdateEventValidator.java | 5 ++--
.../hadoop/ozone/debug/DBDefinitionFactory.java | 30 ++++++++++------------
.../ozone/debug/TestDBDefinitionFactory.java | 21 ++++++++-------
10 files changed, 62 insertions(+), 46 deletions(-)
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBDefinition.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBDefinition.java
index 968d62f0dd..461bd35f41 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBDefinition.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBDefinition.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdds.utils.db;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.server.ServerUtils;
+import org.apache.ratis.util.MemoizedSupplier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,6 +29,9 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
/**
* Simple interface to provide information to create a DBStore..
@@ -55,6 +59,16 @@ public interface DBDefinition {
getLocationConfigKey(), getName());
}
+ static List<String>
getColumnFamilyNames(Iterable<DBColumnFamilyDefinition<?, ?>> columnFamilies) {
+ return
Collections.unmodifiableList(StreamSupport.stream(columnFamilies.spliterator(),
false)
+ .map(DBColumnFamilyDefinition::getName)
+ .collect(Collectors.toList()));
+ }
+
+ default List<String> getColumnFamilyNames() {
+ return getColumnFamilyNames(getColumnFamilies());
+ }
+
/**
* @return The column families present in the DB.
*/
@@ -109,9 +123,17 @@ public interface DBDefinition {
*/
abstract class WithMap implements WithMapInterface {
private final Map<String, DBColumnFamilyDefinition<?, ?>> map;
+ private final Supplier<List<String>> columnFamilyNames;
protected WithMap(Map<String, DBColumnFamilyDefinition<?, ?>> map) {
this.map = map;
+ this.columnFamilyNames = MemoizedSupplier.valueOf(
+ () -> DBDefinition.getColumnFamilyNames(getColumnFamilies()));
+ }
+
+ @Override
+ public final List<String> getColumnFamilyNames() {
+ return columnFamilyNames.get();
}
@Override
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
index de567447ae..d48b413a2a 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java
@@ -50,7 +50,7 @@ import java.util.Map;
/**
* Class defines the structure and types of the om.db.
*/
-public class OMDBDefinition extends DBDefinition.WithMap {
+public final class OMDBDefinition extends DBDefinition.WithMap {
public static final DBColumnFamilyDefinition<String, RepeatedOmKeyInfo>
DELETED_TABLE =
@@ -284,7 +284,13 @@ public class OMDBDefinition extends DBDefinition.WithMap {
USER_TABLE,
VOLUME_TABLE);
- public OMDBDefinition() {
+ private static final OMDBDefinition INSTANCE = new OMDBDefinition();
+
+ public static OMDBDefinition get() {
+ return INSTANCE;
+ }
+
+ private OMDBDefinition() {
super(COLUMN_FAMILIES);
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java
index 753088183b..42ae90b918 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java
@@ -40,7 +40,6 @@ import java.util.stream.Collectors;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.hdds.utils.TransactionInfo;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
-import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.S3SecretManager;
import org.apache.hadoop.ozone.om.codec.OMDBDefinition;
@@ -478,10 +477,7 @@ public final class OzoneManagerDoubleBuffer {
if (cleanupTableInfo != null) {
final List<String> cleanupTables;
if (cleanupTableInfo.cleanupAll()) {
- cleanupTables = new OMDBDefinition().getColumnFamilies()
- .stream()
- .map(DBColumnFamilyDefinition::getName)
- .collect(Collectors.toList());
+ cleanupTables = OMDBDefinition.get().getColumnFamilyNames();
} else {
cleanupTables = Arrays.asList(cleanupTableInfo.cleanupTables());
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
index 5a1612e021..226a547532 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java
@@ -439,8 +439,7 @@ public final class OzoneManagerRatisUtils {
*/
public static TransactionInfo getTrxnInfoFromCheckpoint(
OzoneConfiguration conf, Path dbPath) throws Exception {
- return HAUtils
- .getTrxnInfoFromCheckpoint(conf, dbPath, new OMDBDefinition());
+ return HAUtils.getTrxnInfoFromCheckpoint(conf, dbPath,
OMDBDefinition.get());
}
/**
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBDefinition.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBDefinition.java
index 36245dc874..9ae85b0fcb 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBDefinition.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOMDBDefinition.java
@@ -45,11 +45,10 @@ public class TestOMDBDefinition {
OzoneConfiguration configuration = new OzoneConfiguration();
File metaDir = folder.toFile();
DBStore store = OmMetadataManagerImpl.loadDB(configuration, metaDir);
- OMDBDefinition dbDef = new OMDBDefinition();
// Get list of tables from DB Definitions
final Collection<DBColumnFamilyDefinition<?, ?>> columnFamilyDefinitions
- = dbDef.getColumnFamilies();
+ = OMDBDefinition.get().getColumnFamilies();
final int countOmDefTables = columnFamilyDefinitions.size();
ArrayList<String> missingDBDefTables = new ArrayList<>();
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OMDBUpdatesHandler.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OMDBUpdatesHandler.java
index 41e6bf962a..d1f98c49bd 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OMDBUpdatesHandler.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/OMDBUpdatesHandler.java
@@ -49,14 +49,12 @@ public class OMDBUpdatesHandler extends
ManagedWriteBatch.Handler {
private OMMetadataManager omMetadataManager;
private List<OMDBUpdateEvent> omdbUpdateEvents = new ArrayList<>();
private Map<String, Map<Object, OMDBUpdateEvent>> omdbLatestUpdateEvents =
new HashMap<>();
- private OMDBDefinition omdbDefinition;
- private OmUpdateEventValidator omUpdateEventValidator;
+ private final OMDBDefinition omdbDefinition = OMDBDefinition.get();
+ private final OmUpdateEventValidator omUpdateEventValidator = new
OmUpdateEventValidator(omdbDefinition);
public OMDBUpdatesHandler(OMMetadataManager metadataManager) {
omMetadataManager = metadataManager;
tablesNames = metadataManager.getStore().getTableNames();
- omdbDefinition = new OMDBDefinition();
- omUpdateEventValidator = new OmUpdateEventValidator(omdbDefinition);
}
@Override
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java
index 3831f03bfd..f4f0bfe9ac 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOMDBUpdatesHandler.java
@@ -68,7 +68,7 @@ public class TestOMDBUpdatesHandler {
private OMMetadataManager omMetadataManager;
private OMMetadataManager reconOmMetadataManager;
- private OMDBDefinition omdbDefinition = new OMDBDefinition();
+ private final OMDBDefinition omdbDefinition = OMDBDefinition.get();
private Random random = new Random();
private OzoneConfiguration createNewTestPath(String folderName)
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOmUpdateEventValidator.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOmUpdateEventValidator.java
index 0adb44e87c..7da98acb38 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOmUpdateEventValidator.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/TestOmUpdateEventValidator.java
@@ -53,7 +53,7 @@ import static org.mockito.Mockito.times;
public class TestOmUpdateEventValidator {
private OmUpdateEventValidator eventValidator;
- private OMDBDefinition omdbDefinition;
+ private final OMDBDefinition omdbDefinition = OMDBDefinition.get();
private OMMetadataManager omMetadataManager;
private Logger logger;
@TempDir
@@ -63,11 +63,10 @@ public class TestOmUpdateEventValidator {
public void setUp() throws IOException {
omMetadataManager = initializeNewOmMetadataManager(
temporaryFolder.toFile());
- omdbDefinition = new OMDBDefinition();
eventValidator = new OmUpdateEventValidator(omdbDefinition);
// Create a mock logger
logger = mock(Logger.class);
- eventValidator.setLogger(logger);
+ OmUpdateEventValidator.setLogger(logger);
}
@Test
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBDefinitionFactory.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBDefinitionFactory.java
index a163cda250..98cb69d60a 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBDefinitionFactory.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/debug/DBDefinitionFactory.java
@@ -20,7 +20,10 @@ package org.apache.hadoop.ozone.debug;
import java.nio.file.Path;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
@@ -48,17 +51,14 @@ public final class DBDefinitionFactory {
private DBDefinitionFactory() {
}
- private static HashMap<String, DBDefinition> dbMap;
-
- private static String dnDBSchemaVersion;
+ private static final AtomicReference<String> DATANODE_DB_SCHEMA_VERSION =
new AtomicReference<>();
+ private static final Map<String, DBDefinition> DB_MAP;
static {
- dbMap = new HashMap<>();
- Arrays.asList(
- new SCMDBDefinition(),
- new OMDBDefinition(),
- new ReconSCMDBDefinition()
- ).forEach(dbDefinition -> dbMap.put(dbDefinition.getName(), dbDefinition));
+ final Map<String, DBDefinition> map = new HashMap<>();
+ Arrays.asList(new SCMDBDefinition(), OMDBDefinition.get(), new
ReconSCMDBDefinition())
+ .forEach(dbDefinition -> map.put(dbDefinition.getName(),
dbDefinition));
+ DB_MAP = Collections.unmodifiableMap(map);
}
public static DBDefinition getDefinition(String dbName) {
@@ -66,10 +66,8 @@ public final class DBDefinitionFactory {
if (!dbName.equals(OM_DB_NAME) && dbName.startsWith(OM_DB_NAME)) {
dbName = OM_DB_NAME;
}
- if (dbMap.containsKey(dbName)) {
- return dbMap.get(dbName);
- }
- return getReconDBDefinition(dbName);
+ final DBDefinition definition = DB_MAP.get(dbName);
+ return definition != null ? definition : getReconDBDefinition(dbName);
}
public static DBDefinition getDefinition(Path dbPath,
@@ -83,7 +81,7 @@ public final class DBDefinitionFactory {
}
String dbName = fileName.toString();
if (dbName.endsWith(OzoneConsts.CONTAINER_DB_SUFFIX)) {
- switch (dnDBSchemaVersion) {
+ switch (DATANODE_DB_SCHEMA_VERSION.get()) {
case "V1":
return new DatanodeSchemaOneDBDefinition(
dbPath.toAbsolutePath().toString(), config);
@@ -102,12 +100,12 @@ public final class DBDefinitionFactory {
if (dbName.startsWith(RECON_CONTAINER_KEY_DB)) {
return new ReconDBDefinition(dbName);
} else if (dbName.startsWith(RECON_OM_SNAPSHOT_DB)) {
- return new OMDBDefinition();
+ return OMDBDefinition.get();
}
return null;
}
public static void setDnDBSchemaVersion(String dnDBSchemaVersion) {
- DBDefinitionFactory.dnDBSchemaVersion = dnDBSchemaVersion;
+ DATANODE_DB_SCHEMA_VERSION.set(dnDBSchemaVersion);
}
}
diff --git
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestDBDefinitionFactory.java
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestDBDefinitionFactory.java
index 5e25901293..9152e357ce 100644
---
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestDBDefinitionFactory.java
+++
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/debug/TestDBDefinitionFactory.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.debug;
+import java.nio.file.Path;
import java.nio.file.Paths;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -43,8 +44,7 @@ public class TestDBDefinitionFactory {
@Test
public void testGetDefinition() {
- DBDefinition definition =
- DBDefinitionFactory.getDefinition(new OMDBDefinition().getName());
+ DBDefinition definition =
DBDefinitionFactory.getDefinition(OMDBDefinition.get().getName());
assertInstanceOf(OMDBDefinition.class, definition);
definition = DBDefinitionFactory.getDefinition(
@@ -62,20 +62,19 @@ public class TestDBDefinitionFactory {
definition = DBDefinitionFactory.getDefinition(
RECON_CONTAINER_KEY_DB + "_1");
assertInstanceOf(ReconDBDefinition.class, definition);
+
DBDefinitionFactory.setDnDBSchemaVersion("V2");
- definition =
- DBDefinitionFactory.getDefinition(Paths.get("/tmp/test-container.db"),
- new OzoneConfiguration());
+ final Path dbPath = Paths.get("/tmp/test-container.db");
+ final OzoneConfiguration conf = new OzoneConfiguration();
+ definition = DBDefinitionFactory.getDefinition(dbPath, conf);
assertInstanceOf(DatanodeSchemaTwoDBDefinition.class, definition);
+
DBDefinitionFactory.setDnDBSchemaVersion("V1");
- definition =
- DBDefinitionFactory.getDefinition(Paths.get("/tmp/test-container.db"),
- new OzoneConfiguration());
+ definition = DBDefinitionFactory.getDefinition(dbPath, conf);
assertInstanceOf(DatanodeSchemaOneDBDefinition.class, definition);
+
DBDefinitionFactory.setDnDBSchemaVersion("V3");
- definition =
- DBDefinitionFactory.getDefinition(Paths.get("/tmp/test-container.db"),
- new OzoneConfiguration());
+ definition = DBDefinitionFactory.getDefinition(dbPath, conf);
assertInstanceOf(DatanodeSchemaThreeDBDefinition.class, definition);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
