This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new f087d0b8e4 HDDS-12994. Use DatanodeID in ReconSCMDBDefinition. (#8417)
f087d0b8e4 is described below
commit f087d0b8e4e2ff8a1a5bcbc8fbd136e090f4df40
Author: Tsz-Wo Nicholas Sze <[email protected]>
AuthorDate: Thu May 8 19:53:09 2025 -0700
HDDS-12994. Use DatanodeID in ReconSCMDBDefinition. (#8417)
---
.../hadoop/ozone/recon/scm/ReconContainerManager.java | 7 ++++---
.../hadoop/ozone/recon/scm/ReconNodeManager.java | 19 ++++++++++---------
.../hadoop/ozone/recon/scm/ReconSCMDBDefinition.java | 12 ++++++------
.../recon/scm/ReconStorageContainerManagerFacade.java | 13 +++++--------
.../hadoop/ozone/recon/api/TestContainerEndpoint.java | 3 ++-
.../hadoop/ozone/recon/scm/TestReconNodeManager.java | 13 ++++---------
6 files changed, 31 insertions(+), 36 deletions(-)
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java
index fa3e2d1957..37ed11069c 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconContainerManager.java
@@ -31,6 +31,7 @@
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.DatanodeID;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.scm.container.ContainerID;
@@ -66,7 +67,7 @@ public class ReconContainerManager extends
ContainerManagerImpl {
private final PipelineManager pipelineManager;
private final ContainerHealthSchemaManager containerHealthSchemaManager;
private final ReconContainerMetadataManager cdbServiceProvider;
- private final Table<UUID, DatanodeDetails> nodeDB;
+ private final Table<DatanodeID, DatanodeDetails> nodeDB;
// Container ID -> Datanode UUID -> Timestamp
private final Map<Long, Map<UUID, ContainerReplicaHistory>>
replicaHistoryMap;
// Pipeline -> # of open containers
@@ -376,7 +377,7 @@ public List<ContainerHistory> getAllContainerHistory(long
containerID) {
// Attempt to retrieve hostname from NODES table
if (nodeDB != null) {
try {
- DatanodeDetails dnDetails = nodeDB.get(uuid);
+ final DatanodeDetails dnDetails = nodeDB.get(DatanodeID.of(uuid));
if (dnDetails != null) {
hostname = dnDetails.getHostName();
}
@@ -446,7 +447,7 @@ public void upsertContainerHistory(long containerID, UUID
uuid, long time,
}
}
- public Table<UUID, DatanodeDetails> getNodeDB() {
+ public Table<DatanodeID, DatanodeDetails> getNodeDB() {
return nodeDB;
}
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java
index 612d3a93a4..119db2cf88 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconNodeManager.java
@@ -32,6 +32,7 @@
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.DatanodeID;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandQueueReportProto;
@@ -71,7 +72,7 @@ public class ReconNodeManager extends SCMNodeManager {
private static final Logger LOG = LoggerFactory
.getLogger(ReconNodeManager.class);
- private Table<UUID, DatanodeDetails> nodeDB;
+ private Table<DatanodeID, DatanodeDetails> nodeDB;
private ReconContext reconContext;
private static final Set<Type> ALLOWED_COMMANDS =
ImmutableSet.of(reregisterCommand);
@@ -93,7 +94,7 @@ public ReconNodeManager(OzoneConfiguration conf,
SCMStorageConfig scmStorageConfig,
EventPublisher eventPublisher,
NetworkTopology networkTopology,
- Table<UUID, DatanodeDetails> nodeDB,
+ Table<DatanodeID, DatanodeDetails> nodeDB,
HDDSLayoutVersionManager scmLayoutVersionManager) {
super(conf, scmStorageConfig, eventPublisher, networkTopology,
SCMContext.emptyContext(), scmLayoutVersionManager);
@@ -103,7 +104,7 @@ public ReconNodeManager(OzoneConfiguration conf,
}
public ReconNodeManager(OzoneConfiguration conf, SCMStorageConfig
scmStorageConfig, EventQueue eventQueue,
- NetworkTopology clusterMap, Table<UUID,
DatanodeDetails> table,
+ NetworkTopology clusterMap, Table<DatanodeID,
DatanodeDetails> table,
HDDSLayoutVersionManager scmLayoutVersionManager,
ReconContext reconContext) {
this(conf, scmStorageConfig, eventQueue, clusterMap, table,
scmLayoutVersionManager);
this.reconContext = reconContext;
@@ -111,7 +112,7 @@ public ReconNodeManager(OzoneConfiguration conf,
SCMStorageConfig scmStorageConf
}
private void loadExistingNodes() {
- try (TableIterator<UUID, ? extends Table.KeyValue<UUID, DatanodeDetails>>
+ try (TableIterator<DatanodeID, ? extends Table.KeyValue<DatanodeID,
DatanodeDetails>>
iterator = nodeDB.iterator()) {
int nodeCount = 0;
while (iterator.hasNext()) {
@@ -143,7 +144,7 @@ public VersionResponse getVersion(SCMVersionRequestProto
versionRequest) {
* @param datanodeDetails Datanode details.
*/
public void addNodeToDB(DatanodeDetails datanodeDetails) throws IOException {
- nodeDB.put(datanodeDetails.getUuid(), datanodeDetails);
+ nodeDB.put(datanodeDetails.getID(), datanodeDetails);
LOG.info("Adding new node {} to Node DB.", datanodeDetails.getUuid());
}
@@ -268,7 +269,7 @@ public RegisteredCommand register(
inMemDatanodeDetails.put(datanodeDetails.getUuid(), datanodeDetails);
if (isNodeRegistered(datanodeDetails)) {
try {
- nodeDB.put(datanodeDetails.getUuid(), datanodeDetails);
+ nodeDB.put(datanodeDetails.getID(), datanodeDetails);
LOG.info("Updating nodeDB for " + datanodeDetails.getHostName());
} catch (IOException e) {
LOG.error("Can not update node {} to Node DB.",
@@ -318,7 +319,7 @@ private boolean needUpdate(DatanodeDetails datanodeDetails,
reconDatanodeOutdatedTime;
}
- public void reinitialize(Table<UUID, DatanodeDetails> nodeTable) {
+ public void reinitialize(Table<DatanodeID, DatanodeDetails> nodeTable) {
this.nodeDB = nodeTable;
loadExistingNodes();
}
@@ -326,7 +327,7 @@ public void reinitialize(Table<UUID, DatanodeDetails>
nodeTable) {
@VisibleForTesting
public long getNodeDBKeyCount() throws IOException {
long nodeCount = 0;
- try (TableIterator<UUID, ? extends Table.KeyValue<UUID, DatanodeDetails>>
+ try (TableIterator<DatanodeID, ? extends Table.KeyValue<DatanodeID,
DatanodeDetails>>
iterator = nodeDB.iterator()) {
while (iterator.hasNext()) {
iterator.next();
@@ -347,7 +348,7 @@ public long getNodeDBKeyCount() throws IOException {
public void removeNode(DatanodeDetails datanodeDetails) throws
NodeNotFoundException, IOException {
try {
super.removeNode(datanodeDetails);
- nodeDB.delete(datanodeDetails.getUuid());
+ nodeDB.delete(datanodeDetails.getID());
} catch (IOException ioException) {
LOG.error("Node {} deletion fails from Node DB.",
datanodeDetails.getUuid());
throw ioException;
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSCMDBDefinition.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSCMDBDefinition.java
index c50bc91d7a..3694f7b140 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSCMDBDefinition.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconSCMDBDefinition.java
@@ -18,8 +18,8 @@
package org.apache.hadoop.ozone.recon.scm;
import java.util.Map;
-import java.util.UUID;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.DatanodeID;
import org.apache.hadoop.hdds.scm.metadata.SCMDBDefinition;
import org.apache.hadoop.hdds.utils.db.Codec;
import org.apache.hadoop.hdds.utils.db.DBColumnFamilyDefinition;
@@ -31,14 +31,14 @@
* Recon SCM db file for ozone.
*/
public final class ReconSCMDBDefinition extends SCMDBDefinition {
- private static final Codec<UUID> UUID_CODEC = new DelegatedCodec<>(
- StringCodec.get(), UUID::fromString, UUID::toString,
- UUID.class, DelegatedCodec.CopyType.SHALLOW);
+ private static final Codec<DatanodeID> DATANODE_ID_CODEC = new
DelegatedCodec<>(
+ StringCodec.get(), DatanodeID::fromUuidString, DatanodeID::toString,
+ DatanodeID.class, DelegatedCodec.CopyType.SHALLOW);
public static final String RECON_SCM_DB_NAME = "recon-scm.db";
- public static final DBColumnFamilyDefinition<UUID, DatanodeDetails> NODES
- = new DBColumnFamilyDefinition<>("nodes", UUID_CODEC,
DatanodeDetails.getCodec());
+ public static final DBColumnFamilyDefinition<DatanodeID, DatanodeDetails>
NODES
+ = new DBColumnFamilyDefinition<>("nodes", DATANODE_ID_CODEC,
DatanodeDetails.getCodec());
private static final Map<String, DBColumnFamilyDefinition<?, ?>>
COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap(
diff --git
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
index ead10bcb6c..c956436fd5 100644
---
a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
+++
b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java
@@ -49,7 +49,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
@@ -63,6 +62,7 @@
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.ReconfigurationHandler;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.DatanodeID;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.scm.PlacementPolicy;
import org.apache.hadoop.hdds.scm.ScmUtils;
@@ -619,14 +619,11 @@ private void initializeNewRdbStore(File dbFile) throws
IOException {
try {
final DBStore newStore = DBStoreBuilder.newBuilder(ozoneConfiguration,
ReconSCMDBDefinition.get(), dbFile)
.build();
- Table<UUID, DatanodeDetails> nodeTable =
- ReconSCMDBDefinition.NODES.getTable(dbStore);
- Table<UUID, DatanodeDetails> newNodeTable =
- ReconSCMDBDefinition.NODES.getTable(newStore);
- try (TableIterator<UUID, ? extends KeyValue<UUID,
- DatanodeDetails>> iterator = nodeTable.iterator()) {
+ final Table<DatanodeID, DatanodeDetails> nodeTable =
ReconSCMDBDefinition.NODES.getTable(dbStore);
+ final Table<DatanodeID, DatanodeDetails> newNodeTable =
ReconSCMDBDefinition.NODES.getTable(newStore);
+ try (TableIterator<DatanodeID, ? extends KeyValue<DatanodeID,
DatanodeDetails>> iterator = nodeTable.iterator()) {
while (iterator.hasNext()) {
- KeyValue<UUID, DatanodeDetails> keyValue = iterator.next();
+ final KeyValue<DatanodeID, DatanodeDetails> keyValue =
iterator.next();
newNodeTable.put(keyValue.getKey(), keyValue.getValue());
}
}
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
index c990815936..be74d6ee8f 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerEndpoint.java
@@ -57,6 +57,7 @@
import org.apache.hadoop.hdds.client.StandaloneReplicationConfig;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.DatanodeID;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.scm.container.ContainerID;
@@ -1053,7 +1054,7 @@ public void testGetReplicaHistoryForContainer() throws
IOException {
UUID newDatanode(String hostName, String ipAddress) throws IOException {
final UUID uuid = UUID.randomUUID();
- reconContainerManager.getNodeDB().put(uuid,
+ reconContainerManager.getNodeDB().put(DatanodeID.of(uuid),
DatanodeDetails.newBuilder()
.setUuid(uuid)
.setHostName(hostName)
diff --git
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java
index 748a778fb3..0eae6f5483 100644
---
a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java
+++
b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/scm/TestReconNodeManager.java
@@ -36,7 +36,6 @@
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
-import java.util.UUID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.DatanodeID;
@@ -101,8 +100,7 @@ public void
testReconNodeManagerInitWithInvalidNetworkTopology() throws IOExcept
new ReconStorageConfig(conf, reconUtils);
EventQueue eventQueue = new EventQueue();
NetworkTopology clusterMap = new NetworkTopologyImpl(conf);
- Table<UUID, DatanodeDetails> nodeTable =
- ReconSCMDBDefinition.NODES.getTable(store);
+ final Table<DatanodeID, DatanodeDetails> nodeTable =
ReconSCMDBDefinition.NODES.getTable(store);
ReconNodeManager reconNodeManager = new ReconNodeManager(conf,
scmStorageConfig, eventQueue, clusterMap, nodeTable, versionManager,
reconContext);
assertThat(reconNodeManager.getAllNodes()).isEmpty();
@@ -130,8 +128,7 @@ public void testReconNodeDB() throws IOException,
NodeNotFoundException {
new ReconStorageConfig(conf, new ReconUtils());
EventQueue eventQueue = new EventQueue();
NetworkTopology clusterMap = new NetworkTopologyImpl(conf);
- Table<UUID, DatanodeDetails> nodeTable =
- ReconSCMDBDefinition.NODES.getTable(store);
+ final Table<DatanodeID, DatanodeDetails> nodeTable =
ReconSCMDBDefinition.NODES.getTable(store);
ReconNodeManager reconNodeManager = new ReconNodeManager(conf,
scmStorageConfig, eventQueue, clusterMap, nodeTable, versionManager,
reconContext);
ReconNewNodeHandler reconNewNodeHandler =
@@ -212,8 +209,7 @@ public void testUpdateNodeOperationalStateFromScm() throws
Exception {
new ReconStorageConfig(conf, new ReconUtils());
EventQueue eventQueue = new EventQueue();
NetworkTopology clusterMap = new NetworkTopologyImpl(conf);
- Table<UUID, DatanodeDetails> nodeTable =
- ReconSCMDBDefinition.NODES.getTable(store);
+ final Table<DatanodeID, DatanodeDetails> nodeTable =
ReconSCMDBDefinition.NODES.getTable(store);
ReconNodeManager reconNodeManager = new ReconNodeManager(conf,
scmStorageConfig, eventQueue, clusterMap, nodeTable, versionManager,
reconContext);
@@ -246,8 +242,7 @@ public void testDatanodeUpdate() throws IOException {
new ReconStorageConfig(conf, new ReconUtils());
EventQueue eventQueue = new EventQueue();
NetworkTopology clusterMap = new NetworkTopologyImpl(conf);
- Table<UUID, DatanodeDetails> nodeTable =
- ReconSCMDBDefinition.NODES.getTable(store);
+ final Table<DatanodeID, DatanodeDetails> nodeTable =
ReconSCMDBDefinition.NODES.getTable(store);
ReconNodeManager reconNodeManager = new ReconNodeManager(conf,
scmStorageConfig, eventQueue, clusterMap, nodeTable, versionManager,
reconContext);
ReconNewNodeHandler reconNewNodeHandler =
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
