This is an automated email from the ASF dual-hosted git repository.
erose pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 4cbf5ce0b8 HDDS-13101. Remove duplicate information in datanode list
output (#8523)
4cbf5ce0b8 is described below
commit 4cbf5ce0b8f4bf4d25bc8444fd7c594615fdc1e5
Author: sreejasahithi <[email protected]>
AuthorDate: Thu Aug 7 20:54:35 2025 +0530
HDDS-13101. Remove duplicate information in datanode list output (#8523)
---
.../hdds/scm/cli/datanode/BasicDatanodeInfo.java | 165 +++++++++++++++++++++
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 131 +++++-----------
.../scm/cli/datanode/TestListInfoSubcommand.java | 5 +-
.../src/main/smoketest/admincli/datanode.robot | 3 +-
.../src/main/smoketest/balancer/testBalancer.robot | 2 +-
5 files changed, 207 insertions(+), 99 deletions(-)
diff --git
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfo.java
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfo.java
new file mode 100644
index 0000000000..4a925ffad5
--- /dev/null
+++
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/BasicDatanodeInfo.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.cli.datanode;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+
+/**
+ * Represents filtered Datanode information for json use.
+ */
+public class BasicDatanodeInfo {
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private Long used = null;
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private Long capacity = null;
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ private Double percentUsed = null;
+ private final DatanodeDetails dn;
+ private final HddsProtos.NodeOperationalState opState;
+ private final HddsProtos.NodeState healthState;
+
+ public BasicDatanodeInfo(DatanodeDetails dnDetails,
HddsProtos.NodeOperationalState opState,
+ HddsProtos.NodeState healthState) {
+ this.dn = dnDetails;
+ this.opState = opState;
+ this.healthState = healthState;
+ }
+
+ public BasicDatanodeInfo(DatanodeDetails dnDetails,
HddsProtos.NodeOperationalState opState,
+ HddsProtos.NodeState healthState, long used, long capacity, double
percentUsed) {
+ this(dnDetails, opState, healthState);
+ this.used = used;
+ this.capacity = capacity;
+ this.percentUsed = percentUsed;
+ }
+
+ @JsonProperty(index = 5)
+ public String getId() {
+ return dn.getUuidString();
+ }
+
+ @JsonProperty(index = 10)
+ public String getHostName() {
+ return dn.getHostName();
+ }
+
+ @JsonProperty(index = 15)
+ public String getIpAddress() {
+ return dn.getIpAddress();
+ }
+
+ @JsonProperty(index = 20)
+ public List<DatanodeDetails.Port> getPorts() {
+ return dn.getPorts();
+ }
+
+ @JsonProperty(index = 25)
+ public long getSetupTime() {
+ return dn.getSetupTime();
+ }
+
+ @JsonProperty(index = 30)
+ public int getCurrentVersion() {
+ return dn.getCurrentVersion();
+ }
+
+ @JsonProperty(index = 35)
+ public HddsProtos.NodeOperationalState getOpState() {
+ return opState;
+ }
+
+ @JsonProperty(index = 40)
+ public HddsProtos.NodeOperationalState getPersistedOpState() {
+ return dn.getPersistedOpState();
+ }
+
+ @JsonProperty(index = 45)
+ public long getPersistedOpStateExpiryEpochSec() {
+ return dn.getPersistedOpStateExpiryEpochSec();
+ }
+
+ @JsonProperty(index = 50)
+ public HddsProtos.NodeState getHealthState() {
+ return healthState;
+ }
+
+ @JsonProperty(index = 55)
+ public boolean isDecommissioned() {
+ return dn.isDecommissioned();
+ }
+
+ @JsonProperty(index = 60)
+ public boolean isMaintenance() {
+ return dn.isMaintenance();
+ }
+
+ @JsonProperty(index = 65)
+ public int getLevel() {
+ return dn.getLevel();
+ }
+
+ @JsonProperty(index = 70)
+ public int getCost() {
+ return dn.getCost();
+ }
+
+ @JsonProperty(index = 75)
+ public int getNumOfLeaves() {
+ return dn.getNumOfLeaves();
+ }
+
+ @JsonProperty(index = 80)
+ public String getNetworkFullPath() {
+ return dn.getNetworkFullPath();
+ }
+
+ @JsonProperty(index = 85)
+ public String getNetworkLocation() {
+ return dn.getNetworkLocation();
+ }
+
+ @JsonProperty(index = 90)
+ public String getNetworkName() {
+ return dn.getNetworkName();
+ }
+
+ @JsonProperty(index = 95)
+ public Long getUsed() {
+ return used;
+ }
+
+ @JsonProperty(index = 100)
+ public Long getCapacity() {
+ return capacity;
+ }
+
+ @JsonProperty(index = 105)
+ public Double getPercentUsed() {
+ return percentUsed;
+ }
+
+ @JsonIgnore
+ public DatanodeDetails getDatanodeDetails() {
+ return dn;
+ }
+}
diff --git
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
index a30e91150e..49fb032d6c 100644
---
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
+++
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/ListInfoSubcommand.java
@@ -17,10 +17,10 @@
package org.apache.hadoop.hdds.scm.cli.datanode;
-import com.fasterxml.jackson.annotation.JsonInclude;
import com.google.common.base.Strings;
import java.io.IOException;
import java.util.Collections;
+import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
@@ -88,26 +88,23 @@ public void execute(ScmClient scmClient) throws IOException
{
pipelines = scmClient.listPipelines();
if (exclusiveNodeOptions != null &&
!Strings.isNullOrEmpty(exclusiveNodeOptions.getNodeId())) {
HddsProtos.Node node =
scmClient.queryNode(UUID.fromString(exclusiveNodeOptions.getNodeId()));
- DatanodeWithAttributes dwa = new DatanodeWithAttributes(DatanodeDetails
- .getFromProtoBuf(node.getNodeID()),
- node.getNodeOperationalStates(0),
- node.getNodeStates(0));
-
+ BasicDatanodeInfo singleNodeInfo = new
BasicDatanodeInfo(DatanodeDetails.getFromProtoBuf(node.getNodeID()),
+ node.getNodeOperationalStates(0), node.getNodeStates(0));
if (json) {
- List<DatanodeWithAttributes> singleList =
Collections.singletonList(dwa);
-
System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(singleList));
+ List<BasicDatanodeInfo> dtoList =
Collections.singletonList(singleNodeInfo);
+
System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(dtoList));
} else {
- printDatanodeInfo(dwa);
+ printDatanodeInfo(singleNodeInfo);
}
return;
}
- Stream<DatanodeWithAttributes> allNodes = getAllNodes(scmClient).stream();
+ Stream<BasicDatanodeInfo> allNodes = getAllNodes(scmClient).stream();
if (exclusiveNodeOptions != null &&
!Strings.isNullOrEmpty(exclusiveNodeOptions.getIp())) {
- allNodes = allNodes.filter(p -> p.getDatanodeDetails().getIpAddress()
+ allNodes = allNodes.filter(p -> p.getIpAddress()
.compareToIgnoreCase(exclusiveNodeOptions.getIp()) == 0);
}
if (exclusiveNodeOptions != null &&
!Strings.isNullOrEmpty(exclusiveNodeOptions.getHostname())) {
- allNodes = allNodes.filter(p -> p.getDatanodeDetails().getHostName()
+ allNodes = allNodes.filter(p -> p.getHostName()
.compareToIgnoreCase(exclusiveNodeOptions.getHostname()) == 0);
}
if (!Strings.isNullOrEmpty(nodeOperationalState)) {
@@ -124,42 +121,49 @@ public void execute(ScmClient scmClient) throws
IOException {
}
if (json) {
- List<DatanodeWithAttributes> datanodeList = allNodes.collect(
- Collectors.toList());
- System.out.println(
- JsonUtils.toJsonStringWithDefaultPrettyPrinter(datanodeList));
+ List<BasicDatanodeInfo> datanodeList =
allNodes.collect(Collectors.toList());
+
System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(datanodeList));
} else {
allNodes.forEach(this::printDatanodeInfo);
}
}
- private List<DatanodeWithAttributes> getAllNodes(ScmClient scmClient)
+ private List<BasicDatanodeInfo> getAllNodes(ScmClient scmClient)
throws IOException {
// If sorting is requested
if (exclusiveNodeOptions != null && (exclusiveNodeOptions.mostUsed ||
exclusiveNodeOptions.leastUsed)) {
boolean sortByMostUsed = exclusiveNodeOptions.mostUsed;
- List<HddsProtos.DatanodeUsageInfoProto> usageInfos =
scmClient.getDatanodeUsageInfo(sortByMostUsed,
- Integer.MAX_VALUE);
+ List<HddsProtos.DatanodeUsageInfoProto> usageInfos;
+ try {
+ usageInfos = scmClient.getDatanodeUsageInfo(sortByMostUsed,
Integer.MAX_VALUE);
+ } catch (Exception e) {
+ System.err.println("Failed to get datanode usage info: " +
e.getMessage());
+ return Collections.emptyList();
+ }
return usageInfos.stream()
.map(p -> {
- String uuidStr = p.getNode().getUuid();
- UUID parsedUuid = UUID.fromString(uuidStr);
-
try {
+ String uuidStr = p.getNode().getUuid();
+ UUID parsedUuid = UUID.fromString(uuidStr);
HddsProtos.Node node = scmClient.queryNode(parsedUuid);
long capacity = p.getCapacity();
long used = capacity - p.getRemaining();
double percentUsed = (capacity > 0) ? (used * 100.0) / capacity
: 0.0;
- return new DatanodeWithAttributes(
+ return new BasicDatanodeInfo(
DatanodeDetails.getFromProtoBuf(node.getNodeID()),
node.getNodeOperationalStates(0),
node.getNodeStates(0),
used,
capacity,
percentUsed);
- } catch (IOException e) {
+ } catch (Exception e) {
+ String reason = "Could not process info for an unknown datanode";
+ if (p != null && p.getNode() != null &&
!Strings.isNullOrEmpty(p.getNode().getUuid())) {
+ reason = "Could not process node info for " +
p.getNode().getUuid();
+ }
+ System.err.printf("Error: %s: %s%n", reason, e.getMessage());
return null;
}
})
@@ -170,17 +174,16 @@ private List<DatanodeWithAttributes>
getAllNodes(ScmClient scmClient)
List<HddsProtos.Node> nodes = scmClient.queryNode(null,
null, HddsProtos.QueryScope.CLUSTER, "");
- return nodes.stream()
- .map(p -> new DatanodeWithAttributes(
+ return nodes.stream().map(p -> new BasicDatanodeInfo(
DatanodeDetails.getFromProtoBuf(p.getNodeID()),
p.getNodeOperationalStates(0), p.getNodeStates(0)))
- .sorted((o1, o2) -> o1.healthState.compareTo(o2.healthState))
+ .sorted(Comparator.comparing(BasicDatanodeInfo::getHealthState))
.collect(Collectors.toList());
}
- private void printDatanodeInfo(DatanodeWithAttributes dna) {
+ private void printDatanodeInfo(BasicDatanodeInfo dn) {
StringBuilder pipelineListInfo = new StringBuilder();
- DatanodeDetails datanode = dna.getDatanodeDetails();
+ DatanodeDetails datanode = dn.getDatanodeDetails();
int relatedPipelineNum = 0;
if (!pipelines.isEmpty()) {
List<Pipeline> relatedPipelines = pipelines.stream().filter(
@@ -206,72 +209,14 @@ private void printDatanodeInfo(DatanodeWithAttributes
dna) {
" (" + datanode.getNetworkLocation() + "/" + datanode.getIpAddress()
+ "/" + datanode.getHostName() + "/" + relatedPipelineNum +
" pipelines)");
- System.out.println("Operational State: " + dna.getOpState());
- System.out.println("Health State: " + dna.getHealthState());
+ System.out.println("Operational State: " + dn.getOpState());
+ System.out.println("Health State: " + dn.getHealthState());
System.out.println("Related pipelines:\n" + pipelineListInfo);
- if (dna.getUsed() != null && dna.getCapacity() != null && dna.getUsed() >=
0 && dna.getCapacity() > 0) {
- System.out.println("Capacity: " + dna.getCapacity());
- System.out.println("Used: " + dna.getUsed());
- System.out.printf("Percentage Used : %.2f%%%n%n", dna.getPercentUsed());
- }
- }
-
- private static class DatanodeWithAttributes {
- private DatanodeDetails datanodeDetails;
- private HddsProtos.NodeOperationalState operationalState;
- private HddsProtos.NodeState healthState;
- @JsonInclude(JsonInclude.Include.NON_NULL)
- private Long used = null;
- @JsonInclude(JsonInclude.Include.NON_NULL)
- private Long capacity = null;
- @JsonInclude(JsonInclude.Include.NON_NULL)
- private Double percentUsed = null;
-
- DatanodeWithAttributes(DatanodeDetails dn,
- HddsProtos.NodeOperationalState opState,
- HddsProtos.NodeState healthState) {
- this.datanodeDetails = dn;
- this.operationalState = opState;
- this.healthState = healthState;
- }
-
- DatanodeWithAttributes(DatanodeDetails dn,
- HddsProtos.NodeOperationalState opState,
- HddsProtos.NodeState healthState,
- long used,
- long capacity,
- double percentUsed) {
- this.datanodeDetails = dn;
- this.operationalState = opState;
- this.healthState = healthState;
- this.used = used;
- this.capacity = capacity;
- this.percentUsed = percentUsed;
- }
-
- public DatanodeDetails getDatanodeDetails() {
- return datanodeDetails;
- }
-
- public HddsProtos.NodeOperationalState getOpState() {
- return operationalState;
- }
-
- public HddsProtos.NodeState getHealthState() {
- return healthState;
- }
-
- public Long getUsed() {
- return used;
- }
-
- public Long getCapacity() {
- return capacity;
- }
-
- public Double getPercentUsed() {
- return percentUsed;
+ if (dn.getUsed() != null && dn.getCapacity() != null && dn.getUsed() >= 0
&& dn.getCapacity() > 0) {
+ System.out.println("Capacity: " + dn.getCapacity());
+ System.out.println("Used: " + dn.getUsed());
+ System.out.printf("Percentage Used : %.2f%%%n%n", dn.getPercentUsed());
}
}
}
diff --git
a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java
b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java
index a63d0c925f..fd6834068e 100644
---
a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java
+++
b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestListInfoSubcommand.java
@@ -207,9 +207,8 @@ public void testDataNodeByUuidOutput()
assertEquals(1, root.size(), "Expected 1 node in JSON output");
JsonNode node = root.get(0);
- assertTrue(node.has("datanodeDetails"), "Missing datanodeDetails");
String opState = node.get("opState").asText();
- String uuid = node.get("datanodeDetails").get("uuid").asText();
+ String uuid = node.get("id").asText();
assertEquals("IN_SERVICE", opState, "Expected opState IN_SERVICE but got:
" + opState);
assertEquals(nodes.get(0).getNodeID().getUuid(), uuid,
@@ -365,8 +364,6 @@ private List<HddsProtos.Node> getNodeDetails() {
dnd.setIpAddress("1.2.3." + i + 1);
dnd.setNetworkLocation("/default");
dnd.setNetworkName("host" + i);
- dnd.addPorts(HddsProtos.Port.newBuilder()
- .setName("ratis").setValue(5678).build());
dnd.setUuid(UUID.randomUUID().toString());
HddsProtos.Node.Builder builder = HddsProtos.Node.newBuilder();
diff --git a/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot
b/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot
index d4e7e7a2b4..2795ae5ac4 100644
--- a/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/admincli/datanode.robot
@@ -99,9 +99,10 @@ Incomplete command
List datanodes as JSON
${output} = Execute ozone admin datanode list --json | jq
-r '.'
- Should contain ${output} datanodeDetails
+ Should contain ${output} id
Should contain ${output} healthState
Should contain ${output} opState
+ Should contain ${output} persistedOpState
Get usage info as JSON
${output} = Execute ozone admin datanode usageinfo -m
--json | jq -r '.'
diff --git a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot
b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot
index 999aecb2fc..0436348239 100644
--- a/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot
+++ b/hadoop-ozone/dist/src/main/smoketest/balancer/testBalancer.robot
@@ -150,7 +150,7 @@ All container is closed
Get Datanode Ozone Used Bytes Info
[arguments] ${uuid}
- ${output} = Execute export DATANODES=$(ozone admin datanode list
--json) && for datanode in $(echo "$\{DATANODES\}" | jq -r
'.[].datanodeDetails.uuid'); do ozone admin datanode usageinfo
--uuid=$\{datanode\} --json | jq '{(.[0].datanodeDetails.uuid) :
.[0].ozoneUsed}'; done | jq -s add
+ ${output} = Execute export DATANODES=$(ozone admin datanode list
--json) && for datanode in $(echo "$\{DATANODES\}" | jq -r '.[].id'); do ozone
admin datanode usageinfo --uuid=$\{datanode\} --json | jq
'{(.[0].datanodeDetails.uuid) : .[0].ozoneUsed}'; done | jq -s add
${result} = Execute echo '${output}' | jq '. | to_entries | .[] |
select(.key == "${uuid}") | .value'
[return] ${result}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
