This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-5713 by this push:
new 0f1b034ffc HDDS-13547. [DiskBalancer] Change VolumeChoosingPolicy to
DiskBalancerVolumeChoosingPolicy (#8913)
0f1b034ffc is described below
commit 0f1b034ffc0a8e1914da4041a30d16d06978b009
Author: Gargi Jaiswal <[email protected]>
AuthorDate: Thu Aug 7 13:57:23 2025 +0530
HDDS-13547. [DiskBalancer] Change VolumeChoosingPolicy to
DiskBalancerVolumeChoosingPolicy (#8913)
---
.../ozone/container/diskbalancer/DiskBalancerService.java | 10 +++++-----
.../diskbalancer/policy/DefaultVolumeChoosingPolicy.java | 6 ++++--
...oosingPolicy.java => DiskBalancerVolumeChoosingPolicy.java} | 2 +-
.../hadoop/ozone/container/keyvalue/KeyValueContainer.java | 1 +
.../ozone/container/diskbalancer/TestDiskBalancerService.java | 4 ++--
.../apache/hadoop/ozone/scm/node/TestVolumeChoosingPolicy.java | 8 ++++----
6 files changed, 17 insertions(+), 14 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
index efc06463d6..724ec8f9ff 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
@@ -55,7 +55,7 @@
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import
org.apache.hadoop.ozone.container.diskbalancer.policy.ContainerChoosingPolicy;
-import
org.apache.hadoop.ozone.container.diskbalancer.policy.VolumeChoosingPolicy;
+import
org.apache.hadoop.ozone.container.diskbalancer.policy.DiskBalancerVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import
org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerLocationUtil;
import
org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil;
@@ -109,7 +109,7 @@ public class DiskBalancerService extends BackgroundService {
private Map<HddsVolume, Long> deltaSizes;
private MutableVolumeSet volumeSet;
- private VolumeChoosingPolicy volumeChoosingPolicy;
+ private DiskBalancerVolumeChoosingPolicy volumeChoosingPolicy;
private ContainerChoosingPolicy containerChoosingPolicy;
private final File diskBalancerInfoFile;
@@ -158,7 +158,7 @@ public DiskBalancerService(OzoneContainer ozoneContainer,
volumeSet = ozoneContainer.getVolumeSet();
try {
- volumeChoosingPolicy = (VolumeChoosingPolicy)
+ volumeChoosingPolicy = (DiskBalancerVolumeChoosingPolicy)
conf.getObject(DiskBalancerConfiguration.class)
.getVolumeChoosingPolicyClass().newInstance();
containerChoosingPolicy = (ContainerChoosingPolicy)
@@ -686,7 +686,7 @@ public ContainerChoosingPolicy getContainerChoosingPolicy()
{
return containerChoosingPolicy;
}
- public VolumeChoosingPolicy getVolumeChoosingPolicy() {
+ public DiskBalancerVolumeChoosingPolicy getVolumeChoosingPolicy() {
return volumeChoosingPolicy;
}
@@ -700,7 +700,7 @@ public DiskBalancerTask
createDiskBalancerTask(ContainerData containerData, Hdds
}
@VisibleForTesting
- public void setVolumeChoosingPolicy(VolumeChoosingPolicy
volumeChoosingPolicy) {
+ public void setVolumeChoosingPolicy(DiskBalancerVolumeChoosingPolicy
volumeChoosingPolicy) {
this.volumeChoosingPolicy = volumeChoosingPolicy;
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultVolumeChoosingPolicy.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultVolumeChoosingPolicy.java
index a0166f9e2a..d867b3b0f2 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultVolumeChoosingPolicy.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultVolumeChoosingPolicy.java
@@ -34,7 +34,7 @@
* Destination volumes use committedBytes to account for space already
reserved.
* Both deltaMap and committedBytes are considered to calculate usage.
*/
-public class DefaultVolumeChoosingPolicy implements VolumeChoosingPolicy {
+public class DefaultVolumeChoosingPolicy implements
DiskBalancerVolumeChoosingPolicy {
public static final Logger LOG = LoggerFactory.getLogger(
DefaultVolumeChoosingPolicy.class);
@@ -44,6 +44,8 @@ public Pair<HddsVolume, HddsVolume>
chooseVolume(MutableVolumeSet volumeSet,
double threshold, Map<HddsVolume, Long> deltaMap) {
double idealUsage = volumeSet.getIdealUsage();
+ // Threshold is given as a percentage
+ double normalizedThreshold = threshold / 100;
List<HddsVolume> volumes = StorageVolumeUtil
.getHddsVolumesList(volumeSet.getVolumesList())
.stream()
@@ -51,7 +53,7 @@ public Pair<HddsVolume, HddsVolume>
chooseVolume(MutableVolumeSet volumeSet,
Math.abs(
((double)((volume.getCurrentUsage().getCapacity() -
volume.getCurrentUsage().getAvailable())
+ deltaMap.getOrDefault(volume, 0L) +
volume.getCommittedBytes()))
- / volume.getCurrentUsage().getCapacity() - idealUsage) >=
threshold)
+ / volume.getCurrentUsage().getCapacity() - idealUsage) >=
normalizedThreshold)
.sorted((v1, v2) ->
Double.compare(
(double) ((v2.getCurrentUsage().getCapacity() -
v2.getCurrentUsage().getAvailable())
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/VolumeChoosingPolicy.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DiskBalancerVolumeChoosingPolicy.java
similarity index 96%
rename from
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/VolumeChoosingPolicy.java
rename to
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DiskBalancerVolumeChoosingPolicy.java
index 2f5a202a5e..7bb805f464 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/VolumeChoosingPolicy.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DiskBalancerVolumeChoosingPolicy.java
@@ -25,7 +25,7 @@
/**
* This interface specifies the policy for choosing volumes to balance.
*/
-public interface VolumeChoosingPolicy {
+public interface DiskBalancerVolumeChoosingPolicy {
/**
* Choose a pair of volumes for balancing.
*
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
index ad28798953..3b952c5952 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
@@ -975,6 +975,7 @@ public void copyContainerData(Path destination) throws
IOException {
} catch (Exception e) {
LOG.error("Got exception when copying container {} to {}",
containerData.getContainerID(), destination, e);
+ throw e;
} finally {
if (lock.isWriteLockedByCurrentThread()) {
writeUnlock();
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
index 2795e031b6..07d05fcb0b 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
@@ -52,7 +52,7 @@
import
org.apache.hadoop.ozone.container.diskbalancer.policy.ContainerChoosingPolicy;
import
org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultContainerChoosingPolicy;
import
org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultVolumeChoosingPolicy;
-import
org.apache.hadoop.ozone.container.diskbalancer.policy.VolumeChoosingPolicy;
+import
org.apache.hadoop.ozone.container.diskbalancer.policy.DiskBalancerVolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.keyvalue.ContainerTestVersionInfo;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
@@ -275,7 +275,7 @@ public void testConcurrentTasksNotExceedThreadLimit()
throws Exception {
false, DiskBalancerVersion.DEFAULT_VERSION);
svc.refresh(info);
- VolumeChoosingPolicy volumePolicy = mock(VolumeChoosingPolicy.class);
+ DiskBalancerVolumeChoosingPolicy volumePolicy =
mock(DiskBalancerVolumeChoosingPolicy.class);
ContainerChoosingPolicy containerPolicy =
mock(ContainerChoosingPolicy.class);
svc.setVolumeChoosingPolicy(volumePolicy);
svc.setContainerChoosingPolicy(containerPolicy);
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestVolumeChoosingPolicy.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestVolumeChoosingPolicy.java
index 9f07588c20..e300f8da6e 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestVolumeChoosingPolicy.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestVolumeChoosingPolicy.java
@@ -49,7 +49,7 @@
import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet;
import org.apache.hadoop.ozone.container.common.volume.StorageVolume;
import
org.apache.hadoop.ozone.container.diskbalancer.policy.DefaultVolumeChoosingPolicy;
-import
org.apache.hadoop.ozone.container.diskbalancer.policy.VolumeChoosingPolicy;
+import
org.apache.hadoop.ozone.container.diskbalancer.policy.DiskBalancerVolumeChoosingPolicy;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -64,7 +64,7 @@ public class TestVolumeChoosingPolicy {
private static final int NUM_VOLUMES = 20;
private static final int NUM_THREADS = 10;
private static final int NUM_ITERATIONS = 10000;
- private static final double THRESHOLD = 0.1; // 10% threshold
+ private static final double THRESHOLD = 10; // 10% threshold
private static final OzoneConfiguration CONF = new OzoneConfiguration();
@@ -73,7 +73,7 @@ public class TestVolumeChoosingPolicy {
private MutableVolumeSet volumeSet;
private List<HddsVolume> hddsVolumes;
- private VolumeChoosingPolicy volumeChoosingPolicy;
+ private DiskBalancerVolumeChoosingPolicy volumeChoosingPolicy;
private ExecutorService executor;
// delta sizes for source volumes
@@ -118,7 +118,7 @@ public void testConcurrentVolumeChoosingPerformance()
throws Exception {
* pairChosenCount: Number of successful volume pair choices from the policy.
* FailureCount: Failures due to any exceptions thrown during volume choice
or null return.
*/
- private void testPolicyPerformance(String policyName, VolumeChoosingPolicy
policy) throws Exception {
+ private void testPolicyPerformance(String policyName,
DiskBalancerVolumeChoosingPolicy policy) throws Exception {
CountDownLatch latch = new CountDownLatch(NUM_THREADS);
AtomicInteger pairChosenCount = new AtomicInteger(0);
AtomicInteger pairNotChosenCount = new AtomicInteger(0);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
