This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
from ce54200b066 HDDS-14345. Use ThreadLocalRandom in
RandomPipelineChoosePolicy (#9583)
add aee5b689229 HDDS-7106. [DiskBalancer] Client-SCM interface (#3663)
add 1665d86f9b0 HDDS-7155. [DiskBalancer] Create interface between SCM and
DN (#3701)
add eb3021a5392 HDDS-7205. DiskBalancer CLI (#3739)
add de8b4807d01 HDDS-7234. Add a common option for DiskBalancer commands
(#3762)
add f11d1574544 HDDS-7383. Basic framework of DiskBalancerService (#3874)
add ac3949432bc HDDS-8182. Add volume and container choosing policy (#4408)
add a4ddc9e25fd HDDS-8844. Internal move logic for DiskBalancer (#4887)
add 472ae6e58b5 HDDS-10297. [Disk Balancer] Report command should default
to 25 nodes (#6169)
add e6c2cb70555 HDDS-10299. [DiskBalancer] Status command should output
all statuses by default (#6170)
add 3d908e81a09 HDDS-10304. [DiskBalancer] Start command - Fix nodes not
being processed and incorrect config values (#6181)
add e75d3fe50b0 HDDS-10305. [DiskBalancer] Disk balancer command is not
registered on datanode (#6188)
add 6adb32a0370 HDDS-10332. [Diskbalancer] Include Disk Balancer Report in
the heartbeat message (#6201)
add c299005b402 Merge master into disk balancer feature branch
add d2cd69ac348 HDDS-12436. [DiskBalancer] Add metrics for time spent by
container move (#8001)
add 8a16109551e HDDS-12432. [DiskBalancer] Respect property
"hdds.datanode.disk.balancer.max.disk.throughputInMBPerSec" (#8006)
add 87ad327074d HDDS-12435. [DiskBalancer] Add success move count and fail
move count in status report (#8009)
add b3cb5629307 HDDS-12545. [DiskBalancer] Query status and report API
doesn't require SCM admin privilege (#8112)
add 2f2d7304aa3 HDDS-12437. [DiskBalancer] Estimate the total size pending
to move before disk usage becomes even (#8056)
add ddaeefbd647 HDDS-12567. [DiskBalancer] Log success container move
action (#8165)
add b8bff8ae114 HDDS-12546. [DiskBalancer] Display bytesMoved instead of
VolumeDensity in status output (#8150)
add 010ab2e81bb Merge remote-tracking branch 'origin/master' into HDDS-5713
add 03c90697505 HDDS-12678. [DiskBalancer] Use ( capacity - free ) /
capacity to calculate disk utilization (#8278)
add 891f2fb83c2 HDDS-12796. [DiskBalancer] Exclude UNHEALTHY DNs from
status subcommand output (#8286)
add b26d09ec9f4 Merge remote-tracking branch 'origin/master' into HDDS-5713
add 801111c14a8 HDDS-12879. Use AtomicFileOutputStream to write
DiskBalancerYAML file (#8317)
add 73a813b5bfa HDDS-12431. [DiskBalancer] Use committedBytes to reserve
the space pre-allocated for container (#8297)
add f57c1b1bb5c Merge remote-tracking branch 'origin/master' into HDDS-5713
add bf4f26c8710 HDDS-12654. [DiskBalancer]Add a
hdds.datanode.disk.balancer.stop.after.disk.even property (#8239)
add ade29461cce Merge remote-tracking branch 'origin/master' into HDDS-5713
add 232d133e3be HDDS-13055. [DiskBalancer] Optimize
DefaultContainerChoosingPolicy performance (#8512)
add 8eb944a65a8 HDDS-13212. [DiskBalancer] Fix Inconsistent Health Check
in DiskBalancer Status for Specific Hosts (#8610)
add ee9b8efd24b Merge remote-tracking branch 'origin/master' into HDDS-5713
add fb38964fe3b Merge remote-tracking branch 'origin/master' into HDDS-5713
add 63689bd24bf HDDS-13054. [DiskBalancer] Stop DiskBalancer when DN
enters DECOMMISSIONING or MAINTENANCE states (#8488)
add 21dcb9c1400 HDDS-13291. [DiskBalancer] Add Performance Test for
VolumeChoosingPolicy in DiskBalancer (#8661)
add 7f2931707c3 Merge remote-tracking branch 'apacheupstream' into
HDDS-5713
add a6db12ca302 HDDS-13278. [DiskBalancer] Incorrect tracking of
DiskBalancer inProgressTask and balancedByteMap initialisation (#8650)
add 2334f410497 HDDS-12598. [DiskBalancer] Add design and feature document
(#8837)
add 575ca9ead66 HDDS-12434. [DiskBalancer] Refactor DiskBalancerTask#call
to improve the atomicity of container move (#8693)
add 3623f030eff HDDS-13497. [DiskBalancer] Add new property
"hdds.datanode.disk.balancer.enabled" (#8869)
add 0f1b034ffc0 HDDS-13547. [DiskBalancer] Change VolumeChoosingPolicy to
DiskBalancerVolumeChoosingPolicy (#8913)
add c531e7519a0 HDDS-12439. [DiskBalancer] Reserve committed space during
volume choosing process (#8882)
add 261de892c12 Merge master into HDDS-5713 (#8925)
add fd3823a931e HDDS-13586. [DiskBalancer] Refactor DiskBalancer to remove
unused methods and improve naming (#8956)
add 4e9bb80c601 HDDS-13665. [DiskBalancer] Cleanup for multiple
DiskBalancerReport sent to SCM (#9034)
add 3e6362f2fe3 HDDS-13649. [DiskBalancer] Refine Disk Balancer CLI,
Documentation, and Naming for Improved Usability (#9005)
add d1646190389 Merge remote-tracking branch 'apacheupstream' into
HDDS-5713-upstream
add c00288af23d HDDS-13611. [DiskBalancer] Inconsistent VolumeDataDensity
calculations between SCM and DN and incorrect EstBytesToMove (#8972)
add 6f43e7c5985 HDDS-13666. [DiskBalancer]Avoid reverse looping in
destination selection when thresholds are low (#9068)
add 2fa932c10c9 HDDS-13667. [DiskBalancer] Improve DiskBalancer CLI
message for failed commands (#9091)
add 1f320ef9400 HDDS-13598. Send disk balancer command directly to
datanode (#9256)
add c07a4a7d56f HDDS-13878. Cleanup SCM dependency and add enhancements
for direct Client–DN flow with updated testing (#9310)
add 6ab87743145 HDDS-13602. Delay delete source container replica to avoid
read failure due to read thread holds the old containerData (#8965)
add 5a1997fcb66 HDDS-14049. [DiskBalancer] Missing stop-after-disk-even in
DiskBalancer status output and doc updation (#9415)
add 4b52037fdeb HDDS-14102. [DiskBalancer] Fix incorrect default port
documentation in DiskBalancer(9858 -> 19864) (#9459)
add a534ac2f388 Merge branch 'master' of github.com:apache/ozone into
HDDS-5713-upstream
add 64fa9380217 HDDS-14164. Remove DatanodeDiskBalancerInfoType. (#9492)
add df08861f9d5 Merge branch 'master' into HDDS-5713
add 388084e44ac Merge branch 'am' into HDDS-5713
add 2abd75118ed HDDS-14175. DiskBalancer should not call getCurrentUsage()
multiple times. (#9505)
add 1cc4a874805 HDDS-14186. [DiskBalancer] Don't pick size 0 container
when choosing container to move (#9514)
add fc16787c351 HDDS-14110. [DiskBalancer] Show EstimatedBytesToMove only
during active balancing and improve threshold check message (#9465)
add 0b04d77ba54 HDDS-14214. [DiskBalancer] Include ConfigGroup prefix in
Config Key for DiskBalancerConfiguration (#9573)
add 65a0cd0475b HDDS-14279. Double check selected container state before
move process starts (#9575)
add 216cd287ae3 Merge remote-tracking branch 'origin/master' into HDDS-5713
add 27dcc200399 HDDS-14114. [DiskBalancer] Fix DiskBalancer tmp directory
to be under hdds/<cluster-id>/tmp (#9515)
add 4232d2163c8 HDDS-14213. Fix DiskBalancer VolumeChoosingPolicy to
balance when dest volume exists but no src volume exceeds threshold (#9587)
add fd181c69464 Merge remote-tracking branch 'origin/HDDS-5713' into
master-new
No new revisions were added by this update.
Summary of changes:
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 7 +
.../org/apache/hadoop/hdds/cli/ItemsFromStdin.java | 2 +-
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 3 +
.../common/src/main/resources/ozone-default.xml | 29 +-
.../org/apache/hadoop/hdds/conf/ConfigTag.java | 1 +
.../ozone/HddsDatanodeClientProtocolServer.java | 28 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 7 +-
.../apache/hadoop/ozone/HddsPolicyProvider.java | 11 +-
.../ozone/container/common/impl/ContainerSet.java | 23 +
.../container/common/interfaces/Container.java | 6 +
.../ozone/container/common/interfaces/Handler.java | 13 +
.../common/statemachine/DatanodeStateMachine.java | 24 +-
.../SetNodeOperationalStateCommandHandler.java | 11 +-
.../container/common/utils/ContainerLogger.java | 25 +
.../common/volume/AvailableSpaceFilter.java | 2 +-
.../volume/CapacityVolumeChoosingPolicy.java | 86 ++-
.../container/common/volume/MutableVolumeSet.java | 2 +-
.../volume/RoundRobinVolumeChoosingPolicy.java | 46 +-
.../container/common/volume/StorageVolume.java | 10 +-
.../common/volume/VolumeChoosingPolicyFactory.java | 13 +-
.../ozone/container/common/volume/VolumeUsage.java | 7 +-
.../diskbalancer/DiskBalancerConfiguration.java | 305 ++++++++
.../container/diskbalancer/DiskBalancerInfo.java | 233 ++++++
.../diskbalancer/DiskBalancerProtocolServer.java | 167 +++++
.../diskbalancer/DiskBalancerService.java | 830 +++++++++++++++++++++
.../diskbalancer/DiskBalancerServiceMetrics.java | 153 ++++
.../diskbalancer/DiskBalancerVersion.java | 77 ++
.../DiskBalancerVolumeCalculation.java | 171 +++++
.../container/diskbalancer/DiskBalancerYaml.java | 177 +++++
.../container/diskbalancer}/package-info.java | 4 +-
.../policy/ContainerChoosingPolicy.java | 49 ++
.../policy/DefaultContainerChoosingPolicy.java | 105 +++
.../policy/DefaultVolumeChoosingPolicy.java | 156 ++++
.../policy/DiskBalancerVolumeChoosingPolicy.java} | 33 +-
.../diskbalancer/policy}/package-info.java | 4 +-
.../container/keyvalue/KeyValueContainer.java | 128 +++-
.../ozone/container/keyvalue/KeyValueHandler.java | 41 +-
.../container/ozoneimpl/ContainerController.java | 13 +
.../ozone/container/ozoneimpl/OzoneContainer.java | 40 +
.../DiskBalancerServiceTestImpl.java} | 32 +-
.../TestDefaultContainerChoosingPolicy.java | 250 +++++++
.../TestDefaultVolumeChoosingPolicy.java | 652 ++++++++++++++++
.../TestDiskBalancerProtocolServer.java | 263 +++++++
.../diskbalancer/TestDiskBalancerService.java | 518 +++++++++++++
.../diskbalancer/TestDiskBalancerTask.java | 707 ++++++++++++++++++
.../diskbalancer/TestDiskBalancerYaml.java | 57 ++
...stContainerReconciliationWithMockDatanodes.java | 28 +-
.../container/ozoneimpl/TestContainerReader.java | 4 +-
hadoop-hdds/docs/content/design/diskbalancer.md | 177 +++++
hadoop-hdds/docs/content/feature/DiskBalancer.md | 253 +++++++
.../docs/content/feature/DiskBalancer.zh.md | 245 ++++++
hadoop-hdds/docs/content/feature/diskBalancer.png | Bin 0 -> 116124 bytes
.../om-bootstrapping-with-snapshots.md | 0
.../hadoop/hdds/protocol/DiskBalancerProtocol.java | 88 +++
...DiskBalancerProtocolClientSideTranslatorPB.java | 185 +++++
...DatanodePB.java => DiskBalancerProtocolPB.java} | 11 +-
...DiskBalancerProtocolServerSideTranslatorPB.java | 94 +++
.../apache/hadoop/hdds/scm/client/ScmClient.java | 1 +
.../hadoop/hdds/fs/MockSpaceUsageCheckFactory.java | 16 +
.../hadoop/hdds/utils/MapBackedTableIterator.java | 12 +-
.../src/main/proto/DiskBalancerProtocol.proto | 94 +++
.../interface-client/src/main/proto/hdds.proto | 24 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 -
.../apache/hadoop/hdds/scm/node/NodeManager.java | 12 +-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 27 +-
...inerLocationProtocolServerSideTranslatorPB.java | 3 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 1 -
.../hdds/scm/server/StorageContainerManager.java | 2 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 72 +-
.../hdds/scm/container/SimpleMockNodeManager.java | 12 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 6 +-
.../datanode/AbstractDiskBalancerSubCommand.java | 237 ++++++
.../hdds/scm/cli/datanode/DatanodeCommands.java | 3 +-
...NameParameters.java => DatanodeParameters.java} | 26 +-
.../scm/cli/datanode/DiskBalancerCommands.java | 171 +++++
.../cli/datanode/DiskBalancerCommonOptions.java | 56 ++
.../cli/datanode/DiskBalancerReportSubcommand.java | 130 ++++
.../cli/datanode/DiskBalancerStartSubcommand.java | 149 ++++
.../cli/datanode/DiskBalancerStatusSubcommand.java | 182 +++++
.../cli/datanode/DiskBalancerStopSubcommand.java | 85 +++
.../cli/datanode/DiskBalancerSubCommandUtil.java | 123 +++
.../cli/datanode/DiskBalancerUpdateSubcommand.java | 159 ++++
.../cli/datanode/TestDiskBalancerSubCommands.java | 695 +++++++++++++++++
.../diskbalancer.yaml} | 12 +-
.../{test-ec.sh => test-diskbalancer.sh} | 13 +-
.../smoketest/diskbalancer/testdiskbalancer.robot | 128 ++++
.../scm/node/TestContainerChoosingPolicy.java | 300 ++++++++
.../hadoop/ozone/scm/node/TestDiskBalancer.java | 255 +++++++
...skBalancerDuringDecommissionAndMaintenance.java | 441 +++++++++++
.../ozone/scm/node/TestVolumeChoosingPolicy.java | 285 +++++++
.../ozone/om/service/SnapshotDeletingService.java | 5 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 6 +-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 2 +-
.../hadoop/ozone/recon/tasks/NSSummaryTask.java | 6 +-
.../recon/tasks/NSSummaryTaskDbEventHandler.java | 12 +-
95 files changed, 10132 insertions(+), 238 deletions(-)
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerConfiguration.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerInfo.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerProtocolServer.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerService.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerServiceMetrics.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerVersion.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerVolumeCalculation.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/DiskBalancerYaml.java
copy
{hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/checksum =>
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer}/package-info.java
(91%)
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/ContainerChoosingPolicy.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultContainerChoosingPolicy.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy/DefaultVolumeChoosingPolicy.java
copy
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/{common/interfaces/VolumeChoosingPolicy.java
=> diskbalancer/policy/DiskBalancerVolumeChoosingPolicy.java} (50%)
copy hadoop-hdds/{client/src/main/java/org/apache/hadoop/hdds/scm/client =>
container-service/src/main/java/org/apache/hadoop/ozone/container/diskbalancer/policy}/package-info.java
(85%)
copy
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/{common/BlockDeletingServiceTestImpl.java
=> diskbalancer/DiskBalancerServiceTestImpl.java} (77%)
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDefaultContainerChoosingPolicy.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDefaultVolumeChoosingPolicy.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerProtocolServer.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerService.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerTask.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/diskbalancer/TestDiskBalancerYaml.java
create mode 100644 hadoop-hdds/docs/content/design/diskbalancer.md
create mode 100644 hadoop-hdds/docs/content/feature/DiskBalancer.md
create mode 100644 hadoop-hdds/docs/content/feature/DiskBalancer.zh.md
create mode 100644 hadoop-hdds/docs/content/feature/diskBalancer.png
copy hadoop-hdds/docs/content/{design =>
feature}/om-bootstrapping-with-snapshots.md (100%)
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocol/DiskBalancerProtocol.java
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/DiskBalancerProtocolClientSideTranslatorPB.java
copy
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/{ReconfigureProtocolDatanodePB.java
=> DiskBalancerProtocolPB.java} (74%)
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/DiskBalancerProtocolServerSideTranslatorPB.java
create mode 100644
hadoop-hdds/interface-client/src/main/proto/DiskBalancerProtocol.proto
create mode 100644
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/AbstractDiskBalancerSubCommand.java
copy
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/{HostNameParameters.java
=> DatanodeParameters.java} (58%)
create mode 100644
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommands.java
create mode 100644
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerCommonOptions.java
create mode 100644
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerReportSubcommand.java
create mode 100644
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStartSubcommand.java
create mode 100644
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStatusSubcommand.java
create mode 100644
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerStopSubcommand.java
create mode 100644
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerSubCommandUtil.java
create mode 100644
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/datanode/DiskBalancerUpdateSubcommand.java
create mode 100644
hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestDiskBalancerSubCommands.java
copy hadoop-ozone/dist/src/main/compose/{ozonesecure-ha/s3g-port-forward.yml
=> ozonesecure/diskbalancer.yaml} (79%)
copy hadoop-ozone/dist/src/main/compose/ozonesecure/{test-ec.sh =>
test-diskbalancer.sh} (88%)
mode change 100755 => 100644
create mode 100644
hadoop-ozone/dist/src/main/smoketest/diskbalancer/testdiskbalancer.robot
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestContainerChoosingPolicy.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancer.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDiskBalancerDuringDecommissionAndMaintenance.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestVolumeChoosingPolicy.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
