This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a change to branch HDDS-5713
in repository https://gitbox.apache.org/repos/asf/ozone.git
from 64fa9380217 HDDS-14164. Remove DatanodeDiskBalancerInfoType. (#9492)
add a55efa17c9b HDDS-13573. Intermittent failure in
TestNSSummaryUnifiedControl.testMultipleConcurrentAttempts (#9416).
add ad891ec2e44 HDDS-12607. Parallelize recon tasks to speed up OM rocksdb
reading tasks (#9243).
add 67c35ff0d9f HDDS-14112. Remove jakarta.validation-api (#9467)
add 17913d99351 HDDS-10522. Make OmPrefixInfo immutable (#9427)
add d4570c50a80 HDDS-14111. Make OmBucketInfo ACL list immutable (#9466)
add 3e217f5f4db HDDS-7544. Cache replica set in ContainerEntry (#8990)
add 66705728573 HDDS-13877. Fix and tweak Apple silicon protobuf 2.5.0
build section (#9242)
add b4d7e79b280 HDDS-14098. MiniOzoneCluster not shutdown in some tests
(#9451)
add 4c713042169 HDDS-14100. Close OutputStream properly in
TestFailureHandlingByClient (#9455)
add 37d75c54d3d HDDS-14148. Reduce ContainerEntry map initial size (#9475)
add 4155451d9c3 HDDS-13825. Support custom OM DB name in FSO repair tool
(#9216)
add 074094262dc HDDS-14152. CodecBufferCodec should implement
fromCodecBuffer deserialization method (#9480)
add f490ece49b0 HDDS-13466. Intermittent timeout in TestOmSnapshot (#9471)
add 98ad4e45aba HDDS-14153. Change
SnapshotLocalDataProvider#previousSnapshotLocalData to Optional (#9481)
add 89df7ed2722 HDDS-14037. Create DBDefinition and corresponding
MetadataManager for SnapshotDiff DB (#9398)
add 897c610e650 HDDS-14126. Introduce builder for metadata map (#9474)
add efcb7785384 HDDS-14157. Change SCM ha and pipeline code to throw more
specific exceptions (#9482)
add ffb08af1bb0 HDDS-14053. Extract generic MinHeapMergeIterator from
SstFileSetReader (#9409)
add d6026b238e0 HDDS-14159. Have an option to read only Key in
ManagedRawSSTFileIterator (#9485)
add d03ef0d29f3 HDDS-14122. Make OmKeyInfo ACL list immutable (#9486)
add e9fafe3979f HDDS-14158. Parameter 'format' is unknown for
maven-checkstyle-plugin:3.6.0 (#9483)
add 59ad0a1d7fb HDDS-14104. Refactor RequestContext creation (#9493)
add 9ed1c821eac HDDS-14165. Bump awssdk to 2.40.8 (#9496)
add d5b926745c3 HDDS-14147. Use AclListBuilder in OmDirectoryInfo and
OmKeyArgs (#9494)
add 114d24260b0 HDDS-14036. StreamRead: Make preReadSize, responseDataSize
and read timeout configurable (#9461)
add 97086589ea3 HDDS-14173. Selective checks: handle change in check.yml
(#9499)
add fd8948134ef HDDS-13890. Datanode supports dynamic configuration of SCM
(#9385)
add 4710ac91c7c HDDS-13866. Use component-specific default directory for
Ratis (#9318)
add ed9a6e3604e HDDS-14099. CI tools are reinstalled if not on path (#9454)
add 33544def1cc HDDS-14174. RandomAccessFileChannel.close() should not
throw RuntimeException (#9501)
add cafd9dd62ae HDDS-14176. Do not use Optional for volumeUsage (#9503)
new df08861f9d5 Merge branch 'master' into HDDS-5713
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/intermittent-test-check.yml | 4 +-
dev-support/ci/selective_ci_checks.bats | 11 +
dev-support/ci/selective_ci_checks.sh | 1 +
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 73 ++
.../hdds/scm/storage/StreamBlockInputStream.java | 35 +-
.../hadoop/hdds/scm/TestOzoneClientConfig.java | 15 +
.../scm/storage/TestStreamBlockInputStream.java | 69 ++
.../java/org/apache/hadoop/hdds/HddsUtils.java | 35 +
.../org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java | 7 +-
.../db/{CodecException.java => IteratorType.java} | 38 +-
.../apache/hadoop/hdds/utils/db/StringCodec.java | 0
.../hdds/utils/io/RandomAccessFileChannel.java | 17 +-
.../apache/hadoop/ozone/util/ClosableIterator.java | 3 +-
.../common/src/main/resources/ozone-default.xml | 37 +
.../hadoop/hdds/conf/ConfigurationSource.java | 4 +
.../apache/hadoop/ozone/HddsDatanodeService.java | 128 ++++
.../common/statemachine/DatanodeQueueMetrics.java | 22 +
.../common/statemachine/DatanodeStateMachine.java | 16 +
.../common/statemachine/SCMConnectionManager.java | 20 +-
.../common/statemachine/StateContext.java | 11 +
.../container/common/states/DatanodeState.java | 3 +-
.../states/datanode/RunningDatanodeState.java | 14 +-
.../ozone/container/common/volume/HddsVolume.java | 32 +-
.../container/common/volume/StorageVolume.java | 44 +-
.../container/common/volume/VolumeInfoMetrics.java | 5 +-
.../ozone/container/common/volume/VolumeUsage.java | 14 +-
.../ozone/container/metadata/DatanodeTable.java | 3 +-
.../container/common/impl/TestHddsDispatcher.java | 3 +-
.../states/datanode/TestRunningDatanodeState.java | 1 +
.../common/volume/TestReservedVolumeSpace.java | 19 +-
.../volume/TestStorageVolumeHealthChecks.java | 2 +-
.../container/common/volume/TestVolumeSet.java | 2 +-
hadoop-hdds/docs/content/start/FromSource.md | 37 +-
hadoop-hdds/framework/pom.xml | 4 -
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 29 +-
.../hdds/scm/metadata/DBTransactionBuffer.java | 12 +-
.../scm/metadata/SCMDBTransactionBufferImpl.java | 9 +-
.../org/apache/hadoop/hdds/server/ServerUtils.java | 132 +++-
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 3 +-
.../hdds/utils/db/BatchOperationHandler.java | 5 +-
.../hadoop/hdds/utils/db/CodecBufferCodec.java | 5 +
.../org/apache/hadoop/hdds/utils/db/DBStore.java | 45 ++
.../hdds/utils/db/RDBStoreAbstractIterator.java | 6 +-
.../hdds/utils/db/RDBStoreByteArrayIterator.java | 2 +-
.../hdds/utils/db/RDBStoreCodecBufferIterator.java | 2 +-
.../org/apache/hadoop/hdds/utils/db/RDBTable.java | 8 +-
.../hadoop/hdds/utils/db/RocksDBCheckpoint.java | 4 +-
.../org/apache/hadoop/hdds/utils/db/Table.java | 29 +-
.../apache/hadoop/hdds/utils/db/TypedTable.java | 4 +-
.../apache/hadoop/hdds/server/TestServerUtils.java | 176 +++++
.../hadoop/hdds/utils/db/InMemoryTestTable.java | 2 +-
.../hdds/utils/db/StringInMemoryTestTable.java | 2 +-
.../hadoop/hdds/utils/db/TestCodecBufferCodec.java | 53 ++
.../utils/db/TestRDBStoreByteArrayIterator.java | 8 +-
.../utils/db/TestRDBStoreCodecBufferIterator.java | 2 +-
.../hdds/utils/db/TestTypedRDBTableStore.java | 2 +-
.../hadoop/hdds/utils/db/TestTypedTable.java | 8 +-
.../db/managed/ManagedRawSSTFileIterator.java | 17 +-
.../utils/db/managed/ManagedRawSSTFileReader.java | 7 +-
.../db/managed/TestManagedRawSSTFileIterator.java | 35 +-
.../hadoop/hdds/utils/db/MinHeapMergeIterator.java | 207 ++++++
.../ozone/rocksdb/util/SstFileSetReader.java | 188 ++---
.../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 3 +-
.../hdds/utils/db/TestMinHeapMergeIterator.java | 288 ++++++++
.../ozone/rocksdb/util/TestSstFileSetReader.java | 9 +-
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 2 +-
.../hdds/scm/container/states/ContainerEntry.java | 23 +-
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 2 +-
.../hdds/scm/ha/SCMHADBTransactionBuffer.java | 9 +-
.../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 19 +-
.../hdds/scm/ha/SCMHADBTransactionBufferStub.java | 17 +-
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 2 +-
.../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 8 +-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 17 +-
.../hdds/scm/pipeline/PipelineManagerImpl.java | 20 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 27 +-
.../scm/pipeline/PipelineStateManagerImpl.java | 55 +-
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 16 +-
.../hdds/scm/server/StorageContainerManager.java | 21 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 41 +-
.../pipeline/TestWritableECContainerProvider.java | 6 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 2 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 7 +-
.../hadoop/ozone/om/helpers/AclListBuilder.java | 31 +-
.../apache/hadoop/ozone/om/helpers/MapBuilder.java | 108 +++
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 54 +-
.../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 54 +-
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 65 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 44 +-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 6 -
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 18 +-
.../hadoop/ozone/om/helpers/WithMetadata.java | 22 +-
.../hadoop/ozone/security/acl/RequestContext.java | 87 +--
.../ozone/om/helpers/TestAclListBuilder.java | 84 +--
.../hadoop/ozone/om/helpers/TestMapBuilder.java | 160 ++++
.../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 21 -
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 8 +-
.../hadoop/ozone/om/helpers/TestOmVolumeArgs.java | 4 +-
hadoop-ozone/dev-support/checks/_lib.sh | 59 +-
hadoop-ozone/dev-support/checks/install/hugo.sh | 2 +-
.../dist/src/main/smoketest/omha/testOMHA.robot | 2 +-
.../ozone/recon/TestReconAndAdminContainerCLI.java | 2 +
.../ozone/recon/TestReconContainerEndpoint.java | 2 +
.../scm/TestDatanodeSCMNodesReconfiguration.java | 446 ++++++++++++
.../hadoop/hdds/scm/TestSecretKeySnapshot.java | 5 +-
.../apache/hadoop/hdds/scm/TestSecretKeysApi.java | 5 +-
.../hdds/scm/TestStorageContainerManager.java | 2 +-
.../org/apache/hadoop/ozone/TestBlockTokens.java | 5 +-
.../apache/hadoop/ozone/TestBlockTokensCLI.java | 5 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 2 +-
.../rpc/TestContainerStateMachineFailures.java | 23 +-
.../client/rpc/TestFailureHandlingByClient.java | 335 ++++-----
.../rpc/TestFailureHandlingByClientFlushDelay.java | 70 +-
.../TestRefreshVolumeUsageHandler.java | 2 +-
.../TestContainerCommandReconciliation.java | 2 +-
.../ozone/om/TestOzoneManagerHAWithAllRunning.java | 4 +-
.../hadoop/ozone/om/snapshot/TestOmSnapshot.java | 31 +-
.../snapshot/TestOmSnapshotFsoWithNativeLib.java | 2 -
...mSnapshotFsoWithNativeLibWithLinkedBuckets.java | 2 -
...apshotFsoWithoutNativeLibWithLinkedBuckets.java | 3 -
.../om/snapshot/TestOmSnapshotObjectStore.java | 3 -
...TestOmSnapshotObjectStoreWithLinkedBuckets.java | 3 -
.../TestOmSnapshotWithBucketLinkingLegacy.java | 3 -
.../TestOmSnapshotWithoutBucketLinkingLegacy.java | 3 -
.../ozone/parser/TestOzoneHARatisLogParser.java | 9 +-
.../hadoop/ozone/repair/om/TestFSORepairTool.java | 65 +-
hadoop-ozone/interface-storage/pom.xml | 4 +
.../hadoop/ozone/om/helpers/OmPrefixInfo.java | 68 +-
.../src/main/proto/OmStorageProtocol.proto | 5 +
.../hadoop/ozone/om/helpers/TestOmPrefixInfo.java | 31 +-
.../ozone/om/helpers/TestOmPrefixInfoCodec.java | 3 +-
hadoop-ozone/mini-cluster/pom.xml | 4 +
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 203 ++++++
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 49 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 12 +-
.../apache/hadoop/ozone/om/PrefixManagerImpl.java | 92 +--
.../om/ratis/utils/OzoneManagerRatisUtils.java | 15 +-
.../om/request/bucket/OMBucketCreateRequest.java | 32 +-
.../om/request/bucket/acl/OMBucketAclRequest.java | 13 +-
.../request/bucket/acl/OMBucketAddAclRequest.java | 8 +-
.../bucket/acl/OMBucketRemoveAclRequest.java | 9 +-
.../request/bucket/acl/OMBucketSetAclRequest.java | 10 +-
.../ozone/om/request/key/acl/OMKeyAclRequest.java | 16 +-
.../om/request/key/acl/OMKeyAclRequestWithFSO.java | 18 +-
.../om/request/key/acl/OMKeyAddAclRequest.java | 4 +-
.../request/key/acl/OMKeyAddAclRequestWithFSO.java | 4 +-
.../om/request/key/acl/OMKeyRemoveAclRequest.java | 4 +-
.../key/acl/OMKeyRemoveAclRequestWithFSO.java | 4 +-
.../om/request/key/acl/OMKeySetAclRequest.java | 4 +-
.../request/key/acl/OMKeySetAclRequestWithFSO.java | 4 +-
.../S3MultipartUploadCompleteRequest.java | 4 +-
.../request/util/{package-info.java => AclOp.java} | 14 +-
.../om/request/volume/acl/OMVolumeAclRequest.java | 13 +-
.../request/volume/acl/OMVolumeAddAclRequest.java | 1 +
.../volume/acl/OMVolumeRemoveAclRequest.java | 1 +
.../request/volume/acl/OMVolumeSetAclRequest.java | 1 +
.../ozone/om/service/DirectoryDeletingService.java | 8 +-
.../hadoop/ozone/om/service/QuotaRepairTask.java | 4 +-
.../om/snapshot/OmSnapshotLocalDataManager.java | 16 +-
.../ozone/om/snapshot/SnapshotDiffManager.java | 19 +-
.../om/snapshot/db/SnapshotDiffDBDefinition.java | 145 ++++
.../snapshot/db/SnapshotDiffMetadataManager.java | 51 ++
.../db/SnapshotDiffMetadataManagerImpl.java | 153 ++++
.../hadoop/ozone/om/snapshot/db}/package-info.java | 4 +-
.../om/snapshot/defrag/SnapshotDefragService.java | 19 +-
.../om/snapshot/diff/delta/RDBDifferComputer.java | 3 +-
.../diff/helper/SnapshotDiffObjectInfo.java | 61 ++
.../om/snapshot/diff/helper}/package-info.java | 4 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 1 -
.../om/request/key/TestOMKeyCommitRequest.java | 8 +-
.../ozone/om/response/TestCleanupTableInfo.java | 2 -
.../ozone/om/response/TestOMResponseUtils.java | 14 +-
.../snapshot/TestOmSnapshotLocalDataManager.java | 9 +-
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 4 +-
.../snapshot/defrag/TestSnapshotDefragService.java | 5 +-
.../snapshot/diff/delta/TestRDBDifferComputer.java | 20 +-
.../ozone/security/acl/OzoneNativeAclTestUtil.java | 28 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 2 +-
.../hadoop/ozone/security/acl/TestParentAcl.java | 5 +-
.../ozone/security/acl/TestRequestContext.java | 102 +--
.../hadoop/ozone/security/acl/TestVolumeOwner.java | 9 +-
hadoop-ozone/recon/pom.xml | 4 -
.../apache/hadoop/ozone/recon/ReconConstants.java | 6 +-
.../hadoop/ozone/recon/ReconServerConfigKeys.java | 22 +
.../ozone/recon/scm/ReconPipelineManager.java | 4 +-
.../spi/impl/OzoneManagerServiceProviderImpl.java | 4 +-
.../impl/StorageContainerServiceProviderImpl.java | 4 +-
.../recon/tasks/ContainerKeyMapperHelper.java | 296 +++++---
.../recon/tasks/ContainerKeyMapperTaskFSO.java | 12 +-
.../recon/tasks/ContainerKeyMapperTaskOBS.java | 11 +-
.../recon/tasks/DeletedKeysInsightHandler.java | 14 +-
.../ozone/recon/tasks/FileSizeCountTaskFSO.java | 19 +-
.../ozone/recon/tasks/FileSizeCountTaskHelper.java | 171 ++---
.../ozone/recon/tasks/FileSizeCountTaskOBS.java | 19 +-
.../recon/tasks/MultipartInfoInsightHandler.java | 14 +-
.../hadoop/ozone/recon/tasks/OmTableHandler.java | 14 +-
.../ozone/recon/tasks/OmTableInsightTask.java | 129 +++-
.../ozone/recon/tasks/OpenKeysInsightHandler.java | 13 +-
.../ozone/recon/tasks/ReconTaskControllerImpl.java | 4 +
.../tasks/util/ParallelTableIteratorOperation.java | 296 ++++++++
.../ozone/recon/tasks/util}/package-info.java | 8 +-
.../hadoop/ozone/recon/api/NSSummaryTests.java | 3 +-
.../ozone/recon/api/TestContainerEndpoint.java | 44 +-
.../recon/tasks/TestContainerKeyMapperTask.java | 5 +
.../ozone/recon/tasks/TestFileSizeCountTask.java | 2 +
.../recon/tasks/TestNSSummaryUnifiedControl.java | 805 ++++++++++++++-------
.../ozone/recon/tasks/TestOmTableInsightTask.java | 27 +
.../hadoop/ozone/repair/om/FSORepairTool.java | 160 ++--
pom.xml | 11 +-
209 files changed, 5526 insertions(+), 2129 deletions(-)
create mode 100644
hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestStreamBlockInputStream.java
copy
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/{CodecException.java
=> IteratorType.java} (63%)
rename hadoop-hdds/{framework =>
common}/src/main/java/org/apache/hadoop/hdds/utils/db/StringCodec.java (100%)
create mode 100644
hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestCodecBufferCodec.java
create mode 100644
hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/hadoop/hdds/utils/db/MinHeapMergeIterator.java
create mode 100644
hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/hadoop/hdds/utils/db/TestMinHeapMergeIterator.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/MapBuilder.java
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestMapBuilder.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestDatanodeSCMNodesReconfiguration.java
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/{package-info.java
=> AclOp.java} (71%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffDBDefinition.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffMetadataManager.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffMetadataManagerImpl.java
copy {hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client =>
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db}/package-info.java
(87%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/helper/SnapshotDiffObjectInfo.java
copy
{hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client
=>
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/helper}/package-info.java
(87%)
create mode 100644
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/util/ParallelTableIteratorOperation.java
copy {hadoop-hdds/erasurecode/src/test/java/org/apache/ozone/erasurecode =>
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/util}/package-info.java
(88%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
