This is an automated email from the ASF dual-hosted git repository. szetszwo pushed a commit to branch HDDS-5713 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit df08861f9d554e37816ba62c3f78674e33ec3536 Merge: 64fa9380217 cafd9dd62ae Author: Tsz-Wo Nicholas Sze <[email protected]> AuthorDate: Mon Dec 15 11:47:46 2025 -0800 Merge branch 'master' into HDDS-5713 .github/workflows/intermittent-test-check.yml | 4 +- dev-support/ci/selective_ci_checks.bats | 11 + dev-support/ci/selective_ci_checks.sh | 1 + .../apache/hadoop/hdds/scm/OzoneClientConfig.java | 73 ++ .../hdds/scm/storage/StreamBlockInputStream.java | 35 +- .../hadoop/hdds/scm/TestOzoneClientConfig.java | 15 + .../scm/storage/TestStreamBlockInputStream.java | 69 ++ .../java/org/apache/hadoop/hdds/HddsUtils.java | 35 + .../org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java | 7 +- .../apache/hadoop/hdds/utils/db/IteratorType.java} | 41 +- .../apache/hadoop/hdds/utils/db/StringCodec.java | 0 .../hdds/utils/io/RandomAccessFileChannel.java | 17 +- .../apache/hadoop/ozone/util/ClosableIterator.java | 3 +- .../common/src/main/resources/ozone-default.xml | 37 + .../hadoop/hdds/conf/ConfigurationSource.java | 4 + .../apache/hadoop/ozone/HddsDatanodeService.java | 128 ++++ .../common/statemachine/DatanodeQueueMetrics.java | 22 + .../common/statemachine/DatanodeStateMachine.java | 16 + .../common/statemachine/SCMConnectionManager.java | 20 +- .../common/statemachine/StateContext.java | 11 + .../container/common/states/DatanodeState.java | 3 +- .../states/datanode/RunningDatanodeState.java | 14 +- .../ozone/container/common/volume/HddsVolume.java | 32 +- .../container/common/volume/StorageVolume.java | 44 +- .../container/common/volume/VolumeInfoMetrics.java | 5 +- .../ozone/container/common/volume/VolumeUsage.java | 14 +- .../ozone/container/metadata/DatanodeTable.java | 3 +- .../container/common/impl/TestHddsDispatcher.java | 3 +- .../states/datanode/TestRunningDatanodeState.java | 1 + .../common/volume/TestReservedVolumeSpace.java | 19 +- .../volume/TestStorageVolumeHealthChecks.java | 2 +- .../container/common/volume/TestVolumeSet.java | 2 +- hadoop-hdds/docs/content/start/FromSource.md | 37 +- hadoop-hdds/framework/pom.xml | 4 - .../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 29 +- .../hdds/scm/metadata/DBTransactionBuffer.java | 12 +- .../scm/metadata/SCMDBTransactionBufferImpl.java | 9 +- .../org/apache/hadoop/hdds/server/ServerUtils.java | 132 +++- .../apache/hadoop/hdds/utils/HddsServerUtil.java | 3 +- .../hdds/utils/db/BatchOperationHandler.java | 5 +- .../hadoop/hdds/utils/db/CodecBufferCodec.java | 5 + .../org/apache/hadoop/hdds/utils/db/DBStore.java | 45 ++ .../hdds/utils/db/RDBStoreAbstractIterator.java | 6 +- .../hdds/utils/db/RDBStoreByteArrayIterator.java | 2 +- .../hdds/utils/db/RDBStoreCodecBufferIterator.java | 2 +- .../org/apache/hadoop/hdds/utils/db/RDBTable.java | 8 +- .../hadoop/hdds/utils/db/RocksDBCheckpoint.java | 4 +- .../org/apache/hadoop/hdds/utils/db/Table.java | 29 +- .../apache/hadoop/hdds/utils/db/TypedTable.java | 4 +- .../apache/hadoop/hdds/server/TestServerUtils.java | 176 +++++ .../hadoop/hdds/utils/db/InMemoryTestTable.java | 2 +- .../hdds/utils/db/StringInMemoryTestTable.java | 2 +- .../hadoop/hdds/utils/db/TestCodecBufferCodec.java | 53 ++ .../utils/db/TestRDBStoreByteArrayIterator.java | 8 +- .../utils/db/TestRDBStoreCodecBufferIterator.java | 2 +- .../hdds/utils/db/TestTypedRDBTableStore.java | 2 +- .../hadoop/hdds/utils/db/TestTypedTable.java | 8 +- .../db/managed/ManagedRawSSTFileIterator.java | 17 +- .../utils/db/managed/ManagedRawSSTFileReader.java | 7 +- .../db/managed/TestManagedRawSSTFileIterator.java | 35 +- .../hadoop/hdds/utils/db/MinHeapMergeIterator.java | 207 ++++++ .../ozone/rocksdb/util/SstFileSetReader.java | 188 ++--- .../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 3 +- .../hdds/utils/db/TestMinHeapMergeIterator.java | 288 ++++++++ .../ozone/rocksdb/util/TestSstFileSetReader.java | 9 +- .../rocksdiff/TestRocksDBCheckpointDiffer.java | 2 +- .../hdds/scm/container/states/ContainerEntry.java | 23 +- .../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 2 +- .../hdds/scm/ha/SCMHADBTransactionBuffer.java | 9 +- .../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 19 +- .../hdds/scm/ha/SCMHADBTransactionBufferStub.java | 17 +- .../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 2 +- .../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 8 +- .../hadoop/hdds/scm/pipeline/PipelineManager.java | 17 +- .../hdds/scm/pipeline/PipelineManagerImpl.java | 20 +- .../hdds/scm/pipeline/PipelineStateManager.java | 27 +- .../scm/pipeline/PipelineStateManagerImpl.java | 55 +- .../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 16 +- .../hdds/scm/server/StorageContainerManager.java | 21 +- .../hdds/scm/pipeline/MockPipelineManager.java | 41 +- .../pipeline/TestWritableECContainerProvider.java | 6 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 2 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 7 +- .../hadoop/ozone/om/helpers/AclListBuilder.java | 31 +- .../apache/hadoop/ozone/om/helpers/MapBuilder.java | 108 +++ .../hadoop/ozone/om/helpers/OmBucketInfo.java | 54 +- .../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 54 +- .../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 65 +- .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 44 +- .../hadoop/ozone/om/helpers/OmVolumeArgs.java | 6 - .../hadoop/ozone/om/helpers/OzoneAclUtil.java | 18 +- .../hadoop/ozone/om/helpers/WithMetadata.java | 22 +- .../hadoop/ozone/security/acl/RequestContext.java | 87 +-- .../ozone/om/helpers/TestAclListBuilder.java | 84 +-- .../hadoop/ozone/om/helpers/TestMapBuilder.java | 160 ++++ .../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 21 - .../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 8 +- .../hadoop/ozone/om/helpers/TestOmVolumeArgs.java | 4 +- hadoop-ozone/dev-support/checks/_lib.sh | 59 +- hadoop-ozone/dev-support/checks/install/hugo.sh | 2 +- .../dist/src/main/smoketest/omha/testOMHA.robot | 2 +- .../ozone/recon/TestReconAndAdminContainerCLI.java | 2 + .../ozone/recon/TestReconContainerEndpoint.java | 2 + .../scm/TestDatanodeSCMNodesReconfiguration.java | 446 ++++++++++++ .../hadoop/hdds/scm/TestSecretKeySnapshot.java | 5 +- .../apache/hadoop/hdds/scm/TestSecretKeysApi.java | 5 +- .../hdds/scm/TestStorageContainerManager.java | 2 +- .../org/apache/hadoop/ozone/TestBlockTokens.java | 5 +- .../apache/hadoop/ozone/TestBlockTokensCLI.java | 5 +- .../apache/hadoop/ozone/TestMiniOzoneCluster.java | 2 +- .../rpc/TestContainerStateMachineFailures.java | 23 +- .../client/rpc/TestFailureHandlingByClient.java | 335 ++++----- .../rpc/TestFailureHandlingByClientFlushDelay.java | 70 +- .../TestRefreshVolumeUsageHandler.java | 2 +- .../TestContainerCommandReconciliation.java | 2 +- .../ozone/om/TestOzoneManagerHAWithAllRunning.java | 4 +- .../hadoop/ozone/om/snapshot/TestOmSnapshot.java | 31 +- .../snapshot/TestOmSnapshotFsoWithNativeLib.java | 2 - ...mSnapshotFsoWithNativeLibWithLinkedBuckets.java | 2 - ...apshotFsoWithoutNativeLibWithLinkedBuckets.java | 3 - .../om/snapshot/TestOmSnapshotObjectStore.java | 3 - ...TestOmSnapshotObjectStoreWithLinkedBuckets.java | 3 - .../TestOmSnapshotWithBucketLinkingLegacy.java | 3 - .../TestOmSnapshotWithoutBucketLinkingLegacy.java | 3 - .../ozone/parser/TestOzoneHARatisLogParser.java | 9 +- .../hadoop/ozone/repair/om/TestFSORepairTool.java | 65 +- hadoop-ozone/interface-storage/pom.xml | 4 + .../hadoop/ozone/om/helpers/OmPrefixInfo.java | 68 +- .../src/main/proto/OmStorageProtocol.proto | 5 + .../hadoop/ozone/om/helpers/TestOmPrefixInfo.java | 31 +- .../ozone/om/helpers/TestOmPrefixInfoCodec.java | 3 +- hadoop-ozone/mini-cluster/pom.xml | 4 + .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 203 ++++++ .../apache/hadoop/ozone/om/OmSnapshotManager.java | 49 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 12 +- .../apache/hadoop/ozone/om/PrefixManagerImpl.java | 92 +-- .../om/ratis/utils/OzoneManagerRatisUtils.java | 15 +- .../om/request/bucket/OMBucketCreateRequest.java | 32 +- .../om/request/bucket/acl/OMBucketAclRequest.java | 13 +- .../request/bucket/acl/OMBucketAddAclRequest.java | 8 +- .../bucket/acl/OMBucketRemoveAclRequest.java | 9 +- .../request/bucket/acl/OMBucketSetAclRequest.java | 10 +- .../ozone/om/request/key/acl/OMKeyAclRequest.java | 16 +- .../om/request/key/acl/OMKeyAclRequestWithFSO.java | 18 +- .../om/request/key/acl/OMKeyAddAclRequest.java | 4 +- .../request/key/acl/OMKeyAddAclRequestWithFSO.java | 4 +- .../om/request/key/acl/OMKeyRemoveAclRequest.java | 4 +- .../key/acl/OMKeyRemoveAclRequestWithFSO.java | 4 +- .../om/request/key/acl/OMKeySetAclRequest.java | 4 +- .../request/key/acl/OMKeySetAclRequestWithFSO.java | 4 +- .../S3MultipartUploadCompleteRequest.java | 4 +- .../apache/hadoop/ozone/om/request/util/AclOp.java | 14 +- .../om/request/volume/acl/OMVolumeAclRequest.java | 13 +- .../request/volume/acl/OMVolumeAddAclRequest.java | 1 + .../volume/acl/OMVolumeRemoveAclRequest.java | 1 + .../request/volume/acl/OMVolumeSetAclRequest.java | 1 + .../ozone/om/service/DirectoryDeletingService.java | 8 +- .../hadoop/ozone/om/service/QuotaRepairTask.java | 4 +- .../om/snapshot/OmSnapshotLocalDataManager.java | 16 +- .../ozone/om/snapshot/SnapshotDiffManager.java | 19 +- .../om/snapshot/db/SnapshotDiffDBDefinition.java | 145 ++++ .../snapshot/db/SnapshotDiffMetadataManager.java | 51 ++ .../db/SnapshotDiffMetadataManagerImpl.java | 153 ++++ .../hadoop/ozone/om/snapshot/db/package-info.java | 11 +- .../om/snapshot/defrag/SnapshotDefragService.java | 19 +- .../om/snapshot/diff/delta/RDBDifferComputer.java | 3 +- .../diff/helper/SnapshotDiffObjectInfo.java | 61 ++ .../om/snapshot/diff/helper/package-info.java | 11 +- .../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 1 - .../om/request/key/TestOMKeyCommitRequest.java | 8 +- .../ozone/om/response/TestCleanupTableInfo.java | 2 - .../ozone/om/response/TestOMResponseUtils.java | 14 +- .../snapshot/TestOmSnapshotLocalDataManager.java | 9 +- .../ozone/om/snapshot/TestSnapshotDiffManager.java | 4 +- .../snapshot/defrag/TestSnapshotDefragService.java | 5 +- .../snapshot/diff/delta/TestRDBDifferComputer.java | 20 +- .../ozone/security/acl/OzoneNativeAclTestUtil.java | 28 +- .../security/acl/TestOzoneNativeAuthorizer.java | 2 +- .../hadoop/ozone/security/acl/TestParentAcl.java | 5 +- .../ozone/security/acl/TestRequestContext.java | 102 +-- .../hadoop/ozone/security/acl/TestVolumeOwner.java | 9 +- hadoop-ozone/recon/pom.xml | 4 - .../apache/hadoop/ozone/recon/ReconConstants.java | 6 +- .../hadoop/ozone/recon/ReconServerConfigKeys.java | 22 + .../ozone/recon/scm/ReconPipelineManager.java | 4 +- .../spi/impl/OzoneManagerServiceProviderImpl.java | 4 +- .../impl/StorageContainerServiceProviderImpl.java | 4 +- .../recon/tasks/ContainerKeyMapperHelper.java | 296 +++++--- .../recon/tasks/ContainerKeyMapperTaskFSO.java | 12 +- .../recon/tasks/ContainerKeyMapperTaskOBS.java | 11 +- .../recon/tasks/DeletedKeysInsightHandler.java | 14 +- .../ozone/recon/tasks/FileSizeCountTaskFSO.java | 19 +- .../ozone/recon/tasks/FileSizeCountTaskHelper.java | 171 ++--- .../ozone/recon/tasks/FileSizeCountTaskOBS.java | 19 +- .../recon/tasks/MultipartInfoInsightHandler.java | 14 +- .../hadoop/ozone/recon/tasks/OmTableHandler.java | 14 +- .../ozone/recon/tasks/OmTableInsightTask.java | 129 +++- .../ozone/recon/tasks/OpenKeysInsightHandler.java | 13 +- .../ozone/recon/tasks/ReconTaskControllerImpl.java | 4 + .../tasks/util/ParallelTableIteratorOperation.java | 296 ++++++++ .../ozone/recon/tasks/util/package-info.java | 16 +- .../hadoop/ozone/recon/api/NSSummaryTests.java | 3 +- .../ozone/recon/api/TestContainerEndpoint.java | 44 +- .../recon/tasks/TestContainerKeyMapperTask.java | 5 + .../ozone/recon/tasks/TestFileSizeCountTask.java | 2 + .../recon/tasks/TestNSSummaryUnifiedControl.java | 805 ++++++++++++++------- .../ozone/recon/tasks/TestOmTableInsightTask.java | 27 + .../hadoop/ozone/repair/om/FSORepairTool.java | 160 ++-- pom.xml | 11 +- 209 files changed, 5525 insertions(+), 2155 deletions(-) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
