This is an automated email from the ASF dual-hosted git repository.
tanvipenumudy pushed a change to branch HDDS-13177
in repository https://gitbox.apache.org/repos/asf/ozone.git
from 23c04bd804 HDDS-13240. Add newly added metrics into grafana dashboard.
(#8656)
add c1e84d9061 HDDS-13324. Optimize memory footprint for Recon listKeys
API (#8680)
add 278bcdda94 HDDS-13314. Remove unused maven-pdf-plugin (#8686)
add fd97f743e2 HDDS-13295. Remove jackson1 exclusions for hadoop-common
(#8687)
add 84da9cee92 HDDS-13319. Simplify KeyPrefixFilter (#8692)
add 96390ac142 HDDS-12984. Use InodeID to identify the SST files inside
the tarball. (#8477)
add 29fc8def65 HDDS-12070. Bump Ratis to 3.2.0 (#8689)
add d0d09aacf0 HDDS-13251. Support dynamic Byteman scripts via bmsubmit in
ozonesecure-ha (#8654)
add 665574c663 HDDS-13251. Update Byteman usage README license (#8700)
add 99c163ad37 HDDS-12151. Fail write when volume is full considering min
free space (#8642)
add 7a237fe7fb HDDS-12468. Check for space availability for all dns while
container creation in pipeline (#8663)
add bb3d2872e9 HDDS-12890. Add acceptance robot tests for ozone debug
replicas verify (#8613)
add da4d1ddbf1 HDDS-13280. Smoketest Ozone with Ranger authorizer (#8635)
add 97a89f1147 HDDS-13347. Bump maven-core to 3.9.10 (#8711)
add 65088798ba HDDS-13348. Bump maven-deploy-plugin to 3.1.4 (#8712)
add 4fa3654029 HDDS-13349. Bump junit to 5.13.2 (#8714)
add 0f1b93e085 HDDS-13350. Bump commons-codec to 1.18.0 (#8715)
add 2aa71ac27a HDDS-13351. Bump commons-configuration2 to 2.12.0 (#8713)
add 351a6594a1 HDDS-12953. Directly create DataNodeSafeModeRule in
TestDataNodeSafeModeRule (#8716)
add 18845e0954 HDDS-13058. Fix dependency warnings in
ozone-filesystem-hadoop* (#8694)
add fa3b3684b1 HDDS-12957. Remove unused SafeModeStatus#forceExitSafeMode
field (#8717)
add f5019b2d2d HDDS-12945. Let Robot test check config whether security is
enabled (#8632)
add b1c27bad3f HDDS-13359. [Docs] Accessing Ozone using Cyberduck. (#8718)
add c48fc26940 HDDS-13355. Remove unused class OzoneManagerLockUtil (#8720)
add 9a6428569e HDDS-13221. Remove unused flushFuture from OMClientResponse
(#8723)
add d79ea9c47a HDDS-13281. Disable Ratis metadata write to Raft Log on OM
& SCM. (#8637)
add f2ddbf6a47 HDDS-13149. Add details on Performance Optimization to
Transparent Data Encryption doc (#8622)
add ba7b3cfcd9 HDDS-13330. DBStore and related classes should not throw
IOException (#8691)
add 5e72dd4e1e HDDS-13208. [Docs] Add volume management section under
Architecture/Datanodes. (#8585)
add b9d4e6585b HDDS-13371. [Docs] Protect in-transit traffic (#8729)
add 627bebe774 HDDS-13368. [Docs] Add user doc for Ozone Trash. (#8727)
add d80589388e HDDS-13269. [Docs] Multi-raft user doc. (#8641)
add 1ca2f2a2aa HDDS-13358. Refactor SafeModeStatus to an enum (#8721)
add d9daba0ee8 HDDS-12727. Document high level architecture of S3 Gateway
(#8184)
add e582a3908f HDDS-13366. Add unique key to ACL table rows. (#8725)
add 9e5f8371b8 HDDS-13260. Add smoke test for bucket owner condition
(#8702)
add d6d9253870 HDDS-13362. Abstract OM response class should not be
annotated with CleanupTableInfo (#8732)
add 380f31d425 HDDS-13341. Rename ScanResult.isHealthy() to hasErrors()
(#8746)
add 30db6fc211 HDDS-13390. Bump license-maven-plugin to 2.6.0 (#8750)
add 72ffc27a9d HDDS-13340. Fix minor items identified during
reconciliation branch merge process. (#8705)
add 550503c47a HDDS-13392. Bump awssdk to 2.31.77 (#8748)
add 65f0c09a46 HDDS-13381. Docs: Add user documentation for Volumes,
Buckets, and Keys (#8739)
add 07f0810423 HDDS-13389. Bump junit to 5.13.3 (#8749)
add afcdebdc8f HDDS-13391. Bump sqlite-jdbc to 3.50.2.0 (#8751)
add 0b7bb92fd3 HDDS-13394. Bump zstd-jni to 1.5.7-4 (#8747)
add f51bb1bed2 HDDS-13087. [snapshot] Expand user doc (#8496)
add 39def5d50c HDDS-13268. [Docs] Advanced topics: troubleshooting OM
snapshot installation. (#8719)
add 3a64135760 HDDS-13198. Avoid warning due to wrong
min.free.space.percent default value (#8745)
add f7270513a5 HDDS-12929. Datanode Should Immediately Trigger Container
Close when Volume Full (#8460)
add 5af3467a42 HDDS-13375. Add Ozone logo to the Github README.md (#8741)
No new revisions were added by this update.
Summary of changes:
README.md | 13 +
...s-no-flush.btm => skip-notify-group-remove.btm} | 15 +-
.../{ratis-no-flush.btm => skip-put-block.btm} | 15 +-
.../checksum/ContainerChecksumTreeManager.java | 7 +-
.../container/checksum/ContainerDiffReport.java | 41 +-
.../checksum/ContainerMerkleTreeMetrics.java | 33 ++
.../container/common/helpers/ContainerUtils.java | 2 +-
.../common/impl/BlockDeletingService.java | 5 +-
.../container/common/impl/HddsDispatcher.java | 36 +-
.../container/common/interfaces/DBHandle.java | 4 +-
.../container/common/interfaces/ScanResult.java | 2 +-
.../common/statemachine/DatanodeConfiguration.java | 2 +-
.../container/common/utils/DatanodeStoreCache.java | 14 +-
.../hadoop/ozone/container/common/utils/RawDB.java | 3 +-
.../container/common/utils/ReferenceCountedDB.java | 12 +-
.../ozone/container/common/volume/HddsVolume.java | 6 -
.../container/keyvalue/KeyValueContainerCheck.java | 2 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 9 +-
.../container/metadata/AbstractDatanodeStore.java | 20 +-
.../ozone/container/metadata/AbstractRDBStore.java | 26 +-
.../ozone/container/metadata/DBStoreManager.java | 16 +-
.../WitnessedContainerMetadataStoreImpl.java | 7 +-
.../BackgroundContainerMetadataScanner.java | 2 +-
.../container/ozoneimpl/ContainerScanHelper.java | 2 +-
.../ozone/container/ozoneimpl/DataScanResult.java | 2 +-
.../container/ozoneimpl/MetadataScanResult.java | 4 +-
.../checksum/ContainerMerkleTreeTestUtils.java | 113 +++---
.../checksum/TestContainerChecksumTreeManager.java | 77 ++--
.../common/BlockDeletingServiceTestImpl.java | 3 +-
.../container/common/TestBlockDeletingService.java | 4 +-
.../impl/TestContainerDeletionChoosingPolicy.java | 3 +-
.../container/common/impl/TestHddsDispatcher.java | 3 +-
.../statemachine/TestDatanodeConfiguration.java | 9 +
.../keyvalue/TestKeyValueContainerCheck.java | 30 +-
.../impl/TestKeyValueStreamDataChannel.java | 1 -
.../container/ozoneimpl/TestDataScanResult.java | 8 +-
.../ozoneimpl/TestMetadataScanResult.java | 6 +-
hadoop-hdds/docs/content/concept/Datanodes.md | 34 ++
hadoop-hdds/docs/content/concept/OzoneS3Gateway.md | 112 ++++++
.../docs/content/concept/VolumesBucketsKeys.md | 183 +++++++++
.../docs/content/design/full-volume-handling.md | 163 ++++++++
hadoop-hdds/docs/content/feature/OM-HA.md | 1 +
hadoop-hdds/docs/content/feature/Snapshot.md | 219 +++++++++--
hadoop-hdds/docs/content/feature/Trash.md | 74 ++++
.../docs/content/feature/multi-raft-support.md | 114 ++++++
.../docs/content/interface/CyberDuckOzoneS3.md | 122 ++++++
hadoop-hdds/docs/content/interface/S3.md | 1 +
hadoop-hdds/docs/content/security/SecuringTDE.md | 42 ++
.../content/security/protect-in-transit-traffic.md | 60 +++
.../_index.zh.md => troubleshooting/_index.md} | 11 +-
.../troubleshooting/om-ha-snapshot-installation.md | 39 ++
.../org/apache/hadoop/hdds/utils/Archiver.java | 43 ++
.../hadoop/hdds/utils/DBCheckpointServlet.java | 12 +-
.../hadoop/hdds/utils/MetadataKeyFilters.java | 123 ++----
.../hdds/utils/db/DBColumnFamilyDefinition.java | 6 +-
.../org/apache/hadoop/hdds/utils/db/DBStore.java | 86 +---
.../hadoop/hdds/utils/db/DBStoreBuilder.java | 19 +-
.../org/apache/hadoop/hdds/utils/db/RDBStore.java | 92 ++---
.../org/apache/hadoop/hdds/utils/TestArchiver.java | 89 +++++
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 44 +--
.../hdds/utils/db/RocksDatabaseException.java | 10 +-
.../hdds/scm/container/ContainerManager.java | 5 +-
.../hdds/scm/container/ContainerManagerImpl.java | 22 +-
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 6 +
.../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 18 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 4 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 3 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 10 +
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 9 +
.../hdds/scm/pipeline/PipelineManagerImpl.java | 16 +
.../scm/pipeline/WritableECContainerProvider.java | 6 +
.../hdds/scm/safemode/SCMSafeModeManager.java | 91 ++---
.../apache/hadoop/hdds/scm/server/SCMMXBean.java | 7 -
.../hdds/scm/server/StorageContainerManager.java | 8 +-
.../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 14 +
.../hadoop/hdds/scm/block/TestBlockManager.java | 5 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 16 +
.../hdds/scm/container/SimpleMockNodeManager.java | 7 +
.../scm/container/TestContainerManagerImpl.java | 63 ++-
.../hdds/scm/ha/TestBackgroundSCMService.java | 5 +-
.../apache/hadoop/hdds/scm/ha/TestSCMContext.java | 7 +-
.../hadoop/hdds/scm/ha/TestSCMServiceManager.java | 20 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 6 +-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 6 +-
.../hdds/scm/pipeline/MockPipelineManager.java | 5 +
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 90 ++++-
.../scm/safemode/TestDataNodeSafeModeRule.java | 23 +-
.../hadoop/ozone/om/lock/OzoneManagerLockUtil.java | 96 -----
hadoop-ozone/dist/pom.xml | 2 +-
.../dist/src/main/compose/common/hadoop-test.sh | 2 +-
.../dist/src/main/compose/common/init-kdc.sh | 5 +
.../dist/src/main/compose/common/ranger.yaml | 4 +
.../dist/src/main/compose/common/replicas-test.sh | 64 +--
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 1 -
.../dist/src/main/compose/ozonesecure-ha/.env | 2 +
.../{s3g-virtual-host.yaml => byteman.yaml} | 38 +-
.../src/main/compose/ozonesecure-ha/docker-config | 2 +
.../src/main/compose/ozonesecure-ha/ranger.yaml | 13 +-
.../{test-s3g-virtual-host.sh => test-byteman.sh} | 6 +-
.../src/main/compose/ozonesecure-ha/test-ranger.sh | 4 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 1 -
.../dist/src/main/smoketest/__init__.robot | 2 +-
.../dist/src/main/smoketest/admincli/cert.robot | 1 +
.../src/main/smoketest/admincli/container.robot | 2 +-
.../src/main/smoketest/admincli/datanode.robot | 2 +-
.../src/main/smoketest/admincli/pipeline.robot | 2 +-
.../src/main/smoketest/admincli/reconfigure.robot | 3 +-
.../smoketest/admincli/replicationmanager.robot | 2 +-
.../src/main/smoketest/balancer/testBalancer.robot | 3 +-
.../dist/src/main/smoketest/basic/basic.robot | 1 +
.../dist/src/main/smoketest/basic/links.robot | 1 +
.../src/main/smoketest/basic/ozone-shell-lib.robot | 1 +
.../src/main/smoketest/basic/ozone-shell.robot | 3 +-
.../src/main/smoketest/basic/single_node.robot | 1 +
.../smoketest/certrotation/cert-rotation.robot | 2 +-
.../dist/src/main/smoketest/commonlib.robot | 11 +-
.../src/main/smoketest/debug/auditparser.robot | 1 +
...-datanode.robot => block-existence-check.robot} | 17 +-
...datanode.robot => corrupt-block-checksum.robot} | 26 +-
.../debug/ozone-debug-corrupt-block.robot | 46 ---
.../smoketest/debug/ozone-debug-keywords.robot | 63 +++
.../src/main/smoketest/debug/ozone-debug-ldb.robot | 2 +-
.../debug/ozone-debug-stale-datanode.robot | 46 ---
.../smoketest/debug/ozone-debug-tests-ec3-2.robot | 49 +--
.../smoketest/debug/ozone-debug-tests-ec6-3.robot | 100 -----
.../main/smoketest/debug/ozone-debug-tests.robot | 8 +-
.../src/main/smoketest/debug/ozone-debug.robot | 98 -----
...atanode.robot => stale-datanode-checksum.robot} | 25 +-
.../dist/src/main/smoketest/ec/lib.resource | 1 +
.../dist/src/main/smoketest/ec/rewrite.robot | 2 +-
.../src/main/smoketest/ec/upgrade-ec-check.robot | 2 +-
.../dist/src/main/smoketest/fcq/s3_om_fcq.robot | 1 +
.../src/main/smoketest/freon/generate-chunk.robot | 2 +
.../src/main/smoketest/freon/validate-chunk.robot | 3 +
.../main/smoketest/grpc/grpc-om-s3-metrics.robot | 2 +-
.../main/smoketest/hsync/upgrade-hsync-check.robot | 2 +-
.../main/smoketest/httpfs/operations_tests.robot | 4 +-
.../dist/src/main/smoketest/lib/BytemanLibrary.py | 77 ++++
.../omha/data-creation-before-om-bootstrap.robot | 1 +
.../omha/data-validation-after-om-bootstrap.robot | 1 +
.../main/smoketest/omha/om-cancel-prepare.robot | 3 +-
.../src/main/smoketest/omha/om-fetch-key.robot | 2 +-
.../dist/src/main/smoketest/omha/om-prepare.robot | 3 +-
.../dist/src/main/smoketest/omha/om-prepared.robot | 1 +
.../dist/src/main/smoketest/omha/om-roles.robot | 3 +-
.../main/smoketest/ozone-fi/BytemanKeywords.robot | 119 ++++++
.../dist/src/main/smoketest/ozone-fi/README.md | 211 ++++++++++
.../smoketest/ozone-fi/byteman_faults_sample.robot | 60 +++
.../dist/src/main/smoketest/ozone-lib/freon.robot | 1 -
.../dist/src/main/smoketest/recon/recon-api.robot | 1 +
.../src/main/smoketest/recon/recon-nssummary.robot | 1 +
.../main/smoketest/recon/recon-taskstatus.robot | 1 +
.../src/main/smoketest/s3/MultipartUpload.robot | 32 ++
.../dist/src/main/smoketest/s3/bucketcreate.robot | 8 +
.../dist/src/main/smoketest/s3/bucketdelete.robot | 7 +
.../dist/src/main/smoketest/s3/buckethead.robot | 6 +
.../dist/src/main/smoketest/s3/commonawslib.robot | 30 ++
.../dist/src/main/smoketest/s3/objectcopy.robot | 9 +
.../dist/src/main/smoketest/s3/objectdelete.robot | 18 +
.../dist/src/main/smoketest/s3/objecthead.robot | 10 +
.../dist/src/main/smoketest/s3/objectlist.robot | 6 +
.../dist/src/main/smoketest/s3/objectputget.robot | 15 +-
.../dist/src/main/smoketest/s3/objecttagging.robot | 15 +
.../dist/src/main/smoketest/s3/s3_getsecret.robot | 2 +-
.../src/main/smoketest/s3/secretgenerate.robot | 4 +-
.../dist/src/main/smoketest/s3/secretrevoke.robot | 4 +-
.../main/smoketest/scmha/scm-decommission.robot | 4 +-
.../smoketest/security/ozone-secure-token.robot | 1 +
.../main/smoketest/snapshot/snapshot-acls.robot | 3 +-
.../snapshot/upgrade-snapshot-check.robot | 2 +-
.../smoketest/upgrade/check-finalization.robot | 1 +
.../dist/src/main/smoketest/upgrade/finalize.robot | 1 +
.../dist/src/main/smoketest/upgrade/generate.robot | 2 +-
.../dist/src/main/smoketest/upgrade/prepare.robot | 1 +
.../dist/src/main/smoketest/upgrade/validate.robot | 2 +-
.../hdds/scm/TestSCMDbCheckpointServlet.java | 2 +
...estStorageContainerManagerHAWithAllRunning.java | 2 +-
.../hadoop/hdds/upgrade/TestScmHAFinalization.java | 87 ++--
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 2 +
.../TestOMDbCheckpointServletInodeBasedXfer.java | 368 +++++++++++++++++
.../hadoop/ozone/shell/TestOzoneTenantShell.java | 2 +-
.../om/OMDBCheckpointServletInodeBasedXfer.java | 436 +++++++++++++++++++++
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 19 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 5 +
.../ozone/om/request/upgrade/OMPrepareRequest.java | 27 +-
.../hadoop/ozone/om/response/OMClientResponse.java | 11 -
.../response/key/AbstractOMKeyDeleteResponse.java | 4 -
.../AbstractS3MultipartAbortResponse.java | 9 -
.../S3MultipartUploadAbortResponseWithFSO.java | 3 +-
.../hadoop/ozone/om/snapshot/OmSnapshotUtils.java | 21 +
.../ozone/om/response/TestCleanupTableInfo.java | 20 +-
.../ozone/om/snapshot/TestSnapshotCache.java | 3 +-
hadoop-ozone/ozonefs-hadoop2/pom.xml | 51 +--
hadoop-ozone/ozonefs-hadoop3-client/pom.xml | 25 ++
hadoop-ozone/ozonefs-hadoop3/pom.xml | 46 ++-
.../org/apache/hadoop/ozone/recon/ReconUtils.java | 24 +-
.../scm/ReconStorageContainerManagerFacade.java | 3 +-
.../src/components/aclDrawer/aclDrawer.tsx | 5 +-
.../src/v2/components/aclDrawer/aclDrawer.tsx | 2 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 6 +-
pom.xml | 58 +--
201 files changed, 3941 insertions(+), 1473 deletions(-)
copy dev-support/byteman/{ratis-no-flush.btm => skip-notify-group-remove.btm}
(72%)
copy dev-support/byteman/{ratis-no-flush.btm => skip-put-block.btm} (75%)
create mode 100644 hadoop-hdds/docs/content/concept/OzoneS3Gateway.md
create mode 100644 hadoop-hdds/docs/content/concept/VolumesBucketsKeys.md
create mode 100644 hadoop-hdds/docs/content/design/full-volume-handling.md
create mode 100644 hadoop-hdds/docs/content/feature/Trash.md
create mode 100644 hadoop-hdds/docs/content/feature/multi-raft-support.md
create mode 100644 hadoop-hdds/docs/content/interface/CyberDuckOzoneS3.md
create mode 100644
hadoop-hdds/docs/content/security/protect-in-transit-traffic.md
copy hadoop-hdds/docs/content/{feature/_index.zh.md =>
troubleshooting/_index.md} (80%)
create mode 100644
hadoop-hdds/docs/content/troubleshooting/om-ha-snapshot-installation.md
delete mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/lock/OzoneManagerLockUtil.java
copy hadoop-ozone/dist/src/main/compose/ozonesecure-ha/{s3g-virtual-host.yaml
=> byteman.yaml} (63%)
copy
hadoop-ozone/dist/src/main/compose/ozonesecure-ha/{test-s3g-virtual-host.sh =>
test-byteman.sh} (88%)
mode change 100755 => 100644
copy
hadoop-ozone/dist/src/main/smoketest/debug/{ozone-debug-dead-datanode.robot =>
block-existence-check.robot} (69%)
copy
hadoop-ozone/dist/src/main/smoketest/debug/{ozone-debug-dead-datanode.robot =>
corrupt-block-checksum.robot} (60%)
delete mode 100644
hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-corrupt-block.robot
create mode 100644
hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-keywords.robot
delete mode 100644
hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-stale-datanode.robot
delete mode 100644
hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug-tests-ec6-3.robot
delete mode 100644 hadoop-ozone/dist/src/main/smoketest/debug/ozone-debug.robot
rename
hadoop-ozone/dist/src/main/smoketest/debug/{ozone-debug-dead-datanode.robot =>
stale-datanode-checksum.robot} (61%)
create mode 100644 hadoop-ozone/dist/src/main/smoketest/lib/BytemanLibrary.py
create mode 100644
hadoop-ozone/dist/src/main/smoketest/ozone-fi/BytemanKeywords.robot
create mode 100644 hadoop-ozone/dist/src/main/smoketest/ozone-fi/README.md
create mode 100644
hadoop-ozone/dist/src/main/smoketest/ozone-fi/byteman_faults_sample.robot
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServletInodeBasedXfer.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
