This is an automated email from the ASF dual-hosted git repository.
aswinshakil pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
from c34f4c1ce8a HDDS-13322. Remove module auto-detection from
flaky-test-check (#8679)
add a4bb94ee639 HDDS-10239. Storage Container Reconciliation. (#6121)
add 930d3f0552b HDDS-10372. SCM and Datanode communication for
reconciliation (#6506)
add 5bde01db4c1 HDDS-10923. Container Scanner should still scan unhealthy
containers. (#6809)
add d585363a7ab HDDS-10887. Implement a basic Merkle Tree Manager. (#6778)
add 8676000a6f6 Merge branch 'master' into
HDDS-10239-container-reconciliation
add bde10ab8369 HDDS-10373. Implement framework for capturing Merkle Tree
Metrics. (#6864)
add d7f302e25f0 HDDS-10926. Block deletion should update container merkle
tree. (#6875)
add ab35173e249 HDDS-10376. Add a Datanode API to supply a merkle tree for
a given container. (#6945)
add 98369a8343f Merge branch 'master' into
HDDS-10239-container-reconciliation
add 0baba47daad HDDS-11254. Reconcile commands should be handled by
datanode ReplicationSupervisor (#7076)
add 3355701853c Merge branch 'master' into
HDDS-10239-container-reconciliation
add 5cd6a072b20 Merge branch 'master' into
HDDS-10239-container-reconciliation
add c6860db56ce HDDS-10379. Datanodes should generate initial container
merkle tree during container close. (#7065)
add 9765a6a1347 HDDS-11253. Handle corrupted merkle tree files (#7083)
add 01b44374345 Merge branch 'master' into
HDDS-10239-container-reconciliation
add 04c196c9f40 HDDS-10377. Allow datanodes to do chunk level
modifications to closed containers. (#7111)
add 445eaf1f4a8 HDDS-11290. Container scanner should keep scanning after
non-fatal errors (#7127)
add 986e233599b HDDS-11471. Add new tests for container scanner detecting
multiple errors in one container (#7396)
add d17c41c0a97 HDDS-10928. Implement container comparison logic within
datanodes. (#7293)
add 7bcc9430770 Revert "Merge branch 'master' into
HDDS-10239-container-reconciliation"
add cfcabf14223 Merge branch 'master' of https://github.com/apache/ozone
into HDDS-10239-container-reconciliation-merge
add 2cfd22a407c Merge branch 'master' into
HDDS-10239-container-reconciliation
add 31875f39ae0 HDDS-12079. Improve container merkle tree write handling
(#7694)
add fa8e0a422ce HDDS-11345. Add metrics specific tests for reconciliation
tasks. (#7732)
add 1aef378d70f Merge branch 'master' of https://github.com/apache/ozone
into HDDS-10239-container-reconciliation-merge
add 8fc98b6b340 Merge branch 'master' into
HDDS-10239-container-reconciliation
add be53ed7115e HDDS-12397. Persist putBlock for closed container. (#7943)
add f47b4076464 HDDS-11887. Recon - Identify container replicas difference
based on content checksums (#7942)
add b34c537940b HDDS-11763. Implement container repair logic within
datanodes. (#7474)
add 8768d048160 HDDS-12745. Container checksum should be reported during
container close and DN restart (#8204)
add 989746a0de3 Merge branch 'master' of https://github.com/apache/ozone
into HDDS-10239-container-reconciliation
add 7c591b41984 Merge branch 'master' of https://github.com/apache/ozone
into HDDS-10239-container-reconciliation
add 2b4708b70e5 Merge branch 'master' of https://github.com/apache/ozone
into HDDS-10239-container-reconciliation
add 229e5c94eaf HDDS-12849. On demand container scanner should not be
static (#8296)
add a355664093c HDDS-12980. Add unit test framework for reconciliation.
(#8402)
add 47c1eaa1eb9 Merge branch 'HDDS-10239-container-reconciliation' of
https://github.com/apache/ozone into HDDS-10239-container-reconciliation
add 9a445ed01eb HDDS-10374. Make container scanner generate merkle trees
during the scan (#7490)
add 0a53c73de21 Merge branch 'master' of https://github.com/apache/ozone
into HDDS-10239-container-reconciliation
add 3c4b3bec36e Merge branch 'master' of https://github.com/apache/ozone
into HDDS-10239-container-reconciliation
add e60f22745cc Merge branch 'master' of https://github.com/apache/ozone
into HDDS-10239-container-reconciliation
add 38673465d1a HDDS-13288. Container checksum file proto changes to
account for deleted blocks. (#8649)
add fa02d7d3e6a Merge branch 'master' of https://github.com/apache/ozone
into HDDS-10239-container-reconciliation
No new revisions were added by this update.
Summary of changes:
.../hadoop/hdds/scm/storage/BlockInputStream.java | 11 +-
.../hadoop/hdds/scm/storage/ChunkInputStream.java | 4 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 6 +
.../hadoop/hdds/client/ReplicationConfig.java | 1 +
.../apache/hadoop/hdds/scm/client/ScmClient.java | 8 +
.../hdds/scm/container/ContainerReplicaInfo.java | 28 +-
.../hadoop/hdds/scm/exceptions/SCMException.java | 3 +-
.../apache/hadoop/hdds/scm/net/NetConstants.java | 2 +-
.../protocol/StorageContainerLocationProtocol.java | 8 +
.../ContainerCommandResponseBuilders.java | 11 +
.../hdds/scm/storage/ContainerProtocolCalls.java | 39 ++
.../org/apache/hadoop/ozone/audit/DNAction.java | 3 +-
.../checksum/ContainerChecksumTreeManager.java | 449 ++++++++++++++
.../container/checksum/ContainerDiffReport.java | 84 +++
.../checksum/ContainerMerkleTreeMetrics.java | 122 ++++
.../checksum/ContainerMerkleTreeWriter.java | 245 ++++++++
.../checksum/DNContainerOperationClient.java | 134 +++++
.../container/checksum/ReconcileContainerTask.java | 96 +++
.../ozone/container/checksum}/package-info.java | 4 +-
.../container/common/helpers/ContainerMetrics.java | 20 +-
.../container/common/helpers/ContainerUtils.java | 37 +-
.../helpers}/TokenHelper.java | 14 +-
.../common/impl/BlockDeletingService.java | 19 +-
.../ozone/container/common/impl/ContainerData.java | 39 +-
.../container/common/impl/ContainerDataYaml.java | 4 +-
.../ozone/container/common/impl/ContainerSet.java | 42 ++
.../container/common/impl/HddsDispatcher.java | 16 +-
.../container/common/interfaces/Container.java | 91 +--
.../ozone/container/common/interfaces/Handler.java | 31 +-
.../{package-info.java => ScanResult.java} | 15 +-
.../common/statemachine/DatanodeConfiguration.java | 76 ++-
.../common/statemachine/DatanodeStateMachine.java | 7 +
....java => ReconcileContainerCommandHandler.java} | 62 +-
.../states/endpoint/HeartbeatEndpointTask.java | 6 +
.../container/common/utils/ContainerLogger.java | 41 +-
.../ECReconstructionCoordinator.java | 3 +-
.../container/keyvalue/KeyValueContainer.java | 35 +-
.../container/keyvalue/KeyValueContainerCheck.java | 534 ++++++++--------
.../container/keyvalue/KeyValueContainerData.java | 2 +
.../ozone/container/keyvalue/KeyValueHandler.java | 660 +++++++++++++++++++-
.../container/keyvalue/TarContainerPacker.java | 2 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 23 +-
.../container/keyvalue/impl/BlockManagerImpl.java | 85 +++
.../keyvalue/impl/FilePerBlockStrategy.java | 26 +-
.../keyvalue/interfaces/BlockManager.java | 24 +
.../statemachine/background/BlockDeletingTask.java | 45 +-
.../ozoneimpl/BackgroundContainerDataScanner.java | 60 +-
.../BackgroundContainerMetadataScanner.java | 37 +-
.../container/ozoneimpl/ContainerController.java | 48 +-
.../container/ozoneimpl/ContainerScanError.java | 71 +++
.../container/ozoneimpl/ContainerScanHelper.java | 164 +++++
.../ozoneimpl/ContainerScannerConfiguration.java | 4 +
.../ozone/container/ozoneimpl/DataScanResult.java | 69 +++
.../container/ozoneimpl/MetadataScanResult.java | 90 +++
.../ozoneimpl/OnDemandContainerDataScanner.java | 150 +----
.../ozone/container/ozoneimpl/OzoneContainer.java | 17 +-
.../container/replication/ContainerImporter.java | 2 +-
.../replication/ReplicationSupervisor.java | 4 +-
.../commands/ReconcileContainerCommand.java | 105 ++++
.../checksum/ContainerMerkleTreeTestUtils.java | 356 +++++++++++
.../checksum/TestContainerChecksumTreeManager.java | 562 +++++++++++++++++
.../checksum/TestContainerMerkleTreeWriter.java | 317 ++++++++++
.../checksum/TestReconcileContainerTask.java | 124 ++++
.../ozone/container/common/ContainerTestUtils.java | 129 +++-
.../container/common/TestBlockDeletingService.java | 112 +++-
.../common/TestKeyValueContainerData.java | 4 +
.../TestSchemaOneBackwardsCompatibility.java | 13 +-
.../TestSchemaTwoBackwardsCompatibility.java | 4 +-
.../common/impl/TestContainerDataYaml.java | 34 +-
.../common/impl/TestContainerPersistence.java | 3 +-
.../container/common/impl/TestContainerSet.java | 57 ++
.../container/common/impl/TestHddsDispatcher.java | 15 +-
.../container/common/interfaces/TestHandler.java | 4 +-
.../common/statemachine/TestStateContext.java | 4 +
.../TestReconcileContainerCommandHandler.java | 188 ++++++
.../states/endpoint/TestHeartbeatEndpointTask.java | 40 ++
.../keyvalue/TestContainerCorruptions.java | 233 +++++++
...stContainerReconciliationWithMockDatanodes.java | 623 +++++++++++++++++++
.../container/keyvalue/TestKeyValueContainer.java | 4 +-
.../keyvalue/TestKeyValueContainerCheck.java | 309 +++++++---
.../container/keyvalue/TestKeyValueHandler.java | 252 +++++++-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 57 +-
.../keyvalue/impl/TestBlockManagerImpl.java | 75 ++-
.../keyvalue/impl/TestFilePerBlockStrategy.java | 204 +++++++
.../TestBackgroundContainerDataScanner.java | 73 ++-
.../TestBackgroundContainerMetadataScanner.java | 55 +-
.../ozoneimpl/TestContainerScannersAbstract.java | 33 +-
.../container/ozoneimpl/TestDataScanResult.java | 83 +++
.../ozoneimpl/TestMetadataScanResult.java | 58 ++
.../TestOnDemandContainerDataScanner.java | 157 +++--
.../replication/TestContainerImporter.java | 4 +-
.../replication/TestGrpcReplicationService.java | 8 +-
.../replication/TestReplicationSupervisor.java | 68 +++
.../content/design/container-reconciliation.md | 344 +++++++++++
...inerLocationProtocolClientSideTranslatorPB.java | 10 +
.../src/main/proto/ScmAdminProtocol.proto | 10 +
.../src/main/proto/DatanodeClientProtocol.proto | 41 +-
.../interface-client/src/main/proto/hdds.proto | 2 +
.../proto/ScmServerDatanodeHeartbeatProtocol.proto | 14 +-
.../src/main/proto/ScmServerProtocol.proto | 1 +
.../container/AbstractContainerReportHandler.java | 1 +
.../hdds/scm/container/ContainerReplica.java | 13 +
.../ReconcileContainerEventHandler.java | 92 +++
.../ReconciliationEligibilityHandler.java | 139 +++++
.../container/reconciliation}/package-info.java | 5 +-
.../apache/hadoop/hdds/scm/events/SCMEvents.java | 7 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 1 -
...inerLocationProtocolServerSideTranslatorPB.java | 13 +
.../hdds/scm/server/SCMClientProtocolServer.java | 44 +-
.../hdds/scm/server/SCMDatanodeProtocolServer.java | 8 +
.../hdds/scm/server/StorageContainerManager.java | 5 +
.../org/apache/hadoop/ozone/audit/SCMAction.java | 3 +-
.../scm/container/TestContainerReportHandler.java | 151 ++++-
.../TestIncrementalContainerReportHandler.java | 138 ++++-
.../TestReconcileContainerEventHandler.java | 312 ++++++++++
.../hdds/scm/cli/ContainerOperationClient.java | 4 +
.../hdds/scm/cli/container/ContainerCommands.java | 1 +
...oseSubcommand.java => ReconcileSubcommand.java} | 19 +-
.../src/main/smoketest/admincli/container.robot | 50 +-
.../ozone/freon/ClosedContainerReplicator.java | 7 +-
.../apache/hadoop/hdds/scm/TestCloseContainer.java | 134 ++++-
.../ozone/client/rpc/TestECKeyOutputStream.java | 4 +-
.../apache/hadoop/ozone/container/TestHelper.java | 14 +
.../container/metrics/TestContainerMetrics.java | 3 +-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 75 +++
.../container/server/TestContainerServer.java | 4 +-
.../server/TestSecureContainerServer.java | 3 +-
.../TestContainerCommandReconciliation.java | 670 +++++++++++++++++++++
...tBackgroundContainerDataScannerIntegration.java | 42 +-
...kgroundContainerMetadataScannerIntegration.java | 47 +-
.../TestContainerScannerIntegrationAbstract.java | 207 +------
...estOnDemandContainerDataScannerIntegration.java | 50 +-
.../recon/schema/ContainerSchemaDefinition.java | 3 +-
.../api/types/UnhealthyContainersResponse.java | 13 +
.../ozone/recon/fsck/ContainerHealthStatus.java | 15 +-
.../ozone/recon/fsck/ContainerHealthTask.java | 34 +-
.../ozone/recon/persistence/ContainerHistory.java | 13 +-
.../ozone/recon/scm/ContainerReplicaHistory.java | 16 +-
.../ozone/recon/scm/ReconContainerManager.java | 18 +-
.../ozone/recon/upgrade/ReconLayoutFeature.java | 3 +-
...> UnhealthyContainerReplicaMismatchAction.java} | 34 +-
.../src/__tests__/datanodes/Datanodes.test.tsx | 19 +-
.../__tests__/datanodes/DatanodesTable.test.tsx | 15 +-
.../mocks/pipelineMocks/pipelinesServer.ts | 4 +-
.../src/__tests__/pipelines/Pipelines.test.tsx | 19 +-
.../__tests__/pipelines/PipelinesTable.test.tsx | 8 +-
.../src/__tests__/utils/pipelines.utils.tsx | 4 +-
.../src/components/navBar/navBar.tsx | 8 +-
.../src/v2/components/duMetadata/duMetadata.tsx | 12 +-
.../src/v2/components/navBar/navBar.tsx | 12 +-
.../src/v2/components/tables/containersTable.tsx | 28 +-
.../src/v2/components/tables/pipelinesTable.tsx | 17 +-
.../src/v2/pages/diskUsage/diskUsage.tsx | 22 +-
.../src/views/diskUsage/diskUsage.tsx | 29 +-
.../ozone-recon-web/src/views/insights/om/om.tsx | 20 +-
.../views/missingContainers/missingContainers.tsx | 12 +-
.../ozone/recon/api/TestContainerEndpoint.java | 74 ++-
.../recon/fsck/TestContainerHealthStatus.java | 51 ++
.../ozone/recon/fsck/TestContainerHealthTask.java | 42 +-
.../TestContainerHealthTaskRecordGenerator.java | 51 ++
.../ozone/recon/scm/TestReconContainerManager.java | 6 +-
.../datanode/container/ContainerCommands.java | 5 +-
162 files changed, 10068 insertions(+), 1527 deletions(-)
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerChecksumTreeManager.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerDiffReport.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerMerkleTreeMetrics.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ContainerMerkleTreeWriter.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/DNContainerOperationClient.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/checksum/ReconcileContainerTask.java
copy hadoop-hdds/{client/src/main/java/org/apache/hadoop/hdds/scm/client =>
container-service/src/main/java/org/apache/hadoop/ozone/container/checksum}/package-info.java
(87%)
rename
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/{ec/reconstruction
=> common/helpers}/TokenHelper.java (88%)
copy
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/{package-info.java
=> ScanResult.java} (76%)
copy
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/{ReconstructECContainersCommandHandler.java
=> ReconcileContainerCommandHandler.java} (56%)
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanError.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/DataScanResult.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/MetadataScanResult.java
create mode 100644
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReconcileContainerCommand.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/ContainerMerkleTreeTestUtils.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/TestContainerChecksumTreeManager.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/TestContainerMerkleTreeWriter.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/checksum/TestReconcileContainerTask.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestReconcileContainerCommandHandler.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestContainerCorruptions.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestContainerReconciliationWithMockDatanodes.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestDataScanResult.java
create mode 100644
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestMetadataScanResult.java
create mode 100644 hadoop-hdds/docs/content/design/container-reconciliation.md
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/reconciliation/ReconcileContainerEventHandler.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/reconciliation/ReconciliationEligibilityHandler.java
copy
{hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/execution
=>
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/reconciliation}/package-info.java
(86%)
create mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/reconciliation/TestReconcileContainerEventHandler.java
copy
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/container/{CloseSubcommand.java
=> ReconcileSubcommand.java} (66%)
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/checksum/TestContainerCommandReconciliation.java
copy
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/upgrade/{InitialConstraintUpgradeAction.java
=> UnhealthyContainerReplicaMismatchAction.java} (77%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
