This is an automated email from the ASF dual-hosted git repository.
sammichen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 9c932b01221 HDDS-13664. Persist pendingDeleteBytes only when feature
is finalized. (#9330)
9c932b01221 is described below
commit 9c932b01221300510eadd746ebad9e95cc523b2f
Author: Priyesh Karatha <[email protected]>
AuthorDate: Thu Nov 20 18:30:51 2025 +0530
HDDS-13664. Persist pendingDeleteBytes only when feature is finalized.
(#9330)
---
.../commandhandler/DeleteBlocksCommandHandler.java | 23 +-
.../container/keyvalue/KeyValueContainerData.java | 12 +-
.../KeyValueContainerMetadataInspector.java | 54 ++--
.../ozone/container/keyvalue/PendingDelete.java | 2 +-
.../keyvalue/helpers/KeyValueContainerUtil.java | 143 +++++----
.../TestDNDataDistributionFinalization.java | 326 +++++++++++++++++++++
6 files changed, 467 insertions(+), 93 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
index 6d59e812f93..78a8db03c6b 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
@@ -45,6 +45,7 @@
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto;
import
org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.metrics2.lib.MetricsRegistry;
@@ -64,6 +65,7 @@
import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.metadata.DeleteTransactionStore;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
+import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
import org.apache.hadoop.ozone.protocol.commands.CommandStatus;
import org.apache.hadoop.ozone.protocol.commands.DeleteBlockCommandStatus;
import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
@@ -643,14 +645,19 @@ private void updateMetaData(KeyValueContainerData
containerData,
containerData.getPendingDeleteBlockCountKey(),
pendingDeleteBlocks);
- // update pending deletion blocks count and delete transaction ID in
- // in-memory container status
- long pendingBytes = containerData.getBlockPendingDeletionBytes() +
delTX.getTotalBlockSize();
- metadataTable
- .putWithBatch(batchOperation,
- containerData.getPendingDeleteBlockBytesKey(),
- pendingBytes);
- containerData.incrPendingDeletionBlocks(newDeletionBlocks,
delTX.getTotalBlockSize());
+ // Update pending deletion blocks count, blocks bytes and delete
transaction ID in in-memory container status.
+ // Persist pending bytes only if the feature is finalized.
+ if (VersionedDatanodeFeatures.isFinalized(
+ HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION) &&
delTX.hasTotalBlockSize()) {
+ long pendingBytes = containerData.getBlockPendingDeletionBytes();
+ pendingBytes += delTX.getTotalBlockSize();
+ metadataTable
+ .putWithBatch(batchOperation,
+ containerData.getPendingDeleteBlockBytesKey(),
+ pendingBytes);
+ }
+ containerData.incrPendingDeletionBlocks(newDeletionBlocks,
+ delTX.hasTotalBlockSize() ? delTX.getTotalBlockSize() : 0);
containerData.updateDeleteTransactionId(delTX.getTxID());
}
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
index 800424076f9..e80655e0248 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
@@ -51,6 +51,7 @@
import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import
org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.hdds.utils.db.Table;
@@ -58,6 +59,7 @@
import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
import org.apache.hadoop.ozone.container.common.interfaces.DBHandle;
import
org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil;
+import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
import org.yaml.snakeyaml.nodes.Tag;
/**
@@ -385,8 +387,10 @@ public void updateAndCommitDBCounters(DBHandle db,
metadataTable.putWithBatch(batchOperation, getBlockCountKey(),
b.getCount() - deletedBlockCount);
metadataTable.putWithBatch(batchOperation, getPendingDeleteBlockCountKey(),
b.getPendingDeletion() - deletedBlockCount);
- metadataTable.putWithBatch(batchOperation, getPendingDeleteBlockBytesKey(),
- b.getPendingDeletionBytes() - releasedBytes);
+ if
(VersionedDatanodeFeatures.isFinalized(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION))
{
+ metadataTable.putWithBatch(batchOperation,
getPendingDeleteBlockBytesKey(),
+ b.getPendingDeletionBytes() - releasedBytes);
+ }
db.getStore().getBatchHandler().commitBatchOperation(batchOperation);
}
@@ -397,7 +401,9 @@ public void resetPendingDeleteBlockCount(DBHandle db)
throws IOException {
// Reset the metadata on disk.
Table<String, Long> metadataTable = db.getStore().getMetadataTable();
metadataTable.put(getPendingDeleteBlockCountKey(), 0L);
- metadataTable.put(getPendingDeleteBlockBytesKey(), 0L);
+ if
(VersionedDatanodeFeatures.isFinalized(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION))
{
+ metadataTable.put(getPendingDeleteBlockBytesKey(), 0L);
+ }
}
// NOTE: Below are some helper functions to format keys according
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java
index bf3c3909f1c..08e6b40039a 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java
@@ -34,6 +34,7 @@
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
import org.apache.hadoop.hdds.server.JsonUtils;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -45,6 +46,7 @@
import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaThreeImpl;
import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaTwoImpl;
import
org.apache.hadoop.ozone.container.metadata.DatanodeStoreWithIncrementalChunkList;
+import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -239,8 +241,10 @@ static ObjectNode getDBMetadataJson(Table<String, Long>
metadataTable,
metadataTable.get(containerData.getBytesUsedKey()));
dBMetadata.put(OzoneConsts.PENDING_DELETE_BLOCK_COUNT,
metadataTable.get(containerData.getPendingDeleteBlockCountKey()));
- dBMetadata.put(OzoneConsts.PENDING_DELETE_BLOCK_BYTES,
- metadataTable.get(containerData.getPendingDeleteBlockBytesKey()));
+ if (metadataTable.get(containerData.getPendingDeleteBlockBytesKey()) !=
null) {
+ dBMetadata.put(OzoneConsts.PENDING_DELETE_BLOCK_BYTES,
+ metadataTable.get(containerData.getPendingDeleteBlockBytesKey()));
+ }
dBMetadata.put(OzoneConsts.DELETE_TRANSACTION_KEY,
metadataTable.get(containerData.getLatestDeleteTxnKey()));
dBMetadata.put(OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID,
@@ -434,28 +438,30 @@ private boolean checkAndRepair(ObjectNode parent,
errors.add(deleteCountError);
}
- // check and repair if db delete bytes mismatches delete transaction
- JsonNode pendingDeletionBlockSize = dBMetadata.path(
- OzoneConsts.PENDING_DELETE_BLOCK_BYTES);
- final long dbDeleteBytes = jsonToLong(pendingDeletionBlockSize);
- final JsonNode pendingDeleteBytesAggregate =
aggregates.path(PendingDelete.BYTES);
- final long deleteTransactionBytes =
jsonToLong(pendingDeleteBytesAggregate);
- if (dbDeleteBytes != deleteTransactionBytes) {
- passed = false;
- final BooleanSupplier deleteBytesRepairAction = () -> {
- final String key = containerData.getPendingDeleteBlockBytesKey();
- try {
- metadataTable.put(key, deleteTransactionBytes);
- } catch (IOException ex) {
- LOG.error("Failed to reset {} for container {}.",
- key, containerData.getContainerID(), ex);
- }
- return false;
- };
- final ObjectNode deleteBytesError = buildErrorAndRepair(
- "dBMetadata." + OzoneConsts.PENDING_DELETE_BLOCK_BYTES,
- pendingDeleteBytesAggregate, pendingDeletionBlockSize,
deleteBytesRepairAction);
- errors.add(deleteBytesError);
+ if
(VersionedDatanodeFeatures.isFinalized(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION))
{
+ // check and repair if db delete bytes mismatches delete transaction
+ JsonNode pendingDeletionBlockSize = dBMetadata.path(
+ OzoneConsts.PENDING_DELETE_BLOCK_BYTES);
+ final long dbDeleteBytes = jsonToLong(pendingDeletionBlockSize);
+ final JsonNode pendingDeleteBytesAggregate =
aggregates.path(PendingDelete.BYTES);
+ final long deleteTransactionBytes =
jsonToLong(pendingDeleteBytesAggregate);
+ if (dbDeleteBytes != deleteTransactionBytes) {
+ passed = false;
+ final BooleanSupplier deleteBytesRepairAction = () -> {
+ final String key = containerData.getPendingDeleteBlockBytesKey();
+ try {
+ metadataTable.put(key, deleteTransactionBytes);
+ } catch (IOException ex) {
+ LOG.error("Failed to reset {} for container {}.",
+ key, containerData.getContainerID(), ex);
+ }
+ return false;
+ };
+ final ObjectNode deleteBytesError = buildErrorAndRepair(
+ "dBMetadata." + OzoneConsts.PENDING_DELETE_BLOCK_BYTES,
+ pendingDeleteBytesAggregate, pendingDeletionBlockSize,
deleteBytesRepairAction);
+ errors.add(deleteBytesError);
+ }
}
// check and repair chunks dir.
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/PendingDelete.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/PendingDelete.java
index 0f72d3f37c8..f3d518ae6cc 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/PendingDelete.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/PendingDelete.java
@@ -29,7 +29,7 @@ public class PendingDelete {
private final long count;
private final long bytes;
- PendingDelete(long count, long bytes) {
+ public PendingDelete(long count, long bytes) {
this.count = count;
this.bytes = bytes;
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
index 13a01acd491..1dc699b2d2e 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
@@ -31,6 +31,7 @@
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerChecksumInfo;
+import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager;
@@ -47,6 +48,7 @@
import org.apache.hadoop.ozone.container.metadata.DatanodeStore;
import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaOneImpl;
import org.apache.hadoop.ozone.container.metadata.DatanodeStoreSchemaTwoImpl;
+import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -318,69 +320,24 @@ private static void populateContainerMetadata(
throws IOException {
Table<String, Long> metadataTable = store.getMetadataTable();
- // Set pending deleted block count.
- final long blockPendingDeletion;
- long blockPendingDeletionBytes = 0L;
- Long pendingDeletionBlockBytes = metadataTable.get(kvContainerData
- .getPendingDeleteBlockBytesKey());
- Long pendingDeleteBlockCount =
- metadataTable.get(kvContainerData
- .getPendingDeleteBlockCountKey());
- if (pendingDeleteBlockCount != null) {
- blockPendingDeletion = pendingDeleteBlockCount;
- if (pendingDeletionBlockBytes != null) {
- blockPendingDeletionBytes = pendingDeletionBlockBytes;
- } else {
- LOG.warn("Missing pendingDeleteBlocksize from {}: recalculate them
from delete txn tables",
- metadataTable.getName());
- PendingDelete pendingDeletions = getAggregatePendingDelete(
- store, kvContainerData, kvContainerData.getSchemaVersion());
- blockPendingDeletionBytes = pendingDeletions.getBytes();
- }
- } else {
- LOG.warn("Missing pendingDeleteBlockCount/size from {}: recalculate them
from delete txn tables",
- metadataTable.getName());
- PendingDelete pendingDeletions = getAggregatePendingDelete(
- store, kvContainerData, kvContainerData.getSchemaVersion());
- blockPendingDeletion = pendingDeletions.getCount();
- blockPendingDeletionBytes = pendingDeletions.getBytes();
- }
- // Set delete transaction id.
- Long delTxnId =
- metadataTable.get(kvContainerData.getLatestDeleteTxnKey());
+ // Set pending deleted block count and bytes
+ PendingDelete pendingDeletions =
populatePendingDeletionMetadata(kvContainerData, metadataTable, store);
+
+ // Set delete transaction id
+ Long delTxnId = metadataTable.get(kvContainerData.getLatestDeleteTxnKey());
if (delTxnId != null) {
- kvContainerData
- .updateDeleteTransactionId(delTxnId);
+ kvContainerData.updateDeleteTransactionId(delTxnId);
}
- // Set BlockCommitSequenceId.
- Long bcsId = metadataTable.get(
- kvContainerData.getBcsIdKey());
+ // Set BlockCommitSequenceId
+ Long bcsId = metadataTable.get(kvContainerData.getBcsIdKey());
if (bcsId != null) {
- kvContainerData
- .updateBlockCommitSequenceId(bcsId);
- }
-
- // Set bytes used.
- // commitSpace for Open Containers relies on usedBytes
- final long blockBytes;
- final long blockCount;
- final Long metadataTableBytesUsed =
metadataTable.get(kvContainerData.getBytesUsedKey());
- // Set block count.
- final Long metadataTableBlockCount =
metadataTable.get(kvContainerData.getBlockCountKey());
- if (metadataTableBytesUsed != null && metadataTableBlockCount != null) {
- blockBytes = metadataTableBytesUsed;
- blockCount = metadataTableBlockCount;
- } else {
- LOG.warn("Missing bytesUsed={} or blockCount={} from {}: recalculate
them from block table",
- metadataTableBytesUsed, metadataTableBlockCount,
metadataTable.getName());
- final ContainerData.BlockByteAndCounts b =
getUsedBytesAndBlockCount(store, kvContainerData);
- blockBytes = b.getBytes();
- blockCount = b.getCount();
+ kvContainerData.updateBlockCommitSequenceId(bcsId);
}
- kvContainerData.getStatistics().updateBlocks(blockBytes, blockCount);
-
kvContainerData.getStatistics().setBlockPendingDeletion(blockPendingDeletion,
blockPendingDeletionBytes);
+ // Set block statistics
+ populateBlockStatistics(kvContainerData, metadataTable, store);
+
kvContainerData.getStatistics().setBlockPendingDeletion(pendingDeletions.getCount(),
pendingDeletions.getBytes());
// If the container is missing a chunks directory, possibly due to the
// bug fixed by HDDS-6235, create it here.
@@ -404,6 +361,78 @@ private static void populateContainerMetadata(
populateContainerFinalizeBlock(kvContainerData, store);
}
+ private static PendingDelete populatePendingDeletionMetadata(
+ KeyValueContainerData kvContainerData, Table<String, Long> metadataTable,
+ DatanodeStore store) throws IOException {
+
+ Long pendingDeletionBlockBytes =
metadataTable.get(kvContainerData.getPendingDeleteBlockBytesKey());
+ Long pendingDeleteBlockCount =
metadataTable.get(kvContainerData.getPendingDeleteBlockCountKey());
+
+ if
(!VersionedDatanodeFeatures.isFinalized(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION))
{
+ return handlePreDataDistributionFeature(pendingDeleteBlockCount,
metadataTable, store, kvContainerData);
+ } else if (pendingDeleteBlockCount != null) {
+ return handlePostDataDistributionFeature(pendingDeleteBlockCount,
pendingDeletionBlockBytes,
+ metadataTable, store, kvContainerData);
+ } else {
+ LOG.warn("Missing pendingDeleteBlockCount/size from {}: recalculate them
from delete txn tables",
+ metadataTable.getName());
+ return getAggregatePendingDelete(store, kvContainerData,
kvContainerData.getSchemaVersion());
+ }
+ }
+
+ private static PendingDelete handlePreDataDistributionFeature(
+ Long pendingDeleteBlockCount, Table<String, Long> metadataTable,
+ DatanodeStore store, KeyValueContainerData kvContainerData) throws
IOException {
+
+ if (pendingDeleteBlockCount != null) {
+ return new PendingDelete(pendingDeleteBlockCount, 0L);
+ } else {
+ LOG.warn("Missing pendingDeleteBlockCount/size from {}: recalculate them
from delete txn tables",
+ metadataTable.getName());
+ return getAggregatePendingDelete(store, kvContainerData,
kvContainerData.getSchemaVersion());
+ }
+ }
+
+ private static PendingDelete handlePostDataDistributionFeature(
+ Long pendingDeleteBlockCount, Long pendingDeletionBlockBytes,
+ Table<String, Long> metadataTable, DatanodeStore store,
+ KeyValueContainerData kvContainerData) throws IOException {
+
+ if (pendingDeletionBlockBytes != null) {
+ return new PendingDelete(pendingDeleteBlockCount,
pendingDeletionBlockBytes);
+ } else {
+ LOG.warn("Missing pendingDeleteBlockSize from {}: recalculate them from
delete txn tables",
+ metadataTable.getName());
+ PendingDelete pendingDeletions = getAggregatePendingDelete(
+ store, kvContainerData, kvContainerData.getSchemaVersion());
+ return new PendingDelete(pendingDeleteBlockCount,
pendingDeletions.getBytes());
+ }
+ }
+
+ private static void populateBlockStatistics(
+ KeyValueContainerData kvContainerData, Table<String, Long> metadataTable,
+ DatanodeStore store) throws IOException {
+
+ final Long metadataTableBytesUsed =
metadataTable.get(kvContainerData.getBytesUsedKey());
+ final Long metadataTableBlockCount =
metadataTable.get(kvContainerData.getBlockCountKey());
+
+ final long blockBytes;
+ final long blockCount;
+
+ if (metadataTableBytesUsed != null && metadataTableBlockCount != null) {
+ blockBytes = metadataTableBytesUsed;
+ blockCount = metadataTableBlockCount;
+ } else {
+ LOG.warn("Missing bytesUsed={} or blockCount={} from {}: recalculate
them from block table",
+ metadataTableBytesUsed, metadataTableBlockCount,
metadataTable.getName());
+ final ContainerData.BlockByteAndCounts blockData =
getUsedBytesAndBlockCount(store, kvContainerData);
+ blockBytes = blockData.getBytes();
+ blockCount = blockData.getCount();
+ }
+
+ kvContainerData.getStatistics().updateBlocks(blockBytes, blockCount);
+ }
+
/**
* Loads finalizeBlockLocalIds for container in memory.
* @param kvContainerData - KeyValueContainerData
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java
new file mode 100644
index 00000000000..d714a955b0a
--- /dev/null
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.upgrade;
+
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfig;
+import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
+import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
+import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
+import org.apache.hadoop.ozone.HddsDatanodeService;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.hadoop.ozone.UniformDatanodesFactory;
+import org.apache.hadoop.ozone.client.BucketArgs;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import
org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Tests upgrade finalization failure scenarios and corner cases specific to
DN data distribution feature.
+ */
+public class TestDNDataDistributionFinalization {
+ private static final String CLIENT_ID = UUID.randomUUID().toString();
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestDNDataDistributionFinalization.class);
+
+ private StorageContainerLocationProtocol scmClient;
+ private MiniOzoneHAClusterImpl cluster;
+
+ private static final int NUM_DATANODES = 3;
+ private static final int NUM_SCMS = 3;
+ private final String volumeName = UUID.randomUUID().toString();
+ private final String bucketName = UUID.randomUUID().toString();
+ private OzoneBucket bucket;
+
+ @AfterEach
+ public void cleanup() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ public void init(OzoneConfiguration conf) throws Exception {
+
+ SCMConfigurator configurator = new SCMConfigurator();
+ configurator.setUpgradeFinalizationExecutor(null);
+
+ conf.setInt(SCMStorageConfig.TESTING_INIT_LAYOUT_VERSION_KEY,
HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion());
+ conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100,
+ TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100,
+ TimeUnit.MILLISECONDS);
+ conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
+ 100, TimeUnit.MILLISECONDS);
+ ScmConfig scmConfig = conf.getObject(ScmConfig.class);
+ scmConfig.setBlockDeletionInterval(Duration.ofMillis(100));
+ conf.setFromObject(scmConfig);
+ conf.set(HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT, "0s");
+
+ DatanodeConfiguration dnConf =
+ conf.getObject(DatanodeConfiguration.class);
+ dnConf.setBlockDeletionInterval(Duration.ofMillis(100));
+ conf.setFromObject(dnConf);
+
+ MiniOzoneHAClusterImpl.Builder clusterBuilder =
MiniOzoneCluster.newHABuilder(conf);
+ clusterBuilder.setNumOfStorageContainerManagers(NUM_SCMS)
+ .setNumOfActiveSCMs(NUM_SCMS)
+ .setSCMServiceId("scmservice")
+ .setOMServiceId("omServiceId")
+ .setNumOfOzoneManagers(1)
+ .setSCMConfigurator(configurator)
+ .setNumDatanodes(NUM_DATANODES)
+ .setDatanodeFactory(UniformDatanodesFactory.newBuilder()
+
.setLayoutVersion(HDDSLayoutFeature.INITIAL_VERSION.layoutVersion())
+ .build());
+ this.cluster = clusterBuilder.build();
+
+ scmClient = cluster.getStorageContainerLocationClient();
+ cluster.waitForClusterToBeReady();
+ assertEquals(HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion(),
+
cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
+
+ // Create Volume and Bucket
+ try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(conf)) {
+ ObjectStore store = ozoneClient.getObjectStore();
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ BucketArgs.Builder builder = BucketArgs.newBuilder();
+ volume.createBucket(bucketName, builder.build());
+ bucket = volume.getBucket(bucketName);
+ }
+ }
+
+ /**
+ * Test that validates the upgrade scenario for DN data distribution feature.
+ * This test specifically checks the conditions in
populatePendingDeletionMetadata:
+ * 1. Pre-finalization: handlePreDataDistributionFeature path
+ * 2. Post-finalization: handlePostDataDistributionFeature path
+ * 3. Missing metadata: getAggregatePendingDelete path
+ */
+ @Test
+ public void testDataDistributionUpgradeScenario() throws Exception {
+ init(new OzoneConfiguration());
+
+ // Verify initial state - STORAGE_SPACE_DISTRIBUTION should not be
finalized yet
+ assertEquals(HDDSLayoutFeature.HBASE_SUPPORT.layoutVersion(),
+
cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
+
+ // Create some data and delete operations to trigger pending deletion logic
+ String keyName1 = "testKey1";
+ String keyName2 = "testKey2";
+ byte[] data = new byte[1024];
+
+ // Write some keys
+ try (OzoneOutputStream out = bucket.createKey(keyName1, data.length)) {
+ out.write(data);
+ }
+ try (OzoneOutputStream out = bucket.createKey(keyName2, data.length)) {
+ out.write(data);
+ }
+
+ // Delete one key to create pending deletion blocks
+ bucket.deleteKey(keyName1);
+
+ // Validate pre-finalization state
+ validatePreDataDistributionFeatureState();
+
+ // Now trigger finalization
+ Future<?> finalizationFuture = Executors.newSingleThreadExecutor().submit(
+ () -> {
+ try {
+ scmClient.finalizeScmUpgrade(CLIENT_ID);
+ } catch (IOException ex) {
+ LOG.info("finalization client failed. This may be expected if the"
+
+ " test injected failures.", ex);
+ }
+ });
+
+ // Wait for finalization to complete
+ finalizationFuture.get();
+ TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
+
+ // Verify finalization completed
+ assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(),
+
cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
+
+ // Create more data and deletions to test post-finalization behavior
+ String keyName3 = "testKey3";
+ try (OzoneOutputStream out = bucket.createKey(keyName3, data.length)) {
+ out.write(data);
+ }
+ bucket.deleteKey(keyName2);
+ bucket.deleteKey(keyName3);
+
+ // Validate post-finalization state
+ validatePostDataDistributionFeatureState();
+ }
+
+ /**
+ * Test specifically for the missing metadata scenario that triggers
+ * the getAggregatePendingDelete code path.
+ */
+ @Test
+ public void testMissingPendingDeleteMetadataRecalculation() throws Exception
{
+ init(new OzoneConfiguration());
+
+
+ // Create and delete keys to generate some pending deletion data
+ String keyName = "testKeyForRecalc";
+ byte[] data = new byte[2048];
+
+ try (OzoneOutputStream out = bucket.createKey(keyName, data.length)) {
+ out.write(data);
+ }
+ bucket.deleteKey(keyName);
+ Future<?> finalizationFuture = Executors.newSingleThreadExecutor().submit(
+ () -> {
+ try {
+ scmClient.finalizeScmUpgrade(CLIENT_ID);
+ } catch (IOException ex) {
+ LOG.info("finalization client failed. This may be expected if the"
+
+ " test injected failures.", ex);
+ }
+ });
+ // Wait for finalization
+ finalizationFuture.get();
+ TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID);
+
+ assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(),
+
cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion());
+
+ // Verify the system can handle scenarios where pendingDeleteBlockCount
+ // might be missing and needs recalculation
+ validateRecalculationScenario();
+ }
+
+ private void validatePreDataDistributionFeatureState() {
+ // Before finalization, STORAGE_SPACE_DISTRIBUTION should not be finalized
+ boolean isDataDistributionFinalized =
+
VersionedDatanodeFeatures.isFinalized(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION);
+ assertTrue(!isDataDistributionFinalized ||
+ // In test environment, version manager might be null
+ cluster.getHddsDatanodes().get(0).getDatanodeStateMachine()
+ .getLayoutVersionManager() == null,
+ "STORAGE_SPACE_DISTRIBUTION should not be finalized in pre-upgrade
state");
+
+ // Verify containers exist and have pending deletion metadata
+ validateContainerPendingDeletions(false);
+ }
+
+ private void validatePostDataDistributionFeatureState() {
+ // After finalization, STORAGE_SPACE_DISTRIBUTION should be finalized
+ boolean isDataDistributionFinalized =
+
VersionedDatanodeFeatures.isFinalized(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION);
+ assertTrue(isDataDistributionFinalized ||
+ // In test environment, version manager might be null
+ cluster.getHddsDatanodes().get(0).getDatanodeStateMachine()
+ .getLayoutVersionManager() == null,
+ "STORAGE_SPACE_DISTRIBUTION should be finalized in post-upgrade
state");
+
+ // Verify containers can handle post-finalization pending deletion logic
+ validateContainerPendingDeletions(true);
+ }
+
+ private void validateContainerPendingDeletions(boolean isPostFinalization) {
+ // Get containers from datanodes and validate their pending deletion
handling
+ List<HddsDatanodeService> datanodes = cluster.getHddsDatanodes();
+
+ for (HddsDatanodeService datanode : datanodes) {
+ ContainerSet containerSet = datanode.getDatanodeStateMachine()
+ .getContainer().getContainerSet();
+
+ // Iterate through containers
+ for (Container<?> container : containerSet.getContainerMap().values()) {
+ if (container instanceof KeyValueContainer) {
+ KeyValueContainerData containerData =
+ (KeyValueContainerData) container.getContainerData();
+
+ // Verify the container has been processed through the appropriate
+ // code path in populatePendingDeletionMetadata
+ assertNotNull(containerData.getStatistics());
+
+ // The exact validation will depend on whether we're in pre or post
+ // finalization state, but we should always have valid statistics
+ assertTrue(containerData.getStatistics().getBlockPendingDeletion()
>= 0);
+
+ if (isPostFinalization) {
+ // Post-finalization should have both block count and bytes
+
assertTrue(containerData.getStatistics().getBlockPendingDeletionBytes() >= 0);
+ } else {
+ assertEquals(0,
containerData.getStatistics().getBlockPendingDeletionBytes());
+ }
+ }
+ }
+ }
+ }
+
+ private void validateRecalculationScenario() {
+ // This validates that the system properly handles the case where
+ // pendingDeleteBlockCount is null and needs to be recalculated
+ // from delete transaction tables via getAggregatePendingDelete
+
+ List<HddsDatanodeService> datanodes = cluster.getHddsDatanodes();
+
+ for (HddsDatanodeService datanode : datanodes) {
+ ContainerSet containerSet = datanode.getDatanodeStateMachine()
+ .getContainer().getContainerSet();
+
+ // Verify containers have proper pending deletion statistics
+ // even in recalculation scenarios
+ for (Container<?> container : containerSet.getContainerMap().values()) {
+ if (container instanceof KeyValueContainer) {
+ KeyValueContainerData containerData =
+ ((KeyValueContainer) container).getContainerData();
+
+ // Statistics should be valid even after recalculation
+ assertNotNull(containerData.getStatistics());
+ assertTrue(containerData.getStatistics().getBlockPendingDeletion()
>= 0);
+
assertTrue(containerData.getStatistics().getBlockPendingDeletionBytes() >= 0);
+ }
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
