This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new f856adae607 HDDS-8781. Allow on demand metadata scanning of open
containers (#8442)
f856adae607 is described below
commit f856adae607e8587daa5587814c276748c209b5b
Author: Tejaskriya <[email protected]>
AuthorDate: Sat Jul 12 17:22:22 2025 +0530
HDDS-8781. Allow on demand metadata scanning of open containers (#8442)
---
.../ozone/container/common/impl/ContainerSet.java | 6 +--
.../BackgroundContainerMetadataScanner.java | 19 +--------
.../container/ozoneimpl/ContainerScanHelper.java | 34 ++++++++++++++--
...aScanner.java => OnDemandContainerScanner.java} | 18 ++++++---
.../ozone/container/ozoneimpl/OzoneContainer.java | 4 +-
.../container/common/impl/TestContainerSet.java | 6 +--
...stContainerReconciliationWithMockDatanodes.java | 6 +--
.../container/keyvalue/TestKeyValueHandler.java | 4 +-
...nner.java => TestOnDemandContainerScanner.java} | 34 +++++++++++++---
...> TestOnDemandContainerScannerIntegration.java} | 47 ++++++++++++++++++++--
10 files changed, 129 insertions(+), 49 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
index e03e61605eb..8e998c1aef2 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java
@@ -48,7 +48,7 @@
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.utils.ContainerLogger;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
-import
org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerDataScanner;
+import org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerScanner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,7 +69,7 @@ public class ContainerSet implements Iterable<Container<?>> {
private long recoveringTimeout;
private final Table<ContainerID, String> containerIdsTable;
// Handler that will be invoked when a scan of a container in this set is
requested.
- private OnDemandContainerDataScanner containerScanner;
+ private OnDemandContainerScanner containerScanner;
public static ContainerSet newReadOnlyContainerSet(long recoveringTimeout) {
return new ContainerSet(null, recoveringTimeout);
@@ -131,7 +131,7 @@ public void ensureContainerNotMissing(long containerId,
State state) throws Stor
/**
* @param scanner The scanner instance will be invoked when a scan of a
container in this set is requested.
*/
- public void registerOnDemandScanner(OnDemandContainerDataScanner scanner) {
+ public void registerOnDemandScanner(OnDemandContainerScanner scanner) {
this.containerScanner = scanner;
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerMetadataScanner.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerMetadataScanner.java
index cb57fc1929f..ae634a8a62f 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerMetadataScanner.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/BackgroundContainerMetadataScanner.java
@@ -54,24 +54,7 @@ public Iterator<Container<?>> getContainerIterator() {
@Override
public void scanContainer(Container<?> container)
throws IOException, InterruptedException {
- if (!scanHelper.shouldScanMetadata(container)) {
- return;
- }
-
- long containerID = container.getContainerData().getContainerID();
-
- MetadataScanResult result = container.scanMetaData();
- if (result.isDeleted()) {
- LOG.debug("Container [{}] has been deleted during the metadata scan.",
containerID);
- return;
- }
- if (result.hasErrors()) {
- scanHelper.handleUnhealthyScanResult(containerID, result);
- }
-
- // Do not update the scan timestamp after the scan since this was just a
- // metadata scan, not a full data scan.
- metrics.incNumContainersScanned();
+ scanHelper.scanMetadata(container);
}
@Override
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java
index 6cd7b80f02f..c0e16c7de93 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerScanHelper.java
@@ -63,7 +63,7 @@ public void scanData(Container<?> container,
DataTransferThrottler throttler, Ca
}
ContainerData containerData = container.getContainerData();
long containerId = containerData.getContainerID();
- logScanStart(containerData);
+ logScanStart(containerData, "data");
DataScanResult result = container.scanData(throttler, canceler);
if (result.isDeleted()) {
@@ -88,6 +88,32 @@ public void scanData(Container<?> container,
DataTransferThrottler throttler, Ca
logScanCompleted(containerData, now);
}
+ public void scanMetadata(Container<?> container)
+ throws IOException, InterruptedException {
+ if (!shouldScanMetadata(container)) {
+ return;
+ }
+ ContainerData containerData = container.getContainerData();
+ long containerId = containerData.getContainerID();
+ logScanStart(containerData, "only metadata");
+
+ MetadataScanResult result = container.scanMetaData();
+ if (result.isDeleted()) {
+ log.debug("Container [{}] has been deleted during metadata scan.",
containerId);
+ return;
+ }
+ if (result.hasErrors()) {
+ handleUnhealthyScanResult(containerId, result);
+ }
+
+ Instant now = Instant.now();
+ // Do not update the scan timestamp after the scan since this was just a
+ // metadata scan, not a full data scan.
+ metrics.incNumContainersScanned();
+ // Even if the container was deleted, mark the scan as completed since we
already logged it as starting.
+ logScanCompleted(containerData, now);
+ }
+
public void handleUnhealthyScanResult(long containerID, ScanResult result)
throws IOException {
log.error("Corruption detected in container [{}]. Marking it UNHEALTHY.
{}", containerID, result);
@@ -147,12 +173,12 @@ private boolean recentlyScanned(ContainerData
containerData) {
return recentlyScanned;
}
- private void logScanStart(ContainerData containerData) {
+ private void logScanStart(ContainerData containerData, String scanType) {
if (log.isDebugEnabled()) {
Optional<Instant> scanTimestamp = containerData.lastDataScanTime();
Object lastScanTime = scanTimestamp.map(ts -> "at " +
ts).orElse("never");
- log.debug("Scanning container {}, last scanned {}",
- containerData.getContainerID(), lastScanTime);
+ log.debug("Scanning {} of container {}, last scanned {}",
+ scanType, containerData.getContainerID(), lastScanTime);
}
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerDataScanner.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerScanner.java
similarity index 89%
rename from
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerDataScanner.java
rename to
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerScanner.java
index a85358406bd..62f0c08d71e 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerDataScanner.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OnDemandContainerScanner.java
@@ -32,10 +32,12 @@
/**
* Class for performing on demand scans of containers.
+ * Note: [OPEN] containers are scanned only for metadata,
+ * [CLOSED, QUASI_CLOSED] containers are scanned for metadata and data.
*/
-public final class OnDemandContainerDataScanner {
+public final class OnDemandContainerScanner {
private static final Logger LOG =
- LoggerFactory.getLogger(OnDemandContainerDataScanner.class);
+ LoggerFactory.getLogger(OnDemandContainerScanner.class);
private final ExecutorService scanExecutor;
private final DataTransferThrottler throttler;
@@ -46,7 +48,7 @@ public final class OnDemandContainerDataScanner {
private final ContainerScanHelper scannerHelper;
private final ContainerScanHelper scannerHelperWithoutGap;
- public OnDemandContainerDataScanner(
+ public OnDemandContainerScanner(
ContainerScannerConfiguration conf, ContainerController controller) {
throttler = new DataTransferThrottler(
conf.getOnDemandBandwidthPerVolume());
@@ -77,7 +79,7 @@ public Optional<Future<?>>
scanContainerWithoutGap(Container<?> container) {
}
private Optional<Future<?>> scanContainer(Container<?> container,
ContainerScanHelper helper) {
- if (!helper.shouldScanData(container)) {
+ if (!helper.shouldScanMetadata(container)) {
return Optional.empty();
}
@@ -103,7 +105,13 @@ private void removeContainerFromScheduledContainers(
private void performOnDemandScan(Container<?> container, ContainerScanHelper
helper) {
try {
- helper.scanData(container, throttler, canceler);
+ if (helper.shouldScanData(container)) {
+ helper.scanData(container, throttler, canceler);
+ } else {
+ // for containers that qualify for metadata scan and not data scan,
+ // like OPEN containers, trigger a metadata-only scan
+ helper.scanMetadata(container);
+ }
} catch (IOException e) {
LOG.warn("Unexpected exception while scanning container "
+ container.getContainerData().getContainerID(), e);
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index cc46784bbef..985909a294e 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -125,7 +125,7 @@ public class OzoneContainer {
private final XceiverServerSpi readChannel;
private final ContainerController controller;
private BackgroundContainerMetadataScanner metadataScanner;
- private OnDemandContainerDataScanner onDemandScanner;
+ private OnDemandContainerScanner onDemandScanner;
private List<BackgroundContainerDataScanner> dataScanners;
private List<AbstractBackgroundContainerScanner> backgroundScanners;
private final BlockDeletingService blockDeletingService;
@@ -439,7 +439,7 @@ private void
initOnDemandContainerScanner(ContainerScannerConfiguration c) {
"so the on-demand container data scanner will not start.");
return;
}
- onDemandScanner = new OnDemandContainerDataScanner(c, controller);
+ onDemandScanner = new OnDemandContainerScanner(c, controller);
containerSet.registerOnDemandScanner(onDemandScanner);
}
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
index 10c897b1f54..ad12e1be8ed 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerSet.java
@@ -51,7 +51,7 @@
import org.apache.hadoop.ozone.container.keyvalue.ContainerLayoutTestInfo;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
-import
org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerDataScanner;
+import org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerScanner;
/**
* Class used to test ContainerSet operations.
@@ -296,7 +296,7 @@ public void testContainerScanHandler(ContainerLayoutVersion
layout) throws Excep
containerSet.scanContainer(FIRST_ID);
AtomicLong invocationCount = new AtomicLong();
- OnDemandContainerDataScanner mockScanner =
mock(OnDemandContainerDataScanner.class);
+ OnDemandContainerScanner mockScanner =
mock(OnDemandContainerScanner.class);
when(mockScanner.scanContainer(any())).then(inv -> {
KeyValueContainer c = inv.getArgument(0);
// If the handler was incorrectly triggered for a non-existent
container, this assert would fail.
@@ -323,7 +323,7 @@ public void
testContainerScanHandlerWithoutGap(ContainerLayoutVersion layout) th
containerSet.scanContainer(FIRST_ID);
AtomicLong invocationCount = new AtomicLong();
- OnDemandContainerDataScanner mockScanner =
mock(OnDemandContainerDataScanner.class);
+ OnDemandContainerScanner mockScanner =
mock(OnDemandContainerScanner.class);
when(mockScanner.scanContainerWithoutGap(any())).then(inv -> {
KeyValueContainer c = inv.getArgument(0);
// If the handler was incorrectly triggered for a non-existent
container, this assert would fail.
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestContainerReconciliationWithMockDatanodes.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestContainerReconciliationWithMockDatanodes.java
index e95fc3ab3ec..7fce76a0e4d 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestContainerReconciliationWithMockDatanodes.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestContainerReconciliationWithMockDatanodes.java
@@ -81,7 +81,7 @@
import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import
org.apache.hadoop.ozone.container.ozoneimpl.ContainerScannerConfiguration;
-import
org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerDataScanner;
+import org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerScanner;
import org.apache.ozone.test.GenericTestUtils;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.junit.jupiter.api.AfterAll;
@@ -299,7 +299,7 @@ private static void mockContainerProtocolCalls() {
private static class MockDatanode {
private final KeyValueHandler handler;
private final DatanodeDetails dnDetails;
- private final OnDemandContainerDataScanner onDemandScanner;
+ private final OnDemandContainerScanner onDemandScanner;
private final ContainerSet containerSet;
private final OzoneConfiguration conf;
@@ -322,7 +322,7 @@ private static class MockDatanode {
ContainerController controller = new ContainerController(containerSet,
Collections.singletonMap(ContainerProtos.ContainerType.KeyValueContainer,
handler));
- onDemandScanner = new OnDemandContainerDataScanner(
+ onDemandScanner = new OnDemandContainerScanner(
conf.getObject(ContainerScannerConfiguration.class), controller);
// Register the on-demand container scanner with the container set used
by the KeyValueHandler.
containerSet.registerOnDemandScanner(onDemandScanner);
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
index 503d8c0855d..97fd5a4b8c1 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@ -98,7 +98,7 @@
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import
org.apache.hadoop.ozone.container.ozoneimpl.ContainerScannerConfiguration;
-import
org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerDataScanner;
+import org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerScanner;
import org.apache.hadoop.util.Sets;
import org.apache.hadoop.util.Time;
import org.apache.ozone.test.GenericTestUtils;
@@ -818,7 +818,7 @@ private KeyValueHandler createKeyValueHandler(Path path)
throws IOException {
// Register the on-demand container scanner with the container set used by
the KeyValueHandler.
ContainerController controller = new ContainerController(containerSet,
Collections.singletonMap(ContainerType.KeyValueContainer, kvHandler));
- OnDemandContainerDataScanner onDemandScanner = new
OnDemandContainerDataScanner(
+ OnDemandContainerScanner onDemandScanner = new OnDemandContainerScanner(
conf.getObject(ContainerScannerConfiguration.class), controller);
containerSet.registerOnDemandScanner(onDemandScanner);
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerDataScanner.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java
similarity index 91%
rename from
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerDataScanner.java
rename to
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java
index 42b3da8050d..98a65ca761d 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerDataScanner.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOnDemandContainerScanner.java
@@ -64,16 +64,16 @@
* Unit tests for the on-demand container scanner.
*/
@MockitoSettings(strictness = Strictness.LENIENT)
-public class TestOnDemandContainerDataScanner extends
+public class TestOnDemandContainerScanner extends
TestContainerScannersAbstract {
-
- private OnDemandContainerDataScanner onDemandScanner;
+
+ private OnDemandContainerScanner onDemandScanner;
@Override
@BeforeEach
public void setup() {
super.setup();
- onDemandScanner = new OnDemandContainerDataScanner(conf, controller);
+ onDemandScanner = new OnDemandContainerScanner(conf, controller);
}
@Test
@@ -169,6 +169,28 @@ public void testSameContainerQueuedMultipleTimes() throws
Exception {
eq(corruptData.getContainerData().getContainerID()), any());
}
+ @Test
+ public void testSameOpenContainerQueuedMultipleTimes() throws Exception {
+ //Given a container that has not finished scanning
+ CountDownLatch latch = new CountDownLatch(1);
+ when(openCorruptMetadata.scanMetaData())
+ .thenAnswer((Answer<ScanResult>) invocation -> {
+ latch.await();
+ return getUnhealthyDataScanResult();
+ });
+ Optional<Future<?>> onGoingScan =
onDemandScanner.scanContainer(openCorruptMetadata);
+ assertTrue(onGoingScan.isPresent());
+ assertFalse(onGoingScan.get().isDone());
+ //When scheduling the same container again
+ Optional<Future<?>> secondScan =
onDemandScanner.scanContainer(openCorruptMetadata);
+ //Then the second scan is not scheduled and the first scan can still finish
+ assertFalse(secondScan.isPresent());
+ latch.countDown();
+ onGoingScan.get().get();
+ verify(controller, atLeastOnce()).markContainerUnhealthy(
+ eq(openCorruptMetadata.getContainerData().getContainerID()), any());
+ }
+
@Test
@Override
public void testScannerMetrics() throws Exception {
@@ -184,7 +206,7 @@ public void testScannerMetrics() throws Exception {
//Containers with shouldScanData = false shouldn't increase
// the number of scanned containers
assertEquals(0, metrics.getNumUnHealthyContainers());
- assertEquals(2, metrics.getNumContainersScanned());
+ assertEquals(4, metrics.getNumContainersScanned());
}
@Test
@@ -209,7 +231,7 @@ public void testUnhealthyContainersDetected() throws
Exception {
scanContainer(corruptData);
verifyContainerMarkedUnhealthy(corruptData, atLeastOnce());
scanContainer(openCorruptMetadata);
- verifyContainerMarkedUnhealthy(openCorruptMetadata, never());
+ verifyContainerMarkedUnhealthy(openCorruptMetadata, atLeastOnce());
scanContainer(openContainer);
verifyContainerMarkedUnhealthy(openContainer, never());
// Deleted containers should not be marked unhealthy
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerDataScannerIntegration.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerScannerIntegration.java
similarity index 79%
rename from
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerDataScannerIntegration.java
rename to
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerScannerIntegration.java
index d3b3ed46fde..136a307e0ab 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerDataScannerIntegration.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/dn/scanner/TestOnDemandContainerScannerIntegration.java
@@ -26,6 +26,8 @@
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.Collection;
+import java.util.EnumSet;
+import java.util.Set;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State;
@@ -33,7 +35,7 @@
import org.apache.hadoop.ozone.container.common.utils.ContainerLogger;
import org.apache.hadoop.ozone.container.keyvalue.TestContainerCorruptions;
import
org.apache.hadoop.ozone.container.ozoneimpl.ContainerScannerConfiguration;
-import
org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerDataScanner;
+import org.apache.hadoop.ozone.container.ozoneimpl.OnDemandContainerScanner;
import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.params.ParameterizedTest;
@@ -44,7 +46,7 @@
* is triggered when there is an error while a client interacts with a
* container.
*/
-class TestOnDemandContainerDataScannerIntegration
+class TestOnDemandContainerScannerIntegration
extends TestContainerScannerIntegrationAbstract {
private final GenericTestUtils.LogCapturer logCapturer =
@@ -72,6 +74,13 @@ static Collection<TestContainerCorruptions>
supportedCorruptionTypes() {
TestContainerCorruptions.TRUNCATED_BLOCK);
}
+ static Collection<TestContainerCorruptions>
supportedCorruptionTypesForOpen() {
+ Set<TestContainerCorruptions> set =
EnumSet.copyOf(supportedCorruptionTypes());
+ // Open containers will be checked only for metadata corruption, so
missing block is not a valid corruption type.
+ set.remove(TestContainerCorruptions.MISSING_BLOCK);
+ return set;
+ }
+
@BeforeAll
static void init() throws Exception {
OzoneConfiguration ozoneConfig = new OzoneConfiguration();
@@ -90,7 +99,7 @@ static void init() throws Exception {
}
/**
- * {@link OnDemandContainerDataScanner} should detect corrupted blocks
+ * {@link OnDemandContainerScanner} should detect corrupted blocks
* in a closed container when a client reads from it.
*/
@ParameterizedTest
@@ -137,4 +146,36 @@ void testCorruptionDetected(TestContainerCorruptions
corruption)
assertEquals(newReportedDataChecksum,
updatedChecksumInfo.getContainerMerkleTree().getDataChecksum());
}
}
+
+ /**
+ * {@link OnDemandContainerScanner} should detect corrupted blocks
+ * in an open container when a client reads from it.
+ */
+ @ParameterizedTest
+ @MethodSource("supportedCorruptionTypesForOpen")
+ void testCorruptionDetectedForOpenContainers(TestContainerCorruptions
corruption)
+ throws Exception {
+ String keyName = "keyName";
+
+ long openContainerID = writeDataToOpenContainer();
+ Container<?> openContainer = getDnContainer(openContainerID);
+ assertEquals(State.OPEN, openContainer.getContainerState());
+
+ // Corrupt the container.
+ corruption.applyTo(openContainer);
+ // This method will check that reading from the corrupted key returns an
+ // error to the client.
+ readFromCorruptedKey(keyName);
+
+ // Reading from the corrupted key should have triggered an on-demand scan
+ // of the container, which will detect the corruption.
+ GenericTestUtils.waitFor(
+ () -> openContainer.getContainerState() == State.UNHEALTHY,
+ 500, 5000);
+
+ // Wait for SCM to get a report of the unhealthy replica.
+ waitForScmToSeeReplicaState(openContainerID, UNHEALTHY);
+ corruption.assertLogged(openContainerID, 1, logCapturer);
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
