jojochuang commented on a change in pull request #2889:
URL: https://github.com/apache/hadoop/pull/2889#discussion_r612221274
##########
File path:
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
##########
@@ -562,4 +565,57 @@ void writeBlock(ExtendedBlock block,
BlockConstructionStage stage,
checksum, CachingStrategy.newDefaultStrategy(), false, false,
null, null, new String[0]);
}
+
+ @Test
+ public void testReleaseVolumeRefIfExceptionThrown() throws IOException {
+ Path file = new Path("dataprotocol.dat");
+ int numDataNodes = 1;
+
+ Configuration conf = new HdfsConfiguration();
+ conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, numDataNodes);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
+ numDataNodes).build();
+ try {
+ cluster.waitActive();
+ datanode = cluster.getFileSystem().getDataNodeStats(
+ DatanodeReportType.LIVE)[0];
+ dnAddr = NetUtils.createSocketAddr(datanode.getXferAddr());
+ FileSystem fileSys = cluster.getFileSystem();
+
+ int fileLen = Math.min(
+ conf.getInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 4096), 4096);
+
+ DFSTestUtil.createFile(fileSys, file, fileLen, fileLen,
+ fileSys.getDefaultBlockSize(file),
+ fileSys.getDefaultReplication(file), 0L);
+
+ // get the first blockid for the file
+ final ExtendedBlock firstBlock = DFSTestUtil.getFirstBlock(fileSys,
file);
+
+ String bpid = cluster.getNamesystem().getBlockPoolId();
+ ExtendedBlock blk = new ExtendedBlock(bpid, firstBlock.getLocalBlock());
+ sendBuf.reset();
+ recvBuf.reset();
+
+ // delete the meta file to create a exception in BlockSender constructor
+ DataNode dn = cluster.getDataNodes().get(0);
+ cluster.getMaterializedReplica(0, blk).deleteMeta();
+
+ FsVolumeImpl volume = (FsVolumeImpl) DataNodeTestUtils.getFSDataset(
+ dn).getVolume(blk);
+ int beforeCnt = volume.getReferenceCount();
+
+ sender.copyBlock(blk, BlockTokenSecretManager.DUMMY_TOKEN);
+ sendRecvData("Copy a block.", false);
+ Thread.sleep(1000);
+
+ int afterCnt = volume.getReferenceCount();
+ assertEquals(beforeCnt, afterCnt);
+
+ } catch (InterruptedException e) {
Review comment:
Sorry I wasn't being clear myself. You don't have to catch the
exception. Instead, add InterruptedException to the test method signature.
##########
File path:
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
##########
@@ -280,4 +283,40 @@ public void run() {
}
}
}
+ @Test
+ public void testReleaseVolumeRefIfExceptionThrown() throws IOException {
+ getClusterBuilder().setRamDiskReplicaCapacity(2).build();
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ final int SEED = 0xFADED;
+ Path path = new Path("/" + METHOD_NAME + ".Writer.File.dat");
+
+ DataNode dn = cluster.getDataNodes().get(0);
+ FsDatasetSpi.FsVolumeReferences volumes =
+ DataNodeTestUtils.getFSDataset(dn).getFsVolumeReferences();
+ int[] beforeCnts = new int[volumes.size()];
+ try {
+ FsDatasetImpl ds = (FsDatasetImpl) DataNodeTestUtils.getFSDataset(dn);
+
+ // Create a runtime exception
+ ds.asyncLazyPersistService.shutdown();
+ for (int i = 0; i < volumes.size(); ++i) {
+ beforeCnts[i] = ((FsVolumeImpl) volumes.get(i)).getReferenceCount();
+ }
+
+ makeRandomTestFile(path, BLOCK_SIZE, true, SEED);
+ Thread.sleep(3 * LAZY_WRITER_INTERVAL_SEC * 1000);
+
+ for (int i = 0; i < volumes.size(); ++i) {
+ int afterCnt = ((FsVolumeImpl) volumes.get(i)).getReferenceCount();
+ // LazyWriter keeps trying to save copies even if
+ // asyncLazyPersistService is already shutdown.
+ // If we do not release references, the number of
+ // references will increase infinitely.
+ Assert.assertTrue(
+ beforeCnts[i] == afterCnt || beforeCnts[i] == (afterCnt - 1));
+ }
+ } catch (InterruptedException e) {
Review comment:
Sorry I wasn't being clear myself. You don't have to catch the
exception. Instead, add InterruptedException to the test method signature.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
