mehakmeet commented on a change in pull request #1899: HADOOP-16914 Adding
Output Stream Counters in ABFS
URL: https://github.com/apache/hadoop/pull/1899#discussion_r394228752
##########
File path:
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsOutputStream.java
##########
@@ -133,58 +133,60 @@ public void testAbfsOutputStreamTimeSpendOnWaitTask()
throws IOException {
public void testAbfsOutputStreamQueueShrink() throws IOException {
describe("Testing Queue Shrink calls in AbfsOutputStream");
final AzureBlobFileSystem fs = getFileSystem();
- Path TEST_PATH = new Path("AbfsOutputStreamStatsPath");
+ Path queueShrinkFilePath = new Path("AbfsOutputStreamStatsPath");
AzureBlobFileSystemStore abfss = fs.getAbfsStore();
abfss.getAbfsConfiguration().setDisableOutputStreamFlush(false);
FileSystem.Statistics statistics = fs.getFsStatistics();
String testQueueShrink = "testQueue";
-
AbfsOutputStream outForOneOp = null;
try {
- outForOneOp = (AbfsOutputStream) abfss.createFile(TEST_PATH, statistics,
- true,
- FsPermission.getDefault(), FsPermission.getUMask(fs.getConf()));
+ outForOneOp =
+ (AbfsOutputStream) abfss.createFile(queueShrinkFilePath, statistics,
+ true,
+ FsPermission.getDefault(), FsPermission.getUMask(fs.getConf()));
//Test for shrinking Queue zero time
- Assert.assertEquals("Mismatch in number of queueShrink() Calls", 0,
+ assertValues("number of queueShrink() Calls", 0,
outForOneOp.getOutputStreamStatistics().queueShrink);
outForOneOp.write(testQueueShrink.getBytes());
// Queue is shrunk 2 times when outStream is flushed
outForOneOp.flush();
//Test for shrinking Queue 2 times
- Assert.assertEquals("Mismatch in number of queueShrink() Calls", 2,
+ assertValues("number of queueShrink() Calls", 2,
outForOneOp.getOutputStreamStatistics().queueShrink);
} finally {
- if(outForOneOp != null){
+ if (outForOneOp != null) {
outForOneOp.close();
}
}
AbfsOutputStream outForLargeOps = null;
try {
- outForLargeOps = (AbfsOutputStream) abfss.createFile(TEST_PATH,
+ outForLargeOps = (AbfsOutputStream) abfss.createFile(queueShrinkFilePath,
statistics, true,
FsPermission.getDefault(), FsPermission.getUMask(fs.getConf()));
+ int largeValue = 1000;
//QueueShrink is called 2 times in 1 flush(), hence 1000 flushes must
// give 2000 QueueShrink calls
- for (int i = 0; i < 1000; i++) {
+ for (int i = 0; i < largeValue; i++) {
outForLargeOps.write(testQueueShrink.getBytes());
//Flush is quite expensive so 1000 calls only which takes 1 min+
outForLargeOps.flush();
Review comment:
any way around flush() to get queueShrink() calls after writing ?
flush() is quite expensive as it takes some time even at 1000 calls to test.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
