[
https://issues.apache.org/jira/browse/HADOOP-19604?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=18015159#comment-18015159
]
ASF GitHub Bot commented on HADOOP-19604:
-----------------------------------------
anmolanmol1234 commented on code in PR #7853:
URL: https://github.com/apache/hadoop/pull/7853#discussion_r2288074252
##########
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsOutputStream.java:
##########
@@ -481,6 +482,103 @@ public void testResetCalledOnExceptionInRemoteFlush()
throws Exception {
//expected exception
}
// Verify that reset was called on the message digest
- Mockito.verify(mockMessageDigest, Mockito.times(1)).reset();
+ if (spiedClient.isFullBlobChecksumValidationEnabled()) {
+
Assertions.assertThat(Mockito.mockingDetails(mockMessageDigest).getInvocations()
+ .stream()
+ .filter(i -> i.getMethod().getName().equals("reset"))
+ .count())
+ .as("Expected MessageDigest.reset() to be called exactly once when
checksum validation is enabled")
+ .isEqualTo(1);
+ }
+ }
+
+ /**
+ * Tests that the message digest is reset when an exception occurs during
remote flush.
+ * Simulates a failure in the flush operation and verifies reset is called
on MessageDigest.
+ */
+ @Test
+ public void testNoChecksumComputedWhenConfigFalse() throws Exception {
+ Configuration conf = getRawConfiguration();
+ conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, false);
+ FileSystem fileSystem = FileSystem.newInstance(conf);
+ AzureBlobFileSystem fs = (AzureBlobFileSystem) fileSystem;
+ Assume.assumeTrue(!getIsNamespaceEnabled(fs));
+ AzureBlobFileSystemStore store = Mockito.spy(fs.getAbfsStore());
+ assumeBlobServiceType();
+ Assume.assumeFalse("Not valid for APPEND BLOB", isAppendBlobEnabled());
+
+ // Create spies for the client handler and blob client
+ AbfsClientHandler clientHandler = Mockito.spy(store.getClientHandler());
+ AbfsBlobClient blobClient = Mockito.spy(clientHandler.getBlobClient());
+
+ // Set up the spies to return the mocked objects
+ Mockito.doReturn(clientHandler).when(store).getClientHandler();
+ Mockito.doReturn(blobClient).when(clientHandler).getBlobClient();
+ Mockito.doReturn(blobClient).when(clientHandler).getIngressClient();
+ AbfsOutputStream abfsOutputStream = Mockito.spy(
+ (AbfsOutputStream) fs.create(new
Path("/test/file")).getWrappedStream());
+ AzureIngressHandler ingressHandler = Mockito.spy(
+ abfsOutputStream.getIngressHandler());
+
Mockito.doReturn(ingressHandler).when(abfsOutputStream).getIngressHandler();
+ Mockito.doReturn(blobClient).when(ingressHandler).getClient();
+ FSDataOutputStream os = Mockito.spy(
+ new FSDataOutputStream(abfsOutputStream, null));
+ AbfsOutputStream out = (AbfsOutputStream) os.getWrappedStream();
+ byte[] bytes = new byte[1024 * 1024 * 4];
+ new Random().nextBytes(bytes);
+ // Write some bytes and attempt to flush, which should retry
+ out.write(bytes);
+ out.hsync();
+ Assertions.assertThat(Mockito.mockingDetails(blobClient).getInvocations()
+ .stream()
+ .filter(i ->
i.getMethod().getName().equals("addCheckSumHeaderForWrite"))
+ .count())
+ .as("Expected addCheckSumHeaderForWrite() to be called exactly 0
times")
+ .isZero();
+ }
+
+ /**
+ * Tests that the message digest is reset when an exception occurs during
remote flush.
+ * Simulates a failure in the flush operation and verifies reset is called
on MessageDigest.
+ */
+ @Test
+ public void testChecksumComputedWhenConfigTrue() throws Exception {
+ Configuration conf = getRawConfiguration();
+ conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, true);
+ FileSystem fileSystem = FileSystem.newInstance(conf);
+ AzureBlobFileSystem fs = (AzureBlobFileSystem) fileSystem;
Review Comment:
taken
##########
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsOutputStream.java:
##########
@@ -481,6 +482,103 @@ public void testResetCalledOnExceptionInRemoteFlush()
throws Exception {
//expected exception
}
// Verify that reset was called on the message digest
- Mockito.verify(mockMessageDigest, Mockito.times(1)).reset();
+ if (spiedClient.isFullBlobChecksumValidationEnabled()) {
+
Assertions.assertThat(Mockito.mockingDetails(mockMessageDigest).getInvocations()
+ .stream()
+ .filter(i -> i.getMethod().getName().equals("reset"))
+ .count())
+ .as("Expected MessageDigest.reset() to be called exactly once when
checksum validation is enabled")
+ .isEqualTo(1);
+ }
+ }
+
+ /**
+ * Tests that the message digest is reset when an exception occurs during
remote flush.
+ * Simulates a failure in the flush operation and verifies reset is called
on MessageDigest.
+ */
+ @Test
+ public void testNoChecksumComputedWhenConfigFalse() throws Exception {
+ Configuration conf = getRawConfiguration();
+ conf.setBoolean(FS_AZURE_ABFS_ENABLE_CHECKSUM_VALIDATION, false);
+ FileSystem fileSystem = FileSystem.newInstance(conf);
Review Comment:
taken
> ABFS: Fix WASB ABFS compatibility issues
> ----------------------------------------
>
> Key: HADOOP-19604
> URL: https://issues.apache.org/jira/browse/HADOOP-19604
> Project: Hadoop Common
> Issue Type: Sub-task
> Affects Versions: 3.4.1
> Reporter: Anmol Asrani
> Assignee: Anmol Asrani
> Priority: Major
> Labels: pull-request-available
> Fix For: 3.4.1
>
>
> Fix WASB ABFS compatibility issues. Fix issues such as:-
> # BlockId computation to be consistent across clients for PutBlock and
> PutBlockList
> # Restrict url encoding of certain json metadata during setXAttr calls.
> # Maintain the md5 hash of whole block to validate data integrity during
> flush.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
