steveloughran commented on code in PR #5881:
URL: https://github.com/apache/hadoop/pull/5881#discussion_r1279758355
##########
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java:
##########
@@ -40,6 +40,8 @@ public final class FileSystemConfigurations {
// Retry parameter defaults.
public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3 * 1000; // 3s
public static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30 * 1000; // 30s
+ public static final boolean
DEFAULT_STATIC_RETRY_FOR_CONNECTION_TIMEOUT_ENABLED = true;
+ public static final int DEFAULT_STATIC_RETRY_INTERVAL = 1 * 1000; // 1s
Review Comment:
use 1_000 now we can.
##########
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java:
##########
@@ -269,12 +274,13 @@ String getClientLatency() {
private boolean executeHttpOperation(final int retryCount,
TracingContext tracingContext) throws AzureBlobFileSystemException {
AbfsHttpOperation httpOperation;
+ boolean iOExceptionThrown = false;
Review Comment:
prefer `wasIOExceptionThrown`
##########
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/RetryPolicy.java:
##########
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import java.net.HttpURLConnection;
+
+import static
org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_CONTINUE;
+
+/**
+ * Abstract Class for Retry policy to be used by {@link AbfsClient}
+ * Implementation to be used is based on retry cause.
+ */
+public abstract class RetryPolicy {
Review Comment:
duplicate name; even in different packages it's painful. see
org.apache.hadoop.io.retry.RetryPolicy
##########
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestTracingContext.java:
##########
@@ -226,7 +229,7 @@ fileSystemId, FSOperationType.CREATE_FILESYSTEM,
tracingHeaderFormat, new Tracin
fs.getFileSystemId(), FSOperationType.CREATE_FILESYSTEM, false,
1));
- tracingContext.constructHeader(abfsHttpOperation, "RT");
+ tracingContext.constructHeader(abfsHttpOperation, "RT", "E");
Review Comment:
you should still refer to the constant in the production code; makes it
easier to find usages/make changes
##########
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestTracingContext.java:
##########
@@ -270,4 +273,71 @@ fileSystemId, FSOperationType.CREATE_FILESYSTEM,
tracingHeaderFormat, new Tracin
+ "should be equal to PrimaryRequestId in the original request.")
.isEqualTo(assertionPrimaryId);
}
+
+ @Test
+ public void testTracingContextHeaderForRetrypolicy() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final String fileSystemId = fs.getFileSystemId();
+ final String clientCorrelationId = fs.getClientCorrelationId();
+ final TracingHeaderFormat tracingHeaderFormat =
TracingHeaderFormat.ALL_ID_FORMAT;
+ TracingContext tracingContext = new TracingContext(clientCorrelationId,
+ fileSystemId, FSOperationType.CREATE_FILESYSTEM, tracingHeaderFormat,
new TracingHeaderValidator(
+ fs.getAbfsStore().getAbfsConfiguration().getClientCorrelationId(),
+ fs.getFileSystemId(), FSOperationType.CREATE_FILESYSTEM, false,
+ 0));
+ tracingContext.setPrimaryRequestID();
+ AbfsHttpOperation abfsHttpOperation =
Mockito.mock(AbfsHttpOperation.class);
+
Mockito.doNothing().when(abfsHttpOperation).setRequestProperty(Mockito.anyString(),
Mockito.anyString());
+
+ tracingContext.constructHeader(abfsHttpOperation, null, null);
+ checkHeaderForRetryPolicyAbbreviation(tracingContext.getHeader(), null,
null);
+
+ tracingContext.constructHeader(abfsHttpOperation, null,
STATIC_RETRY_POLICY_ABBREVIATION);
+ checkHeaderForRetryPolicyAbbreviation(tracingContext.getHeader(), null,
null);
+
+ tracingContext.constructHeader(abfsHttpOperation, null,
EXPONENTIAL_RETRY_POLICY_ABBREVIATION);
+ checkHeaderForRetryPolicyAbbreviation(tracingContext.getHeader(), null,
null);
+
+ tracingContext.constructHeader(abfsHttpOperation,
CONNECTION_TIMEOUT_ABBREVIATION, null);
+ checkHeaderForRetryPolicyAbbreviation(tracingContext.getHeader(),
CONNECTION_TIMEOUT_ABBREVIATION, null);
+
+ tracingContext.constructHeader(abfsHttpOperation,
CONNECTION_TIMEOUT_ABBREVIATION, STATIC_RETRY_POLICY_ABBREVIATION);
+ checkHeaderForRetryPolicyAbbreviation(tracingContext.getHeader(),
CONNECTION_TIMEOUT_ABBREVIATION, STATIC_RETRY_POLICY_ABBREVIATION);
+
+ tracingContext.constructHeader(abfsHttpOperation,
CONNECTION_TIMEOUT_ABBREVIATION, EXPONENTIAL_RETRY_POLICY_ABBREVIATION);
+ checkHeaderForRetryPolicyAbbreviation(tracingContext.getHeader(),
CONNECTION_TIMEOUT_ABBREVIATION, EXPONENTIAL_RETRY_POLICY_ABBREVIATION);
+
+ tracingContext.constructHeader(abfsHttpOperation, "503", null);
+ checkHeaderForRetryPolicyAbbreviation(tracingContext.getHeader(), "503",
null);
+
+ tracingContext.constructHeader(abfsHttpOperation, "503",
STATIC_RETRY_POLICY_ABBREVIATION);
+ checkHeaderForRetryPolicyAbbreviation(tracingContext.getHeader(), "503",
null);
+
+ tracingContext.constructHeader(abfsHttpOperation, "503",
EXPONENTIAL_RETRY_POLICY_ABBREVIATION);
+ checkHeaderForRetryPolicyAbbreviation(tracingContext.getHeader(), "503",
null);
+ }
+
+ private void checkHeaderForRetryPolicyAbbreviation(String header, String
expectedFailureReason, String expectedRetryPolicyAbbreviation) {
+ String headerContents[] = header.split(":");
+ String previousReqContext = headerContents[6];
+ if (expectedFailureReason != null) {
+ Assertions.assertThat(previousReqContext.split("_")[1])
+ .isEqualTo(expectedFailureReason);
+ Assertions.assertThat(previousReqContext.split("_").length)
+ .isGreaterThanOrEqualTo(2);
+ } else {
+ Assertions.assertThat(previousReqContext.split("_").length)
+ .isEqualTo(1);
Review Comment:
assertThat(previousReqContext.split("_")).hasSizeEqualTo(1)
##########
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java:
##########
@@ -91,7 +92,8 @@ public class AbfsClient implements Closeable {
private final URL baseUrl;
private final SharedKeyCredentials sharedKeyCredentials;
private final String xMsVersion = "2019-12-12";
- private final ExponentialRetryPolicy retryPolicy;
+ private final ExponentialRetryPolicy exponentialRetryPolicy;
Review Comment:
i think these should both be of the type of the baseclass, and rather than
name exponential static, be `apiRetryPolicy` and `connectivityRetryPolicy`.
that way the policy can be changed without break all field/accessor method
names.
##########
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestStaticRetryPolicy.java:
##########
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.services;
+
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
+import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
+
+import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_MAX_IO_RETRIES;
+import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_STATIC_RETRY_FOR_CONNECTION_TIMEOUT_ENABLED;
+import static
org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_STATIC_RETRY_INTERVAL;
+import static
org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_RESET_ABBREVIATION;
+import static
org.apache.hadoop.fs.azurebfs.services.RetryReasonConstants.CONNECTION_TIMEOUT_ABBREVIATION;
+
+/**
+ * Class to test the behavior of Static Retry policy as well the inheritance
+ * between {@link RetryPolicy}, {@link ExponentialRetryPolicy}, {@link
StaticRetryPolicy}
+ */
+public class TestStaticRetryPolicy extends AbstractAbfsIntegrationTest {
Review Comment:
This is an integration test which only works with credentials.
is it possible to make a unit test? If not, change the classname to start
with ITest
##########
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java:
##########
@@ -217,8 +220,24 @@ protected AbfsPerfTracker getAbfsPerfTracker() {
return abfsPerfTracker;
}
- ExponentialRetryPolicy getRetryPolicy() {
- return retryPolicy;
+ ExponentialRetryPolicy getExponentialRetryPolicy() {
+ return exponentialRetryPolicy;
+ }
+
+ StaticRetryPolicy getStaticRetryPolicy() {
+ return staticRetryPolicy;
+ }
+
+ public RetryPolicy getRetryPolicy(final String failureReason) {
Review Comment:
javadocs for this
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
