jugomezv commented on code in PR #10777:
URL: https://github.com/apache/pinot/pull/10777#discussion_r1204820751
##########
pinot-common/src/main/java/org/apache/pinot/common/utils/fetcher/HttpSegmentFetcher.java:
##########
@@ -97,54 +100,67 @@ public void fetchSegmentToLocal(URI downloadURI, File dest)
}
@Override
- public File fetchUntarSegmentToLocalStreamed(URI downloadURI, File dest,
long maxStreamRateInByte)
+ public File fetchUntarSegmentToLocalStreamed(URI downloadURI, File dest,
long maxStreamRateInByte,
+ AtomicInteger attempts)
throws Exception {
- // Create a RoundRobinURIProvider to round robin IP addresses when retry
uploading. Otherwise may always try to
+ // Create a RoundRobinURIProvider to round robin IP addresses when retry
uploading. Otherwise, may always try to
// download from a same broken host as: 1) DNS may not RR the IP addresses
2) OS cache the DNS resolution result.
RoundRobinURIProvider uriProvider = new RoundRobinURIProvider(downloadURI);
int retryCount = Math.max(_retryCount, uriProvider.numAddresses());
AtomicReference<File> ret = new AtomicReference<>(); // return the untared
segment directory
_logger.info("Retry downloading for {} times. retryCount from pinot server
config: {}, number of IP addresses for "
+ "download URI: {}", retryCount, _retryCount,
uriProvider.numAddresses());
- RetryPolicies.exponentialBackoffRetryPolicy(retryCount, _retryWaitMs,
_retryDelayScaleFactor).attempt(() -> {
- URI uri = uriProvider.next();
- try {
- String hostName = downloadURI.getHost();
- int port = downloadURI.getPort();
- // If the original download address is specified as host name, need
add a "HOST" HTTP header to the HTTP
- // request. Otherwise, if the download address is a LB address, when
the LB be configured as "disallow direct
- // access by IP address", downloading will fail.
- List<Header> httpHeaders = new LinkedList<>();
- if (!InetAddresses.isInetAddress(hostName)) {
- httpHeaders.add(new BasicHeader(HttpHeaders.HOST, hostName + ":" +
port));
- }
- ret.set(_httpClient.downloadUntarFileStreamed(uri, dest,
_authProvider, httpHeaders, maxStreamRateInByte));
+ int tries = 0;
+ try {
+ tries = RetryPolicies.exponentialBackoffRetryPolicy(retryCount,
_retryWaitMs, _retryDelayScaleFactor).attempt(
+ () -> {
+ URI uri = uriProvider.next();
+ try {
+ String hostName = downloadURI.getHost();
+ int port = downloadURI.getPort();
+ // If the original download address is specified as host name, need
add a "HOST" HTTP header to the HTTP
+ // request. Otherwise, if the download address is a LB address, when
the LB be configured as "disallow direct
+ // access by IP address", downloading will fail.
+ List<Header> httpHeaders = new LinkedList<>();
+ if (!InetAddresses.isInetAddress(hostName)) {
+ httpHeaders.add(new BasicHeader(HttpHeaders.HOST, hostName + ":" +
port));
+ }
+ ret.set(_httpClient.downloadUntarFileStreamed(uri, dest,
_authProvider, httpHeaders, maxStreamRateInByte));
- return true;
- } catch (HttpErrorStatusException e) {
- int statusCode = e.getStatusCode();
- if (statusCode == HttpStatus.SC_NOT_FOUND || statusCode >= 500) {
- // Temporary exception
- // 404 is treated as a temporary exception, as the downloadURI may
be backed by multiple hosts,
- // if singe host is down, can retry with another host.
- _logger.warn("Got temporary error status code: {} while downloading
segment from: {} to: {}", statusCode, uri,
+ return true;
+ } catch (HttpErrorStatusException e) {
+ int statusCode = e.getStatusCode();
+ if (statusCode == HttpStatus.SC_NOT_FOUND || statusCode >= 500) {
+ // Temporary exception
+ // 404 is treated as a temporary exception, as the downloadURI may
be backed by multiple hosts,
+ // if singe host is down, can retry with another host.
+ _logger.warn("Got temporary error status code: {} while
downloading segment from: {} to: {}", statusCode,
+ uri,
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
