This is an automated email from the ASF dual-hosted git repository.
sshenoy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new d5df7e43c7 HDDS-12288. Improve bootstrap logging to indicate progress
of snapshot download. (#7861)
d5df7e43c7 is described below
commit d5df7e43c794a577656bf4173346eab1cba0cc8c
Author: Gargi Jaiswal <[email protected]>
AuthorDate: Mon Feb 24 10:32:00 2025 +0530
HDDS-12288. Improve bootstrap logging to indicate progress of snapshot
download. (#7861)
---
.../om/ratis_snapshot/OmRatisSnapshotProvider.java | 35 ++++++++++++++++++++--
1 file changed, 32 insertions(+), 3 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java
index 0189f2dc7f..38724a5b75 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java
@@ -32,6 +32,7 @@
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.io.FileOutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.List;
@@ -139,8 +140,8 @@ public void downloadSnapshot(String leaderNodeID, File
targetFile)
OMNodeDetails leader = peerNodesMap.get(leaderNodeID);
URL omCheckpointUrl = leader.getOMDBCheckpointEndpointUrl(
httpPolicy.isHttpEnabled(), true);
- LOG.info("Downloading latest checkpoint from Leader OM {}. Checkpoint " +
- "URL: {}", leaderNodeID, omCheckpointUrl);
+ LOG.info("Downloading latest checkpoint from Leader OM {}. Checkpoint: {}
URL: {}",
+ leaderNodeID, targetFile.getName(), omCheckpointUrl);
SecurityUtil.doAsCurrentUser(() -> {
HttpURLConnection connection = (HttpURLConnection)
connectionFactory.openConnection(omCheckpointUrl, spnegoEnabled);
@@ -162,7 +163,7 @@ public void downloadSnapshot(String leaderNodeID, File
targetFile)
}
try (InputStream inputStream = connection.getInputStream()) {
- FileUtils.copyInputStreamToFile(inputStream, targetFile);
+ downloadFileWithProgress(inputStream, targetFile);
} catch (IOException ex) {
boolean deleted = FileUtils.deleteQuietly(targetFile);
if (!deleted) {
@@ -177,6 +178,34 @@ public void downloadSnapshot(String leaderNodeID, File
targetFile)
});
}
+ /**
+ * Writes data from the given InputStream to the target file while logging
download progress every 30 seconds.
+ */
+ public static void downloadFileWithProgress(InputStream inputStream, File
targetFile)
+ throws IOException {
+ try (FileOutputStream outputStream = new FileOutputStream(targetFile)) {
+ byte[] buffer = new byte[8 * 1024];
+ long totalBytesRead = 0;
+ int bytesRead;
+ long lastLoggedTime = System.currentTimeMillis();
+
+ while ((bytesRead = inputStream.read(buffer)) != -1) {
+ outputStream.write(buffer, 0, bytesRead);
+ totalBytesRead += bytesRead;
+
+ // Log progress every 30 seconds
+ if (System.currentTimeMillis() - lastLoggedTime >= 30000) {
+ LOG.info("Downloading '{}': {} KB downloaded so far...",
+ targetFile.getName(), totalBytesRead / (1024));
+ lastLoggedTime = System.currentTimeMillis();
+ }
+ }
+
+ LOG.info("Download completed for '{}'. Total size: {} KB",
+ targetFile.getName(), totalBytesRead / (1024));
+ }
+ }
+
/**
* Writes form data to output stream as any HTTP client would for a
* multipart/form-data request.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
