This is an automated email from the ASF dual-hosted git repository.
jtao pushed a commit to branch hotfix-upload-20250130
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/hotfix-upload-20250130 by this
push:
new 99051be29f safe copy in local pinot fs (#14883)
99051be29f is described below
commit 99051be29f4717afb92e13d65d0850698b9d72a0
Author: Jia Guo <jia...@linkedin.com>
AuthorDate: Thu Jan 23 16:06:17 2025 -0800
safe copy in local pinot fs (#14883)
* safe copy in local pinot fs
* make suffix a constant
* fix backup del
* add restoration
* format
* code style
* address comments
* address comments
* add special exception
* add special exception
* copy to tmp file and rename
* use a backup file and delete it
* refactoring
* refactoring
* refactoring
* remove crc
* remove crc
* refactor time stamp
---
.../apache/pinot/spi/filesystem/LocalPinotFS.java | 68 +++++++++++++++++-----
1 file changed, 54 insertions(+), 14 deletions(-)
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/LocalPinotFS.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/LocalPinotFS.java
index b82490b5e0..68ae7d013f 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/LocalPinotFS.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/filesystem/LocalPinotFS.java
@@ -29,8 +29,11 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.text.SimpleDateFormat;
import java.util.Arrays;
+import java.util.Date;
import java.util.List;
+import java.util.TimeZone;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.io.FileUtils;
@@ -43,6 +46,9 @@ import org.apache.pinot.spi.env.PinotConfiguration;
*/
public class LocalPinotFS extends BasePinotFS {
+ public static final String BACKUP = ".backup";
+ public static final String TMP = ".tmp";
+
@Override
public void init(PinotConfiguration configuration) {
}
@@ -127,15 +133,19 @@ public class LocalPinotFS extends BasePinotFS {
return Arrays.stream(file.list()).map(s -> getFileMetadata(new
File(file, s))).collect(Collectors.toList());
} else {
try (Stream<Path> pathStream = Files.walk(Paths.get(fileUri))) {
- return pathStream.filter(s -> !s.equals(file.toPath())).map(p ->
getFileMetadata(p.toFile()))
+ return pathStream.filter(s -> !s.equals(file.toPath()))
+ .map(p -> getFileMetadata(p.toFile()))
.collect(Collectors.toList());
}
}
}
private static FileMetadata getFileMetadata(File file) {
- return new
FileMetadata.Builder().setFilePath(file.getAbsolutePath()).setLastModifiedTime(file.lastModified())
- .setLength(file.length()).setIsDirectory(file.isDirectory()).build();
+ return new FileMetadata.Builder().setFilePath(file.getAbsolutePath())
+ .setLastModifiedTime(file.lastModified())
+ .setLength(file.length())
+ .setIsDirectory(file.isDirectory())
+ .build();
}
@Override
@@ -193,19 +203,49 @@ public class LocalPinotFS extends BasePinotFS {
private static void copy(File srcFile, File dstFile, boolean recursive)
throws IOException {
- if (dstFile.exists()) {
- FileUtils.deleteQuietly(dstFile);
- }
- if (srcFile.isDirectory()) {
- if (recursive) {
- FileUtils.copyDirectory(srcFile, dstFile);
+
+ // Create a temporary file in the same directory as dstFile
+ File tmpFile = new File(dstFile.getParent(), dstFile.getName() + TMP);
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss.SSS");
+ sdf.setTimeZone(TimeZone.getTimeZone("UTC")); // Set time zone to UTC
+ File backupFile = new File(
+ dstFile.getParent(),
+ dstFile.getName() + BACKUP + "_" + sdf.format(new Date())
+ );
+
+ try {
+ // Step 1: Copy the file or directory into the temporary file
+ if (srcFile.isDirectory()) {
+ if (recursive) {
+ FileUtils.copyDirectory(srcFile, tmpFile);
+ } else {
+ throw new IOException(
+ srcFile.getAbsolutePath() + " is a directory and recursive copy
is not enabled.");
+ }
} else {
- // Throws Exception on failure
- throw new IOException(srcFile.getAbsolutePath() + " is a directory and
recursive copy is not enabled.");
+ FileUtils.copyFile(srcFile, tmpFile);
}
- } else {
- // Will create parent directories, throws Exception on failure
- FileUtils.copyFile(srcFile, dstFile);
+
+ if (dstFile.exists() && !dstFile.renameTo(backupFile)) {
+ throw new IOException("Failed to rename destination file to backup
file.");
+ }
+
+ // Step 2: Rename the temporary file to the destination file
+ if (!tmpFile.renameTo(dstFile)) {
+ throw new IOException("Failed to rename temporary file to destination
file.");
+ }
+ FileUtils.deleteQuietly(backupFile);
+ } catch (Exception e) {
+ // Cleanup the temporary file in case of errors
+ // intentionally not deleting the backup file if anything goes wrong
+ FileUtils.deleteQuietly(tmpFile);
+ throw new LocalPinotFSException(e);
+ }
+ }
+
+ public static class LocalPinotFSException extends IOException {
+ LocalPinotFSException(Throwable e) {
+ super(e);
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
