This is an automated email from the ASF dual-hosted git repository.
swamirishi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 9b261560d6 HDDS-11811. rocksdbjni deleted on exit could be used by
other components #7493
9b261560d6 is described below
commit 9b261560d651a39875026da6bbf5364b7dbd50a4
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Wed Nov 27 16:12:23 2024 +0100
HDDS-11811. rocksdbjni deleted on exit could be used by other components
#7493
---
.../hadoop/hdds/utils/NativeLibraryLoader.java | 29 ++++++++-----
.../hadoop/hdds/utils/TestNativeLibraryLoader.java | 50 ++++++++++++++++------
2 files changed, 57 insertions(+), 22 deletions(-)
diff --git
a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java
b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java
index d93933dee3..08e397d068 100644
---
a/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java
+++
b/hadoop-hdds/rocks-native/src/main/java/org/apache/hadoop/hdds/utils/NativeLibraryLoader.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdds.utils;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.ozone.util.ShutdownHookManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,6 +29,7 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
+import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.List;
@@ -170,31 +172,38 @@ public class NativeLibraryLoader {
getSystemProperty(NATIVE_LIB_TMP_DIR) : "";
final File dir = new File(nativeLibDir).getAbsoluteFile();
- // create a temporary file to copy the library to
- final File temp = File.createTempFile(libraryName, getLibOsSuffix(),
dir);
- if (!temp.exists()) {
+ // create a temporary dir to copy the library to
+ final Path tempPath = Files.createTempDirectory(dir.toPath(),
libraryName);
+ final File tempDir = tempPath.toFile();
+ if (!tempDir.exists()) {
return Pair.of(Optional.empty(), null);
- } else {
- temp.deleteOnExit();
}
- Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ Path libPath = tempPath.resolve(libraryFileName);
+ Files.copy(is, libPath, StandardCopyOption.REPLACE_EXISTING);
+ File libFile = libPath.toFile();
+ if (libFile.exists()) {
+ libFile.deleteOnExit();
+ }
+
List<File> dependentFiles = new ArrayList<>();
for (String fileName : dependentFileNames) {
if (is != null) {
is.close();
}
is = getResourceStream(fileName);
- File file = new File(dir, fileName);
- Files.copy(is, file.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ Path path = tempPath.resolve(fileName);
+ Files.copy(is, path, StandardCopyOption.REPLACE_EXISTING);
+ File file = path.toFile();
if (file.exists()) {
file.deleteOnExit();
}
dependentFiles.add(file);
}
- ShutdownHookManager.get().addShutdownHook(temp::delete,
+ ShutdownHookManager.get().addShutdownHook(
+ () -> FileUtil.fullyDelete(tempDir),
LIBRARY_SHUTDOWN_HOOK_PRIORITY);
- return Pair.of(Optional.of(temp), dependentFiles);
+ return Pair.of(Optional.of(libFile), dependentFiles);
} finally {
if (is != null) {
is.close();
diff --git
a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java
b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java
index f0074e0a1a..6e1622ebd7 100644
---
a/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java
+++
b/hadoop-hdds/rocks-native/src/test/java/org/apache/hadoop/hdds/utils/TestNativeLibraryLoader.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdds.utils;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.utils.db.managed.ManagedRawSSTFileReader;
import org.apache.ozone.test.tag.Native;
import org.junit.jupiter.api.io.TempDir;
@@ -28,15 +29,16 @@ import org.mockito.MockedStatic;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.nio.file.Path;
-import java.util.Collections;
+import java.util.Arrays;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import static
org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_LIBRARY_NAME;
import static
org.apache.hadoop.hdds.utils.NativeLibraryLoader.NATIVE_LIB_TMP_DIR;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static
org.apache.hadoop.hdds.utils.NativeLibraryLoader.getJniLibraryFileName;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.CALLS_REAL_METHODS;
import static org.mockito.Mockito.anyString;
@@ -68,21 +70,45 @@ public class TestNativeLibraryLoader {
mockedNativeLibraryLoader.when(() ->
NativeLibraryLoader.getInstance()).thenReturn(loader);
ManagedRawSSTFileReader.loadLibrary();
assertTrue(NativeLibraryLoader.isLibraryLoaded(ROCKS_TOOLS_NATIVE_LIBRARY_NAME));
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("nativeLibraryDirectoryLocations")
+ public void testDummyLibrary(String nativeLibraryDirectoryLocation) {
+ Map<String, Boolean> libraryLoadedMap = new HashMap<>();
+ NativeLibraryLoader loader = new NativeLibraryLoader(libraryLoadedMap);
+ try (MockedStatic<NativeLibraryLoader> mockedNativeLibraryLoader =
mockStatic(NativeLibraryLoader.class,
+ CALLS_REAL_METHODS)) {
+ mockedNativeLibraryLoader.when(() ->
NativeLibraryLoader.getSystemProperty(same(NATIVE_LIB_TMP_DIR)))
+ .thenReturn(nativeLibraryDirectoryLocation);
+
mockedNativeLibraryLoader.when(NativeLibraryLoader::getInstance).thenReturn(loader);
// Mocking to force copy random bytes to create a lib file to
// nativeLibraryDirectoryLocation. But load library will fail.
mockedNativeLibraryLoader.when(() ->
NativeLibraryLoader.getResourceStream(anyString()))
.thenReturn(new ByteArrayInputStream(new byte[]{0, 1, 2, 3}));
String dummyLibraryName = "dummy_lib";
- NativeLibraryLoader.getInstance().loadLibrary(dummyLibraryName,
Collections.emptyList());
- NativeLibraryLoader.isLibraryLoaded(dummyLibraryName);
+ List<String> dependencies = Arrays.asList("dep1", "dep2");
+ File absDir = new File(nativeLibraryDirectoryLocation == null ? "" :
nativeLibraryDirectoryLocation)
+ .getAbsoluteFile();
+
+ NativeLibraryLoader.getInstance().loadLibrary(dummyLibraryName,
dependencies);
+
// Checking if the resource with random was copied to a temp file.
- File[] libPath = new File(nativeLibraryDirectoryLocation == null ? "" :
nativeLibraryDirectoryLocation)
- .getAbsoluteFile().listFiles((dir, name) ->
name.startsWith(dummyLibraryName) &&
- name.endsWith(NativeLibraryLoader.getLibOsSuffix()));
- assertNotNull(libPath);
- assertEquals(1, libPath.length);
- assertTrue(libPath[0].delete());
+ File[] libPath = absDir
+ .listFiles((dir, name) -> name.startsWith(dummyLibraryName));
+ assertThat(libPath)
+ .isNotNull()
+ .isNotEmpty();
+ assertThat(libPath[0])
+ .isDirectory();
+ try {
+ assertThat(new File(libPath[0],
getJniLibraryFileName(dummyLibraryName)))
+ .isFile();
+ dependencies.forEach(dep -> assertThat(new File(libPath[0],
dep)).isFile());
+ } finally {
+ FileUtil.fullyDelete(libPath[0]);
+ }
}
-
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
