Updated Branches: refs/heads/1.6.0-SNAPSHOT 8f5dc6107 -> 211ca036c
ACCUMULO-1409 Make Mini support loading native maps Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/211ca036 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/211ca036 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/211ca036 Branch: refs/heads/1.6.0-SNAPSHOT Commit: 211ca036c9cbf2c9aec2a13496dc27db758a19db Parents: 8f5dc61 Author: Christopher Tubbs <ctubb...@apache.org> Authored: Wed Nov 20 17:53:45 2013 -0500 Committer: Christopher Tubbs <ctubb...@apache.org> Committed: Wed Nov 20 17:53:45 2013 -0500 ---------------------------------------------------------------------- .../minicluster/MiniAccumuloCluster.java | 29 ++++++++------------ .../minicluster/MiniAccumuloConfig.java | 22 +++++++++++++++ .../test/functional/ConfigurableMacIT.java | 3 ++ .../accumulo/test/functional/NativeMapIT.java | 10 +++++-- 4 files changed, 43 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/211ca036/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java ---------------------------------------------------------------------- diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java index 8e7e2ba..0a50747 100644 --- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java +++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java @@ -53,6 +53,7 @@ import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.master.thrift.MasterGoalState; import org.apache.accumulo.core.util.Daemon; import org.apache.accumulo.core.util.Pair; +import org.apache.accumulo.core.util.StringUtil; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.gc.SimpleGarbageCollector; import org.apache.accumulo.master.Master; @@ -61,10 +62,9 @@ import org.apache.accumulo.server.init.Initialize; import org.apache.accumulo.server.util.PortUtils; import org.apache.accumulo.server.util.time.SimpleTimer; import org.apache.accumulo.start.Main; -import org.apache.commons.configuration.MapConfiguration; import org.apache.accumulo.start.classloader.vfs.MiniDFSUtil; import org.apache.accumulo.tserver.TabletServer; -import org.apache.commons.io.FileUtils; +import org.apache.commons.configuration.MapConfiguration; import org.apache.commons.vfs2.FileObject; import org.apache.commons.vfs2.impl.VFSClassLoader; import org.apache.hadoop.conf.Configuration; @@ -235,7 +235,6 @@ public class MiniAccumuloCluster { ArrayList<String> argList = new ArrayList<String>(); argList.addAll(Arrays.asList(javaBin, "-Dproc=" + clazz.getSimpleName(), "-cp", classpath)); - argList.add("-Djava.library.path=" + config.getLibDir()); argList.addAll(extraJvmOpts); for (Entry<String,String> sysProp : config.getSystemProperties().entrySet()) { argList.add(String.format("-D%s=%s", sysProp.getKey(), sysProp.getValue())); @@ -248,6 +247,9 @@ public class MiniAccumuloCluster { builder.environment().put("ACCUMULO_HOME", config.getDir().getAbsolutePath()); builder.environment().put("ACCUMULO_LOG_DIR", config.getLogDir().getAbsolutePath()); builder.environment().put("ACCUMULO_CLIENT_CONF_PATH", config.getClientConfFile().getAbsolutePath()); + String ldLibraryPath = StringUtil.join(Arrays.asList(config.getNativeLibPaths()), File.pathSeparator); + builder.environment().put("LD_LIBRARY_PATH", ldLibraryPath); + builder.environment().put("DYLD_LIBRARY_PATH", ldLibraryPath); // if we're running under accumulo.start, we forward these env vars String env = System.getenv("HADOOP_PREFIX"); @@ -353,6 +355,7 @@ public class MiniAccumuloCluster { File clientConfFile = config.getClientConfFile(); // Write only the properties that correspond to ClientConfiguration properties writeConfigProperties(clientConfFile, Maps.filterEntries(config.getSiteConfig(), new Predicate<Entry<String,String>>() { + @Override public boolean apply(Entry<String,String> v) { return ClientConfiguration.ClientProperty.getPropertyByKey(v.getKey()) != null; } @@ -375,18 +378,6 @@ public class MiniAccumuloCluster { zooCfg.store(fileWriter, null); fileWriter.close(); - - File nativeMap = new File(config.getLibDir().getAbsolutePath() + "/native/map"); - nativeMap.mkdirs(); - File testRoot = new File(new File(new File(System.getProperty("user.dir")).getParent() + "/server/src/main/c++/nativeMap").getAbsolutePath()); - - if (testRoot.exists()) { - for (String file : testRoot.list()) { - File src = new File(testRoot, file); - if (src.isFile() && file.startsWith("libNativeMap")) - FileUtils.copyFile(src, new File(nativeMap, file)); - } - } } private void writeConfig(File file, Iterable<Map.Entry<String,String>> settings) throws IOException { @@ -400,6 +391,7 @@ public class MiniAccumuloCluster { fileWriter.append("</configuration>\n"); fileWriter.close(); } + private void writeConfigProperties(File file, Map<String,String> settings) throws IOException { FileWriter fileWriter = new FileWriter(file); @@ -439,7 +431,7 @@ public class MiniAccumuloCluster { if (!initialized) { // sleep a little bit to let zookeeper come up before calling init, seems to work better long startTime = System.currentTimeMillis(); - while (true) { + while (true) { try { Socket s = new Socket("localhost", config.getZooKeeperPort()); s.getOutputStream().write("ruok\n".getBytes()); @@ -449,7 +441,7 @@ public class MiniAccumuloCluster { if (n == 4 && new String(buffer, 0, n).equals("imok")) break; } catch (Exception e) { - if(System.currentTimeMillis() - startTime >= 10000) { + if (System.currentTimeMillis() - startTime >= 10000) { throw new RuntimeException("Zookeeper did not start within 10 seconds . Check the logs in " + config.getLogDir() + " for errors."); } UtilWaitThread.sleep(250); @@ -626,7 +618,8 @@ public class MiniAccumuloCluster { } public ClientConfiguration getClientConfig() { - return new ClientConfiguration(Arrays.asList(new MapConfiguration(config.getSiteConfig()))).withInstance(this.getInstanceName()).withZkHosts(this.getZooKeepers()); + return new ClientConfiguration(Arrays.asList(new MapConfiguration(config.getSiteConfig()))).withInstance(this.getInstanceName()).withZkHosts( + this.getZooKeepers()); } public FileSystem getFileSystem() { http://git-wip-us.apache.org/repos/asf/accumulo/blob/211ca036/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java ---------------------------------------------------------------------- diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java index 1af2a1e..2326f95 100644 --- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java +++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java @@ -61,6 +61,8 @@ public class MiniAccumuloConfig { private String[] classpathItems = null; + private String[] nativePathItems = null; + /** * @param dir * An empty or nonexistant directory that Accumulo and Zookeeper can store data in. Creating the directory is left to the user. Java 7, Guava, and @@ -425,6 +427,26 @@ public class MiniAccumuloConfig { } /** + * @return the paths to use for loading native libraries + * + * @since 1.6.0 + */ + public String[] getNativeLibPaths() { + return this.nativePathItems == null ? new String[0] : this.nativePathItems; + } + + /** + * Sets the path for processes to use for loading native libraries + * + * @param nativePathItems + * the nativePathItems to set + * @since 1.6.0 + */ + public void setNativeLibPaths(String... nativePathItems) { + this.nativePathItems = nativePathItems; + } + + /** * Sets arbitrary configuration properties. * * @since 1.6.0 http://git-wip-us.apache.org/repos/asf/accumulo/blob/211ca036/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java b/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java index 21c2bb7..54585fe 100644 --- a/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java @@ -23,6 +23,7 @@ import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.client.ZooKeeperInstance; +import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.util.MonitorUtil; import org.apache.accumulo.minicluster.MiniAccumuloCluster; import org.apache.accumulo.minicluster.MiniAccumuloConfig; @@ -41,7 +42,9 @@ public class ConfigurableMacIT extends AbstractMacIT { @Before public void setUp() throws Exception { MiniAccumuloConfig cfg = new MiniAccumuloConfig(createTestDir(this.getClass().getName()), ROOT_PASSWORD); + cfg.setNativeLibPaths(NativeMapIT.nativeMapLocation().getAbsolutePath()); configure(cfg); + cfg.setProperty(Property.TSERV_NATIVEMAP_ENABLED, Boolean.TRUE.toString()); configureForEnvironment(cfg, getClass(), createSharedTestDir(this.getClass().getName() + "-ssl")); cluster = new MiniAccumuloCluster(cfg); cluster.start(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/211ca036/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java b/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java index 408d32c..5a79286 100644 --- a/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/NativeMapIT.java @@ -62,11 +62,15 @@ public class NativeMapIT { return new Value(String.format("r%09d", v).getBytes()); } - @BeforeClass - public static void setUp() { + public static File nativeMapLocation() { File projectDir = new File(System.getProperty("user.dir")).getParentFile(); File nativeMapDir = new File(projectDir, "server/native/target/accumulo-native-" + Constants.VERSION + "/accumulo-native-" + Constants.VERSION); - NativeMap.loadNativeLib(Collections.singletonList(nativeMapDir)); + return nativeMapDir; + } + + @BeforeClass + public static void setUp() { + NativeMap.loadNativeLib(Collections.singletonList(nativeMapLocation())); } private void verifyIterator(int start, int end, int valueOffset, Iterator<Entry<Key,Value>> iter) {
