Repository: accumulo Updated Branches: refs/heads/master c2778b8af -> f159ec1fd
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java b/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java index 1c99f7a..2dfd726 100644 --- a/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java +++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java @@ -120,7 +120,7 @@ abstract public class TransformingIterator extends WrappingIterator implements O } if (options.containsKey(MAX_BUFFER_SIZE_OPT)) { - maxBufferSize = AccumuloConfiguration.getMemoryInBytes(options.get(MAX_BUFFER_SIZE_OPT)); + maxBufferSize = AccumuloConfiguration.getFixedMemoryAsBytes(options.get(MAX_BUFFER_SIZE_OPT)); } else { maxBufferSize = DEFAULT_MAX_BUFFER_SIZE; } @@ -150,7 +150,7 @@ abstract public class TransformingIterator extends WrappingIterator implements O if (option.getKey().equals(AUTH_OPT)) { new Authorizations(option.getValue().getBytes(UTF_8)); } else if (option.getKey().equals(MAX_BUFFER_SIZE_OPT)) { - AccumuloConfiguration.getMemoryInBytes(option.getValue()); + AccumuloConfiguration.getFixedMemoryAsBytes(option.getValue()); } } catch (Exception e) { throw new IllegalArgumentException("Failed to parse opt " + option.getKey() + " " + option.getValue(), e); http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java b/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java index 79db306..11270e3 100644 --- a/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java +++ b/core/src/main/java/org/apache/accumulo/core/security/crypto/CryptoModuleFactory.java @@ -260,7 +260,7 @@ public class CryptoModuleFactory { Map<String,String> cryptoOpts = conf.getAllPropertiesWithPrefix(Property.CRYPTO_PREFIX); cryptoOpts.putAll(conf.getAllPropertiesWithPrefix(Property.INSTANCE_PREFIX)); cryptoOpts.remove(Property.INSTANCE_SECRET.getKey()); - cryptoOpts.put(Property.CRYPTO_BLOCK_STREAM_SIZE.getKey(), Integer.toString((int) conf.getMemoryInBytes(Property.CRYPTO_BLOCK_STREAM_SIZE))); + cryptoOpts.put(Property.CRYPTO_BLOCK_STREAM_SIZE.getKey(), Integer.toString((int) conf.getAsBytes(Property.CRYPTO_BLOCK_STREAM_SIZE))); return fillParamsObjectFromStringMap(params, cryptoOpts); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/core/src/main/java/org/apache/accumulo/core/summary/SummaryWriter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/summary/SummaryWriter.java b/core/src/main/java/org/apache/accumulo/core/summary/SummaryWriter.java index 1ebeeae..f9aa054 100644 --- a/core/src/main/java/org/apache/accumulo/core/summary/SummaryWriter.java +++ b/core/src/main/java/org/apache/accumulo/core/summary/SummaryWriter.java @@ -151,7 +151,7 @@ public class SummaryWriter implements FileSKVWriter { factory = new SummarizerFactory(); } - long maxSize = tableConfig.getMemoryInBytes(Property.TABLE_FILE_SUMMARY_MAX_SIZE); + long maxSize = tableConfig.getAsBytes(Property.TABLE_FILE_SUMMARY_MAX_SIZE); return new SummaryWriter(writer, factory, configs, maxSize); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/core/src/main/java/org/apache/accumulo/core/util/Merge.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/util/Merge.java b/core/src/main/java/org/apache/accumulo/core/util/Merge.java index 1af5e37..f15da96 100644 --- a/core/src/main/java/org/apache/accumulo/core/util/Merge.java +++ b/core/src/main/java/org/apache/accumulo/core/util/Merge.java @@ -90,7 +90,7 @@ public class Merge { } if (opts.goalSize == null || opts.goalSize < 1) { AccumuloConfiguration tableConfig = new ConfigurationCopy(conn.tableOperations().getProperties(opts.getTableName())); - opts.goalSize = tableConfig.getMemoryInBytes(Property.TABLE_SPLIT_THRESHOLD); + opts.goalSize = tableConfig.getAsBytes(Property.TABLE_SPLIT_THRESHOLD); } message("Merging tablets in table %s to %d bytes", opts.getTableName(), opts.goalSize); http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java index 367c833..06148f1 100644 --- a/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/mapred/AccumuloFileOutputFormatTest.java @@ -64,9 +64,9 @@ public class AccumuloFileOutputFormatTest { AccumuloConfiguration acuconf = FileOutputConfigurator.getAccumuloConfiguration(AccumuloFileOutputFormat.class, job); assertEquals(7, acuconf.getCount(Property.TABLE_FILE_REPLICATION)); - assertEquals(300l, acuconf.getMemoryInBytes(Property.TABLE_FILE_BLOCK_SIZE)); - assertEquals(50l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE)); - assertEquals(10l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX)); + assertEquals(300l, acuconf.getAsBytes(Property.TABLE_FILE_BLOCK_SIZE)); + assertEquals(50l, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE)); + assertEquals(10l, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX)); assertEquals("snappy", acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE)); assertEquals(new SamplerConfigurationImpl(samplerConfig), SamplerConfigurationImpl.newSamplerConfig(acuconf)); @@ -95,9 +95,9 @@ public class AccumuloFileOutputFormatTest { acuconf = FileOutputConfigurator.getAccumuloConfiguration(AccumuloFileOutputFormat.class, job); assertEquals(17, acuconf.getCount(Property.TABLE_FILE_REPLICATION)); - assertEquals(1300l, acuconf.getMemoryInBytes(Property.TABLE_FILE_BLOCK_SIZE)); - assertEquals(150l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE)); - assertEquals(110l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX)); + assertEquals(1300l, acuconf.getAsBytes(Property.TABLE_FILE_BLOCK_SIZE)); + assertEquals(150l, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE)); + assertEquals(110l, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX)); assertEquals("lzo", acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE)); assertEquals(new SamplerConfigurationImpl(samplerConfig), SamplerConfigurationImpl.newSamplerConfig(acuconf)); http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java index 5143453..a73e501 100644 --- a/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/mapreduce/AccumuloFileOutputFormatTest.java @@ -64,9 +64,9 @@ public class AccumuloFileOutputFormatTest { AccumuloConfiguration acuconf = FileOutputConfigurator.getAccumuloConfiguration(AccumuloFileOutputFormat.class, job1.getConfiguration()); assertEquals(7, acuconf.getCount(Property.TABLE_FILE_REPLICATION)); - assertEquals(300l, acuconf.getMemoryInBytes(Property.TABLE_FILE_BLOCK_SIZE)); - assertEquals(50l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE)); - assertEquals(10l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX)); + assertEquals(300l, acuconf.getAsBytes(Property.TABLE_FILE_BLOCK_SIZE)); + assertEquals(50l, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE)); + assertEquals(10l, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX)); assertEquals("snappy", acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE)); assertEquals(new SamplerConfigurationImpl(samplerConfig), SamplerConfigurationImpl.newSamplerConfig(acuconf)); @@ -95,9 +95,9 @@ public class AccumuloFileOutputFormatTest { acuconf = FileOutputConfigurator.getAccumuloConfiguration(AccumuloFileOutputFormat.class, job2.getConfiguration()); assertEquals(17, acuconf.getCount(Property.TABLE_FILE_REPLICATION)); - assertEquals(1300l, acuconf.getMemoryInBytes(Property.TABLE_FILE_BLOCK_SIZE)); - assertEquals(150l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE)); - assertEquals(110l, acuconf.getMemoryInBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX)); + assertEquals(1300l, acuconf.getAsBytes(Property.TABLE_FILE_BLOCK_SIZE)); + assertEquals(150l, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE)); + assertEquals(110l, acuconf.getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX)); assertEquals("lzo", acuconf.get(Property.TABLE_FILE_COMPRESSION_TYPE)); assertEquals(new SamplerConfigurationImpl(samplerConfig), SamplerConfigurationImpl.newSamplerConfig(acuconf)); http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/core/src/test/java/org/apache/accumulo/core/conf/AccumuloConfigurationTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/conf/AccumuloConfigurationTest.java b/core/src/test/java/org/apache/accumulo/core/conf/AccumuloConfigurationTest.java index f12ba63..0514a82 100644 --- a/core/src/test/java/org/apache/accumulo/core/conf/AccumuloConfigurationTest.java +++ b/core/src/test/java/org/apache/accumulo/core/conf/AccumuloConfigurationTest.java @@ -19,22 +19,31 @@ package org.apache.accumulo.core.conf; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; + import org.junit.Test; public class AccumuloConfigurationTest { @Test public void testGetMemoryInBytes() throws Exception { - assertEquals(42l, AccumuloConfiguration.getMemoryInBytes("42")); - assertEquals(42l, AccumuloConfiguration.getMemoryInBytes("42b")); - assertEquals(42l, AccumuloConfiguration.getMemoryInBytes("42B")); - assertEquals(42l * 1024l, AccumuloConfiguration.getMemoryInBytes("42K")); - assertEquals(42l * 1024l, AccumuloConfiguration.getMemoryInBytes("42k")); - assertEquals(42l * 1024l * 1024l, AccumuloConfiguration.getMemoryInBytes("42M")); - assertEquals(42l * 1024l * 1024l, AccumuloConfiguration.getMemoryInBytes("42m")); - assertEquals(42l * 1024l * 1024l * 1024l, AccumuloConfiguration.getMemoryInBytes("42G")); - assertEquals(42l * 1024l * 1024l * 1024l, AccumuloConfiguration.getMemoryInBytes("42g")); - + List<Function<String,Long>> funcs = Arrays.asList(AccumuloConfiguration::getFixedMemoryAsBytes, + AccumuloConfiguration::getMemoryAsBytes); + for(Function<String,Long> memFunc : funcs) { + assertEquals(42l, memFunc.apply("42").longValue()); + assertEquals(42l, memFunc.apply("42b").longValue()); + assertEquals(42l, memFunc.apply("42B").longValue()); + assertEquals(42l * 1024l, memFunc.apply("42K").longValue()); + assertEquals(42l * 1024l, memFunc.apply("42k").longValue()); + assertEquals(42l * 1024l * 1024l, memFunc.apply("42M").longValue()); + assertEquals(42l * 1024l * 1024l, memFunc.apply("42m").longValue()); + assertEquals(42l * 1024l * 1024l * 1024l, memFunc.apply("42G").longValue()); + assertEquals(42l * 1024l * 1024l * 1024l, memFunc.apply("42g").longValue()); + } + assertEquals(Runtime.getRuntime().maxMemory() / 10, AccumuloConfiguration.getMemoryAsBytes("10%")); + assertEquals(Runtime.getRuntime().maxMemory() / 5, AccumuloConfiguration.getMemoryAsBytes("20%")); } @Test @@ -48,13 +57,28 @@ public class AccumuloConfigurationTest { } @Test(expected = IllegalArgumentException.class) - public void testGetMemoryInBytesFailureCases1() throws Exception { - AccumuloConfiguration.getMemoryInBytes("42x"); + public void testGetFixedMemoryAsBytesFailureCases1() throws Exception { + AccumuloConfiguration.getFixedMemoryAsBytes("42x"); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetFixedMemoryAsBytesFailureCases2() throws Exception { + AccumuloConfiguration.getFixedMemoryAsBytes("FooBar"); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetFixedMemoryAsBytesFailureCases3() throws Exception { + AccumuloConfiguration.getFixedMemoryAsBytes("40%"); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetMemoryAsBytesFailureCases1() throws Exception { + AccumuloConfiguration.getMemoryAsBytes("42x"); } @Test(expected = IllegalArgumentException.class) - public void testGetMemoryInBytesFailureCases2() throws Exception { - AccumuloConfiguration.getMemoryInBytes("FooBar"); + public void testGetMemoryAsBytesFailureCases2() throws Exception { + AccumuloConfiguration.getMemoryAsBytes("FooBar"); } @Test http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java b/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java index 8a01eb8..7a4188c 100644 --- a/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java +++ b/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java @@ -143,8 +143,14 @@ public class PropertyTypeTest { } @Test - public void testTypeMEMORY() { + public void testTypeBYTES() { valid(null, "1024", "20B", "100K", "1500M", "2G"); + invalid("1M500K", "1M 2K", "1MB", "1.5G", "1,024K", "", "a", "10%"); + } + + @Test + public void testTypeMEMORY() { + valid(null, "1024", "20B", "100K", "1500M", "2G", "10%"); invalid("1M500K", "1M 2K", "1MB", "1.5G", "1,024K", "", "a"); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/docs/src/main/asciidoc/chapters/administration.txt ---------------------------------------------------------------------- diff --git a/docs/src/main/asciidoc/chapters/administration.txt b/docs/src/main/asciidoc/chapters/administration.txt index 7091fa1..e780ce4 100644 --- a/docs/src/main/asciidoc/chapters/administration.txt +++ b/docs/src/main/asciidoc/chapters/administration.txt @@ -104,8 +104,7 @@ installed Accumulo using downstream packaging, the +conf/+ could be something el +/etc/accumulo/+. Before starting Accumulo, the configuration files +accumulo-env.sh+ and +accumulo-site.xml+ must -exist in +conf/+ and be properly configured. You can either edit the files in +conf/+ or run -+accumulo-util create-config+ to generate these files. If you are using +accumulo-cluster+ to launch +exist in +conf/+ and be properly configured. If you are using +accumulo-cluster+ to launch a cluster, the `conf/` directory must also contain hosts file for Accumulo services (i.e +gc+, +masters+, +monitor+, +tservers+, +tracers+). You can either create these files manually or run +accumulo-cluster create-config+. @@ -125,6 +124,12 @@ and specify the following: . Enter the location of ZooKeeper for +$ZOOKEEPER_HOME+ . Optionally, choose a different location for Accumulo logs using +$ACCUMULO_LOG_DIR+ +Accumulo uses +HADOOP_PREFIX+ and +ZOOKEEPER_HOME+ to locate Hadoop and Zookeeper jars +and add them the +CLASSPATH+ variable. If you are running a vendor-specific release of Hadoop +or Zookeeper, you may need to change how your +CLASSPATH+ is built in +accumulo-env.sh+. If +Accumulo has problems later on finding jars, run +accumulo classpath -d+ to debug and print +Accumulo's classpath. + You may want to change the default memory settings for Accumulo's TabletServer which are by set in the +JAVA_OPTS+ settings for 'tservers' in +accumulo-env.sh+. Note the syntax is that of the Java JVM command line options. This value should be less than the @@ -244,18 +249,6 @@ Some settings can be modified via the Accumulo shell and take effect immediately some settings require a process restart to take effect. See the configuration documentation (available in the docs directory of the tarball and in <<configuration>>) for details. -One aspect of Accumulo's configuration which is different as compared to the rest of the Hadoop -ecosystem is that the server-process classpath is determined in part by multiple values. A -bootstrap classpath is based soley on the `accumulo-start.jar`, Log4j and `/path/to/accumulo/conf`. - -A second classloader is used to dynamically load all of the resources specified by `general.classpaths` -in `accumulo-site.xml`. This value is a comma-separated list of regular-expression -paths which are all loaded into a secondary classloader. This includes Hadoop, Accumulo and ZooKeeper -jars necessary to run Accumulo. When this value is not defined, a default value is used which attempts -to load Hadoop from multiple potential locations depending on how Hadoop was installed. It is strongly -recommended that `general.classpaths` is defined and limited to only the necessary jars to prevent -extra jars from being unintentionally loaded into Accumulo processes. - ==== Hostnames in configuration files Accumulo has a number of configuration files which can contain references to other hosts in your @@ -364,17 +357,19 @@ consideration. There is no enforcement of these warnings via the API. ==== Configuring the ClassLoader -Accumulo loads classes from the locations specified in the +general.classpaths+ property. Additionally, Accumulo will load classes -from the locations specified in the +general.dynamic.classpaths+ property and will monitor and reload them if they change. The reloading -feature is useful during the development and testing of iterators as new or modified iterator classes can be deployed to Accumulo without -having to restart the database. - +Accumulo builds its Java classpath in +accumulo-env.sh+. After an Accumulo application has started, it will load classes from the locations +specified in the deprecated +general.classpaths+ property. Additionally, Accumulo will load classes from the locations specified in the ++general.dynamic.classpaths+ property and will monitor and reload them if they change. The reloading feature is useful during the development +and testing of iterators as new or modified iterator classes can be deployed to Accumulo without having to restart the database. +/ Accumulo also has an alternate configuration for the classloader which will allow it to load classes from remote locations. This mechanism uses Apache Commons VFS which enables locations such as http and hdfs to be used. This alternate configuration also uses the +general.classpaths+ property in the same manner described above. It differs in that you need to configure the +general.vfs.classpaths+ property instead of the +general.dynamic.classpath+ property. As in the default configuration, this alternate configuration will also monitor the vfs locations for changes and reload if necessary. +The Accumulo classpath can be viewed in human readable format by running +accumulo classpath -d+. + ===== ClassLoader Contexts With the addition of the VFS based classloader, we introduced the notion of classloader contexts. A context is identified @@ -934,8 +929,9 @@ Time Start Service@Location Name ---- === Logging -Accumulo processes each write to a set of log files. By default these are found under -+$ACCUMULO/logs/+. + +Accumulo processes each write to a set of log files. By default, these logs are found at directory +set by +ACCUMULO_LOG_DIR+ in +accumulo-env.sh+. === Recovery http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java ---------------------------------------------------------------------- diff --git a/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java b/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java index bd6af3a..87fbdae 100644 --- a/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java +++ b/proxy/src/main/java/org/apache/accumulo/proxy/Proxy.java @@ -192,7 +192,7 @@ public class Proxy implements KeywordExecutable { public static ServerAddress createProxyServer(HostAndPort address, TProtocolFactory protocolFactory, Properties properties, ClientConfiguration clientConf) throws Exception { final int numThreads = Integer.parseInt(properties.getProperty(THRIFT_THREAD_POOL_SIZE_KEY, THRIFT_THREAD_POOL_SIZE_DEFAULT)); - final long maxFrameSize = AccumuloConfiguration.getMemoryInBytes(properties.getProperty(THRIFT_MAX_FRAME_SIZE_KEY, THRIFT_MAX_FRAME_SIZE_DEFAULT)); + final long maxFrameSize = AccumuloConfiguration.getFixedMemoryAsBytes(properties.getProperty(THRIFT_MAX_FRAME_SIZE_KEY, THRIFT_MAX_FRAME_SIZE_DEFAULT)); final int simpleTimerThreadpoolSize = Integer.parseInt(Property.GENERAL_SIMPLETIMER_THREADPOOL_SIZE.getDefaultValue()); // How frequently to try to resize the thread pool final long threadpoolResizeInterval = 1000l * 5; http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java b/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java index ad64c83..9d2d406 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java +++ b/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java @@ -131,7 +131,7 @@ public class TServerUtils { long maxMessageSize = 10 * 1000 * 1000; if (maxMessageSizeProperty != null) - maxMessageSize = config.getMemoryInBytes(maxMessageSizeProperty); + maxMessageSize = config.getAsBytes(maxMessageSizeProperty); boolean portSearch = false; if (portSearchProperty != null) http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java index 01faacd..44219a5 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java +++ b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java @@ -127,7 +127,7 @@ public class LargestFirstMemoryManager implements MemoryManager { @Override public void init(ServerConfiguration conf) { this.config = conf; - maxMemory = conf.getConfiguration().getMemoryInBytes(Property.TSERV_MAXMEM); + maxMemory = conf.getConfiguration().getAsBytes(Property.TSERV_MAXMEM); maxConcurrentMincs = conf.getConfiguration().getCount(Property.TSERV_MINC_MAXCONCURRENT); numWaitingMultiplier = TSERV_MINC_MAXCONCURRENT_NUMWAITING_MULTIPLIER; } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java ---------------------------------------------------------------------- diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java index 90f3457..923af12 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java @@ -710,7 +710,7 @@ public class SimpleGarbageCollector extends AccumuloServerContext implements Ifa } int port[] = getConfiguration().getPort(Property.GC_PORT); HostAndPort[] addresses = TServerUtils.getHostAndPorts(this.opts.getAddress(), port); - long maxMessageSize = getConfiguration().getMemoryInBytes(Property.GENERAL_MAX_MESSAGE_SIZE); + long maxMessageSize = getConfiguration().getAsBytes(Property.GENERAL_MAX_MESSAGE_SIZE); try { ServerAddress server = TServerUtils.startTServer(getConfiguration(), getThriftServerType(), processor, this.getClass().getSimpleName(), "GC Monitor Service", 2, getConfiguration().getCount(Property.GENERAL_SIMPLETIMER_THREADPOOL_SIZE), 1000, maxMessageSize, getServerSslParams(), http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java index 7187e3d..abfa3e3 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java @@ -372,7 +372,7 @@ public class TabletServer extends AccumuloServerContext implements Runnable { } }, 5000, 5000); - final long walogMaxSize = aconf.getMemoryInBytes(Property.TSERV_WALOG_MAX_SIZE); + final long walogMaxSize = aconf.getAsBytes(Property.TSERV_WALOG_MAX_SIZE); final long walogMaxAge = aconf.getTimeInMillis(Property.TSERV_WALOG_MAX_AGE); final long minBlockSize = CachedConfiguration.getInstance().getLong("dfs.namenode.fs-limits.min-block-size", 0); if (minBlockSize != 0 && minBlockSize > walogMaxSize) @@ -854,7 +854,7 @@ public class TabletServer extends AccumuloServerContext implements Runnable { } us.queuedMutationSize += additionalMutationSize; long totalQueued = updateTotalQueuedMutationSize(additionalMutationSize); - long total = TabletServer.this.getConfiguration().getMemoryInBytes(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX); + long total = TabletServer.this.getConfiguration().getAsBytes(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX); if (totalQueued > total) { try { flush(us); @@ -3257,7 +3257,7 @@ public class TabletServer extends AccumuloServerContext implements Runnable { private final SharedRateLimiterFactory.RateProvider rateProvider = new SharedRateLimiterFactory.RateProvider() { @Override public long getDesiredRate() { - return getConfiguration().getMemoryInBytes(Property.TSERV_MAJC_THROUGHPUT); + return getConfiguration().getAsBytes(Property.TSERV_MAJC_THROUGHPUT); } }; http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java index 411ffa5..b84997d 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java @@ -166,14 +166,14 @@ public class TabletServerResourceManager { this.conf = tserver.getServerConfigurationFactory(); final AccumuloConfiguration acuConf = conf.getConfiguration(); - long maxMemory = acuConf.getMemoryInBytes(Property.TSERV_MAXMEM); + long maxMemory = acuConf.getAsBytes(Property.TSERV_MAXMEM); boolean usingNativeMap = acuConf.getBoolean(Property.TSERV_NATIVEMAP_ENABLED) && NativeMap.isLoaded(); - long blockSize = acuConf.getMemoryInBytes(Property.TSERV_DEFAULT_BLOCKSIZE); - long dCacheSize = acuConf.getMemoryInBytes(Property.TSERV_DATACACHE_SIZE); - long iCacheSize = acuConf.getMemoryInBytes(Property.TSERV_INDEXCACHE_SIZE); - long sCacheSize = acuConf.getMemoryInBytes(Property.TSERV_SUMMARYCACHE_SIZE); - long totalQueueSize = acuConf.getMemoryInBytes(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX); + long blockSize = acuConf.getAsBytes(Property.TSERV_DEFAULT_BLOCKSIZE); + long dCacheSize = acuConf.getAsBytes(Property.TSERV_DATACACHE_SIZE); + long iCacheSize = acuConf.getAsBytes(Property.TSERV_INDEXCACHE_SIZE); + long sCacheSize = acuConf.getAsBytes(Property.TSERV_SUMMARYCACHE_SIZE); + long totalQueueSize = acuConf.getAsBytes(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX); String policy = acuConf.get(Property.TSERV_CACHE_POLICY); if (policy.equalsIgnoreCase("LRU")) { @@ -359,7 +359,7 @@ public class TabletServerResourceManager { MemoryManagementFramework() { tabletReports = Collections.synchronizedMap(new HashMap<KeyExtent,TabletStateImpl>()); memUsageReports = new LinkedBlockingQueue<>(); - maxMem = conf.getConfiguration().getMemoryInBytes(Property.TSERV_MAXMEM); + maxMem = conf.getConfiguration().getAsBytes(Property.TSERV_MAXMEM); Runnable r1 = new Runnable() { @Override http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategy.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategy.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategy.java index 69e3269..6f9233a 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategy.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategy.java @@ -35,7 +35,7 @@ public class SizeLimitCompactionStrategy extends DefaultCompactionStrategy { @Override public void init(Map<String,String> options) { - limit = AccumuloConfiguration.getMemoryInBytes(options.get(SIZE_LIMIT_OPT)); + limit = AccumuloConfiguration.getFixedMemoryAsBytes(options.get(SIZE_LIMIT_OPT)); } private MajorCompactionRequest filterFiles(MajorCompactionRequest mcr) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/TwoTierCompactionStrategy.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/TwoTierCompactionStrategy.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/TwoTierCompactionStrategy.java index a3877b0..a2cc52a 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/TwoTierCompactionStrategy.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/TwoTierCompactionStrategy.java @@ -82,7 +82,7 @@ public class TwoTierCompactionStrategy extends DefaultCompactionStrategy { String threshold = options.get(LARGE_FILE_COMPRESSION_THRESHOLD); largeFileCompressionType = options.get(LARGE_FILE_COMPRESSION_TYPE); verifyRequiredProperties(threshold, largeFileCompressionType); - largeFileCompressionThreshold = AccumuloConfiguration.getMemoryInBytes(threshold); + largeFileCompressionThreshold = AccumuloConfiguration.getFixedMemoryAsBytes(threshold); } @Override http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java index 083909e..4d2fe7e 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java @@ -447,9 +447,9 @@ public class DfsLogger implements Comparable<DfsLogger> { short replication = (short) conf.getConfiguration().getCount(Property.TSERV_WAL_REPLICATION); if (replication == 0) replication = fs.getDefaultReplication(new Path(logPath)); - long blockSize = conf.getConfiguration().getMemoryInBytes(Property.TSERV_WAL_BLOCKSIZE); + long blockSize = conf.getConfiguration().getAsBytes(Property.TSERV_WAL_BLOCKSIZE); if (blockSize == 0) - blockSize = (long) (conf.getConfiguration().getMemoryInBytes(Property.TSERV_WALOG_MAX_SIZE) * 1.1); + blockSize = (long) (conf.getConfiguration().getAsBytes(Property.TSERV_WALOG_MAX_SIZE) * 1.1); if (conf.getConfiguration().getBoolean(Property.TSERV_WAL_SYNC)) logFile = fs.createSyncable(new Path(logPath), 0, replication, blockSize); else http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java index 11097ce..7c89230 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java @@ -129,7 +129,7 @@ public class LogSorter { this.input = inputStreams.getOriginalInput(); this.decryptingInput = inputStreams.getDecryptingInputStream(); - final long bufferSize = conf.getMemoryInBytes(Property.TSERV_SORT_BUFFER_SIZE); + final long bufferSize = conf.getAsBytes(Property.TSERV_SORT_BUFFER_SIZE); Thread.currentThread().setName("Sorting " + name + " for recovery"); while (true) { final ArrayList<Pair<LogFileKey,LogFileValue>> buffer = new ArrayList<>(); @@ -242,7 +242,7 @@ public class LogSorter { RecoveryStatus status = new RecoveryStatus(); status.name = entries.getKey(); try { - status.progress = entries.getValue().getBytesCopied() / (0.0 + conf.getMemoryInBytes(Property.TSERV_WALOG_MAX_SIZE)); + status.progress = entries.getValue().getBytesCopied() / (0.0 + conf.getAsBytes(Property.TSERV_WALOG_MAX_SIZE)); } catch (IOException ex) { log.warn("Error getting bytes read"); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java index 5808f2e..d930775 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java @@ -278,7 +278,7 @@ public class AccumuloReplicaSystem implements ReplicaSystem { // We have a tserver on the remote -- send the data its way. Status finalStatus; - final long sizeLimit = conf.getMemoryInBytes(Property.REPLICATION_MAX_UNIT_SIZE); + final long sizeLimit = conf.getAsBytes(Property.REPLICATION_MAX_UNIT_SIZE); try { if (p.getName().endsWith(RFILE_SUFFIX)) { span = Trace.start("RFile replication"); http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/BatchWriterReplicationReplayer.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/BatchWriterReplicationReplayer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/BatchWriterReplicationReplayer.java index e5e9e80..4a42b56 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/BatchWriterReplicationReplayer.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/BatchWriterReplicationReplayer.java @@ -58,7 +58,7 @@ public class BatchWriterReplicationReplayer implements AccumuloReplicationReplay AccumuloSecurityException { final LogFileKey key = new LogFileKey(); final LogFileValue value = new LogFileValue(); - final long memoryInBytes = context.getConfiguration().getMemoryInBytes(Property.TSERV_REPLICATION_BW_REPLAYER_MEMORY); + final long memoryInBytes = context.getConfiguration().getAsBytes(Property.TSERV_REPLICATION_BW_REPLAYER_MEMORY); BatchWriter bw = null; long mutationsApplied = 0l; http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/server/tserver/src/main/java/org/apache/accumulo/tserver/scan/LookupTask.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/scan/LookupTask.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/scan/LookupTask.java index 139512f..1bdf428 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/scan/LookupTask.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/scan/LookupTask.java @@ -68,7 +68,7 @@ public class LookupTask extends ScanTask<MultiScanResult> { return; TableConfiguration acuTableConf = server.getTableConfiguration(session.threadPoolExtent); - long maxResultsSize = acuTableConf.getMemoryInBytes(Property.TABLE_SCAN_MAXMEM); + long maxResultsSize = acuTableConf.getAsBytes(Property.TABLE_SCAN_MAXMEM); runState.set(ScanRunState.RUNNING); Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Table: "); http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java index ac5243a..b2c917a 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java @@ -299,7 +299,7 @@ class DatafileManager { // find the smallest file long maxFileSize = Long.MAX_VALUE; - maxMergingMinorCompactionFileSize = AccumuloConfiguration.getMemoryInBytes(tablet.getTableConfiguration().get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE)); + maxMergingMinorCompactionFileSize = AccumuloConfiguration.getFixedMemoryAsBytes(tablet.getTableConfiguration().get(Property.TABLE_MINC_MAX_MERGE_FILE_SIZE)); if (maxMergingMinorCompactionFileSize > 0) { maxFileSize = maxMergingMinorCompactionFileSize; } @@ -469,7 +469,7 @@ class DatafileManager { else log.log(TLevel.TABLET_HIST, tablet.getExtent() + " MinC [memory] -> " + newDatafile); log.debug(String.format("MinC finish lock %.2f secs %s", (t2 - t1) / 1000.0, tablet.getExtent().toString())); - long splitSize = tablet.getTableConfiguration().getMemoryInBytes(Property.TABLE_SPLIT_THRESHOLD); + long splitSize = tablet.getTableConfiguration().getAsBytes(Property.TABLE_SPLIT_THRESHOLD); if (dfv.getSize() > splitSize) { log.debug(String.format("Minor Compaction wrote out file larger than split threshold. split threshold = %,d file size = %,d", splitSize, dfv.getSize())); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java index e4d32a6..d675cbc 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java @@ -712,7 +712,7 @@ public class Tablet implements TabletCommitter { long resultSize = 0L; long resultBytes = 0L; - long maxResultsSize = tableConfiguration.getMemoryInBytes(Property.TABLE_SCAN_MAXMEM); + long maxResultsSize = tableConfiguration.getAsBytes(Property.TABLE_SCAN_MAXMEM); if (columns.size() == 0) { iter.seek(range, LocalityGroupUtil.EMPTY_CF_SET, false); @@ -1464,8 +1464,8 @@ public class Tablet implements TabletCommitter { // check if we already decided that we can never split // check to see if we're big enough to split - long splitThreshold = tableConfiguration.getMemoryInBytes(Property.TABLE_SPLIT_THRESHOLD); - long maxEndRow = tableConfiguration.getMemoryInBytes(Property.TABLE_MAX_END_ROW_SIZE); + long splitThreshold = tableConfiguration.getAsBytes(Property.TABLE_SPLIT_THRESHOLD); + long maxEndRow = tableConfiguration.getAsBytes(Property.TABLE_MAX_END_ROW_SIZE); if (extent.isRootTablet() || estimateTabletSize() <= splitThreshold) { return null; @@ -2066,7 +2066,7 @@ public class Tablet implements TabletCommitter { throw new IllegalArgumentException(); } - if (sp != null && sp.length > tableConfiguration.getMemoryInBytes(Property.TABLE_MAX_END_ROW_SIZE)) { + if (sp != null && sp.length > tableConfiguration.getAsBytes(Property.TABLE_MAX_END_ROW_SIZE)) { String msg = "Cannot split tablet " + extent + ", selected split point too long. Length : " + sp.length; log.warn(msg); throw new IOException(msg); http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java index 9f7ba5c..e2bec9d 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java @@ -25,6 +25,8 @@ import java.util.function.Function; import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.DefaultConfiguration; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.conf.SiteConfiguration; import org.apache.accumulo.core.data.impl.KeyExtent; import org.apache.accumulo.server.conf.NamespaceConfiguration; import org.apache.accumulo.server.conf.ServerConfiguration; @@ -62,7 +64,9 @@ public class LargestFirstMemoryManagerTest { @Override public AccumuloConfiguration getConfiguration() { - return DefaultConfiguration.getInstance(); + SiteConfiguration conf = SiteConfiguration.getInstance(); + conf.set(Property.TSERV_MAXMEM, "1g"); + return conf; } @Override http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategyTest.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategyTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategyTest.java index 91e9860..2b56c88 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategyTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/SizeLimitCompactionStrategyTest.java @@ -36,7 +36,7 @@ public class SizeLimitCompactionStrategyTest { HashMap<FileRef,DataFileValue> ret = new HashMap<>(); for (int i = 0; i < sa.length; i += 2) { - ret.put(new FileRef("hdfs://nn1/accumulo/tables/5/t-0001/" + sa[i]), new DataFileValue(AccumuloConfiguration.getMemoryInBytes(sa[i + 1]), 1)); + ret.put(new FileRef("hdfs://nn1/accumulo/tables/5/t-0001/" + sa[i]), new DataFileValue(AccumuloConfiguration.getFixedMemoryAsBytes(sa[i + 1]), 1)); } return ret; http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/TwoTierCompactionStrategyTest.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/TwoTierCompactionStrategyTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/TwoTierCompactionStrategyTest.java index be73cb2..4659193 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/TwoTierCompactionStrategyTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/TwoTierCompactionStrategyTest.java @@ -44,7 +44,7 @@ public class TwoTierCompactionStrategyTest { HashMap<FileRef,DataFileValue> ret = new HashMap<>(); for (int i = 0; i < sa.length; i += 2) { - ret.put(new FileRef("hdfs://nn1/accumulo/tables/5/t-0001/" + sa[i]), new DataFileValue(AccumuloConfiguration.getMemoryInBytes(sa[i + 1]), 1)); + ret.put(new FileRef("hdfs://nn1/accumulo/tables/5/t-0001/" + sa[i]), new DataFileValue(AccumuloConfiguration.getFixedMemoryAsBytes(sa[i + 1]), 1)); } return ret; http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/strategies/ConfigurableCompactionStrategyTest.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/strategies/ConfigurableCompactionStrategyTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/strategies/ConfigurableCompactionStrategyTest.java index ce43665..7498019 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/strategies/ConfigurableCompactionStrategyTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/strategies/ConfigurableCompactionStrategyTest.java @@ -70,9 +70,9 @@ public class ConfigurableCompactionStrategyTest { plan = ccs.getCompactionPlan(mcr); - Assert.assertEquals(AccumuloConfiguration.getMemoryInBytes("64K"), plan.writeParameters.getBlockSize()); - Assert.assertEquals(AccumuloConfiguration.getMemoryInBytes("256M"), plan.writeParameters.getHdfsBlockSize()); - Assert.assertEquals(AccumuloConfiguration.getMemoryInBytes("32K"), plan.writeParameters.getIndexBlockSize()); + Assert.assertEquals(AccumuloConfiguration.getFixedMemoryAsBytes("64K"), plan.writeParameters.getBlockSize()); + Assert.assertEquals(AccumuloConfiguration.getFixedMemoryAsBytes("256M"), plan.writeParameters.getHdfsBlockSize()); + Assert.assertEquals(AccumuloConfiguration.getFixedMemoryAsBytes("32K"), plan.writeParameters.getIndexBlockSize()); Assert.assertEquals(5, plan.writeParameters.getReplication()); Assert.assertEquals("snappy", plan.writeParameters.getCompressType()); http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/BatchWriterReplicationReplayerTest.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/BatchWriterReplicationReplayerTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/BatchWriterReplicationReplayerTest.java index 4584f56..2ec9f53 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/BatchWriterReplicationReplayerTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/BatchWriterReplicationReplayerTest.java @@ -122,7 +122,7 @@ public class BatchWriterReplicationReplayerTest { expectedMutation.put("cf", "cq4", sMutation.getSystemTimestamp(), "value"); expectedMutation.put("cf", "cq5", sMutation.getSystemTimestamp(), "value"); - expect(conf.getMemoryInBytes(Property.TSERV_REPLICATION_BW_REPLAYER_MEMORY)).andReturn(bwCfg.getMaxMemory()); + expect(conf.getAsBytes(Property.TSERV_REPLICATION_BW_REPLAYER_MEMORY)).andReturn(bwCfg.getMaxMemory()); expect(conn.createBatchWriter(tableName, bwCfg)).andReturn(bw); bw.addMutations(Lists.newArrayList(expectedMutation)); @@ -193,7 +193,7 @@ public class BatchWriterReplicationReplayerTest { // We expect our peer name to be preserved in the mutation that gets written expectedMutation.addReplicationSource(peerName); - expect(conf.getMemoryInBytes(Property.TSERV_REPLICATION_BW_REPLAYER_MEMORY)).andReturn(bwCfg.getMaxMemory()); + expect(conf.getAsBytes(Property.TSERV_REPLICATION_BW_REPLAYER_MEMORY)).andReturn(bwCfg.getMaxMemory()); expect(conn.createBatchWriter(tableName, bwCfg)).andReturn(bw); bw.addMutations(Lists.newArrayList(expectedMutation)); http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java index 990d24d..7f7560f 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/DatafileManagerTest.java @@ -43,7 +43,7 @@ public class DatafileManagerTest { private SortedMap<FileRef,DataFileValue> createFileMap(String... sa) { SortedMap<FileRef,DataFileValue> ret = new TreeMap<>(); for (int i = 0; i < sa.length; i += 2) { - ret.put(new FileRef("hdfs://nn1/accumulo/tables/5/t-0001/" + sa[i]), new DataFileValue(AccumuloConfiguration.getMemoryInBytes(sa[i + 1]), 1)); + ret.put(new FileRef("hdfs://nn1/accumulo/tables/5/t-0001/" + sa[i]), new DataFileValue(AccumuloConfiguration.getFixedMemoryAsBytes(sa[i + 1]), 1)); } return ret; } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/shell/src/main/java/org/apache/accumulo/shell/commands/MergeCommand.java ---------------------------------------------------------------------- diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/MergeCommand.java b/shell/src/main/java/org/apache/accumulo/shell/commands/MergeCommand.java index 0da53c0..d40f43a 100644 --- a/shell/src/main/java/org/apache/accumulo/shell/commands/MergeCommand.java +++ b/shell/src/main/java/org/apache/accumulo/shell/commands/MergeCommand.java @@ -49,7 +49,7 @@ public class MergeCommand extends Command { all = true; } if (cl.hasOption(sizeOpt.getOpt())) { - size = AccumuloConfiguration.getMemoryInBytes(cl.getOptionValue(sizeOpt.getOpt())); + size = AccumuloConfiguration.getFixedMemoryAsBytes(cl.getOptionValue(sizeOpt.getOpt())); } if (startRow == null && endRow == null && size < 0 && !all) { shellState.getReader().flush(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java ---------------------------------------------------------------------- diff --git a/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java b/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java index d773fb3..ee611bf 100644 --- a/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java +++ b/start/src/main/java/org/apache/accumulo/start/classloader/AccumuloClassLoader.java @@ -42,50 +42,14 @@ import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; -/** - * - */ public class AccumuloClassLoader { public static final String CLASSPATH_PROPERTY_NAME = "general.classpaths"; - - /* @formatter:off */ - public static final String ACCUMULO_CLASSPATH_VALUE = - "$ACCUMULO_CONF_DIR,\n" + - "$ACCUMULO_HOME/lib/[^.].*.jar,\n" + - "$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n" + - "$HADOOP_CONF_DIR,\n" + - "$HADOOP_PREFIX/[^.].*.jar,\n" + - "$HADOOP_PREFIX/lib/(?!slf4j)[^.].*.jar,\n" + - "$HADOOP_PREFIX/share/hadoop/common/[^.].*.jar,\n" + - "$HADOOP_PREFIX/share/hadoop/common/lib/(?!slf4j)[^.].*.jar,\n" + - "$HADOOP_PREFIX/share/hadoop/hdfs/[^.].*.jar,\n" + - "$HADOOP_PREFIX/share/hadoop/mapreduce/[^.].*.jar,\n" + - "$HADOOP_PREFIX/share/hadoop/yarn/[^.].*.jar,\n" + - "$HADOOP_PREFIX/share/hadoop/yarn/lib/jersey.*.jar,\n" + - "/usr/hdp/current/hadoop-client/[^.].*.jar,\n" + - "/usr/hdp/current/hadoop-client/lib/(?!slf4j)[^.].*.jar,\n" + - "/usr/hdp/current/hadoop-hdfs-client/[^.].*.jar,\n" + - "/usr/hdp/current/hadoop-mapreduce-client/[^.].*.jar,\n" + - "/usr/hdp/current/hadoop-yarn-client/[^.].*.jar,\n" + - "/usr/hdp/current/hadoop-yarn-client/lib/jersey.*.jar,\n" + - "/usr/hdp/current/hive-client/lib/hive-accumulo-handler.jar\n" + - "/usr/lib/hadoop/[^.].*.jar,\n" + - "/usr/lib/hadoop/lib/[^.].*.jar,\n" + - "/usr/lib/hadoop-hdfs/[^.].*.jar,\n" + - "/usr/lib/hadoop-mapreduce/[^.].*.jar,\n" + - "/usr/lib/hadoop-yarn/[^.].*.jar,\n" + - "/usr/lib/hadoop-yarn/lib/jersey.*.jar,\n" - ; - /* @formatter:on */ - public static final String MAVEN_PROJECT_BASEDIR_PROPERTY_NAME = "general.maven.project.basedir"; public static final String DEFAULT_MAVEN_PROJECT_BASEDIR_VALUE = ""; private static String SITE_CONF; - private static URLClassLoader classloader; - private static final Logger log = LoggerFactory.getLogger(AccumuloClassLoader.class); static { @@ -219,7 +183,7 @@ public class AccumuloClassLoader { } private static ArrayList<URL> findAccumuloURLs() throws IOException { - String cp = getAccumuloString(AccumuloClassLoader.CLASSPATH_PROPERTY_NAME, AccumuloClassLoader.ACCUMULO_CLASSPATH_VALUE); + String cp = getAccumuloString(AccumuloClassLoader.CLASSPATH_PROPERTY_NAME, null); if (cp == null) return new ArrayList<>(); String[] cps = replaceEnvVars(cp, System.getenv()).split(","); http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloClasspathTest.java ---------------------------------------------------------------------- diff --git a/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloClasspathTest.java b/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloClasspathTest.java index 4925d67..2efc20f 100644 --- a/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloClasspathTest.java +++ b/start/src/test/java/org/apache/accumulo/start/classloader/vfs/AccumuloClasspathTest.java @@ -25,11 +25,9 @@ public class AccumuloClasspathTest { private static void assertPattern(String output, String pattern, boolean shouldMatch) { if (shouldMatch) { - Assert.assertTrue("Pattern " + pattern + " did not match output: " + output, - output.matches(pattern)); + Assert.assertTrue("Pattern " + pattern + " did not match output: " + output, output.matches(pattern)); } else { - Assert.assertFalse("Pattern " + pattern + " should not match output: " + output, - output.matches(pattern)); + Assert.assertFalse("Pattern " + pattern + " should not match output: " + output, output.matches(pattern)); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/test/src/main/java/org/apache/accumulo/test/LargeSplitRowIT.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/LargeSplitRowIT.java b/test/src/main/java/org/apache/accumulo/test/LargeSplitRowIT.java index 52e331a..4adc621 100644 --- a/test/src/main/java/org/apache/accumulo/test/LargeSplitRowIT.java +++ b/test/src/main/java/org/apache/accumulo/test/LargeSplitRowIT.java @@ -78,7 +78,7 @@ public class LargeSplitRowIT extends ConfigurableMacBase { // Create a split point that is too large to be an end row and fill it with all 'm' SortedSet<Text> partitionKeys = new TreeSet<>(); - byte data[] = new byte[(int) (TableConfiguration.getMemoryInBytes(Property.TABLE_MAX_END_ROW_SIZE.getDefaultValue()) + 2)]; + byte data[] = new byte[(int) (TableConfiguration.getFixedMemoryAsBytes(Property.TABLE_MAX_END_ROW_SIZE.getDefaultValue()) + 2)]; for (int i = 0; i < data.length; i++) { data[i] = 'm'; } @@ -123,7 +123,7 @@ public class LargeSplitRowIT extends ConfigurableMacBase { // Create a BatchWriter and key for a table entry that is longer than the allowed size for an end row // Fill this key with all m's except the last spot BatchWriter batchWriter = conn.createBatchWriter(tableName, new BatchWriterConfig()); - byte data[] = new byte[(int) (TableConfiguration.getMemoryInBytes(Property.TABLE_MAX_END_ROW_SIZE.getDefaultValue()) + 2)]; + byte data[] = new byte[(int) (TableConfiguration.getFixedMemoryAsBytes(Property.TABLE_MAX_END_ROW_SIZE.getDefaultValue()) + 2)]; for (int i = 0; i < data.length - 1; i++) { data[i] = (byte) 'm'; } @@ -233,7 +233,7 @@ public class LargeSplitRowIT extends ConfigurableMacBase { // Create a BatchWriter and key for a table entry that is longer than the allowed size for an end row BatchWriter batchWriter = conn.createBatchWriter(tableName, new BatchWriterConfig()); - byte data[] = new byte[(int) (TableConfiguration.getMemoryInBytes(Property.TABLE_MAX_END_ROW_SIZE.getDefaultValue()) + 2)]; + byte data[] = new byte[(int) (TableConfiguration.getFixedMemoryAsBytes(Property.TABLE_MAX_END_ROW_SIZE.getDefaultValue()) + 2)]; // Fill key with all j's except for last spot which alternates through 1 through 10 for every j value for (int j = 0; j < max; j += spacing) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/f159ec1f/test/src/main/resources/conf/accumulo-site.xml ---------------------------------------------------------------------- diff --git a/test/src/main/resources/conf/accumulo-site.xml b/test/src/main/resources/conf/accumulo-site.xml deleted file mode 100644 index a9d1416..0000000 --- a/test/src/main/resources/conf/accumulo-site.xml +++ /dev/null @@ -1,107 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> - -<configuration> - - <property> - <name>instance.zookeeper.host</name> - <value>localhost:2181</value> - <description>comma separated list of zookeeper servers</description> - </property> - - <property> - <name>instance.secret</name> - <value>DEFAULT</value> - <description>A secret unique to a given instance that all servers must know in order to communicate with one another. - Change it before initialization. To - change it later use ./bin/accumulo org.apache.accumulo.server.util.ChangeSecret [oldpasswd] [newpasswd], - and then update this file. - </description> - </property> - - <property> - <name>tserver.memory.maps.max</name> - <value>80M</value> - </property> - - <property> - <name>tserver.cache.data.size</name> - <value>7M</value> - </property> - - <property> - <name>tserver.cache.index.size</name> - <value>20M</value> - </property> - - <property> - <name>trace.password</name> - <!-- - change this to the root user's password, and/or change the user below - --> - <value>secret</value> - </property> - - <property> - <name>trace.user</name> - <value>root</value> - </property> - - <property> - <name>tserver.sort.buffer.size</name> - <value>50M</value> - </property> - - <property> - <name>tserver.walog.max.size</name> - <value>100M</value> - </property> - - <property> - <name>general.classpaths</name> - <!-- - Add the following for hadoop-2.0 - $HADOOP_PREFIX/share/hadoop/common/.*.jar, - $HADOOP_PREFIX/share/hadoop/common/lib/.*.jar, - $HADOOP_PREFIX/share/hadoop/hdfs/.*.jar, - $HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar, - $HADOOP_PREFIX/share/hadoop/yarn/.*.jar, - --> - <value> - $ACCUMULO_HOME/server/target/classes/, - $ACCUMULO_HOME/lib/accumulo-server.jar, - $ACCUMULO_HOME/core/target/classes/, - $ACCUMULO_HOME/lib/accumulo-core.jar, - $ACCUMULO_HOME/start/target/classes/, - $ACCUMULO_HOME/lib/accumulo-start.jar, - $ACCUMULO_HOME/fate/target/classes/, - $ACCUMULO_HOME/lib/accumulo-fate.jar, - $ACCUMULO_HOME/proxy/target/classes/, - $ACCUMULO_HOME/lib/accumulo-proxy.jar, - $ACCUMULO_HOME/lib/[^.].*.jar, - $ZOOKEEPER_HOME/zookeeper[^.].*.jar, - $HADOOP_CONF_DIR, - $HADOOP_PREFIX/[^.].*.jar, - $HADOOP_PREFIX/lib/[^.].*.jar, - </value> - <description>Classpaths that accumulo checks for updates and class files. - When using the Security Manager, please remove the ".../target/classes/" values. - </description> - </property> -</configuration>
