ACCUMULO-4638 Clean up some AccumuloConfiguration Lots of small refactorings to clean up AccumuloConfiguration, ServerConfigurationFactory, and related classes. This should clean up some of the internal configuration API, to make it easier to improve implementations later.
* Move static type converter methods to helper class * Remove redundant default config accessor * Replace static filter classes with simple lambdas * Remove static helper for client-side get table config (and comment on the now-obvious potential race condition in OfflineIterator for getting this config via the client API) * Improve PropertyTest.testSensitiveKeys() use of DefaultConfiguration * Rename ServerConfigurationFactory.getConfiguration() to ServerConfigurationFactory.getSystemConfiguration() to more accurately reflect what it retrieves * Remove unnecessary KeyExtent version of ServerConfigurationFactory.getTableConfiguration(...) * Use Java 8 streams and lambda syntax to clean up some configuration implementations Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b78edd33 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b78edd33 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b78edd33 Branch: refs/heads/ACCUMULO-4463 Commit: b78edd3329cb7c981329fa2038a9352f79ab466d Parents: 2e171cd Author: Christopher Tubbs <ctubb...@apache.org> Authored: Thu May 11 22:10:44 2017 -0400 Committer: Christopher Tubbs <ctubb...@apache.org> Committed: Thu May 11 22:10:44 2017 -0400 ---------------------------------------------------------------------- .../apache/accumulo/core/cli/ClientOpts.java | 5 +- .../accumulo/core/client/ZooKeeperInstance.java | 4 +- .../core/client/impl/OfflineIterator.java | 5 +- .../lib/impl/FileOutputConfigurator.java | 3 +- .../core/client/mock/MockNamespace.java | 4 +- .../core/client/mock/MockScannerBase.java | 3 +- .../accumulo/core/client/mock/MockTable.java | 6 +- .../core/client/mock/MockTableOperations.java | 4 +- .../core/client/rfile/RFileScanner.java | 6 +- .../client/rfile/RFileSummariesRetriever.java | 3 +- .../core/client/rfile/RFileWriterBuilder.java | 3 +- .../accumulo/core/compaction/SizeType.java | 4 +- .../core/conf/AccumuloConfiguration.java | 208 +------------------ .../accumulo/core/conf/ConfigSanityCheck.java | 2 +- .../core/conf/ConfigurationTypeHelper.java | 139 +++++++++++++ .../core/conf/DefaultConfiguration.java | 24 +-- .../accumulo/core/file/BloomFilterLayer.java | 3 +- .../accumulo/core/file/rfile/CreateEmpty.java | 2 +- .../apache/accumulo/core/file/rfile/RFile.java | 4 +- .../accumulo/core/file/rfile/SplitLarge.java | 2 +- .../iterators/user/RowEncodingIterator.java | 8 +- .../iterators/user/TransformingIterator.java | 6 +- .../accumulo/core/trace/DistributedTrace.java | 3 +- .../client/mock/MockTableOperationsTest.java | 4 +- .../accumulo/core/client/rfile/RFileTest.java | 4 +- .../core/conf/AccumuloConfigurationTest.java | 73 +------ .../core/conf/ConfigurationTypeHelperTest.java | 118 +++++++++++ .../core/conf/DefaultConfigurationTest.java | 6 +- .../apache/accumulo/core/conf/PropertyTest.java | 67 ++---- .../core/file/BloomFilterLayerLookupTest.java | 4 +- .../accumulo/core/file/FileOperationsTest.java | 3 +- .../core/file/rfile/CreateCompatTestFile.java | 4 +- .../core/file/rfile/MultiLevelIndexTest.java | 3 +- .../core/file/rfile/MultiThreadedRFileTest.java | 5 +- .../accumulo/core/file/rfile/RFileTest.java | 13 +- .../iterators/DefaultIteratorEnvironment.java | 3 +- .../iterators/user/IndexedDocIteratorTest.java | 4 +- .../core/security/crypto/CryptoTest.java | 3 +- .../standalone/StandaloneAccumuloCluster.java | 3 +- .../impl/MiniAccumuloClusterImpl.java | 3 +- .../impl/MiniAccumuloConfigImpl.java | 2 +- .../java/org/apache/accumulo/proxy/Proxy.java | 4 +- .../org/apache/accumulo/server/Accumulo.java | 2 +- .../accumulo/server/AccumuloServerContext.java | 2 +- .../server/client/ClientServiceHandler.java | 4 +- .../accumulo/server/conf/ConfigSanityCheck.java | 2 +- .../server/conf/NamespaceConfiguration.java | 24 +-- .../server/conf/ServerConfiguration.java | 5 +- .../server/conf/ServerConfigurationFactory.java | 24 +-- .../accumulo/server/fs/VolumeManagerImpl.java | 2 +- .../apache/accumulo/server/init/Initialize.java | 5 +- .../balancer/HostRegexTableLoadBalancer.java | 12 +- .../master/balancer/RegexGroupBalancer.java | 4 +- .../tabletserver/LargestFirstMemoryManager.java | 4 +- .../org/apache/accumulo/server/util/Admin.java | 2 +- .../accumulo/server/util/LoginProperties.java | 2 +- .../server/AccumuloServerContextTest.java | 2 +- .../conf/ServerConfigurationFactoryTest.java | 2 +- .../BaseHostRegexTableLoadBalancerTest.java | 2 +- .../HostRegexTableLoadBalancerTest.java | 2 +- .../accumulo/server/util/TServerUtilsTest.java | 23 +- .../accumulo/gc/SimpleGarbageCollector.java | 2 +- .../accumulo/gc/SimpleGarbageCollectorTest.java | 2 +- .../java/org/apache/accumulo/master/Master.java | 4 +- .../master/tableOps/WriteExportFiles.java | 3 +- .../org/apache/accumulo/monitor/Monitor.java | 6 +- .../org/apache/accumulo/tracer/TraceServer.java | 4 +- .../apache/accumulo/tserver/FileManager.java | 5 +- .../apache/accumulo/tserver/TabletServer.java | 12 +- .../tserver/TabletServerResourceManager.java | 10 +- .../compaction/SizeLimitCompactionStrategy.java | 4 +- .../compaction/TwoTierCompactionStrategy.java | 4 +- .../replication/AccumuloReplicaSystem.java | 4 +- .../tserver/replication/ReplicationWorker.java | 2 +- .../tserver/tablet/DatafileManager.java | 4 +- .../tserver/LargestFirstMemoryManagerTest.java | 14 +- .../DefaultCompactionStrategyTest.java | 3 +- .../SizeLimitCompactionStrategyTest.java | 7 +- .../TwoTierCompactionStrategyTest.java | 10 +- .../ConfigurableCompactionStrategyTest.java | 8 +- .../tserver/tablet/DatafileManagerTest.java | 4 +- .../accumulo/shell/commands/ConfigCommand.java | 4 +- .../accumulo/shell/commands/DeleteCommand.java | 4 +- .../accumulo/shell/commands/InsertCommand.java | 4 +- .../accumulo/shell/commands/MergeCommand.java | 4 +- .../accumulo/shell/commands/ScanCommand.java | 5 +- .../accumulo/test/BulkImportMonitoringIT.java | 6 +- .../apache/accumulo/test/CreateRandomRFile.java | 4 +- .../org/apache/accumulo/test/ExistingMacIT.java | 4 +- .../apache/accumulo/test/GetFileInfoBulkIT.java | 6 +- .../apache/accumulo/test/LargeSplitRowIT.java | 8 +- .../org/apache/accumulo/test/ShellServerIT.java | 3 +- .../test/TableConfigurationUpdateIT.java | 4 +- .../org/apache/accumulo/test/TestIngest.java | 4 +- .../accumulo/test/TransportCachingIT.java | 4 +- .../accumulo/test/functional/BulkFileIT.java | 2 +- .../test/functional/DeleteEverythingIT.java | 6 +- .../test/functional/DynamicThreadPoolsIT.java | 7 +- .../test/functional/ScanSessionTimeOutIT.java | 7 +- .../accumulo/test/functional/SplitIT.java | 4 +- .../CloseWriteAheadLogReferencesIT.java | 2 +- .../test/mapreduce/AccumuloInputFormatIT.java | 2 +- .../performance/metadata/FastBulkImportIT.java | 4 +- .../performance/scan/CollectTabletStats.java | 6 +- 104 files changed, 537 insertions(+), 581 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java b/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java index 9354f18..2051246 100644 --- a/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java +++ b/core/src/main/java/org/apache/accumulo/core/cli/ClientOpts.java @@ -40,6 +40,7 @@ import org.apache.accumulo.core.client.security.tokens.AuthenticationToken.Prope import org.apache.accumulo.core.client.security.tokens.KerberosToken; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.ConfigurationTypeHelper; import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.security.Authorizations; @@ -65,14 +66,14 @@ public class ClientOpts extends Help { public static class TimeConverter implements IStringConverter<Long> { @Override public Long convert(String value) { - return AccumuloConfiguration.getTimeInMillis(value); + return ConfigurationTypeHelper.getTimeInMillis(value); } } public static class MemoryConverter implements IStringConverter<Long> { @Override public Long convert(String value) { - return AccumuloConfiguration.getFixedMemoryAsBytes(value); + return ConfigurationTypeHelper.getFixedMemoryAsBytes(value); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java index b983a20..2d65bf5 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java +++ b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java @@ -33,7 +33,7 @@ import org.apache.accumulo.core.client.impl.Credentials; import org.apache.accumulo.core.client.impl.InstanceOperationsImpl; import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; import org.apache.accumulo.core.client.security.tokens.PasswordToken; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.ConfigurationTypeHelper; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.util.ByteBufferUtil; import org.apache.accumulo.core.util.OpTimer; @@ -150,7 +150,7 @@ public class ZooKeeperInstance implements Instance { if ((instanceId == null) == (instanceName == null)) throw new IllegalArgumentException("Expected exactly one of instanceName and instanceId to be set"); this.zooKeepers = clientConf.get(ClientProperty.INSTANCE_ZK_HOST); - this.zooKeepersSessionTimeOut = (int) AccumuloConfiguration.getTimeInMillis(clientConf.get(ClientProperty.INSTANCE_ZK_TIMEOUT)); + this.zooKeepersSessionTimeOut = (int) ConfigurationTypeHelper.getTimeInMillis(clientConf.get(ClientProperty.INSTANCE_ZK_TIMEOUT)); zooCache = zcf.getZooCache(zooKeepers, zooKeepersSessionTimeOut); if (null != instanceName) { // Validates that the provided instanceName actually exists http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineIterator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineIterator.java b/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineIterator.java index 9f51704..0546ab9 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineIterator.java +++ b/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineIterator.java @@ -319,7 +319,10 @@ class OfflineIterator implements Iterator<Entry<Key,Value>> { IOException { // TODO share code w/ tablet - ACCUMULO-1303 - AccumuloConfiguration acuTableConf = AccumuloConfiguration.getTableConfiguration(conn, tableId); + + // possible race condition here, if table is renamed + String tableName = Tables.getTableName(conn.getInstance(), tableId); + AccumuloConfiguration acuTableConf = new ConfigurationCopy(conn.tableOperations().getProperties(tableName)); Configuration conf = CachedConfiguration.getInstance(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/FileOutputConfigurator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/FileOutputConfigurator.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/FileOutputConfigurator.java index 5f73e90..1da81fe 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/FileOutputConfigurator.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/impl/FileOutputConfigurator.java @@ -25,6 +25,7 @@ import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.client.summary.SummarizerConfiguration; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl; import org.apache.hadoop.conf.Configuration; @@ -100,7 +101,7 @@ public class FileOutputConfigurator extends ConfiguratorBase { */ public static AccumuloConfiguration getAccumuloConfiguration(Class<?> implementingClass, Configuration conf) { String prefix = enumToConfKey(implementingClass, Opts.ACCUMULO_PROPERTIES) + "."; - ConfigurationCopy acuConf = new ConfigurationCopy(AccumuloConfiguration.getDefaultConfiguration()); + ConfigurationCopy acuConf = new ConfigurationCopy(DefaultConfiguration.getInstance()); for (Entry<String,String> entry : conf) if (entry.getKey().startsWith(prefix)) { String propString = entry.getKey().substring(prefix.length()); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/client/mock/MockNamespace.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mock/MockNamespace.java b/core/src/main/java/org/apache/accumulo/core/client/mock/MockNamespace.java index 456580b..4f57d50 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockNamespace.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockNamespace.java @@ -23,7 +23,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.security.NamespacePermission; @@ -38,7 +38,7 @@ public class MockNamespace { public MockNamespace() { settings = new HashMap<>(); - for (Entry<String,String> entry : AccumuloConfiguration.getDefaultConfiguration()) { + for (Entry<String,String> entry : DefaultConfiguration.getInstance()) { String key = entry.getKey(); if (key.startsWith(Property.TABLE_PREFIX.getKey())) { settings.put(key, entry.getValue()); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/client/mock/MockScannerBase.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mock/MockScannerBase.java b/core/src/main/java/org/apache/accumulo/core/client/mock/MockScannerBase.java index 6dd5737..294e1d3 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockScannerBase.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockScannerBase.java @@ -27,6 +27,7 @@ import org.apache.accumulo.core.client.ScannerBase; import org.apache.accumulo.core.client.impl.ScannerOptions; import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.data.ArrayByteSequence; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Column; @@ -81,7 +82,7 @@ public class MockScannerBase extends ScannerOptions implements ScannerBase { @Override public AccumuloConfiguration getConfig() { - return AccumuloConfiguration.getDefaultConfiguration(); + return DefaultConfiguration.getInstance(); } @Override http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java b/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java index 6a8bd2f..0124ebd 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockTable.java @@ -30,7 +30,7 @@ import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ConcurrentSkipListSet; import org.apache.accumulo.core.client.admin.TimeType; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.ColumnUpdate; import org.apache.accumulo.core.data.Key; @@ -102,7 +102,7 @@ public class MockTable { this.timeType = timeType; this.tableId = tableId; settings = IteratorUtil.generateInitialTableProperties(limitVersion); - for (Entry<String,String> entry : AccumuloConfiguration.getDefaultConfiguration()) { + for (Entry<String,String> entry : DefaultConfiguration.getInstance()) { String key = entry.getKey(); if (key.startsWith(Property.TABLE_PREFIX.getKey())) settings.put(key, entry.getValue()); @@ -129,7 +129,7 @@ public class MockTable { this.timeType = timeType; this.tableId = tableId; settings = properties; - for (Entry<String,String> entry : AccumuloConfiguration.getDefaultConfiguration()) { + for (Entry<String,String> entry : DefaultConfiguration.getInstance()) { String key = entry.getKey(); if (key.startsWith(Property.TABLE_PREFIX.getKey())) settings.put(key, entry.getValue()); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java b/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java index de486d7..72cb939 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java +++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java @@ -48,7 +48,7 @@ import org.apache.accumulo.core.client.impl.TableOperationsHelper; import org.apache.accumulo.core.client.impl.Tables; import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.client.summary.SummarizerConfiguration; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; @@ -292,7 +292,7 @@ class MockTableOperations extends TableOperationsHelper { for (FileStatus importStatus : fs.listStatus(importPath)) { try { FileSKVIterator importIterator = FileOperations.getInstance().newReaderBuilder().forFile(importStatus.getPath().toString(), fs, fs.getConf()) - .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).seekToBeginning().build(); + .withTableConfiguration(DefaultConfiguration.getInstance()).seekToBeginning().build(); while (importIterator.hasTop()) { Key key = importIterator.getTopKey(); Value value = importIterator.getTopValue(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java index 1b12fb6..c95531b 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java +++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java @@ -34,8 +34,8 @@ import org.apache.accumulo.core.client.impl.BaseIteratorEnvironment; import org.apache.accumulo.core.client.impl.ScannerOptions; import org.apache.accumulo.core.client.rfile.RFileScannerBuilder.InputArgs; import org.apache.accumulo.core.client.sample.SamplerConfiguration; -import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Column; @@ -73,7 +73,7 @@ class RFileScanner extends ScannerOptions implements Scanner { private int batchSize = 1000; private long readaheadThreshold = 3; - private static final long CACHE_BLOCK_SIZE = AccumuloConfiguration.getDefaultConfiguration().getAsBytes(Property.TSERV_DEFAULT_BLOCKSIZE); + private static final long CACHE_BLOCK_SIZE = DefaultConfiguration.getInstance().getAsBytes(Property.TSERV_DEFAULT_BLOCKSIZE); static class Opts { InputArgs in; @@ -281,7 +281,7 @@ class RFileScanner extends ScannerOptions implements Scanner { for (int i = 0; i < sources.length; i++) { FSDataInputStream inputStream = (FSDataInputStream) sources[i].getInputStream(); readers.add(new RFile.Reader(new CachableBlockFile.Reader(inputStream, sources[i].getLength(), opts.in.getConf(), dataCache, indexCache, - AccumuloConfiguration.getDefaultConfiguration()))); + DefaultConfiguration.getInstance()))); } if (getSamplerConfiguration() != null) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileSummariesRetriever.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileSummariesRetriever.java b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileSummariesRetriever.java index 367172a..b1f8954 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileSummariesRetriever.java +++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileSummariesRetriever.java @@ -30,6 +30,7 @@ import org.apache.accumulo.core.client.rfile.RFileScannerBuilder.InputArgs; import org.apache.accumulo.core.client.summary.SummarizerConfiguration; import org.apache.accumulo.core.client.summary.Summary; import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.summary.SummaryReader; import org.apache.accumulo.core.summary.Gatherer; import org.apache.accumulo.core.summary.SummarizerFactory; @@ -79,7 +80,7 @@ class RFileSummariesRetriever implements SummaryInputArguments, SummaryFSOptions @Override public Collection<Summary> read() throws IOException { SummarizerFactory factory = new SummarizerFactory(); - AccumuloConfiguration acuconf = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration acuconf = DefaultConfiguration.getInstance(); Configuration conf = in.getFileSystem().getConf(); RFileSource[] sources = in.getSources(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java index a7decb1..9097a8b 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java +++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriterBuilder.java @@ -33,6 +33,7 @@ import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.client.summary.SummarizerConfiguration; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.file.FileOperations; import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl; import org.apache.hadoop.fs.FSDataOutputStream; @@ -83,7 +84,7 @@ class RFileWriterBuilder implements RFile.OutputArguments, RFile.WriterFSOptions @Override public RFileWriter build() throws IOException { FileOperations fileops = FileOperations.getInstance(); - AccumuloConfiguration acuconf = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration acuconf = DefaultConfiguration.getInstance(); HashMap<String,String> userProps = new HashMap<>(); userProps.putAll(tableConfig); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/compaction/SizeType.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/compaction/SizeType.java b/core/src/main/java/org/apache/accumulo/core/compaction/SizeType.java index ec53f87..cc44d51 100644 --- a/core/src/main/java/org/apache/accumulo/core/compaction/SizeType.java +++ b/core/src/main/java/org/apache/accumulo/core/compaction/SizeType.java @@ -18,12 +18,12 @@ package org.apache.accumulo.core.compaction; import static com.google.common.base.Preconditions.checkArgument; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.ConfigurationTypeHelper; class SizeType implements Type { @Override public String convert(String str) { - long size = AccumuloConfiguration.getFixedMemoryAsBytes(str); + long size = ConfigurationTypeHelper.getFixedMemoryAsBytes(str); checkArgument(size > 0); return Long.toString(size); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java index a9ce195..3beeb66 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java @@ -22,14 +22,9 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; import java.util.TreeMap; -import java.util.concurrent.TimeUnit; import java.util.function.Predicate; import org.apache.accumulo.core.Constants; -import org.apache.accumulo.core.client.AccumuloException; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.TableNotFoundException; -import org.apache.accumulo.core.client.impl.Tables; import org.apache.accumulo.core.conf.PropertyType.PortRange; import org.apache.accumulo.core.util.Pair; import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader; @@ -41,52 +36,6 @@ import org.slf4j.LoggerFactory; */ public abstract class AccumuloConfiguration implements Iterable<Entry<String,String>> { - /** - * A filter that accepts properties whose keys are an exact match. - */ - public static class MatchFilter implements Predicate<String> { - - private String match; - - /** - * Creates a new filter. - * - * @param match - * prefix of property keys to accept - */ - public MatchFilter(String match) { - this.match = match; - } - - @Override - public boolean test(String key) { - return Objects.equals(match, key); - } - } - - /** - * A filter that accepts properties whose keys begin with a prefix. - */ - public static class PrefixFilter implements Predicate<String> { - - private String prefix; - - /** - * Creates a new filter. - * - * @param prefix - * prefix of property keys to accept - */ - public PrefixFilter(String prefix) { - this.prefix = prefix; - } - - @Override - public boolean test(String key) { - return key.startsWith(prefix); - } - } - private static final Logger log = LoggerFactory.getLogger(AccumuloConfiguration.class); /** @@ -101,7 +50,7 @@ public abstract class AccumuloConfiguration implements Iterable<Entry<String,Str */ public String get(String property) { Map<String,String> propMap = new HashMap<>(1); - getProperties(propMap, new MatchFilter(property)); + getProperties(propMap, key -> Objects.equals(property, key)); return propMap.get(property); } @@ -138,7 +87,7 @@ public abstract class AccumuloConfiguration implements Iterable<Entry<String,Str return entries.entrySet().iterator(); } - private void checkType(Property property, PropertyType type) { + private static void checkType(Property property, PropertyType type) { if (!property.getType().equals(type)) { String msg = "Configuration method intended for type " + type + " called with a " + property.getType() + " argument (" + property.getKey() + ")"; IllegalArgumentException err = new IllegalArgumentException(msg); @@ -160,7 +109,7 @@ public abstract class AccumuloConfiguration implements Iterable<Entry<String,Str checkType(property, PropertyType.PREFIX); Map<String,String> propMap = new HashMap<>(); - getProperties(propMap, new PrefixFilter(property.getKey())); + getProperties(propMap, key -> key.startsWith(property.getKey())); return propMap; } @@ -176,77 +125,15 @@ public abstract class AccumuloConfiguration implements Iterable<Entry<String,Str public long getAsBytes(Property property) { String memString = get(property); if (property.getType() == PropertyType.MEMORY) { - return getMemoryAsBytes(memString); + return ConfigurationTypeHelper.getMemoryAsBytes(memString); } else if (property.getType() == PropertyType.BYTES) { - return getFixedMemoryAsBytes(memString); + return ConfigurationTypeHelper.getFixedMemoryAsBytes(memString); } else { throw new IllegalArgumentException(property.getKey() + " is not of BYTES or MEMORY type"); } } /** - * Interprets a string specifying bytes. A bytes type is specified as a long integer followed by an optional B (bytes), K (KB), M (MB), or G (GB). - * - * @param str - * String value - * @return interpreted memory size in bytes - */ - static public long getFixedMemoryAsBytes(String str) { - char lastChar = str.charAt(str.length() - 1); - - if (lastChar == 'b') { - log.warn("The 'b' in " + str + " is being considered as bytes. " + "Setting memory by bits is not supported"); - } - try { - int multiplier; - switch (Character.toUpperCase(lastChar)) { - case 'G': - multiplier = 30; - break; - case 'M': - multiplier = 20; - break; - case 'K': - multiplier = 10; - break; - case 'B': - multiplier = 0; - break; - default: - return Long.parseLong(str); - } - return Long.parseLong(str.substring(0, str.length() - 1)) << multiplier; - } catch (Exception ex) { - throw new IllegalArgumentException("The value '" + str + "' is not a valid memory setting. A valid value would a number " - + "possibly followed by an optional 'G', 'M', 'K', or 'B'."); - } - } - - /** - * Interprets a string specifying a Memory type which is specified as a long integer followed by an optional B (bytes), K (KB), M (MB), G (GB) or % - * (percentage). - * - * @param str - * String value - * @return interpreted memory size in bytes - */ - static public long getMemoryAsBytes(String str) { - char lastChar = str.charAt(str.length() - 1); - if (lastChar == '%') { - try { - int percent = Integer.parseInt(str.substring(0, str.length() - 1)); - if (percent <= 0 || percent >= 100) { - throw new IllegalArgumentException("The value '" + str + "' is not a valid memory setting."); - } - return Runtime.getRuntime().maxMemory() * percent / 100; - } catch (Exception ex) { - throw new IllegalArgumentException("The value '" + str + "' is not a valid memory setting."); - } - } - return getFixedMemoryAsBytes(str); - } - - /** * Gets a property of type {@link PropertyType#TIMEDURATION}, interpreting the value properly. * * @param property @@ -254,48 +141,11 @@ public abstract class AccumuloConfiguration implements Iterable<Entry<String,Str * @return property value * @throws IllegalArgumentException * if the property is of the wrong type - * @see #getTimeInMillis(String) */ public long getTimeInMillis(Property property) { checkType(property, PropertyType.TIMEDURATION); - return getTimeInMillis(get(property)); - } - - /** - * Interprets a string specifying a time duration. A time duration is specified as a long integer followed by an optional d (days), h (hours), m (minutes), s - * (seconds), or ms (milliseconds). A value without a unit is interpreted as seconds. - * - * @param str - * string value - * @return interpreted time duration in milliseconds - */ - public static long getTimeInMillis(String str) { - TimeUnit timeUnit; - int unitsLen = 1; - switch (str.charAt(str.length() - 1)) { - case 'd': - timeUnit = TimeUnit.DAYS; - break; - case 'h': - timeUnit = TimeUnit.HOURS; - break; - case 'm': - timeUnit = TimeUnit.MINUTES; - break; - case 's': - timeUnit = TimeUnit.SECONDS; - if (str.endsWith("ms")) { - timeUnit = TimeUnit.MILLISECONDS; - unitsLen = 2; - } - break; - default: - timeUnit = TimeUnit.SECONDS; - unitsLen = 0; - break; - } - return timeUnit.toMillis(Long.parseLong(str.substring(0, str.length() - unitsLen))); + return ConfigurationTypeHelper.getTimeInMillis(get(property)); } /** @@ -320,25 +170,11 @@ public abstract class AccumuloConfiguration implements Iterable<Entry<String,Str * @return property value * @throws IllegalArgumentException * if the property is of the wrong type - * @see #getFraction(String) */ public double getFraction(Property property) { checkType(property, PropertyType.FRACTION); - return getFraction(get(property)); - } - - /** - * Interprets a string specifying a fraction. A fraction is specified as a double. An optional % at the end signifies a percentage. - * - * @param str - * string value - * @return interpreted fraction as a decimal value - */ - public double getFraction(String str) { - if (str.length() > 0 && str.charAt(str.length() - 1) == '%') - return Double.parseDouble(str.substring(0, str.length() - 1)) / 100.0; - return Double.parseDouble(str); + return ConfigurationTypeHelper.getFraction(get(property)); } /** @@ -349,7 +185,6 @@ public abstract class AccumuloConfiguration implements Iterable<Entry<String,Str * @return property value * @throws IllegalArgumentException * if the property is of the wrong type - * @see #getTimeInMillis(String) */ public int[] getPort(Property property) { checkType(property, PropertyType.PORT); @@ -393,7 +228,6 @@ public abstract class AccumuloConfiguration implements Iterable<Entry<String,Str * @return property value * @throws IllegalArgumentException * if the property is of the wrong type - * @see #getTimeInMillis(String) */ public int getCount(Property property) { checkType(property, PropertyType.COUNT); @@ -429,34 +263,6 @@ public abstract class AccumuloConfiguration implements Iterable<Entry<String,Str } /** - * Gets the default configuration. - * - * @return default configuration - * @see DefaultConfiguration#getInstance() - */ - public static synchronized DefaultConfiguration getDefaultConfiguration() { - return DefaultConfiguration.getInstance(); - } - - /** - * Gets the configuration specific to a table. - * - * @param conn - * connector (used to find table name) - * @param tableId - * table ID - * @return configuration containing table properties - * @throws TableNotFoundException - * if the table is not found - * @throws AccumuloException - * if there is a problem communicating to Accumulo - */ - public static AccumuloConfiguration getTableConfiguration(Connector conn, String tableId) throws TableNotFoundException, AccumuloException { - String tableName = Tables.getTableName(conn.getInstance(), tableId); - return new ConfigurationCopy(conn.tableOperations().getProperties(tableName)); - } - - /** * Gets the maximum number of files per tablet from this configuration. * * @return maximum number of files per tablet http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/conf/ConfigSanityCheck.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/ConfigSanityCheck.java b/core/src/main/java/org/apache/accumulo/core/conf/ConfigSanityCheck.java index 615d430..d9b7fc0 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/ConfigSanityCheck.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/ConfigSanityCheck.java @@ -104,7 +104,7 @@ public class ConfigSanityCheck { private static void checkTimeDuration(Property prop, String value, CheckTimeDuration chk) { verifyPropertyTypes(PropertyType.TIMEDURATION, prop); - if (!chk.check(AccumuloConfiguration.getTimeInMillis(value))) + if (!chk.check(ConfigurationTypeHelper.getTimeInMillis(value))) fatal(PREFIX + chk.getDescription(prop)); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationTypeHelper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationTypeHelper.java b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationTypeHelper.java new file mode 100644 index 0000000..0ada9eb --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationTypeHelper.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.conf; + +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ConfigurationTypeHelper { + + private static final Logger log = LoggerFactory.getLogger(ConfigurationTypeHelper.class); + + /** + * Interprets a string specifying bytes. A bytes type is specified as a long integer followed by an optional B (bytes), K (KB), M (MB), or G (GB). + * + * @param str + * String value + * @return interpreted memory size in bytes + */ + public static long getFixedMemoryAsBytes(String str) { + char lastChar = str.charAt(str.length() - 1); + + if (lastChar == 'b') { + log.warn("The 'b' in " + str + " is being considered as bytes. " + "Setting memory by bits is not supported"); + } + try { + int multiplier; + switch (Character.toUpperCase(lastChar)) { + case 'G': + multiplier = 30; + break; + case 'M': + multiplier = 20; + break; + case 'K': + multiplier = 10; + break; + case 'B': + multiplier = 0; + break; + default: + return Long.parseLong(str); + } + return Long.parseLong(str.substring(0, str.length() - 1)) << multiplier; + } catch (Exception ex) { + throw new IllegalArgumentException("The value '" + str + "' is not a valid memory setting. A valid value would a number " + + "possibly followed by an optional 'G', 'M', 'K', or 'B'."); + } + } + + /** + * Interprets a string specifying a Memory type which is specified as a long integer followed by an optional B (bytes), K (KB), M (MB), G (GB) or % + * (percentage). + * + * @param str + * String value + * @return interpreted memory size in bytes + */ + public static long getMemoryAsBytes(String str) { + char lastChar = str.charAt(str.length() - 1); + if (lastChar == '%') { + try { + int percent = Integer.parseInt(str.substring(0, str.length() - 1)); + if (percent <= 0 || percent >= 100) { + throw new IllegalArgumentException("The value '" + str + "' is not a valid memory setting."); + } + return Runtime.getRuntime().maxMemory() * percent / 100; + } catch (Exception ex) { + throw new IllegalArgumentException("The value '" + str + "' is not a valid memory setting."); + } + } + return getFixedMemoryAsBytes(str); + } + + /** + * Interprets a string specifying a time duration. A time duration is specified as a long integer followed by an optional d (days), h (hours), m (minutes), s + * (seconds), or ms (milliseconds). A value without a unit is interpreted as seconds. + * + * @param str + * string value + * @return interpreted time duration in milliseconds + */ + public static long getTimeInMillis(String str) { + TimeUnit timeUnit; + int unitsLen = 1; + switch (str.charAt(str.length() - 1)) { + case 'd': + timeUnit = TimeUnit.DAYS; + break; + case 'h': + timeUnit = TimeUnit.HOURS; + break; + case 'm': + timeUnit = TimeUnit.MINUTES; + break; + case 's': + timeUnit = TimeUnit.SECONDS; + if (str.endsWith("ms")) { + timeUnit = TimeUnit.MILLISECONDS; + unitsLen = 2; + } + break; + default: + timeUnit = TimeUnit.SECONDS; + unitsLen = 0; + break; + } + return timeUnit.toMillis(Long.parseLong(str.substring(0, str.length() - unitsLen))); + } + + /** + * Interprets a string specifying a fraction. A fraction is specified as a double. An optional % at the end signifies a percentage. + * + * @param str + * string value + * @return interpreted fraction as a decimal value + */ + public static double getFraction(String str) { + if (str.length() > 0 && str.charAt(str.length() - 1) == '%') + return Double.parseDouble(str.substring(0, str.length() - 1)) / 100.0; + return Double.parseDouble(str); + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java index 9386f99..756e241 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java @@ -16,26 +16,20 @@ */ package org.apache.accumulo.core.conf; -import java.util.Collections; -import java.util.HashMap; +import java.util.Arrays; import java.util.Map; -import java.util.Map.Entry; import java.util.function.Predicate; +import java.util.stream.Collectors; /** * An {@link AccumuloConfiguration} that contains only default values for properties. This class is a singleton. */ public class DefaultConfiguration extends AccumuloConfiguration { - private final static Map<String,String> resolvedProps; - static { - Map<String,String> m = new HashMap<>(); - for (Property prop : Property.values()) { - if (!prop.getType().equals(PropertyType.PREFIX)) { - m.put(prop.getKey(), prop.getDefaultValue()); - } - } - resolvedProps = Collections.unmodifiableMap(m); - } + + private static final Map<String,String> resolvedProps = Arrays.stream(Property.values()).filter(p -> p.getType() != PropertyType.PREFIX) + .collect(Collectors.toMap(p -> p.getKey(), p -> p.getDefaultValue())); + + private DefaultConfiguration() {} /** * Gets a default configuration. @@ -53,8 +47,6 @@ public class DefaultConfiguration extends AccumuloConfiguration { @Override public void getProperties(Map<String,String> props, Predicate<String> filter) { - for (Entry<String,String> entry : resolvedProps.entrySet()) - if (filter.test(entry.getKey())) - props.put(entry.getKey(), entry.getValue()); + resolvedProps.entrySet().stream().filter(p -> filter.test(p.getKey())).forEach(e -> props.put(e.getKey(), e.getValue())); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java b/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java index 4808da3..01fb699 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java +++ b/core/src/main/java/org/apache/accumulo/core/file/BloomFilterLayer.java @@ -41,6 +41,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.accumulo.core.bloomfilter.DynamicBloomFilter; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.PartialKey; @@ -448,7 +449,7 @@ public class BloomFilterLayer { ArrayList<Integer> vals = new ArrayList<>(valsSet); Collections.sort(vals); - ConfigurationCopy acuconf = new ConfigurationCopy(AccumuloConfiguration.getDefaultConfiguration()); + ConfigurationCopy acuconf = new ConfigurationCopy(DefaultConfiguration.getInstance()); acuconf.set(Property.TABLE_BLOOM_ENABLED, "true"); acuconf.set(Property.TABLE_BLOOM_KEY_FUNCTOR, "accumulo.core.file.keyfunctor.ColumnFamilyFunctor"); acuconf.set(Property.TABLE_FILE_TYPE, RFile.EXTENSION); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/file/rfile/CreateEmpty.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/CreateEmpty.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/CreateEmpty.java index 3a41e95..d48a0eb 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/rfile/CreateEmpty.java +++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/CreateEmpty.java @@ -78,7 +78,7 @@ public class CreateEmpty { Path path = new Path(arg); log.info("Writing to file '" + path + "'"); FileSKVWriter writer = (new RFileOperations()).newWriterBuilder().forFile(arg, path.getFileSystem(conf), conf) - .withTableConfiguration(DefaultConfiguration.getDefaultConfiguration()).withCompression(opts.codec).build(); + .withTableConfiguration(DefaultConfiguration.getInstance()).withCompression(opts.codec).build(); writer.close(); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java index 16db01d..e7d5cd4 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java +++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/RFile.java @@ -41,7 +41,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.accumulo.core.client.SampleNotPresentException; import org.apache.accumulo.core.client.sample.Sampler; import org.apache.accumulo.core.client.sample.SamplerConfiguration; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.ArrayByteSequence; import org.apache.accumulo.core.data.ByteSequence; @@ -518,7 +518,7 @@ public class RFile { private Sampler sampler; public Writer(BlockFileWriter bfw, int blockSize) throws IOException { - this(bfw, blockSize, (int) AccumuloConfiguration.getDefaultConfiguration().getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX), null, null); + this(bfw, blockSize, (int) DefaultConfiguration.getInstance().getAsBytes(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE_INDEX), null, null); } public Writer(BlockFileWriter bfw, int blockSize, int indexBlockSize, SamplerConfigurationImpl samplerConfig, Sampler sampler) throws IOException { http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/file/rfile/SplitLarge.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/SplitLarge.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/SplitLarge.java index 1f414d5..4816990 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/rfile/SplitLarge.java +++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/SplitLarge.java @@ -57,7 +57,7 @@ public class SplitLarge { opts.parseArgs(SplitLarge.class.getName(), args); for (String file : opts.files) { - AccumuloConfiguration aconf = DefaultConfiguration.getDefaultConfiguration(); + AccumuloConfiguration aconf = DefaultConfiguration.getInstance(); Path path = new Path(file); CachableBlockFile.Reader rdr = new CachableBlockFile.Reader(fs, path, conf, null, null, aconf); try (Reader iter = new RFile.Reader(rdr)) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/iterators/user/RowEncodingIterator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/user/RowEncodingIterator.java b/core/src/main/java/org/apache/accumulo/core/iterators/user/RowEncodingIterator.java index 150d1fa..ebfbea9 100644 --- a/core/src/main/java/org/apache/accumulo/core/iterators/user/RowEncodingIterator.java +++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/RowEncodingIterator.java @@ -19,12 +19,12 @@ package org.apache.accumulo.core.iterators.user; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.SortedMap; -import java.util.HashMap; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.ConfigurationTypeHelper; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.PartialKey; @@ -155,7 +155,7 @@ public abstract class RowEncodingIterator implements SortedKeyValueIterator<Key, public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException { sourceIter = source; if (options.containsKey(MAX_BUFFER_SIZE_OPT)) { - maxBufferSize = AccumuloConfiguration.getFixedMemoryAsBytes(options.get(MAX_BUFFER_SIZE_OPT)); + maxBufferSize = ConfigurationTypeHelper.getFixedMemoryAsBytes(options.get(MAX_BUFFER_SIZE_OPT)); } } @@ -172,7 +172,7 @@ public abstract class RowEncodingIterator implements SortedKeyValueIterator<Key, public boolean validateOptions(Map<String,String> options) { String maxBufferSizeStr = options.get(MAX_BUFFER_SIZE_OPT); try { - AccumuloConfiguration.getFixedMemoryAsBytes(maxBufferSizeStr); + ConfigurationTypeHelper.getFixedMemoryAsBytes(maxBufferSizeStr); } catch (Exception e) { throw new IllegalArgumentException("Failed to parse opt " + MAX_BUFFER_SIZE_OPT + " " + maxBufferSizeStr, e); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java b/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java index 2dfd726..bd8ae15 100644 --- a/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java +++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/TransformingIterator.java @@ -30,7 +30,7 @@ import java.util.Map.Entry; import java.util.NoSuchElementException; import org.apache.accumulo.core.client.IteratorSetting; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.ConfigurationTypeHelper; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.PartialKey; @@ -120,7 +120,7 @@ abstract public class TransformingIterator extends WrappingIterator implements O } if (options.containsKey(MAX_BUFFER_SIZE_OPT)) { - maxBufferSize = AccumuloConfiguration.getFixedMemoryAsBytes(options.get(MAX_BUFFER_SIZE_OPT)); + maxBufferSize = ConfigurationTypeHelper.getFixedMemoryAsBytes(options.get(MAX_BUFFER_SIZE_OPT)); } else { maxBufferSize = DEFAULT_MAX_BUFFER_SIZE; } @@ -150,7 +150,7 @@ abstract public class TransformingIterator extends WrappingIterator implements O if (option.getKey().equals(AUTH_OPT)) { new Authorizations(option.getValue().getBytes(UTF_8)); } else if (option.getKey().equals(MAX_BUFFER_SIZE_OPT)) { - AccumuloConfiguration.getFixedMemoryAsBytes(option.getValue()); + ConfigurationTypeHelper.getFixedMemoryAsBytes(option.getValue()); } } catch (Exception e) { throw new IllegalArgumentException("Failed to parse opt " + option.getKey() + " " + option.getValue(), e); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/main/java/org/apache/accumulo/core/trace/DistributedTrace.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/trace/DistributedTrace.java b/core/src/main/java/org/apache/accumulo/core/trace/DistributedTrace.java index f833b11..649cb3b 100644 --- a/core/src/main/java/org/apache/accumulo/core/trace/DistributedTrace.java +++ b/core/src/main/java/org/apache/accumulo/core/trace/DistributedTrace.java @@ -25,6 +25,7 @@ import org.apache.accumulo.core.client.ClientConfiguration; import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty; import org.apache.accumulo.core.client.Instance; import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.ConfigurationTypeHelper; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.fate.zookeeper.ZooReader; import org.apache.hadoop.conf.Configuration; @@ -92,7 +93,7 @@ public class DistributedTrace { public static void enable(String hostname, String service, ClientConfiguration conf) { String spanReceivers = conf.get(ClientProperty.TRACE_SPAN_RECEIVERS); String zookeepers = conf.get(ClientProperty.INSTANCE_ZK_HOST); - long timeout = AccumuloConfiguration.getTimeInMillis(conf.get(ClientProperty.INSTANCE_ZK_TIMEOUT)); + long timeout = ConfigurationTypeHelper.getTimeInMillis(conf.get(ClientProperty.INSTANCE_ZK_TIMEOUT)); String zkPath = conf.get(ClientProperty.TRACE_ZK_PATH); Map<String,String> properties = conf.getAllPropertiesWithPrefix(ClientProperty.TRACE_SPAN_RECEIVER_PREFIX); enableTracing(hostname, service, spanReceivers, zookeepers, timeout, zkPath, properties); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/client/mock/MockTableOperationsTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/client/mock/MockTableOperationsTest.java b/core/src/test/java/org/apache/accumulo/core/client/mock/MockTableOperationsTest.java index 58f3777..e19204a 100644 --- a/core/src/test/java/org/apache/accumulo/core/client/mock/MockTableOperationsTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/mock/MockTableOperationsTest.java @@ -42,7 +42,7 @@ import org.apache.accumulo.core.client.admin.NewTableConfiguration; import org.apache.accumulo.core.client.admin.TableOperations; import org.apache.accumulo.core.client.admin.TimeType; import org.apache.accumulo.core.client.security.tokens.PasswordToken; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; @@ -226,7 +226,7 @@ public class MockTableOperationsTest { fs.mkdirs(failures); fs.mkdirs(tempFile.getParent()); FileSKVWriter writer = FileOperations.getInstance().newWriterBuilder().forFile(tempFile.toString(), fs, defaultConf) - .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).build(); + .withTableConfiguration(DefaultConfiguration.getInstance()).build(); writer.startDefaultLocalityGroup(); List<Pair<Key,Value>> keyVals = new ArrayList<>(); for (int i = 0; i < 5; i++) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java index 3352b0f..98f9983 100644 --- a/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java +++ b/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java @@ -42,7 +42,7 @@ import org.apache.accumulo.core.client.summary.SummarizerConfiguration; import org.apache.accumulo.core.client.summary.Summary; import org.apache.accumulo.core.client.summary.summarizers.FamilySummarizer; import org.apache.accumulo.core.client.summary.summarizers.VisibilitySummarizer; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.ArrayByteSequence; import org.apache.accumulo.core.data.ByteSequence; @@ -774,7 +774,7 @@ public class RFileTest { private Reader getReader(LocalFileSystem localFs, String testFile) throws IOException { Reader reader = (Reader) FileOperations.getInstance().newReaderBuilder().forFile(testFile).inFileSystem(localFs, localFs.getConf()) - .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).build(); + .withTableConfiguration(DefaultConfiguration.getInstance()).build(); return reader; } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/conf/AccumuloConfigurationTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/conf/AccumuloConfigurationTest.java b/core/src/test/java/org/apache/accumulo/core/conf/AccumuloConfigurationTest.java index f7a37bf..81b5791 100644 --- a/core/src/test/java/org/apache/accumulo/core/conf/AccumuloConfigurationTest.java +++ b/core/src/test/java/org/apache/accumulo/core/conf/AccumuloConfigurationTest.java @@ -19,70 +19,13 @@ package org.apache.accumulo.core.conf; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import java.util.Arrays; -import java.util.List; -import java.util.function.Function; - import org.junit.Test; public class AccumuloConfigurationTest { @Test - public void testGetMemoryInBytes() throws Exception { - List<Function<String,Long>> funcs = Arrays.asList(AccumuloConfiguration::getFixedMemoryAsBytes, AccumuloConfiguration::getMemoryAsBytes); - for (Function<String,Long> memFunc : funcs) { - assertEquals(42l, memFunc.apply("42").longValue()); - assertEquals(42l, memFunc.apply("42b").longValue()); - assertEquals(42l, memFunc.apply("42B").longValue()); - assertEquals(42l * 1024l, memFunc.apply("42K").longValue()); - assertEquals(42l * 1024l, memFunc.apply("42k").longValue()); - assertEquals(42l * 1024l * 1024l, memFunc.apply("42M").longValue()); - assertEquals(42l * 1024l * 1024l, memFunc.apply("42m").longValue()); - assertEquals(42l * 1024l * 1024l * 1024l, memFunc.apply("42G").longValue()); - assertEquals(42l * 1024l * 1024l * 1024l, memFunc.apply("42g").longValue()); - } - assertEquals(Runtime.getRuntime().maxMemory() / 10, AccumuloConfiguration.getMemoryAsBytes("10%")); - assertEquals(Runtime.getRuntime().maxMemory() / 5, AccumuloConfiguration.getMemoryAsBytes("20%")); - } - - @Test - public void testGetTimeInMillis() { - assertEquals(42L * 24 * 60 * 60 * 1000, AccumuloConfiguration.getTimeInMillis("42d")); - assertEquals(42L * 60 * 60 * 1000, AccumuloConfiguration.getTimeInMillis("42h")); - assertEquals(42L * 60 * 1000, AccumuloConfiguration.getTimeInMillis("42m")); - assertEquals(42L * 1000, AccumuloConfiguration.getTimeInMillis("42s")); - assertEquals(42L * 1000, AccumuloConfiguration.getTimeInMillis("42")); - assertEquals(42L, AccumuloConfiguration.getTimeInMillis("42ms")); - } - - @Test(expected = IllegalArgumentException.class) - public void testGetFixedMemoryAsBytesFailureCases1() throws Exception { - AccumuloConfiguration.getFixedMemoryAsBytes("42x"); - } - - @Test(expected = IllegalArgumentException.class) - public void testGetFixedMemoryAsBytesFailureCases2() throws Exception { - AccumuloConfiguration.getFixedMemoryAsBytes("FooBar"); - } - - @Test(expected = IllegalArgumentException.class) - public void testGetFixedMemoryAsBytesFailureCases3() throws Exception { - AccumuloConfiguration.getFixedMemoryAsBytes("40%"); - } - - @Test(expected = IllegalArgumentException.class) - public void testGetMemoryAsBytesFailureCases1() throws Exception { - AccumuloConfiguration.getMemoryAsBytes("42x"); - } - - @Test(expected = IllegalArgumentException.class) - public void testGetMemoryAsBytesFailureCases2() throws Exception { - AccumuloConfiguration.getMemoryAsBytes("FooBar"); - } - - @Test public void testGetPropertyByString() { - AccumuloConfiguration c = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration c = DefaultConfiguration.getInstance(); boolean found = false; for (Property p : Property.values()) { if (p.getType() != PropertyType.PREFIX) { @@ -98,7 +41,7 @@ public class AccumuloConfigurationTest { @Test public void testGetSinglePort() { - AccumuloConfiguration c = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration c = DefaultConfiguration.getInstance(); ConfigurationCopy cc = new ConfigurationCopy(c); cc.set(Property.TSERV_CLIENTPORT, "9997"); int[] ports = cc.getPort(Property.TSERV_CLIENTPORT); @@ -108,7 +51,7 @@ public class AccumuloConfigurationTest { @Test public void testGetAnyPort() { - AccumuloConfiguration c = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration c = DefaultConfiguration.getInstance(); ConfigurationCopy cc = new ConfigurationCopy(c); cc.set(Property.TSERV_CLIENTPORT, "0"); int[] ports = cc.getPort(Property.TSERV_CLIENTPORT); @@ -118,7 +61,7 @@ public class AccumuloConfigurationTest { @Test public void testGetInvalidPort() { - AccumuloConfiguration c = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration c = DefaultConfiguration.getInstance(); ConfigurationCopy cc = new ConfigurationCopy(c); cc.set(Property.TSERV_CLIENTPORT, "1020"); int[] ports = cc.getPort(Property.TSERV_CLIENTPORT); @@ -128,7 +71,7 @@ public class AccumuloConfigurationTest { @Test public void testGetPortRange() { - AccumuloConfiguration c = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration c = DefaultConfiguration.getInstance(); ConfigurationCopy cc = new ConfigurationCopy(c); cc.set(Property.TSERV_CLIENTPORT, "9997-9999"); int[] ports = cc.getPort(Property.TSERV_CLIENTPORT); @@ -140,7 +83,7 @@ public class AccumuloConfigurationTest { @Test(expected = IllegalArgumentException.class) public void testGetPortRangeInvalidLow() { - AccumuloConfiguration c = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration c = DefaultConfiguration.getInstance(); ConfigurationCopy cc = new ConfigurationCopy(c); cc.set(Property.TSERV_CLIENTPORT, "1020-1026"); int[] ports = cc.getPort(Property.TSERV_CLIENTPORT); @@ -152,7 +95,7 @@ public class AccumuloConfigurationTest { @Test(expected = IllegalArgumentException.class) public void testGetPortRangeInvalidHigh() { - AccumuloConfiguration c = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration c = DefaultConfiguration.getInstance(); ConfigurationCopy cc = new ConfigurationCopy(c); cc.set(Property.TSERV_CLIENTPORT, "65533-65538"); int[] ports = cc.getPort(Property.TSERV_CLIENTPORT); @@ -164,7 +107,7 @@ public class AccumuloConfigurationTest { @Test(expected = IllegalArgumentException.class) public void testGetPortInvalidSyntax() { - AccumuloConfiguration c = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration c = DefaultConfiguration.getInstance(); ConfigurationCopy cc = new ConfigurationCopy(c); cc.set(Property.TSERV_CLIENTPORT, "[65533,65538]"); cc.getPort(Property.TSERV_CLIENTPORT); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/conf/ConfigurationTypeHelperTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/conf/ConfigurationTypeHelperTest.java b/core/src/test/java/org/apache/accumulo/core/conf/ConfigurationTypeHelperTest.java new file mode 100644 index 0000000..48d1f32 --- /dev/null +++ b/core/src/test/java/org/apache/accumulo/core/conf/ConfigurationTypeHelperTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.core.conf; + +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; +import java.util.function.Function; + +import org.junit.Test; + +public class ConfigurationTypeHelperTest { + + @Test + public void testGetMemoryInBytes() throws Exception { + Arrays.<Function<String,Long>> asList(ConfigurationTypeHelper::getFixedMemoryAsBytes, ConfigurationTypeHelper::getMemoryAsBytes).stream() + .forEach(memFunc -> { + assertEquals(42l, memFunc.apply("42").longValue()); + assertEquals(42l, memFunc.apply("42b").longValue()); + assertEquals(42l, memFunc.apply("42B").longValue()); + assertEquals(42l * 1024l, memFunc.apply("42K").longValue()); + assertEquals(42l * 1024l, memFunc.apply("42k").longValue()); + assertEquals(42l * 1024l * 1024l, memFunc.apply("42M").longValue()); + assertEquals(42l * 1024l * 1024l, memFunc.apply("42m").longValue()); + assertEquals(42l * 1024l * 1024l * 1024l, memFunc.apply("42G").longValue()); + assertEquals(42l * 1024l * 1024l * 1024l, memFunc.apply("42g").longValue()); + }); + assertEquals(Runtime.getRuntime().maxMemory() / 10, ConfigurationTypeHelper.getMemoryAsBytes("10%")); + assertEquals(Runtime.getRuntime().maxMemory() / 5, ConfigurationTypeHelper.getMemoryAsBytes("20%")); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetFixedMemoryAsBytesFailureCases1() throws Exception { + ConfigurationTypeHelper.getFixedMemoryAsBytes("42x"); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetFixedMemoryAsBytesFailureCases2() throws Exception { + ConfigurationTypeHelper.getFixedMemoryAsBytes("FooBar"); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetFixedMemoryAsBytesFailureCases3() throws Exception { + ConfigurationTypeHelper.getFixedMemoryAsBytes("40%"); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetMemoryAsBytesFailureCases1() throws Exception { + ConfigurationTypeHelper.getMemoryAsBytes("42x"); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetMemoryAsBytesFailureCases2() throws Exception { + ConfigurationTypeHelper.getMemoryAsBytes("FooBar"); + } + + @Test + public void testGetTimeInMillis() { + assertEquals(42L * 24 * 60 * 60 * 1000, ConfigurationTypeHelper.getTimeInMillis("42d")); + assertEquals(42L * 60 * 60 * 1000, ConfigurationTypeHelper.getTimeInMillis("42h")); + assertEquals(42L * 60 * 1000, ConfigurationTypeHelper.getTimeInMillis("42m")); + assertEquals(42L * 1000, ConfigurationTypeHelper.getTimeInMillis("42s")); + assertEquals(42L * 1000, ConfigurationTypeHelper.getTimeInMillis("42")); + assertEquals(42L, ConfigurationTypeHelper.getTimeInMillis("42ms")); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetTimeInMillisFailureCase1() { + ConfigurationTypeHelper.getTimeInMillis("abc"); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetTimeInMillisFailureCase2() { + ConfigurationTypeHelper.getTimeInMillis("ms"); + } + + @Test + public void testGetFraction() { + double delta = 0.0000000000001; + assertEquals(0.5d, ConfigurationTypeHelper.getFraction("0.5"), delta); + assertEquals(3.0d, ConfigurationTypeHelper.getFraction("3"), delta); + assertEquals(-0.25d, ConfigurationTypeHelper.getFraction("-25%"), delta); + assertEquals(0.99546d, ConfigurationTypeHelper.getFraction("99.546%"), delta); + assertEquals(0.0d, ConfigurationTypeHelper.getFraction("0%"), delta); + assertEquals(0.0d, ConfigurationTypeHelper.getFraction("-0.000"), delta); + assertEquals(0.001d, ConfigurationTypeHelper.getFraction(".1%"), delta); + assertEquals(1d, ConfigurationTypeHelper.getFraction("1."), delta); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetFractionFailureCase1() { + ConfigurationTypeHelper.getFraction("%"); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetFractionFailureCase2() { + ConfigurationTypeHelper.getFraction("abc0%"); + } + + @Test(expected = IllegalArgumentException.class) + public void testGetFractionFailureCase3() { + ConfigurationTypeHelper.getFraction(".%"); + } +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/conf/DefaultConfigurationTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/conf/DefaultConfigurationTest.java b/core/src/test/java/org/apache/accumulo/core/conf/DefaultConfigurationTest.java index 9e4c5ec..b7e968a 100644 --- a/core/src/test/java/org/apache/accumulo/core/conf/DefaultConfigurationTest.java +++ b/core/src/test/java/org/apache/accumulo/core/conf/DefaultConfigurationTest.java @@ -17,6 +17,8 @@ package org.apache.accumulo.core.conf; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.util.Map; @@ -28,7 +30,7 @@ public class DefaultConfigurationTest { @Before public void setUp() { - c = new DefaultConfiguration(); + c = DefaultConfiguration.getInstance(); } @Test @@ -41,6 +43,8 @@ public class DefaultConfigurationTest { Map<String,String> p = new java.util.HashMap<>(); c.getProperties(p, x -> true); assertEquals(Property.MASTER_CLIENTPORT.getDefaultValue(), p.get(Property.MASTER_CLIENTPORT.getKey())); + assertFalse(p.containsKey(Property.MASTER_PREFIX.getKey())); + assertTrue(p.containsKey(Property.TSERV_DEFAULT_BLOCKSIZE.getKey())); } @Test http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java b/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java index 79f2f21..a80183c 100644 --- a/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java +++ b/core/src/test/java/org/apache/accumulo/core/conf/PropertyTest.java @@ -23,10 +23,12 @@ import static org.junit.Assert.assertTrue; import java.io.File; import java.util.HashSet; -import java.util.Iterator; import java.util.Map.Entry; import java.util.TreeMap; -import java.util.TreeSet; +import java.util.function.Predicate; +import java.util.stream.Collector; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import org.junit.Test; @@ -84,7 +86,7 @@ public class PropertyTest { @Test public void testRawDefaultValues() { - AccumuloConfiguration conf = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration conf = DefaultConfiguration.getInstance(); assertEquals("${java.io.tmpdir}" + File.separator + "accumulo-vfs-cache-${user.name}", Property.VFS_CLASSLOADER_CACHE_DIR.getRawDefaultValue()); assertEquals(new File(System.getProperty("java.io.tmpdir"), "accumulo-vfs-cache-" + System.getProperty("user.name")).getAbsolutePath(), conf.get(Property.VFS_CLASSLOADER_CACHE_DIR)); @@ -96,49 +98,26 @@ public class PropertyTest { assertEquals("", Property.GENERAL_MAVEN_PROJECT_BASEDIR.getDefaultValue()); } + // This test verifies all "sensitive" properties are properly marked as sensitive @Test public void testSensitiveKeys() { - final TreeMap<String,String> extras = new TreeMap<>(); - extras.put("trace.token.property.blah", "something"); - - AccumuloConfiguration conf = new DefaultConfiguration() { - @Override - public Iterator<Entry<String,String>> iterator() { - final Iterator<Entry<String,String>> parent = super.iterator(); - final Iterator<Entry<String,String>> mine = extras.entrySet().iterator(); - - return new Iterator<Entry<String,String>>() { - - @Override - public boolean hasNext() { - return parent.hasNext() || mine.hasNext(); - } - - @Override - public Entry<String,String> next() { - return parent.hasNext() ? parent.next() : mine.next(); - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - }; - } - }; - TreeSet<String> expected = new TreeSet<>(); - for (Entry<String,String> entry : conf) { - String key = entry.getKey(); - if (key.equals(Property.INSTANCE_SECRET.getKey()) || key.toLowerCase().contains("password") || key.toLowerCase().endsWith("secret") - || key.startsWith(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey())) - expected.add(key); - } - TreeSet<String> actual = new TreeSet<>(); - for (Entry<String,String> entry : conf) { - String key = entry.getKey(); - if (Property.isSensitive(key)) - actual.add(key); - } + // add trace token, because it's a sensitive property not in the default configuration + ConfigurationCopy conf = new ConfigurationCopy(DefaultConfiguration.getInstance()); + conf.set("trace.token.property.blah", "something"); + + // ignores duplicates because ConfigurationCopy already de-duplicates + Collector<Entry<String,String>,?,TreeMap<String,String>> treeMapCollector = Collectors.toMap(e -> e.getKey(), e -> e.getValue(), (a, b) -> a, TreeMap::new); + + Predicate<Entry<String,String>> sensitiveNames = e -> e.getKey().equals(Property.INSTANCE_SECRET.getKey()) || e.getKey().toLowerCase().contains("password") + || e.getKey().toLowerCase().endsWith("secret") || e.getKey().startsWith(Property.TRACE_TOKEN_PROPERTY_PREFIX.getKey()); + + Predicate<Entry<String,String>> isMarkedSensitive = e -> Property.isSensitive(e.getKey()); + + TreeMap<String,String> expected = StreamSupport.stream(conf.spliterator(), false).filter(sensitiveNames).collect(treeMapCollector); + TreeMap<String,String> actual = StreamSupport.stream(conf.spliterator(), false).filter(isMarkedSensitive).collect(treeMapCollector); + + // make sure trace token property wasn't excluded from both + assertEquals("something", expected.get("trace.token.property.blah")); assertEquals(expected, actual); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/file/BloomFilterLayerLookupTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/file/BloomFilterLayerLookupTest.java b/core/src/test/java/org/apache/accumulo/core/file/BloomFilterLayerLookupTest.java index 065438c..4547cd4 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/BloomFilterLayerLookupTest.java +++ b/core/src/test/java/org/apache/accumulo/core/file/BloomFilterLayerLookupTest.java @@ -25,8 +25,8 @@ import java.util.Collections; import java.util.HashSet; import java.util.Random; -import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; @@ -67,7 +67,7 @@ public class BloomFilterLayerLookupTest { ArrayList<Integer> vals = new ArrayList<>(valsSet); Collections.sort(vals); - ConfigurationCopy acuconf = new ConfigurationCopy(AccumuloConfiguration.getDefaultConfiguration()); + ConfigurationCopy acuconf = new ConfigurationCopy(DefaultConfiguration.getInstance()); acuconf.set(Property.TABLE_BLOOM_ENABLED, "true"); acuconf.set(Property.TABLE_BLOOM_KEY_FUNCTOR, ColumnFamilyFunctor.class.getName()); acuconf.set(Property.TABLE_FILE_TYPE, RFile.EXTENSION); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/file/FileOperationsTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/file/FileOperationsTest.java b/core/src/test/java/org/apache/accumulo/core/file/FileOperationsTest.java index a8e4b7f..fb05e67 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/FileOperationsTest.java +++ b/core/src/test/java/org/apache/accumulo/core/file/FileOperationsTest.java @@ -23,6 +23,7 @@ import java.io.File; import java.io.IOException; import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.file.rfile.RFile; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; @@ -50,7 +51,7 @@ public class FileOperationsTest { FileOperations fileOperations = FileOperations.getInstance(); Configuration conf = new Configuration(); FileSystem fs = FileSystem.getLocal(conf); - AccumuloConfiguration acuconf = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration acuconf = DefaultConfiguration.getInstance(); writer = fileOperations.newWriterBuilder().forFile(filename, fs, conf).withTableConfiguration(acuconf).build(); writer.close(); } catch (Exception ex) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java index 34d01ad..cf10172 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java +++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/CreateCompatTestFile.java @@ -19,7 +19,7 @@ package org.apache.accumulo.core.file.rfile; import java.util.HashSet; import java.util.Set; -import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.data.ArrayByteSequence; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; @@ -56,7 +56,7 @@ public class CreateCompatTestFile { public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); - CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(fs, new Path(args[0]), "gz", null, conf, AccumuloConfiguration.getDefaultConfiguration()); + CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(fs, new Path(args[0]), "gz", null, conf, DefaultConfiguration.getInstance()); RFile.Writer writer = new RFile.Writer(_cbw, 1000); writer.startNewLocalityGroup("lg1", newColFamSequence(formatStr("cf_", 1), formatStr("cf_", 2))); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiLevelIndexTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiLevelIndexTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiLevelIndexTest.java index 391bea1..1585baf 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiLevelIndexTest.java +++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiLevelIndexTest.java @@ -22,6 +22,7 @@ import java.util.Random; import junit.framework.TestCase; import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.file.blockfile.ABlockWriter; import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile; @@ -52,7 +53,7 @@ public class MultiLevelIndexTest extends TestCase { } private void runTest(int maxBlockSize, int num) throws IOException { - AccumuloConfiguration aconf = AccumuloConfiguration.getDefaultConfiguration(); + AccumuloConfiguration aconf = DefaultConfiguration.getInstance(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); FSDataOutputStream dos = new FSDataOutputStream(baos, new FileSystem.Statistics("a")); CachableBlockFile.Writer _cbw = new CachableBlockFile.Writer(PositionedOutputs.wrap(dos), "gz", CachedConfiguration.getInstance(), aconf); http://git-wip-us.apache.org/repos/asf/accumulo/blob/b78edd33/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java index ba255bf..bf747f6 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java +++ b/core/src/test/java/org/apache/accumulo/core/file/rfile/MultiThreadedRFileTest.java @@ -33,6 +33,7 @@ import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.client.sample.Sampler; import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.data.ArrayByteSequence; import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; @@ -110,7 +111,7 @@ public class MultiThreadedRFileTest { public TestRFile(AccumuloConfiguration accumuloConfiguration) { this.accumuloConfiguration = accumuloConfiguration; if (this.accumuloConfiguration == null) - this.accumuloConfiguration = AccumuloConfiguration.getDefaultConfiguration(); + this.accumuloConfiguration = DefaultConfiguration.getInstance(); } public void close() throws IOException { @@ -174,7 +175,7 @@ public class MultiThreadedRFileTest { Path path = new Path("file://" + rfile.toString()); // the caches used to obfuscate the multithreaded issues - CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(fs, path, conf, null, null, AccumuloConfiguration.getDefaultConfiguration()); + CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(fs, path, conf, null, null, DefaultConfiguration.getInstance()); reader = new RFile.Reader(_cbr); iter = new ColumnFamilySkippingIterator(reader);
