Repository: accumulo Updated Branches: refs/heads/IGNITE 2e992c889 -> c4a18a63e
ACCUMULO-4463: simplified config Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e8ccdddc Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e8ccdddc Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e8ccdddc Branch: refs/heads/IGNITE Commit: e8ccdddc29e69ea51e49720f9c7424ebd6ec00d5 Parents: 61dd036 Author: Keith Turner <ktur...@apache.org> Authored: Wed May 17 18:26:54 2017 -0400 Committer: Keith Turner <ktur...@apache.org> Committed: Thu May 18 11:00:13 2017 -0400 ---------------------------------------------------------------------- .../cache/BlockCacheConfiguration.java | 55 +++--------- .../cache/lru/LruBlockCacheConfiguration.java | 90 ++++++++++++++++++-- .../cache/BlockConfigurationHelperTest.java | 16 +--- .../file/blockfile/cache/TestLruBlockCache.java | 64 +++++++------- 4 files changed, 133 insertions(+), 92 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/e8ccdddc/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheConfiguration.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheConfiguration.java index d879f17..0c07dbd 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheConfiguration.java +++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/BlockCacheConfiguration.java @@ -21,56 +21,23 @@ import java.util.Map; import java.util.Optional; import org.apache.accumulo.core.conf.AccumuloConfiguration; -import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.conf.Property; public class BlockCacheConfiguration { - public static class BlockCacheConfigurationHelper { - - private final ConfigurationCopy conf; - private final String basePropertyName; - - protected BlockCacheConfigurationHelper(ConfigurationCopy conf, CacheType type, String implName) { - this.conf = conf; - this.basePropertyName = BlockCacheManager.CACHE_PROPERTY_BASE + implName + "." + type.name().toLowerCase() + "."; - } - - public String getPropertyPrefix() { - return basePropertyName; - } - - public String getFullPropertyName(String propertySuffix) { - return this.basePropertyName + propertySuffix; - } - - public Optional<String> get(String property) { - return Optional.ofNullable(this.conf.get(this.getFullPropertyName(property))); - } - - public void set(String propertySuffix, String value) { - conf.set(getFullPropertyName(propertySuffix), value); - } - - public ConfigurationCopy getConfiguration() { - return this.conf; - } - } - /** Maximum allowable size of cache (block put if size > max, evict) */ private final long maxSize; /** Approximate block size */ private final long blockSize; - /** Helper object for working with block cache configuration **/ - private final BlockCacheConfigurationHelper helper; + private final Map<String,String> genProps; - public BlockCacheConfiguration(AccumuloConfiguration conf, CacheType type, String implName) { + private final String prefix; - Map<String,String> props = conf.getAllPropertiesWithPrefix(Property.GENERAL_ARBITRARY_PROP_PREFIX); - ConfigurationCopy blockCacheConfiguration = new ConfigurationCopy(props); - this.helper = new BlockCacheConfigurationHelper(blockCacheConfiguration, type, implName); + public BlockCacheConfiguration(AccumuloConfiguration conf, CacheType type, String implName) { + prefix = getPrefix(type, implName); + genProps = conf.getAllPropertiesWithPrefix(Property.GENERAL_ARBITRARY_PROP_PREFIX); switch (type) { case INDEX: @@ -88,10 +55,6 @@ public class BlockCacheConfiguration { this.blockSize = conf.getAsBytes(Property.TSERV_DEFAULT_BLOCKSIZE); } - public BlockCacheConfigurationHelper getHelper() { - return this.helper; - } - public long getMaxSize() { return this.maxSize; } @@ -100,6 +63,14 @@ public class BlockCacheConfiguration { return this.blockSize; } + protected Optional<String> get(String suffix) { + return Optional.ofNullable(genProps.get(prefix + suffix)); + } + + public static String getPrefix(CacheType type, String implName) { + return BlockCacheManager.CACHE_PROPERTY_BASE + implName + "." + type.name().toLowerCase() + "."; + } + @Override public String toString() { return "maxSize: " + getMaxSize() + ", blockSize: " + getBlockSize(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/e8ccdddc/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCacheConfiguration.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCacheConfiguration.java b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCacheConfiguration.java index 7131d89..49790cb 100644 --- a/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCacheConfiguration.java +++ b/core/src/main/java/org/apache/accumulo/core/file/blockfile/cache/lru/LruBlockCacheConfiguration.java @@ -17,10 +17,16 @@ */ package org.apache.accumulo.core.file.blockfile.cache.lru; +import java.util.HashMap; +import java.util.Map; + import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.file.blockfile.cache.BlockCacheConfiguration; import org.apache.accumulo.core.file.blockfile.cache.CacheType; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; + public final class LruBlockCacheConfiguration extends BlockCacheConfiguration { public static final String PROPERTY_PREFIX = "lru"; @@ -76,14 +82,14 @@ public final class LruBlockCacheConfiguration extends BlockCacheConfiguration { public LruBlockCacheConfiguration(AccumuloConfiguration conf, CacheType type) { super(conf, type, PROPERTY_PREFIX); - this.acceptableFactor = this.getHelper().get(ACCEPTABLE_FACTOR_PROPERTY).map(Float::valueOf).filter(f -> f > 0).orElse(DEFAULT_ACCEPTABLE_FACTOR); - this.minFactor = this.getHelper().get(MIN_FACTOR_PROPERTY).map(Float::valueOf).filter(f -> f > 0).orElse(DEFAULT_MIN_FACTOR); - this.singleFactor = this.getHelper().get(SINGLE_FACTOR_PROPERTY).map(Float::valueOf).filter(f -> f > 0).orElse(DEFAULT_SINGLE_FACTOR); - this.multiFactor = this.getHelper().get(MULTI_FACTOR_PROPERTY).map(Float::valueOf).filter(f -> f > 0).orElse(DEFAULT_MULTI_FACTOR); - this.memoryFactor = this.getHelper().get(MEMORY_FACTOR_PROPERTY).map(Float::valueOf).filter(f -> f > 0).orElse(DEFAULT_MEMORY_FACTOR); - this.mapLoadFactor = this.getHelper().get(MAP_LOAD_PROPERTY).map(Float::valueOf).filter(f -> f > 0).orElse(DEFAULT_LOAD_FACTOR); - this.mapConcurrencyLevel = this.getHelper().get(MAP_CONCURRENCY_PROPERTY).map(Integer::valueOf).filter(i -> i > 0).orElse(DEFAULT_CONCURRENCY_LEVEL); - this.useEvictionThread = this.getHelper().get(EVICTION_THREAD_PROPERTY).map(Boolean::valueOf).orElse(true); + this.acceptableFactor = get(ACCEPTABLE_FACTOR_PROPERTY).map(Float::valueOf).filter(f -> f > 0).orElse(DEFAULT_ACCEPTABLE_FACTOR); + this.minFactor = get(MIN_FACTOR_PROPERTY).map(Float::valueOf).filter(f -> f > 0).orElse(DEFAULT_MIN_FACTOR); + this.singleFactor = get(SINGLE_FACTOR_PROPERTY).map(Float::valueOf).filter(f -> f > 0).orElse(DEFAULT_SINGLE_FACTOR); + this.multiFactor = get(MULTI_FACTOR_PROPERTY).map(Float::valueOf).filter(f -> f > 0).orElse(DEFAULT_MULTI_FACTOR); + this.memoryFactor = get(MEMORY_FACTOR_PROPERTY).map(Float::valueOf).filter(f -> f > 0).orElse(DEFAULT_MEMORY_FACTOR); + this.mapLoadFactor = get(MAP_LOAD_PROPERTY).map(Float::valueOf).filter(f -> f > 0).orElse(DEFAULT_LOAD_FACTOR); + this.mapConcurrencyLevel = get(MAP_CONCURRENCY_PROPERTY).map(Integer::valueOf).filter(i -> i > 0).orElse(DEFAULT_CONCURRENCY_LEVEL); + this.useEvictionThread = get(EVICTION_THREAD_PROPERTY).map(Boolean::valueOf).orElse(true); if (this.getSingleFactor() + this.getMultiFactor() + this.getMemoryFactor() != 1) { throw new IllegalArgumentException("Single, multi, and memory factors " + " should total 1.0"); @@ -128,6 +134,74 @@ public final class LruBlockCacheConfiguration extends BlockCacheConfiguration { return useEvictionThread; } + public static class Builder { + private Map<String,String> props = new HashMap<>(); + private String prefix; + + private Builder(String prefix) { + this.prefix = prefix; + } + + private void set(String prop, float val) { + props.put(prefix + prop, Float.toString(val)); + } + + public Builder acceptableFactor(float af) { + Preconditions.checkArgument(af > 0); + set(ACCEPTABLE_FACTOR_PROPERTY, af); + return this; + } + + public Builder minFactor(float mf) { + Preconditions.checkArgument(mf > 0); + set(MIN_FACTOR_PROPERTY, mf); + return this; + } + + public Builder singleFactor(float sf) { + Preconditions.checkArgument(sf > 0); + set(SINGLE_FACTOR_PROPERTY, sf); + return this; + } + + public Builder multiFactor(float mf) { + Preconditions.checkArgument(mf > 0); + set(MULTI_FACTOR_PROPERTY, mf); + return this; + } + + public Builder memoryFactor(float mf) { + Preconditions.checkArgument(mf > 0); + set(MEMORY_FACTOR_PROPERTY, mf); + return this; + } + + public Builder mapLoadFactor(float mlf) { + Preconditions.checkArgument(mlf > 0); + set(MAP_LOAD_PROPERTY, mlf); + return this; + } + + public Builder mapConcurrencyLevel(int mcl) { + Preconditions.checkArgument(mcl > 0); + props.put(prefix + MAP_CONCURRENCY_PROPERTY, mcl + ""); + return this; + } + + public Builder useEvictionThread(boolean uet) { + props.put(prefix + EVICTION_THREAD_PROPERTY, uet + ""); + return this; + } + + public Map<String,String> buildMap() { + return ImmutableMap.copyOf(props); + } + } + + public static Builder builder(CacheType ct) { + return new Builder(getPrefix(ct, PROPERTY_PREFIX)); + } + @Override public String toString() { return super.toString() + ", acceptableFactor: " + this.getAcceptableFactor() + ", minFactor: " + this.getMinFactor() + ", singleFactor: " http://git-wip-us.apache.org/repos/asf/accumulo/blob/e8ccdddc/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/BlockConfigurationHelperTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/BlockConfigurationHelperTest.java b/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/BlockConfigurationHelperTest.java index d2901e3..72ea49c 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/BlockConfigurationHelperTest.java +++ b/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/BlockConfigurationHelperTest.java @@ -17,26 +17,16 @@ */ package org.apache.accumulo.core.file.blockfile.cache; -import org.apache.accumulo.core.conf.ConfigurationCopy; -import org.apache.accumulo.core.conf.DefaultConfiguration; -import org.apache.accumulo.core.file.blockfile.cache.lru.LruBlockCacheConfiguration; import org.junit.Assert; import org.junit.Test; public class BlockConfigurationHelperTest { - private BlockCacheConfiguration bcc = new BlockCacheConfiguration(new ConfigurationCopy(new DefaultConfiguration()), CacheType.DATA, - LruBlockCacheConfiguration.PROPERTY_PREFIX); - - @Test - public void testGetFullPropertyName() throws Exception { - Assert.assertEquals("general.custom.cache.block.lru.data.acceptable.factor", - bcc.getHelper().getFullPropertyName(LruBlockCacheConfiguration.ACCEPTABLE_FACTOR_PROPERTY)); - } - @Test public void testGetPropertyPrefix() throws Exception { - Assert.assertEquals("general.custom.cache.block.lru.data.", bcc.getHelper().getPropertyPrefix()); + Assert.assertEquals("general.custom.cache.block.lru.data.", BlockCacheConfiguration.getPrefix(CacheType.DATA, "lru")); + Assert.assertEquals("general.custom.cache.block.lru.index.", BlockCacheConfiguration.getPrefix(CacheType.INDEX, "lru")); + Assert.assertEquals("general.custom.cache.block.lru.summary.", BlockCacheConfiguration.getPrefix(CacheType.SUMMARY, "lru")); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/e8ccdddc/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/TestLruBlockCache.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/TestLruBlockCache.java b/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/TestLruBlockCache.java index c986843..d3ea575 100644 --- a/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/TestLruBlockCache.java +++ b/core/src/test/java/org/apache/accumulo/core/file/blockfile/cache/TestLruBlockCache.java @@ -25,7 +25,6 @@ import junit.framework.TestCase; import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.file.blockfile.cache.BlockCacheConfiguration.BlockCacheConfigurationHelper; import org.apache.accumulo.core.file.blockfile.cache.lru.LruBlockCache; import org.apache.accumulo.core.file.blockfile.cache.lru.LruBlockCacheConfiguration; import org.apache.accumulo.core.file.blockfile.cache.lru.LruBlockCacheManager; @@ -39,6 +38,29 @@ import org.apache.accumulo.core.file.blockfile.cache.lru.LruBlockCacheManager; */ public class TestLruBlockCache extends TestCase { + public void testConfiguration() { + ConfigurationCopy cc = new ConfigurationCopy(); + cc.set(Property.TSERV_CACHE_FACTORY_IMPL, LruBlockCacheManager.class.getName()); + cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(1019)); + cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(1000023)); + + LruBlockCacheConfiguration.builder(CacheType.INDEX).useEvictionThread(false).minFactor(0.93f).acceptableFactor(0.97f).singleFactor(0.20f) + .multiFactor(0.30f).memoryFactor(0.50f).mapConcurrencyLevel(5).mapLoadFactor(0.53f).buildMap().forEach(cc::set); + + LruBlockCacheConfiguration lbcc = new LruBlockCacheConfiguration(cc, CacheType.INDEX); + + assertEquals(false, lbcc.isUseEvictionThread()); + assertEquals(0.93f, lbcc.getMinFactor()); + assertEquals(0.97f, lbcc.getAcceptableFactor()); + assertEquals(0.20f, lbcc.getSingleFactor()); + assertEquals(0.30f, lbcc.getMultiFactor()); + assertEquals(0.50f, lbcc.getMemoryFactor()); + assertEquals(0.53f, lbcc.getMapLoadFactor()); + assertEquals(5, lbcc.getMapConcurrencyLevel()); + assertEquals(1019, lbcc.getBlockSize()); + assertEquals(1000023, lbcc.getMaxSize()); + } + public void testBackgroundEvictionThread() throws Exception { long maxSize = 100000; @@ -140,9 +162,9 @@ public class TestLruBlockCache extends TestCase { BlockCacheManager manager = BlockCacheManager.getInstance(cc); cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(blockSize)); cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(maxSize)); - BlockCacheConfigurationHelper helper = new BlockCacheConfigurationHelper(cc, CacheType.INDEX, LruBlockCacheConfiguration.PROPERTY_PREFIX); - helper.set(LruBlockCacheConfiguration.EVICTION_THREAD_PROPERTY, Boolean.FALSE.toString()); - manager.start(helper.getConfiguration()); + LruBlockCacheConfiguration.builder(CacheType.INDEX).useEvictionThread(false).buildMap().forEach(cc::set); + manager.start(cc); + LruBlockCache cache = (LruBlockCache) manager.getBlockCache(CacheType.INDEX); Block[] blocks = generateFixedBlocks(10, blockSize, "block"); @@ -187,14 +209,9 @@ public class TestLruBlockCache extends TestCase { BlockCacheManager manager = BlockCacheManager.getInstance(cc); cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(blockSize)); cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(maxSize)); - BlockCacheConfigurationHelper helper = new BlockCacheConfigurationHelper(cc, CacheType.INDEX, LruBlockCacheConfiguration.PROPERTY_PREFIX); - helper.set(LruBlockCacheConfiguration.EVICTION_THREAD_PROPERTY, Boolean.FALSE.toString()); - helper.set(LruBlockCacheConfiguration.MIN_FACTOR_PROPERTY, Float.toString(0.98f)); - helper.set(LruBlockCacheConfiguration.ACCEPTABLE_FACTOR_PROPERTY, Float.toString(0.99f)); - helper.set(LruBlockCacheConfiguration.SINGLE_FACTOR_PROPERTY, Float.toString(0.25f)); - helper.set(LruBlockCacheConfiguration.MULTI_FACTOR_PROPERTY, Float.toString(0.50f)); - helper.set(LruBlockCacheConfiguration.MEMORY_FACTOR_PROPERTY, Float.toString(0.25f)); - manager.start(helper.getConfiguration()); + LruBlockCacheConfiguration.builder(CacheType.INDEX).useEvictionThread(false).minFactor(0.98f).acceptableFactor(0.99f).singleFactor(0.25f) + .multiFactor(0.50f).memoryFactor(0.25f).buildMap().forEach(cc::set); + manager.start(cc); LruBlockCache cache = (LruBlockCache) manager.getBlockCache(CacheType.INDEX); Block[] singleBlocks = generateFixedBlocks(5, 10000, "single"); @@ -256,14 +273,9 @@ public class TestLruBlockCache extends TestCase { BlockCacheManager manager = BlockCacheManager.getInstance(cc); cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(blockSize)); cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(maxSize)); - BlockCacheConfigurationHelper helper = new BlockCacheConfigurationHelper(cc, CacheType.INDEX, LruBlockCacheConfiguration.PROPERTY_PREFIX); - helper.set(LruBlockCacheConfiguration.EVICTION_THREAD_PROPERTY, Boolean.FALSE.toString()); - helper.set(LruBlockCacheConfiguration.MIN_FACTOR_PROPERTY, Float.toString(0.98f)); - helper.set(LruBlockCacheConfiguration.ACCEPTABLE_FACTOR_PROPERTY, Float.toString(0.99f)); - helper.set(LruBlockCacheConfiguration.SINGLE_FACTOR_PROPERTY, Float.toString(0.33f)); - helper.set(LruBlockCacheConfiguration.MULTI_FACTOR_PROPERTY, Float.toString(0.33f)); - helper.set(LruBlockCacheConfiguration.MEMORY_FACTOR_PROPERTY, Float.toString(0.34f)); - manager.start(helper.getConfiguration()); + LruBlockCacheConfiguration.builder(CacheType.INDEX).useEvictionThread(false).minFactor(0.98f).acceptableFactor(0.99f).singleFactor(0.33f) + .multiFactor(0.33f).memoryFactor(0.34f).buildMap().forEach(cc::set); + manager.start(cc); LruBlockCache cache = (LruBlockCache) manager.getBlockCache(CacheType.INDEX); Block[] singleBlocks = generateFixedBlocks(5, blockSize, "single"); @@ -384,15 +396,9 @@ public class TestLruBlockCache extends TestCase { BlockCacheManager manager = BlockCacheManager.getInstance(cc); cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(blockSize)); cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(maxSize)); - BlockCacheConfigurationHelper helper = new BlockCacheConfigurationHelper(cc, CacheType.INDEX, LruBlockCacheConfiguration.PROPERTY_PREFIX); - helper.set(LruBlockCacheConfiguration.EVICTION_THREAD_PROPERTY, Boolean.FALSE.toString()); - helper.set(LruBlockCacheConfiguration.EVICTION_THREAD_PROPERTY, Boolean.FALSE.toString()); - helper.set(LruBlockCacheConfiguration.MIN_FACTOR_PROPERTY, Float.toString(0.66f)); - helper.set(LruBlockCacheConfiguration.ACCEPTABLE_FACTOR_PROPERTY, Float.toString(0.99f)); - helper.set(LruBlockCacheConfiguration.SINGLE_FACTOR_PROPERTY, Float.toString(0.33f)); - helper.set(LruBlockCacheConfiguration.MULTI_FACTOR_PROPERTY, Float.toString(0.33f)); - helper.set(LruBlockCacheConfiguration.MEMORY_FACTOR_PROPERTY, Float.toString(0.34f)); - manager.start(helper.getConfiguration()); + LruBlockCacheConfiguration.builder(CacheType.INDEX).useEvictionThread(false).minFactor(0.66f).acceptableFactor(0.99f).singleFactor(0.33f) + .multiFactor(0.33f).memoryFactor(0.34f).buildMap().forEach(cc::set); + manager.start(cc); LruBlockCache cache = (LruBlockCache) manager.getBlockCache(CacheType.INDEX); Block[] singleBlocks = generateFixedBlocks(20, blockSize, "single");
