ignite-916 Eviction policy should evict cache entries when memory size limit is reached
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/9d89d9c6 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/9d89d9c6 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/9d89d9c6 Branch: refs/heads/ignite-916 Commit: 9d89d9c6aceaabe8d9daa4d8ff89ae9af5f1b2eb Parents: ba7fddb Author: agura <ag...@gridgain.com> Authored: Mon May 18 21:01:36 2015 +0300 Committer: agura <ag...@gridgain.com> Committed: Tue May 19 12:15:06 2015 +0300 ---------------------------------------------------------------------- .../ignite/cache/eviction/EvictableEntry.java | 7 + .../ignite/cache/eviction/EvictionPolicy.java | 2 + .../cache/eviction/fifo/FifoEvictionPolicy.java | 105 ++++- .../cache/CacheEvictableEntryImpl.java | 17 + .../eviction/GridCacheEvictionAbstractTest.java | 17 + .../cache/eviction/GridCacheMockEntry.java | 5 + ...ridCacheFifoBatchEvictionPolicySelfTest.java | 63 ++- .../GridCacheFifoEvictionPolicySelfTest.java | 64 ++- ...idCacheFifoMemoryEvictionPolicySelfTest.java | 413 +++++++++++++++++++ 9 files changed, 661 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/9d89d9c6/modules/core/src/main/java/org/apache/ignite/cache/eviction/EvictableEntry.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/cache/eviction/EvictableEntry.java b/modules/core/src/main/java/org/apache/ignite/cache/eviction/EvictableEntry.java index d87109f..9f1889a 100644 --- a/modules/core/src/main/java/org/apache/ignite/cache/eviction/EvictableEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/cache/eviction/EvictableEntry.java @@ -46,6 +46,13 @@ public interface EvictableEntry<K, V> extends Cache.Entry<K, V> { public boolean isCached(); /** + * Returns entry size in bytes. + * + * @return entry size in bytes. + */ + public int size(); + + /** * Gets metadata added by eviction policy. * * @return Metadata value or {@code null}. http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/9d89d9c6/modules/core/src/main/java/org/apache/ignite/cache/eviction/EvictionPolicy.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/cache/eviction/EvictionPolicy.java b/modules/core/src/main/java/org/apache/ignite/cache/eviction/EvictionPolicy.java index f409e9b..07c269d 100644 --- a/modules/core/src/main/java/org/apache/ignite/cache/eviction/EvictionPolicy.java +++ b/modules/core/src/main/java/org/apache/ignite/cache/eviction/EvictionPolicy.java @@ -20,6 +20,7 @@ package org.apache.ignite.cache.eviction; import org.apache.ignite.cache.eviction.fifo.*; import org.apache.ignite.cache.eviction.lru.*; import org.apache.ignite.cache.eviction.random.*; +import org.apache.ignite.cache.eviction.sorted.*; /** * Pluggable cache eviction policy. Usually, implementations will internally order @@ -32,6 +33,7 @@ import org.apache.ignite.cache.eviction.random.*; * <li>{@link LruEvictionPolicy}</li> * <li>{@link RandomEvictionPolicy}</li> * <li>{@link FifoEvictionPolicy}</li> + * <li>{@link SortedEvictionPolicy}</li> * </ul> * <p> * The eviction policy thread-safety is ensured by Ignition. Implementations of this interface should http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/9d89d9c6/modules/core/src/main/java/org/apache/ignite/cache/eviction/fifo/FifoEvictionPolicy.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/cache/eviction/fifo/FifoEvictionPolicy.java b/modules/core/src/main/java/org/apache/ignite/cache/eviction/fifo/FifoEvictionPolicy.java index bf8cf0d..06f67a2 100644 --- a/modules/core/src/main/java/org/apache/ignite/cache/eviction/fifo/FifoEvictionPolicy.java +++ b/modules/core/src/main/java/org/apache/ignite/cache/eviction/fifo/FifoEvictionPolicy.java @@ -46,6 +46,12 @@ public class FifoEvictionPolicy<K, V> implements EvictionPolicy<K, V>, FifoEvict /** Batch size. */ private volatile int batchSize = 1; + /** Max memory size. */ + private volatile long maxMemSize; + + /** Memory size. */ + private final LongAdder8 memSize = new LongAdder8(); + /** FIFO queue. */ private final ConcurrentLinkedDeque8<EvictableEntry<K, V>> queue = new ConcurrentLinkedDeque8<>(); @@ -83,6 +89,17 @@ public class FifoEvictionPolicy<K, V> implements EvictionPolicy<K, V>, FifoEvict } /** + * Constructs FIFO eviction policy with maximum memory size. + * + * @param maxMemSize Max mem size. + */ + public FifoEvictionPolicy(long maxMemSize) { + A.ensure(maxMemSize > 0, "maxMemSize > 0"); + + this.maxMemSize = maxMemSize; + } + + /** * Gets maximum allowed size of cache before entry will start getting evicted. * * @return Maximum allowed size of cache before entry will start getting evicted. @@ -114,11 +131,25 @@ public class FifoEvictionPolicy<K, V> implements EvictionPolicy<K, V>, FifoEvict this.batchSize = batchSize; } + public long getMaxMemSize() { + return maxMemSize; + } + + public void setMaxMemSize(long maxMemSize) { + A.ensure(maxMemSize > 0, "maxMemSize > 0"); + + this.maxMemSize = maxMemSize; + } + /** {@inheritDoc} */ @Override public int getCurrentSize() { return queue.size(); } + public long getCurrentMemSize() { + return memSize.longValue(); + } + /** * Gets read-only view on internal {@code FIFO} queue in proper order. * @@ -141,8 +172,11 @@ public class FifoEvictionPolicy<K, V> implements EvictionPolicy<K, V>, FifoEvict else { Node<EvictableEntry<K, V>> node = entry.removeMeta(); - if (node != null) + if (node != null) { queue.unlinkx(node); + + memSize.add(-entry.size()); + } } } @@ -173,11 +207,18 @@ public class FifoEvictionPolicy<K, V> implements EvictionPolicy<K, V>, FifoEvict return false; } + memSize.add(entry.size()); + return true; } // If node was unlinked by concurrent shrink() call, we must repeat the whole cycle. else if (!entry.removeMeta(node)) return false; + else { + memSize.add(-entry.size()); + + return true; + } } } @@ -189,38 +230,76 @@ public class FifoEvictionPolicy<K, V> implements EvictionPolicy<K, V>, FifoEvict * Shrinks FIFO queue to maximum allowed size. */ private void shrink() { + long maxMem = this.maxMemSize; + + if (maxMem > 0) { + long startMemSize = memSize.longValue(); + + if (startMemSize >= maxMem) + for (long i = maxMem; i < startMemSize && memSize.longValue() > maxMem;) { + int size = shrink0(); + + if (size == -1) + break; + + i += size; + } + } + int max = this.max; - int batchSize = this.batchSize; + if (max > 0) { + int batchSize = this.batchSize; - int startSize = queue.sizex(); + int startSize = queue.sizex(); - // Shrink only if queue is full. - if (startSize >= max + batchSize) { - for (int i = max; i < startSize && queue.sizex() > max; i++) { - EvictableEntry<K, V> entry = queue.poll(); + // Shrink only if queue is full. + if (startSize >= max + batchSize) + for (int i = max; i < startSize && queue.sizex() > max; i++) + if (shrink0() == -1) + break; + } + } - if (entry == null) - break; + /** + * Tries to remove one item from queue. + * + * @return number of bytes that was free. {@code -1} if queue is empty. + */ + private int shrink0() { + EvictableEntry<K, V> entry = queue.poll(); - Node<EvictableEntry<K, V>> meta = entry.removeMeta(); + if (entry == null) + return -1; - if (meta != null && !entry.evict()) - touch(entry); - } + int size = 0; + + Node<EvictableEntry<K, V>> meta = entry.removeMeta(); + + if (meta != null) { + size = entry.size(); + + memSize.add(-size); + + if (!entry.evict()) + touch(entry); } + + return size; } /** {@inheritDoc} */ @Override public void writeExternal(ObjectOutput out) throws IOException { out.writeInt(max); out.writeInt(batchSize); + out.writeLong(maxMemSize); } /** {@inheritDoc} */ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { max = in.readInt(); batchSize = in.readInt(); + maxMemSize = in.readLong(); } /** {@inheritDoc} */ http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/9d89d9c6/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEvictableEntryImpl.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEvictableEntryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEvictableEntryImpl.java index 5d6062e..8b2aad8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEvictableEntryImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEvictableEntryImpl.java @@ -91,6 +91,23 @@ public class CacheEvictableEntryImpl<K, V> implements EvictableEntry<K, V> { } /** {@inheritDoc} */ + public int size() { + try { + int keySize = cached.key().valueBytes(cached.context().cacheObjectContext()).length; + + CacheObject val = cached.peek(true, false, false, null); + + return val == null ? keySize : keySize + val.valueBytes(cached.context().cacheObjectContext()).length; + } + catch (GridCacheEntryRemovedException e) { + return 0; + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + } + + /** {@inheritDoc} */ @SuppressWarnings("unchecked") @Override public V getValue() { try { http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/9d89d9c6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEvictionAbstractTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEvictionAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEvictionAbstractTest.java index e0dab7d..bc17f14 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEvictionAbstractTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheEvictionAbstractTest.java @@ -66,6 +66,9 @@ public abstract class GridCacheEvictionAbstractTest<T extends EvictionPolicy<?, /** Policy batch size. */ protected int plcBatchSize = 0; + /** Policy max memory size. */ + protected long plcMaxMemSize = 0; + /** Policy max. */ protected int plcMax = 10; @@ -431,6 +434,15 @@ public abstract class GridCacheEvictionAbstractTest<T extends EvictionPolicy<?, */ @SuppressWarnings({"PublicConstructorInNonPublicClass"}) protected static class MockEntry extends GridCacheMockEntry<String, String> { + /** Key size. */ + public static final int KEY_SIZE = 1; + + /** Value size. */ + public static final int VALUE_SIZE = 1; + + /** Entry size. */ + public static final int ENTRY_SIZE = KEY_SIZE + VALUE_SIZE; + /** */ private IgniteCache<String, String> parent; @@ -477,6 +489,11 @@ public abstract class GridCacheEvictionAbstractTest<T extends EvictionPolicy<?, } /** {@inheritDoc} */ + @Override public int size() { + return val == null ? KEY_SIZE : ENTRY_SIZE; + } + + /** {@inheritDoc} */ @Override public String toString() { return S.toString(MockEntry.class, this, super.toString()); } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/9d89d9c6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheMockEntry.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheMockEntry.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheMockEntry.java index 5898ed9..5d183bf 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheMockEntry.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/GridCacheMockEntry.java @@ -83,6 +83,11 @@ public class GridCacheMockEntry<K, V> extends GridMetadataAwareAdapter implement return !evicted; } + /** {@inheritDoc} */ + @Override public int size() { + return 0; + } + /** * * @return Evicted or not. http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/9d89d9c6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/fifo/GridCacheFifoBatchEvictionPolicySelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/fifo/GridCacheFifoBatchEvictionPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/fifo/GridCacheFifoBatchEvictionPolicySelfTest.java index f907c36..d2dd3d2 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/fifo/GridCacheFifoBatchEvictionPolicySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/fifo/GridCacheFifoBatchEvictionPolicySelfTest.java @@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.cache.eviction.fifo; import org.apache.ignite.*; import org.apache.ignite.cache.eviction.*; import org.apache.ignite.cache.eviction.fifo.*; +import org.apache.ignite.internal.processors.cache.*; import org.apache.ignite.internal.processors.cache.eviction.*; import java.util.*; @@ -53,14 +54,17 @@ public class GridCacheFifoBatchEvictionPolicySelfTest extends p.onEntryAccessed(false, e1); check(p.queue(), e1); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); p.onEntryAccessed(false, e2); check(p.queue(), e1, e2); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); p.onEntryAccessed(false, e3); check(p.queue(), e1, e2, e3); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); p.onEntryAccessed(false, e4); @@ -72,6 +76,7 @@ public class GridCacheFifoBatchEvictionPolicySelfTest extends assertFalse(e4.isEvicted()); assertEquals(4, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); p.onEntryAccessed(false, e5); @@ -79,6 +84,7 @@ public class GridCacheFifoBatchEvictionPolicySelfTest extends check(p.queue(), e3, e4, e5); assertEquals(3, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); assertTrue(e1.isEvicted()); assertTrue(e2.isEvicted()); @@ -91,6 +97,7 @@ public class GridCacheFifoBatchEvictionPolicySelfTest extends check(p.queue(), e3, e4, e5, e1); assertEquals(4, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); assertFalse(e3.isEvicted()); assertFalse(e4.isEvicted()); @@ -101,6 +108,8 @@ public class GridCacheFifoBatchEvictionPolicySelfTest extends check(p.queue(), e3, e4, e5, e1); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); + assertFalse(e3.isEvicted()); assertFalse(e4.isEvicted()); assertFalse(e5.isEvicted()); @@ -108,10 +117,11 @@ public class GridCacheFifoBatchEvictionPolicySelfTest extends p.onEntryAccessed(false, e1); - assertEquals(4, p.getCurrentSize()); - check(p.queue(), e3, e4, e5, e1); + assertEquals(4, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); + assertFalse(e3.isEvicted()); assertFalse(e4.isEvicted()); assertFalse(e5.isEvicted()); @@ -120,6 +130,7 @@ public class GridCacheFifoBatchEvictionPolicySelfTest extends p.onEntryAccessed(true, e1); assertEquals(3, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); assertFalse(e3.isEvicted()); assertFalse(e4.isEvicted()); @@ -128,6 +139,7 @@ public class GridCacheFifoBatchEvictionPolicySelfTest extends p.onEntryAccessed(true, e4); assertEquals(2, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); assertFalse(e3.isEvicted()); assertFalse(e5.isEvicted()); @@ -135,12 +147,14 @@ public class GridCacheFifoBatchEvictionPolicySelfTest extends p.onEntryAccessed(true, e5); assertEquals(1, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); assertFalse(e3.isEvicted()); p.onEntryAccessed(true, e3); assertEquals(0, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); assertFalse(e3.isEvicted()); @@ -175,6 +189,7 @@ public class GridCacheFifoBatchEvictionPolicySelfTest extends info(p); assertEquals(cnt - batchSize, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); } finally { stopAllGrids(); @@ -228,6 +243,8 @@ public class GridCacheFifoBatchEvictionPolicySelfTest extends assert curSize < max + batchSize : "curSize < max + batchSize [curSize=" + curSize + ", max=" + max + ", batchSize=" + batchSize + ']'; + + assertEquals(p.queue().size() * MockEntry.KEY_SIZE, p.getCurrentMemSize()); } finally { stopAllGrids(); @@ -242,13 +259,9 @@ public class GridCacheFifoBatchEvictionPolicySelfTest extends startGrid(); MockEntry e1 = new MockEntry("1"); - MockEntry e2 = new MockEntry("2"); - MockEntry e3 = new MockEntry("3"); - MockEntry e4 = new MockEntry("4"); - MockEntry e5 = new MockEntry("5"); FifoEvictionPolicy<String, String> p = policy(); @@ -259,27 +272,37 @@ public class GridCacheFifoBatchEvictionPolicySelfTest extends assertFalse(e1.isEvicted()); + assertEquals(p.queue().size() * MockEntry.KEY_SIZE, p.getCurrentMemSize()); + p.onEntryAccessed(false, e2); assertFalse(e1.isEvicted()); assertFalse(e2.isEvicted()); + assertEquals(p.queue().size() * MockEntry.KEY_SIZE, p.getCurrentMemSize()); + p.onEntryAccessed(false, e3); assertFalse(e1.isEvicted()); assertFalse(e3.isEvicted()); + assertEquals(p.queue().size() * MockEntry.KEY_SIZE, p.getCurrentMemSize()); + p.onEntryAccessed(false, e4); assertFalse(e1.isEvicted()); assertFalse(e3.isEvicted()); assertFalse(e4.isEvicted()); + assertEquals(p.queue().size() * MockEntry.KEY_SIZE, p.getCurrentMemSize()); + p.onEntryAccessed(false, e5); assertFalse(e1.isEvicted()); assertFalse(e3.isEvicted()); assertFalse(e5.isEvicted()); + + assertEquals(p.queue().size() * MockEntry.KEY_SIZE, p.getCurrentMemSize()); } finally { stopAllGrids(); @@ -299,12 +322,19 @@ public class GridCacheFifoBatchEvictionPolicySelfTest extends try { IgniteCache<Object, Object> cache = ignite.cache(null); + GridCacheContext cctx = ((IgniteCacheProxy)cache).context(); + + FifoEvictionPolicy plc = + (FifoEvictionPolicy)cctx.config().getEvictionPolicy(); + int cnt = 500; int min = Integer.MAX_VALUE; int minIdx = 0; + int memSize = 0; + for (int i = 0; i < cnt; i++) { cache.put(i, i); @@ -314,11 +344,19 @@ public class GridCacheFifoBatchEvictionPolicySelfTest extends min = cacheSize; minIdx = i; } + + if (i >= cnt - plcMax) { + memSize += cctx.toCacheKeyObject(i).valueBytes(cctx.cacheObjectContext()).length; + memSize += cctx.toCacheObject(i).valueBytes(cctx.cacheObjectContext()).length; + } } // Batch evicted. assert min >= plcMax : "Min cache size is too small: " + min; + assertEquals(plcMax, plc.getCurrentSize()); + assertEquals(memSize, plc.getCurrentMemSize()); + info("Min cache size [min=" + min + ", idx=" + minIdx + ']'); info("Current cache size " + cache.size()); info("Current cache key size " + cache.size()); @@ -346,6 +384,9 @@ public class GridCacheFifoBatchEvictionPolicySelfTest extends // Batch evicted. assert min >= plcMax : "Min cache size is too small: " + min; + + assertEquals(plcMax, plc.getCurrentSize()); + assertEquals(memSize, plc.getCurrentMemSize()); } finally { stopAllGrids(); @@ -378,7 +419,15 @@ public class GridCacheFifoBatchEvictionPolicySelfTest extends /** {@inheritDoc} */ @Override protected void checkPolicies(int plcMax) { - for (int i = 0; i < gridCnt; i++) + for (int i = 0; i < gridCnt; i++) { assert policy(i).queue().size() <= plcMax + policy(i).getBatchSize(); + + int size = 0; + + for (EvictableEntry<String, String> entry : policy(i).queue()) + size += ((CacheEvictableEntryImpl)entry).size(); + + assertEquals(size, ((FifoEvictionPolicy)policy(i)).getCurrentMemSize()); + } } } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/9d89d9c6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/fifo/GridCacheFifoEvictionPolicySelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/fifo/GridCacheFifoEvictionPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/fifo/GridCacheFifoEvictionPolicySelfTest.java index 64d3831..14b536b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/fifo/GridCacheFifoEvictionPolicySelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/fifo/GridCacheFifoEvictionPolicySelfTest.java @@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.cache.eviction.fifo; import org.apache.ignite.*; import org.apache.ignite.cache.eviction.*; import org.apache.ignite.cache.eviction.fifo.*; +import org.apache.ignite.internal.processors.cache.*; import org.apache.ignite.internal.processors.cache.eviction.*; import java.util.*; @@ -52,26 +53,30 @@ public class GridCacheFifoEvictionPolicySelfTest extends p.onEntryAccessed(false, e1); check(p.queue(), e1); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); p.onEntryAccessed(false, e2); check(p.queue(), e1, e2); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); p.onEntryAccessed(false, e3); check(p.queue(), e1, e2, e3); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); assert !e1.isEvicted(); assert !e2.isEvicted(); assert !e3.isEvicted(); assertEquals(3, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); p.onEntryAccessed(false, e4); check(p.queue(), e2, e3, e4); - assertEquals(3, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); assert e1.isEvicted(); assert !e2.isEvicted(); @@ -81,8 +86,8 @@ public class GridCacheFifoEvictionPolicySelfTest extends p.onEntryAccessed(false, e5); check(p.queue(), e3, e4, e5); - assertEquals(3, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); assert e2.isEvicted(); assert !e3.isEvicted(); @@ -92,8 +97,8 @@ public class GridCacheFifoEvictionPolicySelfTest extends p.onEntryAccessed(false, e1 = new MockEntry("1", "1")); check(p.queue(), e4, e5, e1); - assertEquals(3, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); assert e3.isEvicted(); assert !e1.isEvicted(); @@ -103,6 +108,7 @@ public class GridCacheFifoEvictionPolicySelfTest extends p.onEntryAccessed(false, e5); check(p.queue(), e4, e5, e1); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); assert !e1.isEvicted(); assert !e4.isEvicted(); @@ -110,9 +116,9 @@ public class GridCacheFifoEvictionPolicySelfTest extends p.onEntryAccessed(false, e1); - assertEquals(3, p.getCurrentSize()); - check(p.queue(), e4, e5, e1); + assertEquals(3, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); assert !e1.isEvicted(); assert !e4.isEvicted(); @@ -120,9 +126,9 @@ public class GridCacheFifoEvictionPolicySelfTest extends p.onEntryAccessed(false, e5); - assertEquals(3, p.getCurrentSize()); - check(p.queue(), e4, e5, e1); + assertEquals(3, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); assert !e1.isEvicted(); assert !e4.isEvicted(); @@ -131,6 +137,7 @@ public class GridCacheFifoEvictionPolicySelfTest extends p.onEntryAccessed(true, e1); assertEquals(2, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); assert !e1.isEvicted(); assert !e4.isEvicted(); @@ -139,6 +146,7 @@ public class GridCacheFifoEvictionPolicySelfTest extends p.onEntryAccessed(true, e4); assertEquals(1, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); assert !e4.isEvicted(); assert !e5.isEvicted(); @@ -146,6 +154,7 @@ public class GridCacheFifoEvictionPolicySelfTest extends p.onEntryAccessed(true, e5); assertEquals(0, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); assert !e5.isEvicted(); @@ -177,6 +186,7 @@ public class GridCacheFifoEvictionPolicySelfTest extends info(p); assertEquals(max, p.getCurrentSize()); + assertEquals(max * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); } finally { stopAllGrids(); @@ -223,8 +233,11 @@ public class GridCacheFifoEvictionPolicySelfTest extends info(p); int curSize = p.getCurrentSize(); + long curMemSize = p.getCurrentMemSize(); assert curSize <= max : "curSize <= max [curSize=" + curSize + ", max=" + max + ']'; + assert curMemSize <= max * MockEntry.ENTRY_SIZE : "curMemSize <= maxMemSize [curMemSize=" + curMemSize + + ", maxMemSize =" + max * MockEntry.ENTRY_SIZE + ']'; } finally { stopAllGrids(); @@ -239,13 +252,9 @@ public class GridCacheFifoEvictionPolicySelfTest extends startGrid(); MockEntry e1 = new MockEntry("1"); - MockEntry e2 = new MockEntry("2"); - MockEntry e3 = new MockEntry("3"); - MockEntry e4 = new MockEntry("4"); - MockEntry e5 = new MockEntry("5"); FifoEvictionPolicy<String, String> p = policy(); @@ -255,28 +264,33 @@ public class GridCacheFifoEvictionPolicySelfTest extends p.onEntryAccessed(false, e1); assertFalse(e1.isEvicted()); + assertEquals(p.queue().size() * MockEntry.KEY_SIZE, p.getCurrentMemSize()); p.onEntryAccessed(false, e2); assertFalse(e1.isEvicted()); assertFalse(e2.isEvicted()); + assertEquals(p.queue().size() * MockEntry.KEY_SIZE, p.getCurrentMemSize()); p.onEntryAccessed(false, e3); assertFalse(e1.isEvicted()); assertFalse(e3.isEvicted()); + assertEquals(p.queue().size() * MockEntry.KEY_SIZE, p.getCurrentMemSize()); p.onEntryAccessed(false, e4); assertFalse(e1.isEvicted()); assertFalse(e3.isEvicted()); assertFalse(e4.isEvicted()); + assertEquals(p.queue().size() * MockEntry.KEY_SIZE, p.getCurrentMemSize()); p.onEntryAccessed(false, e5); assertFalse(e1.isEvicted()); assertFalse(e3.isEvicted()); assertFalse(e5.isEvicted()); + assertEquals(p.queue().size() * MockEntry.KEY_SIZE, p.getCurrentMemSize()); } finally { stopAllGrids(); @@ -296,12 +310,19 @@ public class GridCacheFifoEvictionPolicySelfTest extends try { IgniteCache<Object, Object> cache = ignite.cache(null); + GridCacheContext cctx = ((IgniteCacheProxy)cache).context(); + + FifoEvictionPolicy plc = + (FifoEvictionPolicy)cctx.config().getEvictionPolicy(); + int cnt = 500; int min = Integer.MAX_VALUE; int minIdx = 0; + int memSize = 0; + for (int i = 0; i < cnt; i++) { cache.put(i, i); @@ -311,10 +332,18 @@ public class GridCacheFifoEvictionPolicySelfTest extends min = cacheSize; minIdx = i; } + + if (i >= cnt - plcMax) { + memSize += cctx.toCacheKeyObject(i).valueBytes(cctx.cacheObjectContext()).length; + memSize += cctx.toCacheObject(i).valueBytes(cctx.cacheObjectContext()).length; + } } assert min >= plcMax : "Min cache size is too small: " + min; + assertEquals(plcMax, plc.getCurrentSize()); + assertEquals(memSize, plc.getCurrentMemSize()); + info("Min cache size [min=" + min + ", idx=" + minIdx + ']'); info("Current cache size " + cache.size()); info("Current cache key size " + cache.size()); @@ -341,6 +370,9 @@ public class GridCacheFifoEvictionPolicySelfTest extends info("Current cache key size " + cache.size()); assert min >= plcMax : "Min cache size is too small: " + min; + + assertEquals(plcMax, plc.getCurrentSize()); + assertEquals(memSize, plc.getCurrentMemSize()); } finally { stopAllGrids(); @@ -366,7 +398,15 @@ public class GridCacheFifoEvictionPolicySelfTest extends /** {@inheritDoc} */ @Override protected void checkPolicies(int plcMax) { - for (int i = 0; i < gridCnt; i++) + for (int i = 0; i < gridCnt; i++) { assert policy(i).queue().size() <= plcMax; + + int size = 0; + + for (EvictableEntry<String, String> entry : policy(i).queue()) + size += ((CacheEvictableEntryImpl)entry).size(); + + assertEquals(size, ((FifoEvictionPolicy)policy(i)).getCurrentMemSize()); + } } } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/9d89d9c6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/fifo/GridCacheFifoMemoryEvictionPolicySelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/fifo/GridCacheFifoMemoryEvictionPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/fifo/GridCacheFifoMemoryEvictionPolicySelfTest.java new file mode 100644 index 0000000..610db1d --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/fifo/GridCacheFifoMemoryEvictionPolicySelfTest.java @@ -0,0 +1,413 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.eviction.fifo; + +import org.apache.ignite.*; +import org.apache.ignite.cache.eviction.*; +import org.apache.ignite.cache.eviction.fifo.*; +import org.apache.ignite.internal.processors.cache.*; +import org.apache.ignite.internal.processors.cache.eviction.*; + +import java.util.*; + +import static org.apache.ignite.cache.CacheMode.*; + +/** + * FIFO memory size limited eviction test. + */ +public class GridCacheFifoMemoryEvictionPolicySelfTest extends + GridCacheEvictionAbstractTest<FifoEvictionPolicy<String, String>> { + /** + * @throws Exception If failed. + */ + public void testPolicy() throws Exception { + try { + startGrid(); + + MockEntry e1 = new MockEntry("1", "1"); + MockEntry e2 = new MockEntry("2", "2"); + MockEntry e3 = new MockEntry("3", "3"); + MockEntry e4 = new MockEntry("4", "4"); + MockEntry e5 = new MockEntry("5", "5"); + + FifoEvictionPolicy<String, String> p = policy(); + + p.setMaxMemSize(6); + + p.onEntryAccessed(false, e1); + + check(p.queue(), e1); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); + + p.onEntryAccessed(false, e2); + + check(p.queue(), e1, e2); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); + + p.onEntryAccessed(false, e3); + + check(p.queue(), e1, e2, e3); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); + + assert !e1.isEvicted(); + assert !e2.isEvicted(); + assert !e3.isEvicted(); + + assertEquals(3, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); + + p.onEntryAccessed(false, e4); + + check(p.queue(), e2, e3, e4); + assertEquals(3, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); + + assert e1.isEvicted(); + assert !e2.isEvicted(); + assert !e3.isEvicted(); + assert !e4.isEvicted(); + + p.onEntryAccessed(false, e5); + + check(p.queue(), e3, e4, e5); + assertEquals(3, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); + + assert e2.isEvicted(); + assert !e3.isEvicted(); + assert !e4.isEvicted(); + assert !e5.isEvicted(); + + p.onEntryAccessed(false, e1 = new MockEntry("1", "1")); + + check(p.queue(), e4, e5, e1); + assertEquals(3, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); + + assert e3.isEvicted(); + assert !e1.isEvicted(); + assert !e4.isEvicted(); + assert !e5.isEvicted(); + + p.onEntryAccessed(false, e5); + + check(p.queue(), e4, e5, e1); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); + + assert !e1.isEvicted(); + assert !e4.isEvicted(); + assert !e5.isEvicted(); + + p.onEntryAccessed(false, e1); + + check(p.queue(), e4, e5, e1); + assertEquals(3, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); + + assert !e1.isEvicted(); + assert !e4.isEvicted(); + assert !e5.isEvicted(); + + p.onEntryAccessed(false, e5); + + check(p.queue(), e4, e5, e1); + assertEquals(3, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); + + assert !e1.isEvicted(); + assert !e4.isEvicted(); + assert !e5.isEvicted(); + + p.onEntryAccessed(true, e1); + + assertEquals(2, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); + + assert !e1.isEvicted(); + assert !e4.isEvicted(); + assert !e5.isEvicted(); + + p.onEntryAccessed(true, e4); + + assertEquals(1, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); + + assert !e4.isEvicted(); + assert !e5.isEvicted(); + + p.onEntryAccessed(true, e5); + + assertEquals(0, p.getCurrentSize()); + assertEquals(p.queue().size() * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); + + assert !e5.isEvicted(); + + info(p); + } + finally { + stopAllGrids(); + } + } + + /** + * @throws Exception If failed. + */ + public void testMemory() throws Exception { + try { + startGrid(); + + FifoEvictionPolicy<String, String> p = policy(); + + int max = 10; + + p.setMaxMemSize(20); + + int cnt = 11; + + for (int i = 0; i < cnt; i++) + p.onEntryAccessed(false, new MockEntry(Integer.toString(i), Integer.toString(i))); + + info(p); + + assertEquals(max, p.getCurrentSize()); + assertEquals(max * MockEntry.ENTRY_SIZE, p.getCurrentMemSize()); + } + finally { + stopAllGrids(); + } + } + + /** + * @throws Exception If failed. + */ + public void testRandom() throws Exception { + try { + startGrid(); + + FifoEvictionPolicy<String, String> p = policy(); + + int max = 10; + + p.setMaxMemSize(10); + + Random rand = new Random(); + + int keys = 31; + + MockEntry[] fifos = new MockEntry[keys]; + + for (int i = 0; i < fifos.length; i++) + fifos[i] = new MockEntry(Integer.toString(i)); + + int runs = 5000000; + + for (int i = 0; i < runs; i++) { + boolean rmv = rand.nextBoolean(); + + int j = rand.nextInt(fifos.length); + + MockEntry e = entry(fifos, j); + + if (rmv) + fifos[j] = new MockEntry(Integer.toString(j)); + + p.onEntryAccessed(rmv, e); + } + + info(p); + + int curSize = p.getCurrentSize(); + long curMemSize = p.getCurrentMemSize(); + + assert curSize <= max : "curSize <= max [curSize=" + curSize + ", max=" + max + ']'; + assert curMemSize <= max * MockEntry.ENTRY_SIZE : "curMemSize <= maxMemSize [curMemSize=" + curMemSize + + ", maxMemSize =" + max * MockEntry.ENTRY_SIZE + ']'; + } + finally { + stopAllGrids(); + } + } + + /** + * @throws Exception If failed. + */ + public void testAllowEmptyEntries() throws Exception { + try { + startGrid(); + + MockEntry e1 = new MockEntry("1"); + MockEntry e2 = new MockEntry("2"); + MockEntry e3 = new MockEntry("3"); + MockEntry e4 = new MockEntry("4"); + MockEntry e5 = new MockEntry("5"); + + FifoEvictionPolicy<String, String> p = policy(); + + p.setMaxMemSize(10); + + p.onEntryAccessed(false, e1); + + assertFalse(e1.isEvicted()); + assertEquals(p.queue().size() * MockEntry.KEY_SIZE, p.getCurrentMemSize()); + + p.onEntryAccessed(false, e2); + + assertFalse(e1.isEvicted()); + assertFalse(e2.isEvicted()); + assertEquals(p.queue().size() * MockEntry.KEY_SIZE, p.getCurrentMemSize()); + + p.onEntryAccessed(false, e3); + + assertFalse(e1.isEvicted()); + assertFalse(e3.isEvicted()); + assertEquals(p.queue().size() * MockEntry.KEY_SIZE, p.getCurrentMemSize()); + + p.onEntryAccessed(false, e4); + + assertFalse(e1.isEvicted()); + assertFalse(e3.isEvicted()); + assertFalse(e4.isEvicted()); + assertEquals(p.queue().size() * MockEntry.KEY_SIZE, p.getCurrentMemSize()); + + p.onEntryAccessed(false, e5); + + assertFalse(e1.isEvicted()); + assertFalse(e3.isEvicted()); + assertFalse(e5.isEvicted()); + assertEquals(p.queue().size() * MockEntry.KEY_SIZE, p.getCurrentMemSize()); + } + finally { + stopAllGrids(); + } + } + + /** + * @throws Exception If failed. + */ + public void testPut() throws Exception { + mode = LOCAL; + syncCommit = true; + plcMax = 0; + int max = 100; + + Ignite ignite = startGrid(); + + try { + IgniteCache<Object, Object> cache = ignite.cache(null); + + GridCacheContext cctx = ((IgniteCacheProxy)cache).context(); + + FifoEvictionPolicy plc = + (FifoEvictionPolicy)cctx.config().getEvictionPolicy(); + + int cnt = 500; + + int min = Integer.MAX_VALUE; + + int minIdx = 0; + + int memSize = 0; + + for (int i = 0; i < cnt; i++) { + cache.put(i, i); + + int cacheSize = cache.size(); + + if (i > max && cacheSize < min) { + min = cacheSize; + minIdx = i; + } + + if (i >= cnt - max) { + memSize += cctx.toCacheKeyObject(i).valueBytes(cctx.cacheObjectContext()).length; + memSize += cctx.toCacheObject(i).valueBytes(cctx.cacheObjectContext()).length; + } + } + +// assert min >= max : "Min cache size is too small: " + min; + + assertEquals(max, plc.getCurrentSize()); + assertEquals(memSize, plc.getCurrentMemSize()); + + info("Min cache size [min=" + min + ", idx=" + minIdx + ']'); + info("Current cache size " + cache.size()); + info("Current cache key size " + cache.size()); + + min = Integer.MAX_VALUE; + + minIdx = 0; + + // Touch. + for (int i = cnt; --i > cnt - plcMax;) { + cache.get(i); + + int cacheSize = cache.size(); + + if (cacheSize < min) { + min = cacheSize; + minIdx = i; + } + } + + info("----"); + info("Min cache size [min=" + min + ", idx=" + minIdx + ']'); + info("Current cache size " + cache.size()); + info("Current cache key size " + cache.size()); + + assert min >= max : "Min cache size is too small: " + min; + + assertEquals(max, plc.getCurrentSize()); + assertEquals(memSize, plc.getCurrentMemSize()); + } + finally { + stopAllGrids(); + } + } + + /** {@inheritDoc} */ + @Override protected FifoEvictionPolicy<String, String> createPolicy(int plcMax) { + return new FifoEvictionPolicy<>(1000L); + } + + /** {@inheritDoc} */ + @Override protected FifoEvictionPolicy<String, String> createNearPolicy(int nearMax) { + return new FifoEvictionPolicy<>(nearMax); + } + + /** {@inheritDoc} */ + @Override protected void checkNearPolicies(int endNearPlcSize) { + for (int i = 0; i < gridCnt; i++) + for (EvictableEntry<String, String> e : nearPolicy(i).queue()) + assert !e.isCached() : "Invalid near policy size: " + nearPolicy(i).queue(); + } + + /** {@inheritDoc} */ + @Override protected void checkPolicies(int plcMax) { + for (int i = 0; i < gridCnt; i++) { + assert policy(i).queue().size() <= plcMax; + + int size = 0; + + for (EvictableEntry<String, String> entry : policy(i).queue()) + size += ((CacheEvictableEntryImpl)entry).size(); + + assertEquals(size, ((FifoEvictionPolicy)policy(i)).getCurrentMemSize()); + } + } + +}
