This is an automated email from the ASF dual-hosted git repository. ggregory pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/commons-collections.git
The following commit(s) were added to refs/heads/master by this push: new 56da86956 COLLECTIONS-853: Change LayerManager to use List and added generics to LayerdedBloomFilter (#481) 56da86956 is described below commit 56da86956588b45280938180b40eb3aac1d16a86 Author: Claude Warren <cla...@apache.org> AuthorDate: Sun May 12 13:36:17 2024 +0100 COLLECTIONS-853: Change LayerManager to use List and added generics to LayerdedBloomFilter (#481) * Added generics to LayeredBloomFilter, modified WrappedBloomfFilter to requrie copy() implementation and changed LayerManager LinkedList declaration to List. * removed wildcard include * Placed NumberedBloomFilter class into LayeredBloomFilterTest where it is used and fixed implementation * made wrapped Bloom filter private with protected access member * removed null checks from LayerManager first() and last() * fixed generics * removed LayerdBloomFilter.fixed() methods * changed to Deque in API * fixed issue with advanceOnCount implementation --- .../collections4/bloomfilter/BloomFilter.java | 2 +- .../collections4/bloomfilter/LayerManager.java | 115 +++++++++++------- .../bloomfilter/LayeredBloomFilter.java | 27 ++--- .../bloomfilter/WrappedBloomFilter.java | 11 +- .../BitMapProducerFromLayeredBloomFilterTest.java | 4 +- .../BitMapProducerFromWrappedBloomFilterTest.java | 12 ++ .../CellProducerFromLayeredBloomFilterTest.java | 4 +- .../collections4/bloomfilter/LayerManagerTest.java | 69 +++++------ .../bloomfilter/LayeredBloomFilterTest.java | 133 +++++++++++++++------ .../bloomfilter/WrappedBloomFilterTest.java | 15 ++- 10 files changed, 245 insertions(+), 147 deletions(-) diff --git a/src/main/java/org/apache/commons/collections4/bloomfilter/BloomFilter.java b/src/main/java/org/apache/commons/collections4/bloomfilter/BloomFilter.java index d991c0eee..feb32a483 100644 --- a/src/main/java/org/apache/commons/collections4/bloomfilter/BloomFilter.java +++ b/src/main/java/org/apache/commons/collections4/bloomfilter/BloomFilter.java @@ -120,7 +120,7 @@ public interface BloomFilter extends IndexProducer, BitMapProducer { * Creates a new instance of the BloomFilter with the same properties as the current one. * @return a copy of this BloomFilter */ - BloomFilter copy(); + <T extends BloomFilter> T copy(); // update operations diff --git a/src/main/java/org/apache/commons/collections4/bloomfilter/LayerManager.java b/src/main/java/org/apache/commons/collections4/bloomfilter/LayerManager.java index b139d16ce..0e984b0aa 100644 --- a/src/main/java/org/apache/commons/collections4/bloomfilter/LayerManager.java +++ b/src/main/java/org/apache/commons/collections4/bloomfilter/LayerManager.java @@ -16,6 +16,7 @@ */ package org.apache.commons.collections4.bloomfilter; +import java.util.Deque; import java.util.LinkedList; import java.util.NoSuchElementException; import java.util.Objects; @@ -50,15 +51,15 @@ import java.util.function.Supplier; * * @since 4.5 */ -public class LayerManager implements BloomFilterProducer { +public class LayerManager<T extends BloomFilter> implements BloomFilterProducer { /** * Builder to create Layer Manager */ - public static class Builder { - private Predicate<LayerManager> extendCheck; - private Supplier<BloomFilter> supplier; - private Consumer<LinkedList<BloomFilter>> cleanup; + public static class Builder<T extends BloomFilter> { + private Predicate<LayerManager<T>> extendCheck; + private Supplier<T> supplier; + private Consumer<Deque<T>> cleanup; private Builder() { extendCheck = ExtendCheck.neverAdvance(); @@ -70,11 +71,11 @@ public class LayerManager implements BloomFilterProducer { * * @return a new LayerManager. */ - public LayerManager build() { + public LayerManager<T> build() { Objects.requireNonNull(supplier, "Supplier must not be null"); Objects.requireNonNull(extendCheck, "ExtendCheck must not be null"); Objects.requireNonNull(cleanup, "Cleanup must not be null"); - return new LayerManager(supplier, extendCheck, cleanup, true); + return new LayerManager<>(supplier, extendCheck, cleanup, true); } /** @@ -84,7 +85,7 @@ public class LayerManager implements BloomFilterProducer { * dated or stale filters. * @return this */ - public Builder setCleanup(Consumer<LinkedList<BloomFilter>> cleanup) { + public Builder<T> setCleanup(Consumer<Deque<T>> cleanup) { this.cleanup = cleanup; return this; } @@ -97,7 +98,7 @@ public class LayerManager implements BloomFilterProducer { * created. * @return this for chaining. */ - public Builder setExtendCheck(Predicate<LayerManager> extendCheck) { + public Builder<T> setExtendCheck(Predicate<LayerManager<T>> extendCheck) { this.extendCheck = extendCheck; return this; } @@ -109,14 +110,14 @@ public class LayerManager implements BloomFilterProducer { * @param supplier The supplier of new Bloom filter instances. * @return this for chaining. */ - public Builder setSupplier(Supplier<BloomFilter> supplier) { + public Builder<T> setSupplier(Supplier<T> supplier) { this.supplier = supplier; return this; } } /** - * Static methods to create a Consumer of a LinkedList of BloomFilter perform + * Static methods to create a Consumer of a List of BloomFilter perform * tests on whether to reduce the collection of Bloom filters. */ public static final class Cleanup { @@ -124,7 +125,7 @@ public class LayerManager implements BloomFilterProducer { * A Cleanup that never removes anything. * @return A Consumer suitable for the LayerManager {@code cleanup} parameter. */ - public static Consumer<LinkedList<BloomFilter>> noCleanup() { + public static <T extends BloomFilter> Consumer<Deque<T>> noCleanup() { return x -> {}; } @@ -137,7 +138,7 @@ public class LayerManager implements BloomFilterProducer { * @return A Consumer suitable for the LayerManager {@code cleanup} parameter. * @throws IllegalArgumentException if {@code maxSize <= 0}. */ - public static Consumer<LinkedList<BloomFilter>> onMaxSize(int maxSize) { + public static <T extends BloomFilter> Consumer<Deque<T>> onMaxSize(int maxSize) { if (maxSize <= 0) { throw new IllegalArgumentException("'maxSize' must be greater than 0"); } @@ -154,14 +155,24 @@ public class LayerManager implements BloomFilterProducer { * * @return A Consumer suitable for the LayerManager {@code cleanup} parameter. */ - public static Consumer<LinkedList<BloomFilter>> removeEmptyTarget() { + public static <T extends BloomFilter> Consumer<Deque<T>> removeEmptyTarget() { return x -> { - if (x.getLast().cardinality() == 0) { + if (!x.isEmpty() && x.getLast().isEmpty()) { x.removeLast(); } }; } + /** + * Removes any layer identified by the predicate. + * + * @param test Predicate. + * @return A Consumer suitable for the LayerManager {@code cleanup} parameter. + */ + public static <T extends BloomFilter> Consumer<Deque<T>> removeIf(Predicate<? super T> test) { + return x -> x.removeIf(test); + } + private Cleanup() { } } @@ -179,16 +190,20 @@ public class LayerManager implements BloomFilterProducer { * @return A Predicate suitable for the LayerManager {@code extendCheck} parameter. * @throws IllegalArgumentException if {@code breakAt <= 0} */ - public static Predicate<LayerManager> advanceOnCount(int breakAt) { + public static <T extends BloomFilter> Predicate<LayerManager<T>> advanceOnCount(int breakAt) { if (breakAt <= 0) { throw new IllegalArgumentException("'breakAt' must be greater than 0"); } - return new Predicate<LayerManager>() { + return new Predicate<LayerManager<T>>() { int count; @Override - public boolean test(LayerManager filter) { - return ++count % breakAt == 0; + public boolean test(LayerManager<T> filter) { + if (++count == breakAt) { + count = 0; + return true; + } + return false; } }; } @@ -197,8 +212,8 @@ public class LayerManager implements BloomFilterProducer { * Advances the target once a merge has been performed. * @return A Predicate suitable for the LayerManager {@code extendCheck} parameter. */ - public static Predicate<LayerManager> advanceOnPopulated() { - return lm -> !lm.filters.peekLast().isEmpty(); + public static <T extends BloomFilter> Predicate<LayerManager<T>> advanceOnPopulated() { + return lm -> !lm.last().isEmpty(); } /** @@ -212,12 +227,12 @@ public class LayerManager implements BloomFilterProducer { * @return A Predicate suitable for the LayerManager {@code extendCheck} parameter. * @throws IllegalArgumentException if {@code maxN <= 0} */ - public static Predicate<LayerManager> advanceOnSaturation(double maxN) { + public static <T extends BloomFilter> Predicate<LayerManager<T>> advanceOnSaturation(double maxN) { if (maxN <= 0) { throw new IllegalArgumentException("'maxN' must be greater than 0"); } return manager -> { - BloomFilter bf = manager.filters.peekLast(); + BloomFilter bf = manager.last(); return maxN <= bf.getShape().estimateN(bf.cardinality()); }; } @@ -227,7 +242,7 @@ public class LayerManager implements BloomFilterProducer { * perform the advance. * @return A Predicate suitable for the LayerManager {@code extendCheck} parameter. */ - public static Predicate<LayerManager> neverAdvance() { + public static <T extends BloomFilter> Predicate<LayerManager<T>> neverAdvance() { return x -> false; } @@ -242,15 +257,15 @@ public class LayerManager implements BloomFilterProducer { * @see ExtendCheck#neverAdvance() * @see Cleanup#noCleanup() */ - public static Builder builder() { - return new Builder(); + public static <T extends BloomFilter> Builder<T> builder() { + return new Builder<>(); } - private final LinkedList<BloomFilter> filters = new LinkedList<>(); - private final Consumer<LinkedList<BloomFilter>> filterCleanup; + private final LinkedList<T> filters = new LinkedList<>(); + private final Consumer<Deque<T>> filterCleanup; - private final Predicate<LayerManager> extendCheck; + private final Predicate<LayerManager<T>> extendCheck; - private final Supplier<BloomFilter> filterSupplier; + private final Supplier<T> filterSupplier; /** * Constructor. @@ -263,8 +278,8 @@ public class LayerManager implements BloomFilterProducer { * list. * @param initialize true if the filter list should be initialized. */ - private LayerManager(Supplier<BloomFilter> filterSupplier, Predicate<LayerManager> extendCheck, - Consumer<LinkedList<BloomFilter>> filterCleanup, boolean initialize) { + private LayerManager(Supplier<T> filterSupplier, Predicate<LayerManager<T>> extendCheck, + Consumer<Deque<T>> filterCleanup, boolean initialize) { this.filterSupplier = filterSupplier; this.extendCheck = extendCheck; this.filterCleanup = filterCleanup; @@ -277,7 +292,7 @@ public class LayerManager implements BloomFilterProducer { * Adds a new Bloom filter to the list. */ private void addFilter() { - BloomFilter bf = filterSupplier.get(); + T bf = filterSupplier.get(); if (bf == null) { throw new NullPointerException("filterSupplier returned null."); } @@ -302,9 +317,9 @@ public class LayerManager implements BloomFilterProducer { * * @return a copy of this layer Manager. */ - public LayerManager copy() { - LayerManager newMgr = new LayerManager(filterSupplier, extendCheck, filterCleanup, false); - for (BloomFilter bf : filters) { + public LayerManager<T> copy() { + LayerManager<T> newMgr = new LayerManager<>(filterSupplier, extendCheck, filterCleanup, false); + for (T bf : filters) { newMgr.filters.add(bf.copy()); } return newMgr; @@ -337,7 +352,7 @@ public class LayerManager implements BloomFilterProducer { * @throws NoSuchElementException if depth is not in the range * [0,filters.size()) */ - public final BloomFilter get(int depth) { + public final T get(int depth) { if (depth < 0 || depth >= filters.size()) { throw new NoSuchElementException(String.format("Depth must be in the range [0,%s)", filters.size())); } @@ -346,7 +361,7 @@ public class LayerManager implements BloomFilterProducer { /** * Returns the number of filters in the LayerManager. In the default LayerManager implementation - * there is alwasy at least one layer. + * there is always at least one layer. * * @return the current depth. */ @@ -354,17 +369,37 @@ public class LayerManager implements BloomFilterProducer { return filters.size(); } + /** + * Gets the Bloom filter from the first layer. + * No extension check is performed during this call. + * @return The Bloom filter from the first layer. + * @see #getTarget() + */ + public final T first() { + return filters.getFirst(); + } + + /** + * Gets the Bloom filter from the last layer. + * No extension check is performed during this call. + * @return The Bloom filter from the last layer. + * @see #getTarget() + */ + public final T last() { + return filters.getLast(); + } + /** * Returns the current target filter. If a new filter should be created based on * {@code extendCheck} it will be created before this method returns. * * @return the current target filter after any extension. */ - public final BloomFilter getTarget() { + public final T getTarget() { if (extendCheck.test(this)) { next(); } - return filters.peekLast(); + return last(); } /** diff --git a/src/main/java/org/apache/commons/collections4/bloomfilter/LayeredBloomFilter.java b/src/main/java/org/apache/commons/collections4/bloomfilter/LayeredBloomFilter.java index 9ed2688bb..0a79537dc 100644 --- a/src/main/java/org/apache/commons/collections4/bloomfilter/LayeredBloomFilter.java +++ b/src/main/java/org/apache/commons/collections4/bloomfilter/LayeredBloomFilter.java @@ -59,9 +59,10 @@ import java.util.function.Predicate; * removes them. It also checks it a new layer should be added, and if so adds * it and sets the {@code target} before the operation.</li> * </ul> + * @param <T> The type of Bloom Filter that is used for the layers. * @since 4.5 */ -public class LayeredBloomFilter implements BloomFilter, BloomFilterProducer { +public class LayeredBloomFilter<T extends BloomFilter> implements BloomFilter, BloomFilterProducer { /** * A class used to locate matching filters across all the layers. */ @@ -88,24 +89,10 @@ public class LayeredBloomFilter implements BloomFilter, BloomFilterProducer { return true; } } - /** - * Creates a fixed size layered bloom filter that adds new filters to the list, - * but never merges them. List will never exceed maxDepth. As additional filters - * are added earlier filters are removed. - * - * @param shape The shape for the enclosed Bloom filters. - * @param maxDepth The maximum depth of layers. - * @return An empty layered Bloom filter of the specified shape and depth. - */ - public static LayeredBloomFilter fixed(final Shape shape, int maxDepth) { - LayerManager manager = LayerManager.builder().setExtendCheck(LayerManager.ExtendCheck.advanceOnPopulated()) - .setCleanup(LayerManager.Cleanup.onMaxSize(maxDepth)).setSupplier(() -> new SimpleBloomFilter(shape)).build(); - return new LayeredBloomFilter(shape, manager); - } private final Shape shape; - private LayerManager layerManager; + private final LayerManager<T> layerManager; /** * Constructor. @@ -113,7 +100,7 @@ public class LayeredBloomFilter implements BloomFilter, BloomFilterProducer { * @param shape the Shape of the enclosed Bloom filters * @param layerManager the LayerManager to manage the layers. */ - public LayeredBloomFilter(Shape shape, LayerManager layerManager) { + public LayeredBloomFilter(Shape shape, LayerManager<T> layerManager) { this.shape = shape; this.layerManager = layerManager; } @@ -184,8 +171,8 @@ public class LayeredBloomFilter implements BloomFilter, BloomFilterProducer { } @Override - public LayeredBloomFilter copy() { - return new LayeredBloomFilter(shape, layerManager.copy()); + public LayeredBloomFilter<T> copy() { + return new LayeredBloomFilter<>(shape, layerManager.copy()); } /** @@ -329,7 +316,7 @@ public class LayeredBloomFilter implements BloomFilter, BloomFilterProducer { * @return the Bloom filter at the specified depth. * @throws NoSuchElementException if depth is not in the range [0,getDepth()) */ - public BloomFilter get(int depth) { + public T get(int depth) { return layerManager.get(depth); } diff --git a/src/main/java/org/apache/commons/collections4/bloomfilter/WrappedBloomFilter.java b/src/main/java/org/apache/commons/collections4/bloomfilter/WrappedBloomFilter.java index b1d661e89..18aa6f5d8 100644 --- a/src/main/java/org/apache/commons/collections4/bloomfilter/WrappedBloomFilter.java +++ b/src/main/java/org/apache/commons/collections4/bloomfilter/WrappedBloomFilter.java @@ -25,7 +25,7 @@ import java.util.function.LongPredicate; * @since 4.5 */ public abstract class WrappedBloomFilter implements BloomFilter { - final BloomFilter wrapped; + private final BloomFilter wrapped; /** * Wraps a Bloom filter. The wrapped filter is maintained as a reference @@ -36,6 +36,10 @@ public abstract class WrappedBloomFilter implements BloomFilter { this.wrapped = bf; } + protected BloomFilter getWrapped() { + return wrapped; + } + @Override public long[] asBitMapArray() { return wrapped.asBitMapArray(); @@ -81,11 +85,6 @@ public abstract class WrappedBloomFilter implements BloomFilter { return wrapped.contains(indexProducer); } - @Override - public BloomFilter copy() { - return wrapped.copy(); - } - @Override public int estimateIntersection(BloomFilter other) { return wrapped.estimateIntersection(other); diff --git a/src/test/java/org/apache/commons/collections4/bloomfilter/BitMapProducerFromLayeredBloomFilterTest.java b/src/test/java/org/apache/commons/collections4/bloomfilter/BitMapProducerFromLayeredBloomFilterTest.java index bd71067bb..246ef518c 100644 --- a/src/test/java/org/apache/commons/collections4/bloomfilter/BitMapProducerFromLayeredBloomFilterTest.java +++ b/src/test/java/org/apache/commons/collections4/bloomfilter/BitMapProducerFromLayeredBloomFilterTest.java @@ -22,13 +22,13 @@ public class BitMapProducerFromLayeredBloomFilterTest extends AbstractBitMapProd @Override protected BitMapProducer createEmptyProducer() { - return LayeredBloomFilter.fixed(shape, 10); + return LayeredBloomFilterTest.fixed(shape, 10); } @Override protected BitMapProducer createProducer() { final Hasher hasher = new IncrementingHasher(0, 1); - final BloomFilter bf = LayeredBloomFilter.fixed(shape, 10); + final BloomFilter bf = LayeredBloomFilterTest.fixed(shape, 10); bf.merge(hasher); return bf; } diff --git a/src/test/java/org/apache/commons/collections4/bloomfilter/BitMapProducerFromWrappedBloomFilterTest.java b/src/test/java/org/apache/commons/collections4/bloomfilter/BitMapProducerFromWrappedBloomFilterTest.java index ebb78ab43..27befb1e6 100644 --- a/src/test/java/org/apache/commons/collections4/bloomfilter/BitMapProducerFromWrappedBloomFilterTest.java +++ b/src/test/java/org/apache/commons/collections4/bloomfilter/BitMapProducerFromWrappedBloomFilterTest.java @@ -23,6 +23,12 @@ public class BitMapProducerFromWrappedBloomFilterTest extends AbstractBitMapProd @Override protected BitMapProducer createEmptyProducer() { return new WrappedBloomFilter(new DefaultBloomFilterTest.SparseDefaultBloomFilter(shape)) { + @Override + public BloomFilter copy() { + BloomFilter result = new DefaultBloomFilterTest.SparseDefaultBloomFilter(shape); + result.merge(getWrapped()); + return result; + } }; } @@ -30,6 +36,12 @@ public class BitMapProducerFromWrappedBloomFilterTest extends AbstractBitMapProd protected BitMapProducer createProducer() { final Hasher hasher = new IncrementingHasher(0, 1); final BloomFilter bf = new WrappedBloomFilter(new DefaultBloomFilterTest.SparseDefaultBloomFilter(shape)) { + @Override + public BloomFilter copy() { + BloomFilter result = new DefaultBloomFilterTest.SparseDefaultBloomFilter(shape); + result.merge(getWrapped()); + return result; + } }; bf.merge(hasher); return bf; diff --git a/src/test/java/org/apache/commons/collections4/bloomfilter/CellProducerFromLayeredBloomFilterTest.java b/src/test/java/org/apache/commons/collections4/bloomfilter/CellProducerFromLayeredBloomFilterTest.java index 0c340061c..ad3fa8c1c 100644 --- a/src/test/java/org/apache/commons/collections4/bloomfilter/CellProducerFromLayeredBloomFilterTest.java +++ b/src/test/java/org/apache/commons/collections4/bloomfilter/CellProducerFromLayeredBloomFilterTest.java @@ -22,13 +22,13 @@ public class CellProducerFromLayeredBloomFilterTest extends AbstractCellProducer @Override protected CellProducer createEmptyProducer() { - return CellProducer.from(LayeredBloomFilter.fixed(shape, 10)); + return CellProducer.from(LayeredBloomFilterTest.fixed(shape, 10)); } @Override protected CellProducer createProducer() { final Hasher hasher = new IncrementingHasher(3, 2); - final BloomFilter bf = LayeredBloomFilter.fixed(shape, 10); + final BloomFilter bf = LayeredBloomFilterTest.fixed(shape, 10); bf.merge(hasher); return CellProducer.from(bf); } diff --git a/src/test/java/org/apache/commons/collections4/bloomfilter/LayerManagerTest.java b/src/test/java/org/apache/commons/collections4/bloomfilter/LayerManagerTest.java index 4ee438b89..74e8b98e5 100644 --- a/src/test/java/org/apache/commons/collections4/bloomfilter/LayerManagerTest.java +++ b/src/test/java/org/apache/commons/collections4/bloomfilter/LayerManagerTest.java @@ -16,9 +16,9 @@ */ package org.apache.commons.collections4.bloomfilter; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertSame; @@ -27,6 +27,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; import java.util.Arrays; +import java.util.Deque; import java.util.LinkedList; import java.util.List; import java.util.NoSuchElementException; @@ -39,13 +40,13 @@ import org.junit.jupiter.params.provider.ValueSource; public class LayerManagerTest { - private Shape shape = Shape.fromKM(17, 72); + private final Shape shape = Shape.fromKM(17, 72); @ParameterizedTest @ValueSource(ints = {4, 10, 2, 1}) public void testAdvanceOnCount(int breakAt) { - Predicate<LayerManager> underTest = LayerManager.ExtendCheck.advanceOnCount(breakAt); - LayerManager layerManager = testingBuilder().build(); + Predicate<LayerManager<BloomFilter>> underTest = LayerManager.ExtendCheck.advanceOnCount(breakAt); + LayerManager<BloomFilter> layerManager = testingBuilder().build(); for (int i = 0; i < breakAt - 1; i++) { assertFalse(underTest.test(layerManager), "at " + i); layerManager.getTarget().merge(TestingHashers.FROM1); @@ -61,8 +62,8 @@ public class LayerManagerTest { @Test public void testAdvanceOnPopulated() { - Predicate<LayerManager> underTest = LayerManager.ExtendCheck.advanceOnPopulated(); - LayerManager layerManager = testingBuilder().build(); + Predicate<LayerManager<BloomFilter>> underTest = LayerManager.ExtendCheck.advanceOnPopulated(); + LayerManager<BloomFilter> layerManager = testingBuilder().build(); assertFalse(underTest.test(layerManager)); layerManager.getTarget().merge(TestingHashers.FROM1); assertTrue(underTest.test(layerManager)); @@ -70,10 +71,10 @@ public class LayerManagerTest { @Test public void testAdvanceOnSaturation() { - Double maxN = shape.estimateMaxN(); + double maxN = shape.estimateMaxN(); int hashStart = 0; - Predicate<LayerManager> underTest = LayerManager.ExtendCheck.advanceOnSaturation(maxN); - LayerManager layerManager = testingBuilder().build(); + Predicate<LayerManager<BloomFilter>> underTest = LayerManager.ExtendCheck.advanceOnSaturation(maxN); + LayerManager<BloomFilter> layerManager = testingBuilder().build(); while (layerManager.getTarget().getShape().estimateN(layerManager.getTarget().cardinality()) < maxN) { assertFalse(underTest.test(layerManager)); layerManager.getTarget().merge(new IncrementingHasher(hashStart, shape.getNumberOfHashFunctions())); @@ -86,15 +87,15 @@ public class LayerManagerTest { @Test public void testBuilder() { - LayerManager.Builder underTest = LayerManager.builder(); - NullPointerException npe = assertThrows(NullPointerException.class, () -> underTest.build()); + LayerManager.Builder<BloomFilter> underTest = LayerManager.builder(); + NullPointerException npe = assertThrows(NullPointerException.class, underTest::build); assertTrue(npe.getMessage().contains("Supplier must not be null")); underTest.setSupplier(() -> null).setCleanup(null); - npe = assertThrows(NullPointerException.class, () -> underTest.build()); + npe = assertThrows(NullPointerException.class, underTest::build); assertTrue(npe.getMessage().contains("Cleanup must not be null")); underTest.setCleanup(x -> { }).setExtendCheck(null); - npe = assertThrows(NullPointerException.class, () -> underTest.build()); + npe = assertThrows(NullPointerException.class, underTest::build); assertTrue(npe.getMessage().contains("ExtendCheck must not be null")); npe = assertThrows(NullPointerException.class, () -> LayerManager.builder().setSupplier(() -> null).build()); @@ -104,7 +105,7 @@ public class LayerManagerTest { @Test public void testClear() { - LayerManager underTest = LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(shape)).build(); + LayerManager<BloomFilter> underTest = LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(shape)).build(); underTest.getTarget().merge(TestingHashers.randomHasher()); underTest.next(); underTest.getTarget().merge(TestingHashers.randomHasher()); @@ -118,7 +119,7 @@ public class LayerManagerTest { @Test public void testCopy() { - LayerManager underTest = LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(shape)).build(); + LayerManager<BloomFilter> underTest = LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(shape)).build(); underTest.getTarget().merge(TestingHashers.randomHasher()); underTest.next(); underTest.getTarget().merge(TestingHashers.randomHasher()); @@ -126,7 +127,7 @@ public class LayerManagerTest { underTest.getTarget().merge(TestingHashers.randomHasher()); assertEquals(3, underTest.getDepth()); - LayerManager copy = underTest.copy(); + LayerManager<BloomFilter> copy = underTest.copy(); assertNotSame(underTest, copy); // object equals not implemented assertNotEquals(underTest, copy); @@ -138,7 +139,7 @@ public class LayerManagerTest { @Test public void testForEachBloomFilter() { - LayerManager underTest = LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(shape)) + LayerManager<BloomFilter> underTest = LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(shape)) .setExtendCheck(LayerManager.ExtendCheck.advanceOnPopulated()).build(); List<BloomFilter> lst = new ArrayList<>(); @@ -160,21 +161,21 @@ public class LayerManagerTest { @Test public void testGet() { SimpleBloomFilter f = new SimpleBloomFilter(shape); - LayerManager underTest = LayerManager.builder().setSupplier(() -> f).build(); + LayerManager<BloomFilter> underTest = LayerManager.builder().setSupplier(() -> f).build(); assertEquals(1, underTest.getDepth()); assertSame(f, underTest.get(0)); assertThrows(NoSuchElementException.class, () -> underTest.get(-1)); assertThrows(NoSuchElementException.class, () -> underTest.get(1)); } - private LayerManager.Builder testingBuilder() { + private LayerManager.Builder<BloomFilter> testingBuilder() { return LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(shape)); } @Test public void testNeverAdvance() { - Predicate<LayerManager> underTest = LayerManager.ExtendCheck.neverAdvance(); - LayerManager layerManager = testingBuilder().build(); + Predicate<LayerManager<BloomFilter>> underTest = LayerManager.ExtendCheck.neverAdvance(); + LayerManager<BloomFilter> layerManager = testingBuilder().build(); assertFalse(underTest.test(layerManager)); for (int i = 0; i < 10; i++) { layerManager.getTarget().merge(TestingHashers.randomHasher()); @@ -184,7 +185,7 @@ public class LayerManagerTest { @Test public void testNextAndGetDepth() { - LayerManager underTest = LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(shape)).build(); + LayerManager<BloomFilter> underTest = LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(shape)).build(); assertEquals(1, underTest.getDepth()); underTest.getTarget().merge(TestingHashers.randomHasher()); assertEquals(1, underTest.getDepth()); @@ -194,8 +195,8 @@ public class LayerManagerTest { @Test public void testNoCleanup() { - Consumer<LinkedList<BloomFilter>> underTest = LayerManager.Cleanup.noCleanup(); - LinkedList<BloomFilter> list = new LinkedList<>(); + Consumer<Deque<BloomFilter>> underTest = LayerManager.Cleanup.noCleanup(); + Deque<BloomFilter> list = new LinkedList<>(); for (int i = 0; i < 20; i++) { assertEquals(i, list.size()); list.add(new SimpleBloomFilter(shape)); @@ -206,7 +207,7 @@ public class LayerManagerTest { @ParameterizedTest @ValueSource(ints = {5, 100, 2, 1}) public void testOnMaxSize(int maxSize) { - Consumer<LinkedList<BloomFilter>> underTest = LayerManager.Cleanup.onMaxSize(maxSize); + Consumer<Deque<BloomFilter>> underTest = LayerManager.Cleanup.onMaxSize(maxSize); LinkedList<BloomFilter> list = new LinkedList<>(); for (int i = 0; i < maxSize; i++) { assertEquals(i, list.size()); @@ -230,7 +231,7 @@ public class LayerManagerTest { @Test public void testRemoveEmptyTarget() { - Consumer<LinkedList<BloomFilter>> underTest = LayerManager.Cleanup.removeEmptyTarget(); + Consumer<Deque<BloomFilter>> underTest = LayerManager.Cleanup.removeEmptyTarget(); LinkedList<BloomFilter> list = new LinkedList<>(); // removes an empty filter @@ -265,7 +266,6 @@ public class LayerManagerTest { underTest.accept(list); assertEquals(2, list.size()); assertEquals(bf, list.get(0)); - } @Test @@ -273,7 +273,7 @@ public class LayerManagerTest { boolean[] extendCheckCalled = { false }; boolean[] cleanupCalled = { false }; int[] supplierCount = { 0 }; - LayerManager underTest = LayerManager.builder().setSupplier(() -> { + LayerManager<BloomFilter> underTest = LayerManager.builder().setSupplier(() -> { supplierCount[0]++; return new SimpleBloomFilter(shape); }).setExtendCheck(lm -> { @@ -291,13 +291,4 @@ public class LayerManagerTest { assertEquals(2, supplierCount[0]); } - static class NumberedBloomFilter extends WrappedBloomFilter { - int value; - int sequence; - NumberedBloomFilter(Shape shape, int value, int sequence) { - super(new SimpleBloomFilter(shape)); - this.value = value; - this.sequence = sequence; - } - } } diff --git a/src/test/java/org/apache/commons/collections4/bloomfilter/LayeredBloomFilterTest.java b/src/test/java/org/apache/commons/collections4/bloomfilter/LayeredBloomFilterTest.java index 0859e58c6..25f807a80 100644 --- a/src/test/java/org/apache/commons/collections4/bloomfilter/LayeredBloomFilterTest.java +++ b/src/test/java/org/apache/commons/collections4/bloomfilter/LayeredBloomFilterTest.java @@ -16,30 +16,60 @@ */ package org.apache.commons.collections4.bloomfilter; -import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.ArrayList; -import java.util.LinkedList; +import java.util.Deque; +import java.util.Iterator; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Predicate; +import java.util.function.Supplier; import org.apache.commons.collections4.bloomfilter.LayerManager.Cleanup; import org.apache.commons.collections4.bloomfilter.LayerManager.ExtendCheck; -import org.apache.commons.collections4.bloomfilter.LayerManagerTest.NumberedBloomFilter; import org.junit.jupiter.api.Test; -public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloomFilter> { +public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloomFilter<?>> { + + /** + * Creates a fixed size layered bloom filter that adds new filters to the list, + * but never merges them. List will never exceed maxDepth. As additional filters + * are added earlier filters are removed. Uses SimpleBloomFilters. + * + * @param shape The shape for the enclosed Bloom filters. + * @param maxDepth The maximum depth of layers. + * @return An empty layered Bloom filter of the specified shape and depth. + */ + public static LayeredBloomFilter<BloomFilter> fixed(final Shape shape, int maxDepth) { + return fixed(shape, maxDepth, () -> new SimpleBloomFilter(shape)); + } + + /** + * Creates a fixed size layered bloom filter that adds new filters to the list, + * but never merges them. List will never exceed maxDepth. As additional filters + * are added earlier filters are removed. + * + * @param shape The shape for the enclosed Bloom filters. + * @param maxDepth The maximum depth of layers. + * @param supplier A supplier of the Bloom filters to create layers with. + * @return An empty layered Bloom filter of the specified shape and depth. + */ + public static <T extends BloomFilter> LayeredBloomFilter<T> fixed(final Shape shape, int maxDepth, Supplier<T> supplier) { + LayerManager.Builder<T> builder = LayerManager.builder(); + builder.setExtendCheck(LayerManager.ExtendCheck.advanceOnPopulated()) + .setCleanup(LayerManager.Cleanup.onMaxSize(maxDepth)).setSupplier(supplier); + return new LayeredBloomFilter<>(shape, builder.build()); + } /** * A Predicate that advances after a quantum of time. */ - static class AdvanceOnTimeQuanta implements Predicate<LayerManager> { + static class AdvanceOnTimeQuanta implements Predicate<LayerManager<TimestampedBloomFilter>> { long quanta; AdvanceOnTimeQuanta(long quanta, TimeUnit unit) { @@ -47,10 +77,9 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom } @Override - public boolean test(LayerManager lm) { + public boolean test(LayerManager<TimestampedBloomFilter> lm) { // can not use getTarget() as it causes recursion. - TimestampedBloomFilter bf = (TimestampedBloomFilter) lm.get(lm.getDepth() - 1); - return bf.timestamp + quanta < System.currentTimeMillis(); + return lm.last().timestamp + quanta < System.currentTimeMillis(); } } @@ -58,7 +87,7 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom * A Consumer that cleans the list based on how long each filters has been in * the list. */ - static class CleanByTime implements Consumer<LinkedList<BloomFilter>> { + static class CleanByTime<T extends TimestampedBloomFilter> implements Consumer<List<T>> { long elapsedTime; CleanByTime(long duration, TimeUnit unit) { @@ -66,13 +95,18 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom } @Override - public void accept(LinkedList<BloomFilter> t) { + public void accept(List<T> t) { long min = System.currentTimeMillis() - elapsedTime; - while (!t.isEmpty() && ((TimestampedBloomFilter) t.getFirst()).getTimestamp() < min) { - TimestampedBloomFilter bf = (TimestampedBloomFilter) t.getFirst(); - dbgInstrument.add(String.format("Removing old entry: T:%s (Aged: %s) \n", bf.getTimestamp(), - (min - bf.getTimestamp()))); - t.removeFirst(); + Iterator<T> iter = t.iterator(); + while (iter.hasNext()) { + TimestampedBloomFilter bf = iter.next(); + if (bf.getTimestamp() < min) { + dbgInstrument.add(String.format("Removing old entry: T:%s (Aged: %s) \n", bf.getTimestamp(), + (min - bf.getTimestamp()))); + iter.remove(); + } else { + return; + } } } } @@ -80,7 +114,7 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom /** * A Bloomfilter implementation that tracks the creation time. */ - static class TimestampedBloomFilter extends WrappedBloomFilter { + public static class TimestampedBloomFilter extends WrappedBloomFilter { final long timestamp; TimestampedBloomFilter(BloomFilter bf) { @@ -88,13 +122,23 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom this.timestamp = System.currentTimeMillis(); } + TimestampedBloomFilter(BloomFilter bf, long timestamp) { + super(bf); + this.timestamp = timestamp; + } + public long getTimestamp() { return timestamp; } + + @Override + public TimestampedBloomFilter copy() { + return new TimestampedBloomFilter(this.getWrapped().copy(), timestamp); + } } // ***example of instrumentation *** - private static List<String> dbgInstrument = new ArrayList<>(); + private static final List<String> dbgInstrument = new ArrayList<>(); /** * Creates a LayeredBloomFilter that retains enclosed filters for @@ -109,19 +153,21 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom * @param qUnit the unit of time to apply to quanta. * @return LayeredBloomFilter with the above properties. */ - static LayeredBloomFilter createTimedLayeredFilter(Shape shape, long duration, TimeUnit dUnit, long quanta, + static LayeredBloomFilter<TimestampedBloomFilter> createTimedLayeredFilter(Shape shape, long duration, TimeUnit dUnit, long quanta, TimeUnit qUnit) { - LayerManager layerManager = LayerManager.builder() + LayerManager.Builder<TimestampedBloomFilter> builder = LayerManager.builder(); + Consumer<Deque<TimestampedBloomFilter>> cleanup = Cleanup.removeEmptyTarget().andThen(new CleanByTime(duration, dUnit)); + LayerManager<TimestampedBloomFilter> layerManager = builder .setSupplier(() -> new TimestampedBloomFilter(new SimpleBloomFilter(shape))) - .setCleanup(Cleanup.removeEmptyTarget().andThen(new CleanByTime(duration, dUnit))) + .setCleanup(cleanup) .setExtendCheck(new AdvanceOnTimeQuanta(quanta, qUnit) .or(LayerManager.ExtendCheck.advanceOnSaturation(shape.estimateMaxN()))) .build(); - return new LayeredBloomFilter(shape, layerManager); + return new LayeredBloomFilter<>(shape, layerManager); } // instrumentation to record timestamps in dbgInstrument list - private Predicate<BloomFilter> dbg = (bf) -> { + private final Predicate<BloomFilter> dbg = (bf) -> { TimestampedBloomFilter tbf = (TimestampedBloomFilter) bf; long ts = System.currentTimeMillis(); dbgInstrument.add(String.format("T:%s (Elapsed:%s)- EstN:%s (Card:%s)\n", tbf.timestamp, ts - tbf.timestamp, @@ -131,8 +177,8 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom // *** end of instrumentation *** @Override - protected LayeredBloomFilter createEmptyFilter(Shape shape) { - return LayeredBloomFilter.fixed(shape, 10); + protected LayeredBloomFilter<BloomFilter> createEmptyFilter(Shape shape) { + return LayeredBloomFilterTest.fixed(shape, 10); } protected BloomFilter makeFilter(Hasher h) { @@ -151,8 +197,8 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom return makeFilter(IndexProducer.fromIndexArray(values)); } - private LayeredBloomFilter setupFindTest() { - LayeredBloomFilter filter = LayeredBloomFilter.fixed(getTestShape(), 10); + private LayeredBloomFilter<BloomFilter> setupFindTest() { + LayeredBloomFilter<BloomFilter> filter = LayeredBloomFilterTest.fixed(getTestShape(), 10); filter.merge(TestingHashers.FROM1); filter.merge(TestingHashers.FROM11); filter.merge(new IncrementingHasher(11, 2)); @@ -163,9 +209,9 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom @Override @Test public void testCardinalityAndIsEmpty() { - LayerManager layerManager = LayerManager.builder().setExtendCheck(ExtendCheck.neverAdvance()) + LayerManager<BloomFilter> layerManager = LayerManager.builder().setExtendCheck(ExtendCheck.neverAdvance()) .setSupplier(() -> new SimpleBloomFilter(getTestShape())).build(); - testCardinalityAndIsEmpty(new LayeredBloomFilter(getTestShape(), layerManager)); + testCardinalityAndIsEmpty(new LayeredBloomFilter<>(getTestShape(), layerManager)); } /** @@ -194,7 +240,7 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom // create a filter that removes filters that are 4 seconds old // and quantises time to 1 second intervals. - LayeredBloomFilter underTest = createTimedLayeredFilter(shape, 600, TimeUnit.MILLISECONDS, 150, + LayeredBloomFilter<TimestampedBloomFilter> underTest = createTimedLayeredFilter(shape, 600, TimeUnit.MILLISECONDS, 150, TimeUnit.MILLISECONDS); for (int i = 0; i < 10; i++) { @@ -226,7 +272,7 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom } @Test public void testFindBitMapProducer() { - LayeredBloomFilter filter = setupFindTest(); + LayeredBloomFilter<BloomFilter> filter = setupFindTest(); IndexProducer idxProducer = TestingHashers.FROM1.indices(getTestShape()); BitMapProducer producer = BitMapProducer.fromIndexProducer(idxProducer, getTestShape().getNumberOfBits()); @@ -244,7 +290,7 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom @Test public void testFindBloomFilter() { - LayeredBloomFilter filter = setupFindTest(); + LayeredBloomFilter<BloomFilter> filter = setupFindTest(); int[] expected = {0, 3}; int[] result = filter.find(TestingHashers.FROM1); assertArrayEquals(expected, result); @@ -256,7 +302,7 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom @Test public void testFindIndexProducer() { IndexProducer producer = TestingHashers.FROM1.indices(getTestShape()); - LayeredBloomFilter filter = setupFindTest(); + LayeredBloomFilter<BloomFilter> filter = setupFindTest(); int[] expected = {0, 3}; int[] result = filter.find(producer); @@ -272,7 +318,7 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom public final void testGetLayer() { BloomFilter bf = new SimpleBloomFilter(getTestShape()); bf.merge(TestingHashers.FROM11); - LayeredBloomFilter filter = LayeredBloomFilter.fixed(getTestShape(), 10); + LayeredBloomFilter<BloomFilter> filter = LayeredBloomFilterTest.fixed(getTestShape(), 10); filter.merge(TestingHashers.FROM1); filter.merge(TestingHashers.FROM11); filter.merge(new IncrementingHasher(11, 2)); @@ -282,7 +328,7 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom @Test public void testMultipleFilters() { - LayeredBloomFilter filter = LayeredBloomFilter.fixed(getTestShape(), 10); + LayeredBloomFilter<BloomFilter> filter = LayeredBloomFilterTest.fixed(getTestShape(), 10); filter.merge(TestingHashers.FROM1); filter.merge(TestingHashers.FROM11); assertEquals(2, filter.getDepth()); @@ -296,10 +342,10 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom @Test public final void testNext() { - LayerManager layerManager = LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(getTestShape())) + LayerManager<BloomFilter> layerManager = LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(getTestShape())) .build(); - LayeredBloomFilter filter = new LayeredBloomFilter(getTestShape(), layerManager); + LayeredBloomFilter<BloomFilter> filter = new LayeredBloomFilter<>(getTestShape(), layerManager); filter.merge(TestingHashers.FROM1); filter.merge(TestingHashers.FROM11); assertEquals(1, filter.getDepth()); @@ -345,4 +391,19 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom f = (NumberedBloomFilter) underTest.get(0); assertEquals(3, f.sequence); // it is a new one. } + + static class NumberedBloomFilter extends WrappedBloomFilter { + int value; + int sequence; + NumberedBloomFilter(Shape shape, int value, int sequence) { + super(new SimpleBloomFilter(shape)); + this.value = value; + this.sequence = sequence; + } + + @Override + public BloomFilter copy() { + return new NumberedBloomFilter(getShape(), value, sequence); + } + } } diff --git a/src/test/java/org/apache/commons/collections4/bloomfilter/WrappedBloomFilterTest.java b/src/test/java/org/apache/commons/collections4/bloomfilter/WrappedBloomFilterTest.java index eca4a21a3..5e346b95a 100644 --- a/src/test/java/org/apache/commons/collections4/bloomfilter/WrappedBloomFilterTest.java +++ b/src/test/java/org/apache/commons/collections4/bloomfilter/WrappedBloomFilterTest.java @@ -26,6 +26,12 @@ public class WrappedBloomFilterTest extends AbstractBloomFilterTest<WrappedBloom @Override protected WrappedBloomFilter createEmptyFilter(Shape shape) { return new WrappedBloomFilter(new DefaultBloomFilterTest.SparseDefaultBloomFilter(shape)) { + @Override + public BloomFilter copy() { + BloomFilter result = new DefaultBloomFilterTest.SparseDefaultBloomFilter(shape); + result.merge(this.getWrapped()); + return result; + } }; } @@ -39,7 +45,14 @@ public class WrappedBloomFilterTest extends AbstractBloomFilterTest<WrappedBloom return characteristics; } }; - WrappedBloomFilter underTest = new WrappedBloomFilter(inner) {}; + WrappedBloomFilter underTest = new WrappedBloomFilter(inner) { + @Override + public BloomFilter copy() { + BloomFilter result = new DefaultBloomFilterTest.SparseDefaultBloomFilter(shape); + result.merge(this.getWrapped()); + return result; + } + }; assertEquals(characteristics, underTest.characteristics()); } }