COLLECTIONS-853: Change LayerManager to use List and added generics to LayerdedBloomFilter (#481)

* Added generics to LayeredBloomFilter, modified WrappedBloomfFilter to requrie copy() implementation and changed LayerManager LinkedList declaration to List.

* removed wildcard include

* Placed NumberedBloomFilter class into LayeredBloomFilterTest where it is used and fixed implementation

* made wrapped Bloom filter private with protected access member

* removed null checks from LayerManager first() and last()

* fixed generics

* removed LayerdBloomFilter.fixed()  methods

* changed to Deque in API

* fixed issue with advanceOnCount implementation
This commit is contained in:
Claude Warren 2024-05-12 13:36:17 +01:00 committed by GitHub
parent bd8e9506b3
commit 56da869565
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 245 additions and 147 deletions

View File

@ -120,7 +120,7 @@ public interface BloomFilter extends IndexProducer, BitMapProducer {
* Creates a new instance of the BloomFilter with the same properties as the current one.
* @return a copy of this BloomFilter
*/
BloomFilter copy();
<T extends BloomFilter> T copy();
// update operations

View File

@ -16,6 +16,7 @@
*/
package org.apache.commons.collections4.bloomfilter;
import java.util.Deque;
import java.util.LinkedList;
import java.util.NoSuchElementException;
import java.util.Objects;
@ -50,15 +51,15 @@ import java.util.function.Supplier;
*
* @since 4.5
*/
public class LayerManager implements BloomFilterProducer {
public class LayerManager<T extends BloomFilter> implements BloomFilterProducer {
/**
* Builder to create Layer Manager
*/
public static class Builder {
private Predicate<LayerManager> extendCheck;
private Supplier<BloomFilter> supplier;
private Consumer<LinkedList<BloomFilter>> cleanup;
public static class Builder<T extends BloomFilter> {
private Predicate<LayerManager<T>> extendCheck;
private Supplier<T> supplier;
private Consumer<Deque<T>> cleanup;
private Builder() {
extendCheck = ExtendCheck.neverAdvance();
@ -70,11 +71,11 @@ public class LayerManager implements BloomFilterProducer {
*
* @return a new LayerManager.
*/
public LayerManager build() {
public LayerManager<T> build() {
Objects.requireNonNull(supplier, "Supplier must not be null");
Objects.requireNonNull(extendCheck, "ExtendCheck must not be null");
Objects.requireNonNull(cleanup, "Cleanup must not be null");
return new LayerManager(supplier, extendCheck, cleanup, true);
return new LayerManager<>(supplier, extendCheck, cleanup, true);
}
/**
@ -84,7 +85,7 @@ public class LayerManager implements BloomFilterProducer {
* dated or stale filters.
* @return this
*/
public Builder setCleanup(Consumer<LinkedList<BloomFilter>> cleanup) {
public Builder<T> setCleanup(Consumer<Deque<T>> cleanup) {
this.cleanup = cleanup;
return this;
}
@ -97,7 +98,7 @@ public class LayerManager implements BloomFilterProducer {
* created.
* @return this for chaining.
*/
public Builder setExtendCheck(Predicate<LayerManager> extendCheck) {
public Builder<T> setExtendCheck(Predicate<LayerManager<T>> extendCheck) {
this.extendCheck = extendCheck;
return this;
}
@ -109,14 +110,14 @@ public class LayerManager implements BloomFilterProducer {
* @param supplier The supplier of new Bloom filter instances.
* @return this for chaining.
*/
public Builder setSupplier(Supplier<BloomFilter> supplier) {
public Builder<T> setSupplier(Supplier<T> supplier) {
this.supplier = supplier;
return this;
}
}
/**
* Static methods to create a Consumer of a LinkedList of BloomFilter perform
* Static methods to create a Consumer of a List of BloomFilter perform
* tests on whether to reduce the collection of Bloom filters.
*/
public static final class Cleanup {
@ -124,7 +125,7 @@ public class LayerManager implements BloomFilterProducer {
* A Cleanup that never removes anything.
* @return A Consumer suitable for the LayerManager {@code cleanup} parameter.
*/
public static Consumer<LinkedList<BloomFilter>> noCleanup() {
public static <T extends BloomFilter> Consumer<Deque<T>> noCleanup() {
return x -> {};
}
@ -137,7 +138,7 @@ public class LayerManager implements BloomFilterProducer {
* @return A Consumer suitable for the LayerManager {@code cleanup} parameter.
* @throws IllegalArgumentException if {@code maxSize <= 0}.
*/
public static Consumer<LinkedList<BloomFilter>> onMaxSize(int maxSize) {
public static <T extends BloomFilter> Consumer<Deque<T>> onMaxSize(int maxSize) {
if (maxSize <= 0) {
throw new IllegalArgumentException("'maxSize' must be greater than 0");
}
@ -154,14 +155,24 @@ public class LayerManager implements BloomFilterProducer {
*
* @return A Consumer suitable for the LayerManager {@code cleanup} parameter.
*/
public static Consumer<LinkedList<BloomFilter>> removeEmptyTarget() {
public static <T extends BloomFilter> Consumer<Deque<T>> removeEmptyTarget() {
return x -> {
if (x.getLast().cardinality() == 0) {
if (!x.isEmpty() && x.getLast().isEmpty()) {
x.removeLast();
}
};
}
/**
* Removes any layer identified by the predicate.
*
* @param test Predicate.
* @return A Consumer suitable for the LayerManager {@code cleanup} parameter.
*/
public static <T extends BloomFilter> Consumer<Deque<T>> removeIf(Predicate<? super T> test) {
return x -> x.removeIf(test);
}
private Cleanup() {
}
}
@ -179,16 +190,20 @@ public class LayerManager implements BloomFilterProducer {
* @return A Predicate suitable for the LayerManager {@code extendCheck} parameter.
* @throws IllegalArgumentException if {@code breakAt <= 0}
*/
public static Predicate<LayerManager> advanceOnCount(int breakAt) {
public static <T extends BloomFilter> Predicate<LayerManager<T>> advanceOnCount(int breakAt) {
if (breakAt <= 0) {
throw new IllegalArgumentException("'breakAt' must be greater than 0");
}
return new Predicate<LayerManager>() {
return new Predicate<LayerManager<T>>() {
int count;
@Override
public boolean test(LayerManager filter) {
return ++count % breakAt == 0;
public boolean test(LayerManager<T> filter) {
if (++count == breakAt) {
count = 0;
return true;
}
return false;
}
};
}
@ -197,8 +212,8 @@ public class LayerManager implements BloomFilterProducer {
* Advances the target once a merge has been performed.
* @return A Predicate suitable for the LayerManager {@code extendCheck} parameter.
*/
public static Predicate<LayerManager> advanceOnPopulated() {
return lm -> !lm.filters.peekLast().isEmpty();
public static <T extends BloomFilter> Predicate<LayerManager<T>> advanceOnPopulated() {
return lm -> !lm.last().isEmpty();
}
/**
@ -212,12 +227,12 @@ public class LayerManager implements BloomFilterProducer {
* @return A Predicate suitable for the LayerManager {@code extendCheck} parameter.
* @throws IllegalArgumentException if {@code maxN <= 0}
*/
public static Predicate<LayerManager> advanceOnSaturation(double maxN) {
public static <T extends BloomFilter> Predicate<LayerManager<T>> advanceOnSaturation(double maxN) {
if (maxN <= 0) {
throw new IllegalArgumentException("'maxN' must be greater than 0");
}
return manager -> {
BloomFilter bf = manager.filters.peekLast();
BloomFilter bf = manager.last();
return maxN <= bf.getShape().estimateN(bf.cardinality());
};
}
@ -227,7 +242,7 @@ public class LayerManager implements BloomFilterProducer {
* perform the advance.
* @return A Predicate suitable for the LayerManager {@code extendCheck} parameter.
*/
public static Predicate<LayerManager> neverAdvance() {
public static <T extends BloomFilter> Predicate<LayerManager<T>> neverAdvance() {
return x -> false;
}
@ -242,15 +257,15 @@ public class LayerManager implements BloomFilterProducer {
* @see ExtendCheck#neverAdvance()
* @see Cleanup#noCleanup()
*/
public static Builder builder() {
return new Builder();
public static <T extends BloomFilter> Builder<T> builder() {
return new Builder<>();
}
private final LinkedList<BloomFilter> filters = new LinkedList<>();
private final Consumer<LinkedList<BloomFilter>> filterCleanup;
private final LinkedList<T> filters = new LinkedList<>();
private final Consumer<Deque<T>> filterCleanup;
private final Predicate<LayerManager> extendCheck;
private final Predicate<LayerManager<T>> extendCheck;
private final Supplier<BloomFilter> filterSupplier;
private final Supplier<T> filterSupplier;
/**
* Constructor.
@ -263,8 +278,8 @@ public class LayerManager implements BloomFilterProducer {
* list.
* @param initialize true if the filter list should be initialized.
*/
private LayerManager(Supplier<BloomFilter> filterSupplier, Predicate<LayerManager> extendCheck,
Consumer<LinkedList<BloomFilter>> filterCleanup, boolean initialize) {
private LayerManager(Supplier<T> filterSupplier, Predicate<LayerManager<T>> extendCheck,
Consumer<Deque<T>> filterCleanup, boolean initialize) {
this.filterSupplier = filterSupplier;
this.extendCheck = extendCheck;
this.filterCleanup = filterCleanup;
@ -277,7 +292,7 @@ public class LayerManager implements BloomFilterProducer {
* Adds a new Bloom filter to the list.
*/
private void addFilter() {
BloomFilter bf = filterSupplier.get();
T bf = filterSupplier.get();
if (bf == null) {
throw new NullPointerException("filterSupplier returned null.");
}
@ -302,9 +317,9 @@ public class LayerManager implements BloomFilterProducer {
*
* @return a copy of this layer Manager.
*/
public LayerManager copy() {
LayerManager newMgr = new LayerManager(filterSupplier, extendCheck, filterCleanup, false);
for (BloomFilter bf : filters) {
public LayerManager<T> copy() {
LayerManager<T> newMgr = new LayerManager<>(filterSupplier, extendCheck, filterCleanup, false);
for (T bf : filters) {
newMgr.filters.add(bf.copy());
}
return newMgr;
@ -337,7 +352,7 @@ public class LayerManager implements BloomFilterProducer {
* @throws NoSuchElementException if depth is not in the range
* [0,filters.size())
*/
public final BloomFilter get(int depth) {
public final T get(int depth) {
if (depth < 0 || depth >= filters.size()) {
throw new NoSuchElementException(String.format("Depth must be in the range [0,%s)", filters.size()));
}
@ -346,7 +361,7 @@ public class LayerManager implements BloomFilterProducer {
/**
* Returns the number of filters in the LayerManager. In the default LayerManager implementation
* there is alwasy at least one layer.
* there is always at least one layer.
*
* @return the current depth.
*/
@ -354,17 +369,37 @@ public class LayerManager implements BloomFilterProducer {
return filters.size();
}
/**
* Gets the Bloom filter from the first layer.
* No extension check is performed during this call.
* @return The Bloom filter from the first layer.
* @see #getTarget()
*/
public final T first() {
return filters.getFirst();
}
/**
* Gets the Bloom filter from the last layer.
* No extension check is performed during this call.
* @return The Bloom filter from the last layer.
* @see #getTarget()
*/
public final T last() {
return filters.getLast();
}
/**
* Returns the current target filter. If a new filter should be created based on
* {@code extendCheck} it will be created before this method returns.
*
* @return the current target filter after any extension.
*/
public final BloomFilter getTarget() {
public final T getTarget() {
if (extendCheck.test(this)) {
next();
}
return filters.peekLast();
return last();
}
/**

View File

@ -59,9 +59,10 @@ import java.util.function.Predicate;
* removes them. It also checks it a new layer should be added, and if so adds
* it and sets the {@code target} before the operation.</li>
* </ul>
* @param <T> The type of Bloom Filter that is used for the layers.
* @since 4.5
*/
public class LayeredBloomFilter implements BloomFilter, BloomFilterProducer {
public class LayeredBloomFilter<T extends BloomFilter> implements BloomFilter, BloomFilterProducer {
/**
* A class used to locate matching filters across all the layers.
*/
@ -88,24 +89,10 @@ public class LayeredBloomFilter implements BloomFilter, BloomFilterProducer {
return true;
}
}
/**
* Creates a fixed size layered bloom filter that adds new filters to the list,
* but never merges them. List will never exceed maxDepth. As additional filters
* are added earlier filters are removed.
*
* @param shape The shape for the enclosed Bloom filters.
* @param maxDepth The maximum depth of layers.
* @return An empty layered Bloom filter of the specified shape and depth.
*/
public static LayeredBloomFilter fixed(final Shape shape, int maxDepth) {
LayerManager manager = LayerManager.builder().setExtendCheck(LayerManager.ExtendCheck.advanceOnPopulated())
.setCleanup(LayerManager.Cleanup.onMaxSize(maxDepth)).setSupplier(() -> new SimpleBloomFilter(shape)).build();
return new LayeredBloomFilter(shape, manager);
}
private final Shape shape;
private LayerManager layerManager;
private final LayerManager<T> layerManager;
/**
* Constructor.
@ -113,7 +100,7 @@ public class LayeredBloomFilter implements BloomFilter, BloomFilterProducer {
* @param shape the Shape of the enclosed Bloom filters
* @param layerManager the LayerManager to manage the layers.
*/
public LayeredBloomFilter(Shape shape, LayerManager layerManager) {
public LayeredBloomFilter(Shape shape, LayerManager<T> layerManager) {
this.shape = shape;
this.layerManager = layerManager;
}
@ -184,8 +171,8 @@ public class LayeredBloomFilter implements BloomFilter, BloomFilterProducer {
}
@Override
public LayeredBloomFilter copy() {
return new LayeredBloomFilter(shape, layerManager.copy());
public LayeredBloomFilter<T> copy() {
return new LayeredBloomFilter<>(shape, layerManager.copy());
}
/**
@ -329,7 +316,7 @@ public class LayeredBloomFilter implements BloomFilter, BloomFilterProducer {
* @return the Bloom filter at the specified depth.
* @throws NoSuchElementException if depth is not in the range [0,getDepth())
*/
public BloomFilter get(int depth) {
public T get(int depth) {
return layerManager.get(depth);
}

View File

@ -25,7 +25,7 @@ import java.util.function.LongPredicate;
* @since 4.5
*/
public abstract class WrappedBloomFilter implements BloomFilter {
final BloomFilter wrapped;
private final BloomFilter wrapped;
/**
* Wraps a Bloom filter. The wrapped filter is maintained as a reference
@ -36,6 +36,10 @@ public abstract class WrappedBloomFilter implements BloomFilter {
this.wrapped = bf;
}
protected BloomFilter getWrapped() {
return wrapped;
}
@Override
public long[] asBitMapArray() {
return wrapped.asBitMapArray();
@ -81,11 +85,6 @@ public abstract class WrappedBloomFilter implements BloomFilter {
return wrapped.contains(indexProducer);
}
@Override
public BloomFilter copy() {
return wrapped.copy();
}
@Override
public int estimateIntersection(BloomFilter other) {
return wrapped.estimateIntersection(other);

View File

@ -22,13 +22,13 @@ public class BitMapProducerFromLayeredBloomFilterTest extends AbstractBitMapProd
@Override
protected BitMapProducer createEmptyProducer() {
return LayeredBloomFilter.fixed(shape, 10);
return LayeredBloomFilterTest.fixed(shape, 10);
}
@Override
protected BitMapProducer createProducer() {
final Hasher hasher = new IncrementingHasher(0, 1);
final BloomFilter bf = LayeredBloomFilter.fixed(shape, 10);
final BloomFilter bf = LayeredBloomFilterTest.fixed(shape, 10);
bf.merge(hasher);
return bf;
}

View File

@ -23,6 +23,12 @@ public class BitMapProducerFromWrappedBloomFilterTest extends AbstractBitMapProd
@Override
protected BitMapProducer createEmptyProducer() {
return new WrappedBloomFilter(new DefaultBloomFilterTest.SparseDefaultBloomFilter(shape)) {
@Override
public BloomFilter copy() {
BloomFilter result = new DefaultBloomFilterTest.SparseDefaultBloomFilter(shape);
result.merge(getWrapped());
return result;
}
};
}
@ -30,6 +36,12 @@ public class BitMapProducerFromWrappedBloomFilterTest extends AbstractBitMapProd
protected BitMapProducer createProducer() {
final Hasher hasher = new IncrementingHasher(0, 1);
final BloomFilter bf = new WrappedBloomFilter(new DefaultBloomFilterTest.SparseDefaultBloomFilter(shape)) {
@Override
public BloomFilter copy() {
BloomFilter result = new DefaultBloomFilterTest.SparseDefaultBloomFilter(shape);
result.merge(getWrapped());
return result;
}
};
bf.merge(hasher);
return bf;

View File

@ -22,13 +22,13 @@ public class CellProducerFromLayeredBloomFilterTest extends AbstractCellProducer
@Override
protected CellProducer createEmptyProducer() {
return CellProducer.from(LayeredBloomFilter.fixed(shape, 10));
return CellProducer.from(LayeredBloomFilterTest.fixed(shape, 10));
}
@Override
protected CellProducer createProducer() {
final Hasher hasher = new IncrementingHasher(3, 2);
final BloomFilter bf = LayeredBloomFilter.fixed(shape, 10);
final BloomFilter bf = LayeredBloomFilterTest.fixed(shape, 10);
bf.merge(hasher);
return CellProducer.from(bf);
}

View File

@ -16,9 +16,9 @@
*/
package org.apache.commons.collections4.bloomfilter;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotSame;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertSame;
@ -27,6 +27,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Deque;
import java.util.LinkedList;
import java.util.List;
import java.util.NoSuchElementException;
@ -39,13 +40,13 @@ import org.junit.jupiter.params.provider.ValueSource;
public class LayerManagerTest {
private Shape shape = Shape.fromKM(17, 72);
private final Shape shape = Shape.fromKM(17, 72);
@ParameterizedTest
@ValueSource(ints = {4, 10, 2, 1})
public void testAdvanceOnCount(int breakAt) {
Predicate<LayerManager> underTest = LayerManager.ExtendCheck.advanceOnCount(breakAt);
LayerManager layerManager = testingBuilder().build();
Predicate<LayerManager<BloomFilter>> underTest = LayerManager.ExtendCheck.advanceOnCount(breakAt);
LayerManager<BloomFilter> layerManager = testingBuilder().build();
for (int i = 0; i < breakAt - 1; i++) {
assertFalse(underTest.test(layerManager), "at " + i);
layerManager.getTarget().merge(TestingHashers.FROM1);
@ -61,8 +62,8 @@ public class LayerManagerTest {
@Test
public void testAdvanceOnPopulated() {
Predicate<LayerManager> underTest = LayerManager.ExtendCheck.advanceOnPopulated();
LayerManager layerManager = testingBuilder().build();
Predicate<LayerManager<BloomFilter>> underTest = LayerManager.ExtendCheck.advanceOnPopulated();
LayerManager<BloomFilter> layerManager = testingBuilder().build();
assertFalse(underTest.test(layerManager));
layerManager.getTarget().merge(TestingHashers.FROM1);
assertTrue(underTest.test(layerManager));
@ -70,10 +71,10 @@ public class LayerManagerTest {
@Test
public void testAdvanceOnSaturation() {
Double maxN = shape.estimateMaxN();
double maxN = shape.estimateMaxN();
int hashStart = 0;
Predicate<LayerManager> underTest = LayerManager.ExtendCheck.advanceOnSaturation(maxN);
LayerManager layerManager = testingBuilder().build();
Predicate<LayerManager<BloomFilter>> underTest = LayerManager.ExtendCheck.advanceOnSaturation(maxN);
LayerManager<BloomFilter> layerManager = testingBuilder().build();
while (layerManager.getTarget().getShape().estimateN(layerManager.getTarget().cardinality()) < maxN) {
assertFalse(underTest.test(layerManager));
layerManager.getTarget().merge(new IncrementingHasher(hashStart, shape.getNumberOfHashFunctions()));
@ -86,15 +87,15 @@ public class LayerManagerTest {
@Test
public void testBuilder() {
LayerManager.Builder underTest = LayerManager.builder();
NullPointerException npe = assertThrows(NullPointerException.class, () -> underTest.build());
LayerManager.Builder<BloomFilter> underTest = LayerManager.builder();
NullPointerException npe = assertThrows(NullPointerException.class, underTest::build);
assertTrue(npe.getMessage().contains("Supplier must not be null"));
underTest.setSupplier(() -> null).setCleanup(null);
npe = assertThrows(NullPointerException.class, () -> underTest.build());
npe = assertThrows(NullPointerException.class, underTest::build);
assertTrue(npe.getMessage().contains("Cleanup must not be null"));
underTest.setCleanup(x -> {
}).setExtendCheck(null);
npe = assertThrows(NullPointerException.class, () -> underTest.build());
npe = assertThrows(NullPointerException.class, underTest::build);
assertTrue(npe.getMessage().contains("ExtendCheck must not be null"));
npe = assertThrows(NullPointerException.class, () -> LayerManager.builder().setSupplier(() -> null).build());
@ -104,7 +105,7 @@ public class LayerManagerTest {
@Test
public void testClear() {
LayerManager underTest = LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(shape)).build();
LayerManager<BloomFilter> underTest = LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(shape)).build();
underTest.getTarget().merge(TestingHashers.randomHasher());
underTest.next();
underTest.getTarget().merge(TestingHashers.randomHasher());
@ -118,7 +119,7 @@ public class LayerManagerTest {
@Test
public void testCopy() {
LayerManager underTest = LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(shape)).build();
LayerManager<BloomFilter> underTest = LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(shape)).build();
underTest.getTarget().merge(TestingHashers.randomHasher());
underTest.next();
underTest.getTarget().merge(TestingHashers.randomHasher());
@ -126,7 +127,7 @@ public class LayerManagerTest {
underTest.getTarget().merge(TestingHashers.randomHasher());
assertEquals(3, underTest.getDepth());
LayerManager copy = underTest.copy();
LayerManager<BloomFilter> copy = underTest.copy();
assertNotSame(underTest, copy);
// object equals not implemented
assertNotEquals(underTest, copy);
@ -138,7 +139,7 @@ public class LayerManagerTest {
@Test
public void testForEachBloomFilter() {
LayerManager underTest = LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(shape))
LayerManager<BloomFilter> underTest = LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(shape))
.setExtendCheck(LayerManager.ExtendCheck.advanceOnPopulated()).build();
List<BloomFilter> lst = new ArrayList<>();
@ -160,21 +161,21 @@ public class LayerManagerTest {
@Test
public void testGet() {
SimpleBloomFilter f = new SimpleBloomFilter(shape);
LayerManager underTest = LayerManager.builder().setSupplier(() -> f).build();
LayerManager<BloomFilter> underTest = LayerManager.builder().setSupplier(() -> f).build();
assertEquals(1, underTest.getDepth());
assertSame(f, underTest.get(0));
assertThrows(NoSuchElementException.class, () -> underTest.get(-1));
assertThrows(NoSuchElementException.class, () -> underTest.get(1));
}
private LayerManager.Builder testingBuilder() {
private LayerManager.Builder<BloomFilter> testingBuilder() {
return LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(shape));
}
@Test
public void testNeverAdvance() {
Predicate<LayerManager> underTest = LayerManager.ExtendCheck.neverAdvance();
LayerManager layerManager = testingBuilder().build();
Predicate<LayerManager<BloomFilter>> underTest = LayerManager.ExtendCheck.neverAdvance();
LayerManager<BloomFilter> layerManager = testingBuilder().build();
assertFalse(underTest.test(layerManager));
for (int i = 0; i < 10; i++) {
layerManager.getTarget().merge(TestingHashers.randomHasher());
@ -184,7 +185,7 @@ public class LayerManagerTest {
@Test
public void testNextAndGetDepth() {
LayerManager underTest = LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(shape)).build();
LayerManager<BloomFilter> underTest = LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(shape)).build();
assertEquals(1, underTest.getDepth());
underTest.getTarget().merge(TestingHashers.randomHasher());
assertEquals(1, underTest.getDepth());
@ -194,8 +195,8 @@ public class LayerManagerTest {
@Test
public void testNoCleanup() {
Consumer<LinkedList<BloomFilter>> underTest = LayerManager.Cleanup.noCleanup();
LinkedList<BloomFilter> list = new LinkedList<>();
Consumer<Deque<BloomFilter>> underTest = LayerManager.Cleanup.noCleanup();
Deque<BloomFilter> list = new LinkedList<>();
for (int i = 0; i < 20; i++) {
assertEquals(i, list.size());
list.add(new SimpleBloomFilter(shape));
@ -206,7 +207,7 @@ public class LayerManagerTest {
@ParameterizedTest
@ValueSource(ints = {5, 100, 2, 1})
public void testOnMaxSize(int maxSize) {
Consumer<LinkedList<BloomFilter>> underTest = LayerManager.Cleanup.onMaxSize(maxSize);
Consumer<Deque<BloomFilter>> underTest = LayerManager.Cleanup.onMaxSize(maxSize);
LinkedList<BloomFilter> list = new LinkedList<>();
for (int i = 0; i < maxSize; i++) {
assertEquals(i, list.size());
@ -230,7 +231,7 @@ public class LayerManagerTest {
@Test
public void testRemoveEmptyTarget() {
Consumer<LinkedList<BloomFilter>> underTest = LayerManager.Cleanup.removeEmptyTarget();
Consumer<Deque<BloomFilter>> underTest = LayerManager.Cleanup.removeEmptyTarget();
LinkedList<BloomFilter> list = new LinkedList<>();
// removes an empty filter
@ -265,7 +266,6 @@ public class LayerManagerTest {
underTest.accept(list);
assertEquals(2, list.size());
assertEquals(bf, list.get(0));
}
@Test
@ -273,7 +273,7 @@ public class LayerManagerTest {
boolean[] extendCheckCalled = { false };
boolean[] cleanupCalled = { false };
int[] supplierCount = { 0 };
LayerManager underTest = LayerManager.builder().setSupplier(() -> {
LayerManager<BloomFilter> underTest = LayerManager.builder().setSupplier(() -> {
supplierCount[0]++;
return new SimpleBloomFilter(shape);
}).setExtendCheck(lm -> {
@ -291,13 +291,4 @@ public class LayerManagerTest {
assertEquals(2, supplierCount[0]);
}
static class NumberedBloomFilter extends WrappedBloomFilter {
int value;
int sequence;
NumberedBloomFilter(Shape shape, int value, int sequence) {
super(new SimpleBloomFilter(shape));
this.value = value;
this.sequence = sequence;
}
}
}

View File

@ -16,30 +16,60 @@
*/
package org.apache.commons.collections4.bloomfilter;
import static org.junit.Assert.assertEquals;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.Deque;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.function.Supplier;
import org.apache.commons.collections4.bloomfilter.LayerManager.Cleanup;
import org.apache.commons.collections4.bloomfilter.LayerManager.ExtendCheck;
import org.apache.commons.collections4.bloomfilter.LayerManagerTest.NumberedBloomFilter;
import org.junit.jupiter.api.Test;
public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloomFilter> {
public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloomFilter<?>> {
/**
* Creates a fixed size layered bloom filter that adds new filters to the list,
* but never merges them. List will never exceed maxDepth. As additional filters
* are added earlier filters are removed. Uses SimpleBloomFilters.
*
* @param shape The shape for the enclosed Bloom filters.
* @param maxDepth The maximum depth of layers.
* @return An empty layered Bloom filter of the specified shape and depth.
*/
public static LayeredBloomFilter<BloomFilter> fixed(final Shape shape, int maxDepth) {
return fixed(shape, maxDepth, () -> new SimpleBloomFilter(shape));
}
/**
* Creates a fixed size layered bloom filter that adds new filters to the list,
* but never merges them. List will never exceed maxDepth. As additional filters
* are added earlier filters are removed.
*
* @param shape The shape for the enclosed Bloom filters.
* @param maxDepth The maximum depth of layers.
* @param supplier A supplier of the Bloom filters to create layers with.
* @return An empty layered Bloom filter of the specified shape and depth.
*/
public static <T extends BloomFilter> LayeredBloomFilter<T> fixed(final Shape shape, int maxDepth, Supplier<T> supplier) {
LayerManager.Builder<T> builder = LayerManager.builder();
builder.setExtendCheck(LayerManager.ExtendCheck.advanceOnPopulated())
.setCleanup(LayerManager.Cleanup.onMaxSize(maxDepth)).setSupplier(supplier);
return new LayeredBloomFilter<>(shape, builder.build());
}
/**
* A Predicate that advances after a quantum of time.
*/
static class AdvanceOnTimeQuanta implements Predicate<LayerManager> {
static class AdvanceOnTimeQuanta implements Predicate<LayerManager<TimestampedBloomFilter>> {
long quanta;
AdvanceOnTimeQuanta(long quanta, TimeUnit unit) {
@ -47,10 +77,9 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom
}
@Override
public boolean test(LayerManager lm) {
public boolean test(LayerManager<TimestampedBloomFilter> lm) {
// can not use getTarget() as it causes recursion.
TimestampedBloomFilter bf = (TimestampedBloomFilter) lm.get(lm.getDepth() - 1);
return bf.timestamp + quanta < System.currentTimeMillis();
return lm.last().timestamp + quanta < System.currentTimeMillis();
}
}
@ -58,7 +87,7 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom
* A Consumer that cleans the list based on how long each filters has been in
* the list.
*/
static class CleanByTime implements Consumer<LinkedList<BloomFilter>> {
static class CleanByTime<T extends TimestampedBloomFilter> implements Consumer<List<T>> {
long elapsedTime;
CleanByTime(long duration, TimeUnit unit) {
@ -66,13 +95,18 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom
}
@Override
public void accept(LinkedList<BloomFilter> t) {
public void accept(List<T> t) {
long min = System.currentTimeMillis() - elapsedTime;
while (!t.isEmpty() && ((TimestampedBloomFilter) t.getFirst()).getTimestamp() < min) {
TimestampedBloomFilter bf = (TimestampedBloomFilter) t.getFirst();
dbgInstrument.add(String.format("Removing old entry: T:%s (Aged: %s) \n", bf.getTimestamp(),
(min - bf.getTimestamp())));
t.removeFirst();
Iterator<T> iter = t.iterator();
while (iter.hasNext()) {
TimestampedBloomFilter bf = iter.next();
if (bf.getTimestamp() < min) {
dbgInstrument.add(String.format("Removing old entry: T:%s (Aged: %s) \n", bf.getTimestamp(),
(min - bf.getTimestamp())));
iter.remove();
} else {
return;
}
}
}
}
@ -80,7 +114,7 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom
/**
* A Bloomfilter implementation that tracks the creation time.
*/
static class TimestampedBloomFilter extends WrappedBloomFilter {
public static class TimestampedBloomFilter extends WrappedBloomFilter {
final long timestamp;
TimestampedBloomFilter(BloomFilter bf) {
@ -88,13 +122,23 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom
this.timestamp = System.currentTimeMillis();
}
TimestampedBloomFilter(BloomFilter bf, long timestamp) {
super(bf);
this.timestamp = timestamp;
}
public long getTimestamp() {
return timestamp;
}
@Override
public TimestampedBloomFilter copy() {
return new TimestampedBloomFilter(this.getWrapped().copy(), timestamp);
}
}
// ***example of instrumentation ***
private static List<String> dbgInstrument = new ArrayList<>();
private static final List<String> dbgInstrument = new ArrayList<>();
/**
* Creates a LayeredBloomFilter that retains enclosed filters for
@ -109,19 +153,21 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom
* @param qUnit the unit of time to apply to quanta.
* @return LayeredBloomFilter with the above properties.
*/
static LayeredBloomFilter createTimedLayeredFilter(Shape shape, long duration, TimeUnit dUnit, long quanta,
static LayeredBloomFilter<TimestampedBloomFilter> createTimedLayeredFilter(Shape shape, long duration, TimeUnit dUnit, long quanta,
TimeUnit qUnit) {
LayerManager layerManager = LayerManager.builder()
LayerManager.Builder<TimestampedBloomFilter> builder = LayerManager.builder();
Consumer<Deque<TimestampedBloomFilter>> cleanup = Cleanup.removeEmptyTarget().andThen(new CleanByTime(duration, dUnit));
LayerManager<TimestampedBloomFilter> layerManager = builder
.setSupplier(() -> new TimestampedBloomFilter(new SimpleBloomFilter(shape)))
.setCleanup(Cleanup.removeEmptyTarget().andThen(new CleanByTime(duration, dUnit)))
.setCleanup(cleanup)
.setExtendCheck(new AdvanceOnTimeQuanta(quanta, qUnit)
.or(LayerManager.ExtendCheck.advanceOnSaturation(shape.estimateMaxN())))
.build();
return new LayeredBloomFilter(shape, layerManager);
return new LayeredBloomFilter<>(shape, layerManager);
}
// instrumentation to record timestamps in dbgInstrument list
private Predicate<BloomFilter> dbg = (bf) -> {
private final Predicate<BloomFilter> dbg = (bf) -> {
TimestampedBloomFilter tbf = (TimestampedBloomFilter) bf;
long ts = System.currentTimeMillis();
dbgInstrument.add(String.format("T:%s (Elapsed:%s)- EstN:%s (Card:%s)\n", tbf.timestamp, ts - tbf.timestamp,
@ -131,8 +177,8 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom
// *** end of instrumentation ***
@Override
protected LayeredBloomFilter createEmptyFilter(Shape shape) {
return LayeredBloomFilter.fixed(shape, 10);
protected LayeredBloomFilter<BloomFilter> createEmptyFilter(Shape shape) {
return LayeredBloomFilterTest.fixed(shape, 10);
}
protected BloomFilter makeFilter(Hasher h) {
@ -151,8 +197,8 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom
return makeFilter(IndexProducer.fromIndexArray(values));
}
private LayeredBloomFilter setupFindTest() {
LayeredBloomFilter filter = LayeredBloomFilter.fixed(getTestShape(), 10);
private LayeredBloomFilter<BloomFilter> setupFindTest() {
LayeredBloomFilter<BloomFilter> filter = LayeredBloomFilterTest.fixed(getTestShape(), 10);
filter.merge(TestingHashers.FROM1);
filter.merge(TestingHashers.FROM11);
filter.merge(new IncrementingHasher(11, 2));
@ -163,9 +209,9 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom
@Override
@Test
public void testCardinalityAndIsEmpty() {
LayerManager layerManager = LayerManager.builder().setExtendCheck(ExtendCheck.neverAdvance())
LayerManager<BloomFilter> layerManager = LayerManager.builder().setExtendCheck(ExtendCheck.neverAdvance())
.setSupplier(() -> new SimpleBloomFilter(getTestShape())).build();
testCardinalityAndIsEmpty(new LayeredBloomFilter(getTestShape(), layerManager));
testCardinalityAndIsEmpty(new LayeredBloomFilter<>(getTestShape(), layerManager));
}
/**
@ -194,7 +240,7 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom
// create a filter that removes filters that are 4 seconds old
// and quantises time to 1 second intervals.
LayeredBloomFilter underTest = createTimedLayeredFilter(shape, 600, TimeUnit.MILLISECONDS, 150,
LayeredBloomFilter<TimestampedBloomFilter> underTest = createTimedLayeredFilter(shape, 600, TimeUnit.MILLISECONDS, 150,
TimeUnit.MILLISECONDS);
for (int i = 0; i < 10; i++) {
@ -226,7 +272,7 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom
}
@Test
public void testFindBitMapProducer() {
LayeredBloomFilter filter = setupFindTest();
LayeredBloomFilter<BloomFilter> filter = setupFindTest();
IndexProducer idxProducer = TestingHashers.FROM1.indices(getTestShape());
BitMapProducer producer = BitMapProducer.fromIndexProducer(idxProducer, getTestShape().getNumberOfBits());
@ -244,7 +290,7 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom
@Test
public void testFindBloomFilter() {
LayeredBloomFilter filter = setupFindTest();
LayeredBloomFilter<BloomFilter> filter = setupFindTest();
int[] expected = {0, 3};
int[] result = filter.find(TestingHashers.FROM1);
assertArrayEquals(expected, result);
@ -256,7 +302,7 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom
@Test
public void testFindIndexProducer() {
IndexProducer producer = TestingHashers.FROM1.indices(getTestShape());
LayeredBloomFilter filter = setupFindTest();
LayeredBloomFilter<BloomFilter> filter = setupFindTest();
int[] expected = {0, 3};
int[] result = filter.find(producer);
@ -272,7 +318,7 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom
public final void testGetLayer() {
BloomFilter bf = new SimpleBloomFilter(getTestShape());
bf.merge(TestingHashers.FROM11);
LayeredBloomFilter filter = LayeredBloomFilter.fixed(getTestShape(), 10);
LayeredBloomFilter<BloomFilter> filter = LayeredBloomFilterTest.fixed(getTestShape(), 10);
filter.merge(TestingHashers.FROM1);
filter.merge(TestingHashers.FROM11);
filter.merge(new IncrementingHasher(11, 2));
@ -282,7 +328,7 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom
@Test
public void testMultipleFilters() {
LayeredBloomFilter filter = LayeredBloomFilter.fixed(getTestShape(), 10);
LayeredBloomFilter<BloomFilter> filter = LayeredBloomFilterTest.fixed(getTestShape(), 10);
filter.merge(TestingHashers.FROM1);
filter.merge(TestingHashers.FROM11);
assertEquals(2, filter.getDepth());
@ -296,10 +342,10 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom
@Test
public final void testNext() {
LayerManager layerManager = LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(getTestShape()))
LayerManager<BloomFilter> layerManager = LayerManager.builder().setSupplier(() -> new SimpleBloomFilter(getTestShape()))
.build();
LayeredBloomFilter filter = new LayeredBloomFilter(getTestShape(), layerManager);
LayeredBloomFilter<BloomFilter> filter = new LayeredBloomFilter<>(getTestShape(), layerManager);
filter.merge(TestingHashers.FROM1);
filter.merge(TestingHashers.FROM11);
assertEquals(1, filter.getDepth());
@ -345,4 +391,19 @@ public class LayeredBloomFilterTest extends AbstractBloomFilterTest<LayeredBloom
f = (NumberedBloomFilter) underTest.get(0);
assertEquals(3, f.sequence); // it is a new one.
}
static class NumberedBloomFilter extends WrappedBloomFilter {
int value;
int sequence;
NumberedBloomFilter(Shape shape, int value, int sequence) {
super(new SimpleBloomFilter(shape));
this.value = value;
this.sequence = sequence;
}
@Override
public BloomFilter copy() {
return new NumberedBloomFilter(getShape(), value, sequence);
}
}
}

View File

@ -26,6 +26,12 @@ public class WrappedBloomFilterTest extends AbstractBloomFilterTest<WrappedBloom
@Override
protected WrappedBloomFilter createEmptyFilter(Shape shape) {
return new WrappedBloomFilter(new DefaultBloomFilterTest.SparseDefaultBloomFilter(shape)) {
@Override
public BloomFilter copy() {
BloomFilter result = new DefaultBloomFilterTest.SparseDefaultBloomFilter(shape);
result.merge(this.getWrapped());
return result;
}
};
}
@ -39,7 +45,14 @@ public class WrappedBloomFilterTest extends AbstractBloomFilterTest<WrappedBloom
return characteristics;
}
};
WrappedBloomFilter underTest = new WrappedBloomFilter(inner) {};
WrappedBloomFilter underTest = new WrappedBloomFilter(inner) {
@Override
public BloomFilter copy() {
BloomFilter result = new DefaultBloomFilterTest.SparseDefaultBloomFilter(shape);
result.merge(this.getWrapped());
return result;
}
};
assertEquals(characteristics, underTest.characteristics());
}
}