Apache
/
commons-collections
mirror of https://github.com/apache/commons-collections.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #317 from Claudenw/fix_collections_821 

moved IndexFilter to its own file.
This commit is contained in:
Claude Warren 2022-07-06 16:52:06 +01:00 committed by GitHub
parent fcbcb2e746 57b5be9cce
commit c811f541aa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 139 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
src/main/java/org/apache/commons/collections4/bloomfilter/Hasher.java
View File

@ -16,8 +16,6 @@
*/
package org.apache.commons.collections4.bloomfilter;
import java.util.function.IntPredicate;
/**
* A Hasher creates IndexProducer based on the hash implementation and the
* provided Shape.
@ -55,119 +53,4 @@ public interface Hasher {
* @return the iterator of integers
*/
IndexProducer uniqueIndices(Shape shape);
/**
* A convenience class for Hasher implementations to filter out duplicate indices.
*
* <p><em>If the index is negative the behavior is not defined.</em></p>
*
* <p>This is conceptually a unique filter implemented as a {@code IntPredicate}.</p>
* @since 4.5
*/
final class IndexFilter implements IntPredicate {
private final IntPredicate tracker;
private final int size;
private final IntPredicate consumer;
/**
* Creates an instance optimized for the specified shape.
* @param shape The shape that is being generated.
* @param consumer The consumer to accept the values.
* @return an IndexFilter optimized for the specified shape.
*/
public static IndexFilter create(Shape shape, IntPredicate consumer) {
return new IndexFilter(shape, consumer);
}
/**
* Creates an instance optimized for the specified shape.
* @param shape The shape that is being generated.
* @param consumer The consumer to accept the values.
*/
private IndexFilter(Shape shape, IntPredicate consumer) {
this.size = shape.getNumberOfBits();
this.consumer = consumer;
if (BitMap.numberOfBitMaps(shape.getNumberOfBits()) * Long.BYTES < (long) shape.getNumberOfHashFunctions()
* Integer.BYTES) {
this.tracker = new BitMapTracker(shape);
} else {
this.tracker = new ArrayTracker(shape);
}
}
/**
* Test if the number should be processed by the {@code consumer}.
*
* <p>If the number has <em>not</em> been seen before it is passed to the {@code consumer} and the result returned.
* If the number has been seen before the {@code consumer} is not called and {@code true} returned.</p>
*
* <p><em>If the input is not in the range [0,size) an IndexOutOfBoundsException exception is thrown.</em></p>
*
* @param number the number to check.
* @return {@code true} if processing should continue, {@code false} otherwise.
*/
@Override
public boolean test(int number) {
if (number >= size) {
throw new IndexOutOfBoundsException(String.format("number too large %d >= %d", number, size));
}
return tracker.test(number) ? consumer.test(number) : true;
}
/**
* An IndexTracker implementation that uses an array of integers to track whether or not a
* number has been seen. Suitable for Shapes that have few hash functions.
* @since 4.5
*/
static class ArrayTracker implements IntPredicate {
private int[] seen;
private int populated;
/**
* Constructs the tracker based on the shape.
* @param shape the shape to build the tracker for.
*/
ArrayTracker(Shape shape) {
seen = new int[shape.getNumberOfHashFunctions()];
}
@Override
public boolean test(int number) {
if (number < 0) {
throw new IndexOutOfBoundsException("number may not be less than zero. " + number);
}
for (int i = 0; i < populated; i++) {
if (seen[i] == number) {
return false;
}
}
seen[populated++] = number;
return true;
}
}
/**
* An IndexTracker implementation that uses an array of bit maps to track whether or not a
* number has been seen.
* @since 4.5
*/
static class BitMapTracker implements IntPredicate {
private long[] bits;
/**
* Constructs a bit map based tracker for the specified shape.
* @param shape The shape that is being generated.
*/
BitMapTracker(Shape shape) {
bits = new long[BitMap.numberOfBitMaps(shape.getNumberOfBits())];
}
@Override
public boolean test(int number) {
boolean retval = !BitMap.contains(bits, number);
BitMap.set(bits, number);
return retval;
}
}
}
}

133
src/main/java/org/apache/commons/collections4/bloomfilter/IndexFilter.java Normal file
View File

@ -0,0 +1,133 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.commons.collections4.bloomfilter;
import java.util.function.IntPredicate;
/**
* A convenience class for Hasher implementations to filter out duplicate indices.
*
* <p><em>If the index is negative the behavior is not defined.</em></p>
*
* <p>This is conceptually a unique filter implemented as a {@code IntPredicate}.</p>
* @since 4.5
*/
public final class IndexFilter {
private final IntPredicate tracker;
private final int size;
private final IntPredicate consumer;
/**
* Creates an instance optimized for the specified shape.
* @param shape The shape that is being generated.
* @param consumer The consumer to accept the values.
* @return an IndexFilter optimized for the specified shape.
*/
public static IntPredicate create(Shape shape, IntPredicate consumer) {
return new IndexFilter(shape, consumer)::test;
}
/**
* Creates an instance optimized for the specified shape.
* @param shape The shape that is being generated.
* @param consumer The consumer to accept the values.
*/
private IndexFilter(Shape shape, IntPredicate consumer) {
this.size = shape.getNumberOfBits();
this.consumer = consumer;
if (BitMap.numberOfBitMaps(shape.getNumberOfBits()) * Long.BYTES < (long) shape.getNumberOfHashFunctions()
* Integer.BYTES) {
this.tracker = new BitMapTracker(shape);
} else {
this.tracker = new ArrayTracker(shape);
}
}
/**
* Test if the number should be processed by the {@code consumer}.
*
* <p>If the number has <em>not</em> been seen before it is passed to the {@code consumer} and the result returned.
* If the number has been seen before the {@code consumer} is not called and {@code true} returned.</p>
*
* <p><em>If the input is not in the range [0,size) an IndexOutOfBoundsException exception is thrown.</em></p>
*
* @param number the number to check.
* @return {@code true} if processing should continue, {@code false} otherwise.
*/
public boolean test(int number) {
if (number >= size) {
throw new IndexOutOfBoundsException(String.format("number too large %d >= %d", number, size));
}
return tracker.test(number) ? consumer.test(number) : true;
}
/**
* An IndexTracker implementation that uses an array of integers to track whether or not a
* number has been seen. Suitable for Shapes that have few hash functions.
* @since 4.5
*/
static class ArrayTracker implements IntPredicate {
private int[] seen;
private int populated;
/**
* Constructs the tracker based on the shape.
* @param shape the shape to build the tracker for.
*/
ArrayTracker(Shape shape) {
seen = new int[shape.getNumberOfHashFunctions()];
}
@Override
public boolean test(int number) {
if (number < 0) {
throw new IndexOutOfBoundsException("number may not be less than zero. " + number);
}
for (int i = 0; i < populated; i++) {
if (seen[i] == number) {
return false;
}
}
seen[populated++] = number;
return true;
}
}
/**
* An IndexTracker implementation that uses an array of bit maps to track whether or not a
* number has been seen.
* @since 4.5
*/
static class BitMapTracker implements IntPredicate {
private long[] bits;
/**
* Constructs a bit map based tracker for the specified shape.
* @param shape The shape that is being generated.
*/
BitMapTracker(Shape shape) {
bits = new long[BitMap.numberOfBitMaps(shape.getNumberOfBits())];
}
@Override
public boolean test(int number) {
boolean retval = !BitMap.contains(bits, number);
BitMap.set(bits, number);
return retval;
}
}
}

2
src/main/java/org/apache/commons/collections4/bloomfilter/SimpleHasher.java
View File

@ -173,7 +173,7 @@ public class SimpleHasher implements Hasher {
@Override
public boolean forEachIndex(IntPredicate consumer) {
Objects.requireNonNull(consumer, "consumer");
IndexFilter filter = IndexFilter.create(shape, consumer);
IntPredicate filter = IndexFilter.create(shape, consumer);
int bits = shape.getNumberOfBits();

3
src/test/java/org/apache/commons/collections4/bloomfilter/ArrayTrackerTest.java
View File

@ -22,7 +22,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.function.IntPredicate;
import org.apache.commons.collections4.bloomfilter.Hasher.IndexFilter.ArrayTracker;
import org.junit.jupiter.api.Test;
/**
@ -33,7 +32,7 @@ public class ArrayTrackerTest {
@Test
public void testSeen() {
Shape shape = Shape.fromKM(3, 12);
IntPredicate tracker = new ArrayTracker(shape);
IntPredicate tracker = new IndexFilter.ArrayTracker(shape);
assertTrue(tracker.test(0));
assertFalse(tracker.test(0));

3
src/test/java/org/apache/commons/collections4/bloomfilter/BitMapTrackerTest.java
View File

@ -21,7 +21,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.function.IntPredicate;
import org.apache.commons.collections4.bloomfilter.Hasher.IndexFilter.BitMapTracker;
import org.junit.jupiter.api.Test;
/**
@ -32,7 +31,7 @@ public class BitMapTrackerTest {
@Test
public void testSeen() {
Shape shape = Shape.fromKM(3, 12);
IntPredicate tracker = new BitMapTracker(shape);
IntPredicate tracker = new IndexFilter.BitMapTracker(shape);
assertTrue(tracker.test(0));
assertFalse(tracker.test(0));

51
src/test/java/org/apache/commons/collections4/bloomfilter/IndexFilterTest.java
View File

@ -21,16 +21,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
import java.util.SplittableRandom;
import java.util.concurrent.ThreadLocalRandom;
import java.util.function.IntPredicate;
import org.apache.commons.collections4.bloomfilter.Hasher.IndexFilter;
import org.apache.commons.collections4.bloomfilter.Hasher.IndexFilter.ArrayTracker;
import org.apache.commons.collections4.bloomfilter.Hasher.IndexFilter.BitMapTracker;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
@ -44,7 +41,7 @@ public class IndexFilterTest {
public void testFiltering() {
Shape shape = Shape.fromKM(3, 12);
List<Integer> consumer = new ArrayList<Integer>();
IndexFilter filter = IndexFilter.create(shape, consumer::add);
IntPredicate filter = IndexFilter.create(shape, consumer::add);
for (int i = 0; i < 12; i++) {
assertTrue(filter.test(i));
@ -65,7 +62,7 @@ public class IndexFilterTest {
for (int n = 0; n < 10; n++) {
used.clear();
List<Integer> consumer = new ArrayList<>();
IndexFilter filter = IndexFilter.create(shape, consumer::add);
IntPredicate filter = IndexFilter.create(shape, consumer::add);
// Make random indices; these may be duplicates
long seed = ThreadLocalRandom.current().nextLong();
@ -93,46 +90,4 @@ public class IndexFilterTest {
assertThrows(IndexOutOfBoundsException.class, () -> filter.test(-1));
}
}
@Test
public void testConstructor()
throws IllegalArgumentException, IllegalAccessException, NoSuchFieldException, SecurityException {
Field tracker = IndexFilter.class.getDeclaredField("tracker");
tracker.setAccessible(true);
List<Integer> consumer = new ArrayList<Integer>();
// test even split
int k = 2;
int m = Long.SIZE;
Shape shape = Shape.fromKM(k, m);
IndexFilter filter = IndexFilter.create(shape, consumer::add);
assertTrue(tracker.get(filter) instanceof ArrayTracker);
// test k ints < longs for m
k = 1;
shape = Shape.fromKM(k, m);
filter = IndexFilter.create(shape, consumer::add);
assertTrue(tracker.get(filter) instanceof ArrayTracker);
// test k ints > longs for m
k = 3;
shape = Shape.fromKM(k, m);
filter = IndexFilter.create(shape, consumer::add);
assertTrue(tracker.get(filter) instanceof BitMapTracker);
/* test overflows */
shape = Shape.fromKM(2, Integer.MAX_VALUE);
filter = IndexFilter.create(shape, consumer::add);
assertTrue(tracker.get(filter) instanceof ArrayTracker);
// overflow when computing the storage of the int array
shape = Shape.fromKM(Integer.MAX_VALUE, 123);
filter = IndexFilter.create(shape, consumer::add);
// *** fails ***
assertTrue(tracker.get(filter) instanceof BitMapTracker);
shape = Shape.fromKM(Integer.MAX_VALUE, Integer.MAX_VALUE);
filter = IndexFilter.create(shape, consumer::add);
assertTrue(tracker.get(filter) instanceof BitMapTracker);
}
}