Apache/lucene
1
0
Fork 0
mirror of https://github.com/apache/lucene.git synced 2025-03-06 16:29:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Replace Map<Long, Object> by primitive LongObjectHashMap. (#13392)

Browse Source 
Add LongObjectHashMap and replace Map<Long, Object>.
Add LongIntHashMap and replace Map<Long, Int>.
Add HPPC dependency to join and spatial modules for primitive values float and double.
This commit is contained in:
Bruno Roustant 2024-05-21 17:11:34 +02:00 committed by GitHub
parent 5b1a34bddd
commit f70999980c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 3280 additions and 265 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lucene/CHANGES.txt
View File

@ -346,6 +346,8 @@ Optimizations
* GITHUB#13368: Replace Map<Integer, Object> by primitive IntObjectHashMap. (Bruno Roustant)
* GITHUB#13392: Replace Map<Long, Object> by primitive LongObjectHashMap. (Bruno Roustant)
Bug Fixes
---------------------

4
lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BiSegGraph.java
View File

@ -20,6 +20,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.analysis.cn.smart.Utility;
import org.apache.lucene.util.hppc.IntObjectHashMap;
import org.apache.lucene.util.hppc.ObjectCursor;
/**
* Graph representing possible token pairs (bigrams) at each start offset in the sentence.
@ -218,8 +219,7 @@ class BiSegGraph {
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
for (IntObjectHashMap.ObjectCursor<ArrayList<SegTokenPair>> segList :
tokenPairListTable.values()) {
for (ObjectCursor<ArrayList<SegTokenPair>> segList : tokenPairListTable.values()) {
for (SegTokenPair pair : segList.value) {
sb.append(pair).append("\n");
}

9
lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java
View File

@ -22,9 +22,7 @@ import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.NUMERIC_
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesConsumer;
@ -54,6 +52,7 @@ import org.apache.lucene.util.LongsRef;
import org.apache.lucene.util.MathUtil;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.compress.LZ4;
import org.apache.lucene.util.hppc.LongIntHashMap;
import org.apache.lucene.util.packed.DirectMonotonicWriter;
import org.apache.lucene.util.packed.DirectWriter;
@ -273,7 +272,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
meta.writeLong(numValues);
final int numBitsPerValue;
boolean doBlocks = false;
Map<Long, Integer> encode = null;
LongIntHashMap encode = null;
if (min >= max) { // meta[-1]: All values are 0
numBitsPerValue = 0;
meta.writeInt(-1); // tablesize
@ -289,7 +288,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
for (Long v : sortedUniqueValues) {
meta.writeLong(v); // table[] entry
}
encode = new HashMap<>();
encode = new LongIntHashMap();
for (int i = 0; i < sortedUniqueValues.length; ++i) {
encode.put(sortedUniqueValues[i], i);
}
@ -339,7 +338,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
int numBitsPerValue,
long min,
long gcd,
Map<Long, Integer> encode)
LongIntHashMap encode)
throws IOException {
DirectWriter writer = DirectWriter.getInstance(data, numValues, numBitsPerValue);
for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {

16
lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -90,6 +90,8 @@ import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.Version;
import org.apache.lucene.util.hppc.LongObjectHashMap;
import org.apache.lucene.util.hppc.ObjectCursor;
/**
* An <code>IndexWriter</code> creates and maintains an index.
@ -4379,7 +4381,7 @@ public class IndexWriter
final ReadersAndUpdates mergedDeletesAndUpdates = getPooledInstance(merge.info, true);
int numDeletesBefore = mergedDeletesAndUpdates.getDelCount();
// field -> delGen -> dv field updates
Map<String, Map<Long, DocValuesFieldUpdates>> mappedDVUpdates = new HashMap<>();
Map<String, LongObjectHashMap<DocValuesFieldUpdates>> mappedDVUpdates = new HashMap<>();
boolean anyDVUpdates = false;
@ -4412,9 +4414,9 @@ public class IndexWriter
String field = ent.getKey();
Map<Long, DocValuesFieldUpdates> mappedField = mappedDVUpdates.get(field);
LongObjectHashMap<DocValuesFieldUpdates> mappedField = mappedDVUpdates.get(field);
if (mappedField == null) {
mappedField = new HashMap<>();
mappedField = new LongObjectHashMap<>();
mappedDVUpdates.put(field, mappedField);
}
@ -4470,10 +4472,10 @@ public class IndexWriter
if (anyDVUpdates) {
// Persist the merged DV updates onto the RAU for the merged segment:
for (Map<Long, DocValuesFieldUpdates> d : mappedDVUpdates.values()) {
for (DocValuesFieldUpdates updates : d.values()) {
updates.finish();
mergedDeletesAndUpdates.addDVUpdate(updates);
for (LongObjectHashMap<DocValuesFieldUpdates> d : mappedDVUpdates.values()) {
for (ObjectCursor<DocValuesFieldUpdates> updates : d.values()) {
updates.value.finish();
mergedDeletesAndUpdates.addDVUpdate(updates.value);
}
}
}

6
lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java
View File

@ -17,15 +17,14 @@
package org.apache.lucene.index;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.RefCount;
import org.apache.lucene.util.hppc.LongObjectHashMap;
/**
* Manages the {@link DocValuesProducer} held by {@link SegmentReader} and keeps track of their
@ -33,7 +32,8 @@ import org.apache.lucene.util.RefCount;
*/
final class SegmentDocValues {
private final Map<Long, RefCount<DocValuesProducer>> genDVProducers = new HashMap<>();
private final LongObjectHashMap<RefCount<DocValuesProducer>> genDVProducers =
new LongObjectHashMap<>();
private RefCount<DocValuesProducer> newDocValuesProducer(
SegmentCommitInfo si, Directory dir, final Long gen, FieldInfos infos) throws IOException {

45
lucene/core/src/java/org/apache/lucene/util/hppc/HashContainers.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.util.hppc;
import java.util.concurrent.atomic.AtomicInteger;
/** Constants for primitive maps. */
public class HashContainers {
public static final int DEFAULT_EXPECTED_ELEMENTS = 4;
public static final float DEFAULT_LOAD_FACTOR = 0.75f;
/** Minimal sane load factor (99 empty slots per 100). */
public static final float MIN_LOAD_FACTOR = 1 / 100.0f;
/** Maximum sane load factor (1 empty slot per 100). */
public static final float MAX_LOAD_FACTOR = 99 / 100.0f;
/** Minimum hash buffer size. */
public static final int MIN_HASH_ARRAY_LENGTH = 4;
/**
* Maximum array size for hash containers (power-of-two and still allocable in Java, not a
* negative int).
*/
public static final int MAX_HASH_ARRAY_LENGTH = 0x80000000 >>> 1;
static final AtomicInteger ITERATION_SEED = new AtomicInteger();
}

33
lucene/core/src/java/org/apache/lucene/util/hppc/IntIntHashMap.java
View File

@ -18,10 +18,12 @@
package org.apache.lucene.util.hppc;
import static org.apache.lucene.util.BitUtil.nextHighestPowerOfTwo;
import static org.apache.lucene.util.hppc.HashContainers.*;
import java.util.Arrays;
import java.util.Iterator;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.RamUsageEstimator;
/**
* A hash map of <code>int</code> to <code>int</code>, implemented using open addressing with linear
@ -31,28 +33,10 @@ import java.util.concurrent.atomic.AtomicInteger;
*
* <p>github: https://github.com/carrotsearch/hppc release 0.9.0
*/
public class IntIntHashMap implements Iterable<IntIntHashMap.IntIntCursor>, Cloneable {
public class IntIntHashMap implements Iterable<IntIntHashMap.IntIntCursor>, Accountable, Cloneable {
public static final int DEFAULT_EXPECTED_ELEMENTS = 4;
public static final float DEFAULT_LOAD_FACTOR = 0.75f;
private static final AtomicInteger ITERATION_SEED = new AtomicInteger();
/** Minimal sane load factor (99 empty slots per 100). */
public static final float MIN_LOAD_FACTOR = 1 / 100.0f;
/** Maximum sane load factor (1 empty slot per 100). */
public static final float MAX_LOAD_FACTOR = 99 / 100.0f;
/** Minimum hash buffer size. */
public static final int MIN_HASH_ARRAY_LENGTH = 4;
/**
* Maximum array size for hash containers (power-of-two and still allocable in Java, not a
* negative int).
*/
public static final int MAX_HASH_ARRAY_LENGTH = 0x80000000 >>> 1;
private static final long BASE_RAM_BYTES_USED =
RamUsageEstimator.shallowSizeOfInstance(IntIntHashMap.class);
/** The array holding keys. */
public int[] keys;
@ -463,6 +447,11 @@ public class IntIntHashMap implements Iterable<IntIntHashMap.IntIntCursor>, Clon
return iterationSeed = BitMixer.mixPhi(iterationSeed);
}
@Override
public long ramBytesUsed() {
return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(keys) + RamUsageEstimator.sizeOf(values);
}
/** An iterator implementation for {@link #iterator}. */
private final class EntryIterator extends AbstractIterator<IntIntCursor> {
private final IntIntCursor cursor;

62
lucene/core/src/java/org/apache/lucene/util/hppc/IntObjectHashMap.java
View File

@ -18,14 +18,16 @@
package org.apache.lucene.util.hppc;
import static org.apache.lucene.util.BitUtil.nextHighestPowerOfTwo;
import static org.apache.lucene.util.hppc.HashContainers.*;
import java.util.Arrays;
import java.util.Iterator;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.RamUsageEstimator;
/**
* A hash map of <code>int</code> to <code>Object</code>, implemented using open addressing with
* linear probing for collision resolution.
* linear probing for collision resolution. Supports null values.
*
* <p>Mostly forked and trimmed from com.carrotsearch.hppc.IntObjectHashMap
*
@ -33,28 +35,10 @@ import java.util.concurrent.atomic.AtomicInteger;
*/
@SuppressWarnings("unchecked")
public class IntObjectHashMap<VType>
implements Iterable<IntObjectHashMap.IntObjectCursor<VType>>, Cloneable {
implements Iterable<IntObjectHashMap.IntObjectCursor<VType>>, Accountable, Cloneable {
public static final int DEFAULT_EXPECTED_ELEMENTS = 4;
public static final float DEFAULT_LOAD_FACTOR = 0.75f;
private static final AtomicInteger ITERATION_SEED = new AtomicInteger();
/** Minimal sane load factor (99 empty slots per 100). */
public static final float MIN_LOAD_FACTOR = 1 / 100.0f;
/** Maximum sane load factor (1 empty slot per 100). */
public static final float MAX_LOAD_FACTOR = 99 / 100.0f;
/** Minimum hash buffer size. */
public static final int MIN_HASH_ARRAY_LENGTH = 4;
/**
* Maximum array size for hash containers (power-of-two and still allocable in Java, not a
* negative int).
*/
public static final int MAX_HASH_ARRAY_LENGTH = 0x80000000 >>> 1;
private static final long BASE_RAM_BYTES_USED =
RamUsageEstimator.shallowSizeOfInstance(IntObjectHashMap.class);
/** The array holding keys. */
public int[] keys;
@ -304,7 +288,7 @@ public class IntObjectHashMap<VType>
return (VType) values[index];
}
public VType indexReplace(int index, int newValue) {
public VType indexReplace(int index, VType newValue) {
assert index >= 0 : "The index must point at an existing key.";
assert index <= mask || (index == mask + 1 && hasEmptyKey);
@ -436,6 +420,19 @@ public class IntObjectHashMap<VType>
return new EntryIterator();
}
@Override
public long ramBytesUsed() {
return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(keys) + sizeOfValues();
}
private long sizeOfValues() {
long size = RamUsageEstimator.shallowSizeOf(values);
for (ObjectCursor<VType> value : values()) {
size += RamUsageEstimator.sizeOfObject(value);
}
return size;
}
/** An iterator implementation for {@link #iterator}. */
private final class EntryIterator extends AbstractIterator<IntObjectCursor<VType>> {
private final IntObjectCursor<VType> cursor;
@ -869,21 +866,4 @@ public class IntObjectHashMap<VType>
return "[cursor, index: " + index + ", key: " + key + ", value: " + value + "]";
}
}
/** Forked from HPPC, holding int index and Object value */
public static final class ObjectCursor<VType> {
/**
* The current value's index in the container this cursor belongs to. The meaning of this index
* is defined by the container (usually it will be an index in the underlying storage buffer).
*/
public int index;
/** The current value. */
public VType value;
@Override
public String toString() {
return "[cursor, index: " + index + ", value: " + value + "]";
}
}
}

35
lucene/core/src/java/org/apache/lucene/util/hppc/LongCursor.java Normal file
View File

@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.util.hppc;
/** Forked from HPPC, holding int index and long value */
public final class LongCursor {
/**
* The current value's index in the container this cursor belongs to. The meaning of this index is
* defined by the container (usually it will be an index in the underlying storage buffer).
*/
public int index;
/** The current value. */
public long value;
@Override
public String toString() {
return "[cursor, index: " + index + ", value: " + value + "]";
}
}

900
lucene/core/src/java/org/apache/lucene/util/hppc/LongIntHashMap.java Normal file
View File

@ -0,0 +1,900 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.util.hppc;
import static org.apache.lucene.util.BitUtil.nextHighestPowerOfTwo;
import static org.apache.lucene.util.hppc.HashContainers.DEFAULT_EXPECTED_ELEMENTS;
import static org.apache.lucene.util.hppc.HashContainers.DEFAULT_LOAD_FACTOR;
import static org.apache.lucene.util.hppc.HashContainers.ITERATION_SEED;
import static org.apache.lucene.util.hppc.HashContainers.MAX_HASH_ARRAY_LENGTH;
import static org.apache.lucene.util.hppc.HashContainers.MAX_LOAD_FACTOR;
import static org.apache.lucene.util.hppc.HashContainers.MIN_HASH_ARRAY_LENGTH;
import static org.apache.lucene.util.hppc.HashContainers.MIN_LOAD_FACTOR;
import java.util.Arrays;
import java.util.Iterator;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.RamUsageEstimator;
/**
* A hash map of <code>long</code> to <code>int</code>, implemented using open addressing with
* linear probing for collision resolution.
*
* <p>Mostly forked and trimmed from com.carrotsearch.hppc.LongIntHashMap
*
* <p>github: https://github.com/carrotsearch/hppc release 0.9.0
*/
public class LongIntHashMap
implements Iterable<LongIntHashMap.LongIntCursor>, Accountable, Cloneable {
private static final long BASE_RAM_BYTES_USED =
RamUsageEstimator.shallowSizeOfInstance(LongIntHashMap.class);
/** The array holding keys. */
public long[] keys;
/** The array holding values. */
public int[] values;
/**
* The number of stored keys (assigned key slots), excluding the special "empty" key, if any (use
* {@link #size()} instead).
*
* @see #size()
*/
protected int assigned;
/** Mask for slot scans in {@link #keys}. */
protected int mask;
/** Expand (rehash) {@link #keys} when {@link #assigned} hits this value. */
protected int resizeAt;
/** Special treatment for the "empty slot" key marker. */
protected boolean hasEmptyKey;
/** The load factor for {@link #keys}. */
protected double loadFactor;
/** Seed used to ensure the hash iteration order is different from an iteration to another. */
protected int iterationSeed;
/** New instance with sane defaults. */
public LongIntHashMap() {
this(DEFAULT_EXPECTED_ELEMENTS);
}
/**
* New instance with sane defaults.
*
* @param expectedElements The expected number of elements guaranteed not to cause buffer
* expansion (inclusive).
*/
public LongIntHashMap(int expectedElements) {
this(expectedElements, DEFAULT_LOAD_FACTOR);
}
/**
* New instance with the provided defaults.
*
* @param expectedElements The expected number of elements guaranteed not to cause a rehash
* (inclusive).
* @param loadFactor The load factor for internal buffers. Insane load factors (zero, full
* capacity) are rejected by {@link #verifyLoadFactor(double)}.
*/
public LongIntHashMap(int expectedElements, double loadFactor) {
this.loadFactor = verifyLoadFactor(loadFactor);
iterationSeed = ITERATION_SEED.incrementAndGet();
ensureCapacity(expectedElements);
}
/** Create a hash map from all key-value pairs of another container. */
public LongIntHashMap(Iterable<? extends LongIntCursor> container) {
this();
putAll(container);
}
public int put(long key, int value) {
assert assigned < mask + 1;
final int mask = this.mask;
if (((key) == 0)) {
hasEmptyKey = true;
int previousValue = values[mask + 1];
values[mask + 1] = value;
return previousValue;
} else {
final long[] keys = this.keys;
int slot = hashKey(key) & mask;
long existing;
while (!((existing = keys[slot]) == 0)) {
if (((existing) == (key))) {
final int previousValue = values[slot];
values[slot] = value;
return previousValue;
}
slot = (slot + 1) & mask;
}
if (assigned == resizeAt) {
allocateThenInsertThenRehash(slot, key, value);
} else {
keys[slot] = key;
values[slot] = value;
}
assigned++;
return 0;
}
}
public int putAll(Iterable<? extends LongIntCursor> iterable) {
final int count = size();
for (LongIntCursor c : iterable) {
put(c.key, c.value);
}
return size() - count;
}
/**
* <a href="http://trove4j.sourceforge.net">Trove</a>-inspired API method. An equivalent of the
* following code:
*
* <pre>
* if (!map.containsKey(key)) map.put(value);
* </pre>
*
* @param key The key of the value to check.
* @param value The value to put if <code>key</code> does not exist.
* @return <code>true</code> if <code>key</code> did not exist and <code>value</code> was placed
* in the map.
*/
public boolean putIfAbsent(long key, int value) {
int keyIndex = indexOf(key);
if (!indexExists(keyIndex)) {
indexInsert(keyIndex, key, value);
return true;
} else {
return false;
}
}
/**
* If <code>key</code> exists, <code>putValue</code> is inserted into the map, otherwise any
* existing value is incremented by <code>additionValue</code>.
*
* @param key The key of the value to adjust.
* @param putValue The value to put if <code>key</code> does not exist.
* @param incrementValue The value to add to the existing value if <code>key</code> exists.
* @return Returns the current value associated with <code>key</code> (after changes).
*/
public int putOrAdd(long key, int putValue, int incrementValue) {
assert assigned < mask + 1;
int keyIndex = indexOf(key);
if (indexExists(keyIndex)) {
putValue = values[keyIndex] + incrementValue;
indexReplace(keyIndex, putValue);
} else {
indexInsert(keyIndex, key, putValue);
}
return putValue;
}
/**
* Adds <code>incrementValue</code> to any existing value for the given <code>key</code> or
* inserts <code>incrementValue</code> if <code>key</code> did not previously exist.
*
* @param key The key of the value to adjust.
* @param incrementValue The value to put or add to the existing value if <code>key</code> exists.
* @return Returns the current value associated with <code>key</code> (after changes).
*/
public int addTo(long key, int incrementValue) {
return putOrAdd(key, incrementValue, incrementValue);
}
public int remove(long key) {
final int mask = this.mask;
if (((key) == 0)) {
hasEmptyKey = false;
int previousValue = values[mask + 1];
values[mask + 1] = 0;
return previousValue;
} else {
final long[] keys = this.keys;
int slot = hashKey(key) & mask;
long existing;
while (!((existing = keys[slot]) == 0)) {
if (((existing) == (key))) {
final int previousValue = values[slot];
shiftConflictingKeys(slot);
return previousValue;
}
slot = (slot + 1) & mask;
}
return 0;
}
}
public int get(long key) {
if (((key) == 0)) {
return hasEmptyKey ? values[mask + 1] : 0;
} else {
final long[] keys = this.keys;
final int mask = this.mask;
int slot = hashKey(key) & mask;
long existing;
while (!((existing = keys[slot]) == 0)) {
if (((existing) == (key))) {
return values[slot];
}
slot = (slot + 1) & mask;
}
return 0;
}
}
public int getOrDefault(long key, int defaultValue) {
if (((key) == 0)) {
return hasEmptyKey ? values[mask + 1] : defaultValue;
} else {
final long[] keys = this.keys;
final int mask = this.mask;
int slot = hashKey(key) & mask;
long existing;
while (!((existing = keys[slot]) == 0)) {
if (((existing) == (key))) {
return values[slot];
}
slot = (slot + 1) & mask;
}
return defaultValue;
}
}
public boolean containsKey(long key) {
if (((key) == 0)) {
return hasEmptyKey;
} else {
final long[] keys = this.keys;
final int mask = this.mask;
int slot = hashKey(key) & mask;
long existing;
while (!((existing = keys[slot]) == 0)) {
if (((existing) == (key))) {
return true;
}
slot = (slot + 1) & mask;
}
return false;
}
}
public int indexOf(long key) {
final int mask = this.mask;
if (((key) == 0)) {
return hasEmptyKey ? mask + 1 : ~(mask + 1);
} else {
final long[] keys = this.keys;
int slot = hashKey(key) & mask;
long existing;
while (!((existing = keys[slot]) == 0)) {
if (((existing) == (key))) {
return slot;
}
slot = (slot + 1) & mask;
}
return ~slot;
}
}
public boolean indexExists(int index) {
assert index < 0 || (index >= 0 && index <= mask) || (index == mask + 1 && hasEmptyKey);
return index >= 0;
}
public int indexGet(int index) {
assert index >= 0 : "The index must point at an existing key.";
assert index <= mask || (index == mask + 1 && hasEmptyKey);
return values[index];
}
public int indexReplace(int index, int newValue) {
assert index >= 0 : "The index must point at an existing key.";
assert index <= mask || (index == mask + 1 && hasEmptyKey);
int previousValue = values[index];
values[index] = newValue;
return previousValue;
}
public void indexInsert(int index, long key, int value) {
assert index < 0 : "The index must not point at an existing key.";
index = ~index;
if (((key) == 0)) {
assert index == mask + 1;
values[index] = value;
hasEmptyKey = true;
} else {
assert ((keys[index]) == 0);
if (assigned == resizeAt) {
allocateThenInsertThenRehash(index, key, value);
} else {
keys[index] = key;
values[index] = value;
}
assigned++;
}
}
public int indexRemove(int index) {
assert index >= 0 : "The index must point at an existing key.";
assert index <= mask || (index == mask + 1 && hasEmptyKey);
int previousValue = values[index];
if (index > mask) {
hasEmptyKey = false;
values[index] = 0;
} else {
shiftConflictingKeys(index);
}
return previousValue;
}
public void clear() {
assigned = 0;
hasEmptyKey = false;
Arrays.fill(keys, 0);
/* */
}
public void release() {
assigned = 0;
hasEmptyKey = false;
keys = null;
values = null;
ensureCapacity(DEFAULT_EXPECTED_ELEMENTS);
}
public int size() {
return assigned + (hasEmptyKey ? 1 : 0);
}
public boolean isEmpty() {
return size() == 0;
}
@Override
public int hashCode() {
int h = hasEmptyKey ? 0xDEADBEEF : 0;
for (LongIntCursor c : this) {
h += BitMixer.mix(c.key) + BitMixer.mix(c.value);
}
return h;
}
@Override
public boolean equals(Object obj) {
return obj != null && getClass() == obj.getClass() && equalElements(getClass().cast(obj));
}
/** Return true if all keys of some other container exist in this container. */
protected boolean equalElements(LongIntHashMap other) {
if (other.size() != size()) {
return false;
}
for (LongIntCursor c : other) {
long key = c.key;
if (!containsKey(key) || !((get(key)) == (c.value))) {
return false;
}
}
return true;
}
/**
* Ensure this container can hold at least the given number of keys (entries) without resizing its
* buffers.
*
* @param expectedElements The total number of keys, inclusive.
*/
public void ensureCapacity(int expectedElements) {
if (expectedElements > resizeAt || keys == null) {
final long[] prevKeys = this.keys;
final int[] prevValues = this.values;
allocateBuffers(minBufferSize(expectedElements, loadFactor));
if (prevKeys != null && !isEmpty()) {
rehash(prevKeys, prevValues);
}
}
}
/**
* Provides the next iteration seed used to build the iteration starting slot and offset
* increment. This method does not need to be synchronized, what matters is that each thread gets
* a sequence of varying seeds.
*/
protected int nextIterationSeed() {
return iterationSeed = BitMixer.mixPhi(iterationSeed);
}
@Override
public long ramBytesUsed() {
return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(keys) + RamUsageEstimator.sizeOf(values);
}
/** An iterator implementation for {@link #iterator}. */
private final class EntryIterator extends AbstractIterator<LongIntCursor> {
private final LongIntCursor cursor;
private final int increment;
private int index;
private int slot;
public EntryIterator() {
cursor = new LongIntCursor();
int seed = nextIterationSeed();
increment = iterationIncrement(seed);
slot = seed & mask;
}
@Override
protected LongIntCursor fetch() {
final int mask = LongIntHashMap.this.mask;
while (index <= mask) {
long existing;
index++;
slot = (slot + increment) & mask;
if (!((existing = keys[slot]) == 0)) {
cursor.index = slot;
cursor.key = existing;
cursor.value = values[slot];
return cursor;
}
}
if (index == mask + 1 && hasEmptyKey) {
cursor.index = index;
cursor.key = 0;
cursor.value = values[index++];
return cursor;
}
return done();
}
}
@Override
public Iterator<LongIntCursor> iterator() {
return new EntryIterator();
}
/** Returns a specialized view of the keys of this associated container. */
public KeysContainer keys() {
return new KeysContainer();
}
/** A view of the keys inside this hash map. */
public final class KeysContainer implements Iterable<LongCursor> {
@Override
public Iterator<LongCursor> iterator() {
return new KeysIterator();
}
public int size() {
return LongIntHashMap.this.size();
}
public long[] toArray() {
long[] array = new long[size()];
int i = 0;
for (LongCursor cursor : this) {
array[i++] = cursor.value;
}
return array;
}
}
/** An iterator over the set of assigned keys. */
private final class KeysIterator extends AbstractIterator<LongCursor> {
private final LongCursor cursor;
private final int increment;
private int index;
private int slot;
public KeysIterator() {
cursor = new LongCursor();
int seed = nextIterationSeed();
increment = iterationIncrement(seed);
slot = seed & mask;
}
@Override
protected LongCursor fetch() {
final int mask = LongIntHashMap.this.mask;
while (index <= mask) {
long existing;
index++;
slot = (slot + increment) & mask;
if (!((existing = keys[slot]) == 0)) {
cursor.index = slot;
cursor.value = existing;
return cursor;
}
}
if (index == mask + 1 && hasEmptyKey) {
cursor.index = index++;
cursor.value = 0;
return cursor;
}
return done();
}
}
/**
* @return Returns a container with all values stored in this map.
*/
public ValuesContainer values() {
return new ValuesContainer();
}
/** A view over the set of values of this map. */
public final class ValuesContainer implements Iterable<IntCursor> {
@Override
public Iterator<IntCursor> iterator() {
return new ValuesIterator();
}
public int size() {
return LongIntHashMap.this.size();
}
public int[] toArray() {
int[] array = new int[size()];
int i = 0;
for (IntCursor cursor : this) {
array[i++] = cursor.value;
}
return array;
}
}
/** An iterator over the set of assigned values. */
private final class ValuesIterator extends AbstractIterator<IntCursor> {
private final IntCursor cursor;
private final int increment;
private int index;
private int slot;
public ValuesIterator() {
cursor = new IntCursor();
int seed = nextIterationSeed();
increment = iterationIncrement(seed);
slot = seed & mask;
}
@Override
protected IntCursor fetch() {
final int mask = LongIntHashMap.this.mask;
while (index <= mask) {
index++;
slot = (slot + increment) & mask;
if (!((keys[slot]) == 0)) {
cursor.index = slot;
cursor.value = values[slot];
return cursor;
}
}
if (index == mask + 1 && hasEmptyKey) {
cursor.index = index;
cursor.value = values[index++];
return cursor;
}
return done();
}
}
@Override
public LongIntHashMap clone() {
try {
/* */
LongIntHashMap cloned = (LongIntHashMap) super.clone();
cloned.keys = keys.clone();
cloned.values = values.clone();
cloned.hasEmptyKey = hasEmptyKey;
cloned.iterationSeed = nextIterationSeed();
return cloned;
} catch (CloneNotSupportedException e) {
throw new RuntimeException(e);
}
}
/** Convert the contents of this map to a human-friendly string. */
@Override
public String toString() {
final StringBuilder buffer = new StringBuilder();
buffer.append("[");
boolean first = true;
for (LongIntCursor cursor : this) {
if (!first) {
buffer.append(", ");
}
buffer.append(cursor.key);
buffer.append("=>");
buffer.append(cursor.value);
first = false;
}
buffer.append("]");
return buffer.toString();
}
/** Creates a hash map from two index-aligned arrays of key-value pairs. */
public static LongIntHashMap from(long[] keys, int[] values) {
if (keys.length != values.length) {
throw new IllegalArgumentException(
"Arrays of keys and values must have an identical length.");
}
LongIntHashMap map = new LongIntHashMap(keys.length);
for (int i = 0; i < keys.length; i++) {
map.put(keys[i], values[i]);
}
return map;
}
/**
* Returns a hash code for the given key.
*
* <p>The output from this function should evenly distribute keys across the entire integer range.
*/
protected int hashKey(long key) {
assert !((key) == 0); // Handled as a special case (empty slot marker).
return BitMixer.mixPhi(key);
}
/**
* Validate load factor range and return it. Override and suppress if you need insane load
* factors.
*/
protected double verifyLoadFactor(double loadFactor) {
checkLoadFactor(loadFactor, MIN_LOAD_FACTOR, MAX_LOAD_FACTOR);
return loadFactor;
}
/** Rehash from old buffers to new buffers. */
protected void rehash(long[] fromKeys, int[] fromValues) {
assert fromKeys.length == fromValues.length && checkPowerOfTwo(fromKeys.length - 1);
// Rehash all stored key/value pairs into the new buffers.
final long[] keys = this.keys;
final int[] values = this.values;
final int mask = this.mask;
long existing;
// Copy the zero element's slot, then rehash everything else.
int from = fromKeys.length - 1;
keys[keys.length - 1] = fromKeys[from];
values[values.length - 1] = fromValues[from];
while (--from >= 0) {
if (!((existing = fromKeys[from]) == 0)) {
int slot = hashKey(existing) & mask;
while (!((keys[slot]) == 0)) {
slot = (slot + 1) & mask;
}
keys[slot] = existing;
values[slot] = fromValues[from];
}
}
}
/**
* Allocate new internal buffers. This method attempts to allocate and assign internal buffers
* atomically (either allocations succeed or not).
*/
protected void allocateBuffers(int arraySize) {
assert Integer.bitCount(arraySize) == 1;
// Ensure no change is done if we hit an OOM.
long[] prevKeys = this.keys;
int[] prevValues = this.values;
try {
int emptyElementSlot = 1;
this.keys = (new long[arraySize + emptyElementSlot]);
this.values = (new int[arraySize + emptyElementSlot]);
} catch (OutOfMemoryError e) {
this.keys = prevKeys;
this.values = prevValues;
throw new BufferAllocationException(
"Not enough memory to allocate buffers for rehashing: %,d -> %,d",
e, this.mask + 1, arraySize);
}
this.resizeAt = expandAtCount(arraySize, loadFactor);
this.mask = arraySize - 1;
}
/**
* This method is invoked when there is a new key/ value pair to be inserted into the buffers but
* there is not enough empty slots to do so.
*
* <p>New buffers are allocated. If this succeeds, we know we can proceed with rehashing so we
* assign the pending element to the previous buffer (possibly violating the invariant of having
* at least one empty slot) and rehash all keys, substituting new buffers at the end.
*/
protected void allocateThenInsertThenRehash(int slot, long pendingKey, int pendingValue) {
assert assigned == resizeAt && ((keys[slot]) == 0) && !((pendingKey) == 0);
// Try to allocate new buffers first. If we OOM, we leave in a consistent state.
final long[] prevKeys = this.keys;
final int[] prevValues = this.values;
allocateBuffers(nextBufferSize(mask + 1, size(), loadFactor));
assert this.keys.length > prevKeys.length;
// We have succeeded at allocating new data so insert the pending key/value at
// the free slot in the old arrays before rehashing.
prevKeys[slot] = pendingKey;
prevValues[slot] = pendingValue;
// Rehash old keys, including the pending key.
rehash(prevKeys, prevValues);
}
static int nextBufferSize(int arraySize, int elements, double loadFactor) {
assert checkPowerOfTwo(arraySize);
if (arraySize == MAX_HASH_ARRAY_LENGTH) {
throw new BufferAllocationException(
"Maximum array size exceeded for this load factor (elements: %d, load factor: %f)",
elements, loadFactor);
}
return arraySize << 1;
}
static int expandAtCount(int arraySize, double loadFactor) {
assert checkPowerOfTwo(arraySize);
// Take care of hash container invariant (there has to be at least one empty slot to ensure
// the lookup loop finds either the element or an empty slot).
return Math.min(arraySize - 1, (int) Math.ceil(arraySize * loadFactor));
}
static boolean checkPowerOfTwo(int arraySize) {
// These are internals, we can just assert without retrying.
assert arraySize > 1;
assert nextHighestPowerOfTwo(arraySize) == arraySize;
return true;
}
static int minBufferSize(int elements, double loadFactor) {
if (elements < 0) {
throw new IllegalArgumentException("Number of elements must be >= 0: " + elements);
}
long length = (long) Math.ceil(elements / loadFactor);
if (length == elements) {
length++;
}
length = Math.max(MIN_HASH_ARRAY_LENGTH, nextHighestPowerOfTwo(length));
if (length > MAX_HASH_ARRAY_LENGTH) {
throw new BufferAllocationException(
"Maximum array size exceeded for this load factor (elements: %d, load factor: %f)",
elements, loadFactor);
}
return (int) length;
}
static void checkLoadFactor(
double loadFactor, double minAllowedInclusive, double maxAllowedInclusive) {
if (loadFactor < minAllowedInclusive || loadFactor > maxAllowedInclusive) {
throw new BufferAllocationException(
"The load factor should be in range [%.2f, %.2f]: %f",
minAllowedInclusive, maxAllowedInclusive, loadFactor);
}
}
static int iterationIncrement(int seed) {
return 29 + ((seed & 7) << 1); // Small odd integer.
}
/**
* Shift all the slot-conflicting keys and values allocated to (and including) <code>slot</code>.
*/
protected void shiftConflictingKeys(int gapSlot) {
final long[] keys = this.keys;
final int[] values = this.values;
final int mask = this.mask;
// Perform shifts of conflicting keys to fill in the gap.
int distance = 0;
while (true) {
final int slot = (gapSlot + (++distance)) & mask;
final long existing = keys[slot];
if (((existing) == 0)) {
break;
}
final int idealSlot = hashKey(existing);
final int shift = (slot - idealSlot) & mask;
if (shift >= distance) {
// Entry at this position was originally at or before the gap slot.
// Move the conflict-shifted entry to the gap's position and repeat the procedure
// for any entries to the right of the current position, treating it
// as the new gap.
keys[gapSlot] = existing;
values[gapSlot] = values[slot];
gapSlot = slot;
distance = 0;
}
}
// Mark the last found gap slot without a conflict as empty.
keys[gapSlot] = 0;
values[gapSlot] = 0;
assigned--;
}
/** Forked from HPPC, holding int index,key and value */
public final class LongIntCursor {
/**
* The current key and value's index in the container this cursor belongs to. The meaning of
* this index is defined by the container (usually it will be an index in the underlying storage
* buffer).
*/
public int index;
/** The current key. */
public long key;
/** The current value. */
public int value;
@Override
public String toString() {
return "[cursor, index: " + index + ", key: " + key + ", value: " + value + "]";
}
}
}

869
lucene/core/src/java/org/apache/lucene/util/hppc/LongObjectHashMap.java Normal file
View File

@ -0,0 +1,869 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.util.hppc;
import static org.apache.lucene.util.BitUtil.nextHighestPowerOfTwo;
import static org.apache.lucene.util.hppc.HashContainers.*;
import java.util.Arrays;
import java.util.Iterator;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.RamUsageEstimator;
/**
* A hash map of <code>long</code> to <code>Object</code>, implemented using open addressing with
* linear probing for collision resolution. Supports null values.
*
* <p>Mostly forked and trimmed from com.carrotsearch.hppc.LongObjectHashMap
*
* <p>github: https://github.com/carrotsearch/hppc release 0.9.0
*/
@SuppressWarnings("unchecked")
public class LongObjectHashMap<VType>
implements Iterable<LongObjectHashMap.LongObjectCursor<VType>>, Accountable, Cloneable {
private static final long BASE_RAM_BYTES_USED =
RamUsageEstimator.shallowSizeOfInstance(LongObjectHashMap.class);
/** The array holding keys. */
public long[] keys;
/** The array holding values. */
public Object[] values;
/**
* The number of stored keys (assigned key slots), excluding the special "empty" key, if any (use
* {@link #size()} instead).
*
* @see #size()
*/
protected int assigned;
/** Mask for slot scans in {@link #keys}. */
protected int mask;
/** Expand (rehash) {@link #keys} when {@link #assigned} hits this value. */
protected int resizeAt;
/** Special treatment for the "empty slot" key marker. */
protected boolean hasEmptyKey;
/** The load factor for {@link #keys}. */
protected double loadFactor;
/** Seed used to ensure the hash iteration order is different from an iteration to another. */
protected int iterationSeed;
/** New instance with sane defaults. */
public LongObjectHashMap() {
this(DEFAULT_EXPECTED_ELEMENTS);
}
/**
* New instance with sane defaults.
*
* @param expectedElements The expected number of elements guaranteed not to cause buffer
* expansion (inclusive).
*/
public LongObjectHashMap(int expectedElements) {
this(expectedElements, DEFAULT_LOAD_FACTOR);
}
/**
* New instance with the provided defaults.
*
* @param expectedElements The expected number of elements guaranteed not to cause a rehash
* (inclusive).
* @param loadFactor The load factor for internal buffers. Insane load factors (zero, full
* capacity) are rejected by {@link #verifyLoadFactor(double)}.
*/
public LongObjectHashMap(int expectedElements, double loadFactor) {
this.loadFactor = verifyLoadFactor(loadFactor);
iterationSeed = ITERATION_SEED.incrementAndGet();
ensureCapacity(expectedElements);
}
/** Create a hash map from all key-value pairs of another container. */
public LongObjectHashMap(Iterable<? extends LongObjectCursor<? extends VType>> container) {
this();
putAll(container);
}
public VType put(long key, VType value) {
assert assigned < mask + 1;
final int mask = this.mask;
if (((key) == 0)) {
hasEmptyKey = true;
VType previousValue = (VType) values[mask + 1];
values[mask + 1] = value;
return previousValue;
} else {
final long[] keys = this.keys;
int slot = hashKey(key) & mask;
long existing;
while (!((existing = keys[slot]) == 0)) {
if (((existing) == (key))) {
final VType previousValue = (VType) values[slot];
values[slot] = value;
return previousValue;
}
slot = (slot + 1) & mask;
}
if (assigned == resizeAt) {
allocateThenInsertThenRehash(slot, key, value);
} else {
keys[slot] = key;
values[slot] = value;
}
assigned++;
return null;
}
}
public int putAll(Iterable<? extends LongObjectCursor<? extends VType>> iterable) {
final int count = size();
for (LongObjectCursor<? extends VType> c : iterable) {
put(c.key, c.value);
}
return size() - count;
}
/**
* <a href="http://trove4j.sourceforge.net">Trove</a>-inspired API method. An equivalent of the
* following code:
*
* <pre>
* if (!map.containsKey(key)) map.put(value);
* </pre>
*
* @param key The key of the value to check.
* @param value The value to put if <code>key</code> does not exist.
* @return <code>true</code> if <code>key</code> did not exist and <code>value</code> was placed
* in the map.
*/
public boolean putIfAbsent(long key, VType value) {
int keyIndex = indexOf(key);
if (!indexExists(keyIndex)) {
indexInsert(keyIndex, key, value);
return true;
} else {
return false;
}
}
public VType remove(long key) {
final int mask = this.mask;
if (((key) == 0)) {
hasEmptyKey = false;
VType previousValue = (VType) values[mask + 1];
values[mask + 1] = 0;
return previousValue;
} else {
final long[] keys = this.keys;
int slot = hashKey(key) & mask;
long existing;
while (!((existing = keys[slot]) == 0)) {
if (((existing) == (key))) {
final VType previousValue = (VType) values[slot];
shiftConflictingKeys(slot);
return previousValue;
}
slot = (slot + 1) & mask;
}
return null;
}
}
public VType get(long key) {
if (((key) == 0)) {
return hasEmptyKey ? (VType) values[mask + 1] : null;
} else {
final long[] keys = this.keys;
final int mask = this.mask;
int slot = hashKey(key) & mask;
long existing;
while (!((existing = keys[slot]) == 0)) {
if (((existing) == (key))) {
return (VType) values[slot];
}
slot = (slot + 1) & mask;
}
return null;
}
}
public VType getOrDefault(long key, VType defaultValue) {
if (((key) == 0)) {
return hasEmptyKey ? (VType) values[mask + 1] : defaultValue;
} else {
final long[] keys = this.keys;
final int mask = this.mask;
int slot = hashKey(key) & mask;
long existing;
while (!((existing = keys[slot]) == 0)) {
if (((existing) == (key))) {
return (VType) values[slot];
}
slot = (slot + 1) & mask;
}
return defaultValue;
}
}
public boolean containsKey(long key) {
if (((key) == 0)) {
return hasEmptyKey;
} else {
final long[] keys = this.keys;
final int mask = this.mask;
int slot = hashKey(key) & mask;
long existing;
while (!((existing = keys[slot]) == 0)) {
if (((existing) == (key))) {
return true;
}
slot = (slot + 1) & mask;
}
return false;
}
}
public int indexOf(long key) {
final int mask = this.mask;
if (((key) == 0)) {
return hasEmptyKey ? mask + 1 : ~(mask + 1);
} else {
final long[] keys = this.keys;
int slot = hashKey(key) & mask;
long existing;
while (!((existing = keys[slot]) == 0)) {
if (((existing) == (key))) {
return slot;
}
slot = (slot + 1) & mask;
}
return ~slot;
}
}
public boolean indexExists(int index) {
assert index < 0 || (index >= 0 && index <= mask) || (index == mask + 1 && hasEmptyKey);
return index >= 0;
}
public VType indexGet(int index) {
assert index >= 0 : "The index must point at an existing key.";
assert index <= mask || (index == mask + 1 && hasEmptyKey);
return (VType) values[index];
}
public VType indexReplace(int index, VType newValue) {
assert index >= 0 : "The index must point at an existing key.";
assert index <= mask || (index == mask + 1 && hasEmptyKey);
VType previousValue = (VType) values[index];
values[index] = newValue;
return previousValue;
}
public void indexInsert(int index, long key, VType value) {
assert index < 0 : "The index must not point at an existing key.";
index = ~index;
if (((key) == 0)) {
assert index == mask + 1;
values[index] = value;
hasEmptyKey = true;
} else {
assert ((keys[index]) == 0);
if (assigned == resizeAt) {
allocateThenInsertThenRehash(index, key, value);
} else {
keys[index] = key;
values[index] = value;
}
assigned++;
}
}
public VType indexRemove(int index) {
assert index >= 0 : "The index must point at an existing key.";
assert index <= mask || (index == mask + 1 && hasEmptyKey);
VType previousValue = (VType) values[index];
if (index > mask) {
hasEmptyKey = false;
values[index] = 0;
} else {
shiftConflictingKeys(index);
}
return previousValue;
}
public void clear() {
assigned = 0;
hasEmptyKey = false;
Arrays.fill(keys, 0);
/* */
}
public void release() {
assigned = 0;
hasEmptyKey = false;
keys = null;
values = null;
ensureCapacity(DEFAULT_EXPECTED_ELEMENTS);
}
public int size() {
return assigned + (hasEmptyKey ? 1 : 0);
}
public boolean isEmpty() {
return size() == 0;
}
@Override
public int hashCode() {
int h = hasEmptyKey ? 0xDEADBEEF : 0;
for (LongObjectCursor<VType> c : this) {
h += BitMixer.mix(c.key) + BitMixer.mix(c.value);
}
return h;
}
@Override
public boolean equals(Object obj) {
return obj != null && getClass() == obj.getClass() && equalElements(getClass().cast(obj));
}
/** Return true if all keys of some other container exist in this container. */
protected boolean equalElements(LongObjectHashMap<?> other) {
if (other.size() != size()) {
return false;
}
for (LongObjectCursor<?> c : other) {
long key = c.key;
if (!containsKey(key) || !java.util.Objects.equals(c.value, get(key))) {
return false;
}
}
return true;
}
/**
* Ensure this container can hold at least the given number of keys (entries) without resizing its
* buffers.
*
* @param expectedElements The total number of keys, inclusive.
*/
public void ensureCapacity(int expectedElements) {
if (expectedElements > resizeAt || keys == null) {
final long[] prevKeys = this.keys;
final VType[] prevValues = (VType[]) this.values;
allocateBuffers(minBufferSize(expectedElements, loadFactor));
if (prevKeys != null && !isEmpty()) {
rehash(prevKeys, prevValues);
}
}
}
/**
* Provides the next iteration seed used to build the iteration starting slot and offset
* increment. This method does not need to be synchronized, what matters is that each thread gets
* a sequence of varying seeds.
*/
protected int nextIterationSeed() {
return iterationSeed = BitMixer.mixPhi(iterationSeed);
}
@Override
public Iterator<LongObjectCursor<VType>> iterator() {
return new EntryIterator();
}
@Override
public long ramBytesUsed() {
return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(keys) + sizeOfValues();
}
private long sizeOfValues() {
long size = RamUsageEstimator.shallowSizeOf(values);
for (ObjectCursor<VType> value : values()) {
size += RamUsageEstimator.sizeOfObject(value);
}
return size;
}
/** An iterator implementation for {@link #iterator}. */
private final class EntryIterator extends AbstractIterator<LongObjectCursor<VType>> {
private final LongObjectCursor<VType> cursor;
private final int increment;
private int index;
private int slot;
public EntryIterator() {
cursor = new LongObjectCursor<VType>();
int seed = nextIterationSeed();
increment = iterationIncrement(seed);
slot = seed & mask;
}
@Override
protected LongObjectCursor<VType> fetch() {
final int mask = LongObjectHashMap.this.mask;
while (index <= mask) {
long existing;
index++;
slot = (slot + increment) & mask;
if (!((existing = keys[slot]) == 0)) {
cursor.index = slot;
cursor.key = existing;
cursor.value = (VType) values[slot];
return cursor;
}
}
if (index == mask + 1 && hasEmptyKey) {
cursor.index = index;
cursor.key = 0;
cursor.value = (VType) values[index++];
return cursor;
}
return done();
}
}
/** Returns a specialized view of the keys of this associated container. */
public KeysContainer keys() {
return new KeysContainer();
}
/** A view of the keys inside this hash map. */
public final class KeysContainer implements Iterable<LongCursor> {
@Override
public Iterator<LongCursor> iterator() {
return new KeysIterator();
}
public int size() {
return LongObjectHashMap.this.size();
}
public long[] toArray() {
long[] array = new long[size()];
int i = 0;
for (LongCursor cursor : this) {
array[i++] = cursor.value;
}
return array;
}
}
/** An iterator over the set of assigned keys. */
private final class KeysIterator extends AbstractIterator<LongCursor> {
private final LongCursor cursor;
private final int increment;
private int index;
private int slot;
public KeysIterator() {
cursor = new LongCursor();
int seed = nextIterationSeed();
increment = iterationIncrement(seed);
slot = seed & mask;
}
@Override
protected LongCursor fetch() {
final int mask = LongObjectHashMap.this.mask;
while (index <= mask) {
long existing;
index++;
slot = (slot + increment) & mask;
if (!((existing = keys[slot]) == 0)) {
cursor.index = slot;
cursor.value = existing;
return cursor;
}
}
if (index == mask + 1 && hasEmptyKey) {
cursor.index = index++;
cursor.value = 0;
return cursor;
}
return done();
}
}
/**
* @return Returns a container with all values stored in this map.
*/
public ValuesContainer values() {
return new ValuesContainer();
}
/** A view over the set of values of this map. */
public final class ValuesContainer implements Iterable<ObjectCursor<VType>> {
@Override
public Iterator<ObjectCursor<VType>> iterator() {
return new ValuesIterator();
}
public int size() {
return LongObjectHashMap.this.size();
}
public VType[] toArray() {
VType[] array = (VType[]) new Object[size()];
int i = 0;
for (ObjectCursor<VType> cursor : this) {
array[i++] = cursor.value;
}
return array;
}
}
/** An iterator over the set of assigned values. */
private final class ValuesIterator extends AbstractIterator<ObjectCursor<VType>> {
private final ObjectCursor<VType> cursor;
private final int increment;
private int index;
private int slot;
public ValuesIterator() {
cursor = new ObjectCursor<>();
int seed = nextIterationSeed();
increment = iterationIncrement(seed);
slot = seed & mask;
}
@Override
protected ObjectCursor<VType> fetch() {
final int mask = LongObjectHashMap.this.mask;
while (index <= mask) {
index++;
slot = (slot + increment) & mask;
if (!((keys[slot]) == 0)) {
cursor.index = slot;
cursor.value = (VType) values[slot];
return cursor;
}
}
if (index == mask + 1 && hasEmptyKey) {
cursor.index = index;
cursor.value = (VType) values[index++];
return cursor;
}
return done();
}
}
@Override
public LongObjectHashMap<VType> clone() {
try {
/* */
LongObjectHashMap<VType> cloned = (LongObjectHashMap<VType>) super.clone();
cloned.keys = keys.clone();
cloned.values = values.clone();
cloned.hasEmptyKey = hasEmptyKey;
cloned.iterationSeed = nextIterationSeed();
return cloned;
} catch (CloneNotSupportedException e) {
throw new RuntimeException(e);
}
}
/** Convert the contents of this map to a human-friendly string. */
@Override
public String toString() {
final StringBuilder buffer = new StringBuilder();
buffer.append("[");
boolean first = true;
for (LongObjectCursor<VType> cursor : this) {
if (!first) {
buffer.append(", ");
}
buffer.append(cursor.key);
buffer.append("=>");
buffer.append(cursor.value);
first = false;
}
buffer.append("]");
return buffer.toString();
}
/** Creates a hash map from two index-aligned arrays of key-value pairs. */
public static <VType> LongObjectHashMap<VType> from(long[] keys, VType[] values) {
if (keys.length != values.length) {
throw new IllegalArgumentException(
"Arrays of keys and values must have an identical length.");
}
LongObjectHashMap<VType> map = new LongObjectHashMap<>(keys.length);
for (int i = 0; i < keys.length; i++) {
map.put(keys[i], values[i]);
}
return map;
}
/**
* Returns a hash code for the given key.
*
* <p>The output from this function should evenly distribute keys across the entire integer range.
*/
protected int hashKey(long key) {
assert !((key) == 0); // Handled as a special case (empty slot marker).
return BitMixer.mixPhi(key);
}
/**
* Validate load factor range and return it. Override and suppress if you need insane load
* factors.
*/
protected double verifyLoadFactor(double loadFactor) {
checkLoadFactor(loadFactor, MIN_LOAD_FACTOR, MAX_LOAD_FACTOR);
return loadFactor;
}
/** Rehash from old buffers to new buffers. */
protected void rehash(long[] fromKeys, VType[] fromValues) {
assert fromKeys.length == fromValues.length && checkPowerOfTwo(fromKeys.length - 1);
// Rehash all stored key/value pairs into the new buffers.
final long[] keys = this.keys;
final VType[] values = (VType[]) this.values;
final int mask = this.mask;
long existing;
// Copy the zero element's slot, then rehash everything else.
int from = fromKeys.length - 1;
keys[keys.length - 1] = fromKeys[from];
values[values.length - 1] = fromValues[from];
while (--from >= 0) {
if (!((existing = fromKeys[from]) == 0)) {
int slot = hashKey(existing) & mask;
while (!((keys[slot]) == 0)) {
slot = (slot + 1) & mask;
}
keys[slot] = existing;
values[slot] = fromValues[from];
}
}
}
/**
* Allocate new internal buffers. This method attempts to allocate and assign internal buffers
* atomically (either allocations succeed or not).
*/
protected void allocateBuffers(int arraySize) {
assert Integer.bitCount(arraySize) == 1;
// Ensure no change is done if we hit an OOM.
long[] prevKeys = this.keys;
VType[] prevValues = (VType[]) this.values;
try {
int emptyElementSlot = 1;
this.keys = (new long[arraySize + emptyElementSlot]);
this.values = new Object[arraySize + emptyElementSlot];
} catch (OutOfMemoryError e) {
this.keys = prevKeys;
this.values = prevValues;
throw new BufferAllocationException(
"Not enough memory to allocate buffers for rehashing: %,d -> %,d",
e, this.mask + 1, arraySize);
}
this.resizeAt = expandAtCount(arraySize, loadFactor);
this.mask = arraySize - 1;
}
/**
* This method is invoked when there is a new key/ value pair to be inserted into the buffers but
* there is not enough empty slots to do so.
*
* <p>New buffers are allocated. If this succeeds, we know we can proceed with rehashing so we
* assign the pending element to the previous buffer (possibly violating the invariant of having
* at least one empty slot) and rehash all keys, substituting new buffers at the end.
*/
protected void allocateThenInsertThenRehash(int slot, long pendingKey, VType pendingValue) {
assert assigned == resizeAt && ((keys[slot]) == 0) && !((pendingKey) == 0);
// Try to allocate new buffers first. If we OOM, we leave in a consistent state.
final long[] prevKeys = this.keys;
final VType[] prevValues = (VType[]) this.values;
allocateBuffers(nextBufferSize(mask + 1, size(), loadFactor));
assert this.keys.length > prevKeys.length;
// We have succeeded at allocating new data so insert the pending key/value at
// the free slot in the old arrays before rehashing.
prevKeys[slot] = pendingKey;
prevValues[slot] = pendingValue;
// Rehash old keys, including the pending key.
rehash(prevKeys, prevValues);
}
static int nextBufferSize(int arraySize, int elements, double loadFactor) {
assert checkPowerOfTwo(arraySize);
if (arraySize == MAX_HASH_ARRAY_LENGTH) {
throw new BufferAllocationException(
"Maximum array size exceeded for this load factor (elements: %d, load factor: %f)",
elements, loadFactor);
}
return arraySize << 1;
}
static int expandAtCount(int arraySize, double loadFactor) {
assert checkPowerOfTwo(arraySize);
// Take care of hash container invariant (there has to be at least one empty slot to ensure
// the lookup loop finds either the element or an empty slot).
return Math.min(arraySize - 1, (int) Math.ceil(arraySize * loadFactor));
}
static boolean checkPowerOfTwo(int arraySize) {
// These are internals, we can just assert without retrying.
assert arraySize > 1;
assert nextHighestPowerOfTwo(arraySize) == arraySize;
return true;
}
static int minBufferSize(int elements, double loadFactor) {
if (elements < 0) {
throw new IllegalArgumentException("Number of elements must be >= 0: " + elements);
}
long length = (long) Math.ceil(elements / loadFactor);
if (length == elements) {
length++;
}
length = Math.max(MIN_HASH_ARRAY_LENGTH, nextHighestPowerOfTwo(length));
if (length > MAX_HASH_ARRAY_LENGTH) {
throw new BufferAllocationException(
"Maximum array size exceeded for this load factor (elements: %d, load factor: %f)",
elements, loadFactor);
}
return (int) length;
}
static void checkLoadFactor(
double loadFactor, double minAllowedInclusive, double maxAllowedInclusive) {
if (loadFactor < minAllowedInclusive || loadFactor > maxAllowedInclusive) {
throw new BufferAllocationException(
"The load factor should be in range [%.2f, %.2f]: %f",
minAllowedInclusive, maxAllowedInclusive, loadFactor);
}
}
static int iterationIncrement(int seed) {
return 29 + ((seed & 7) << 1); // Small odd integer.
}
/**
* Shift all the slot-conflicting keys and values allocated to (and including) <code>slot</code>.
*/
protected void shiftConflictingKeys(int gapSlot) {
final long[] keys = this.keys;
final VType[] values = (VType[]) this.values;
final int mask = this.mask;
// Perform shifts of conflicting keys to fill in the gap.
int distance = 0;
while (true) {
final int slot = (gapSlot + (++distance)) & mask;
final long existing = keys[slot];
if (((existing) == 0)) {
break;
}
final int idealSlot = hashKey(existing);
final int shift = (slot - idealSlot) & mask;
if (shift >= distance) {
// Entry at this position was originally at or before the gap slot.
// Move the conflict-shifted entry to the gap's position and repeat the procedure
// for any entries to the right of the current position, treating it
// as the new gap.
keys[gapSlot] = existing;
values[gapSlot] = values[slot];
gapSlot = slot;
distance = 0;
}
}
// Mark the last found gap slot without a conflict as empty.
keys[gapSlot] = 0;
values[gapSlot] = null;
assigned--;
}
/** Forked from HPPC, holding int index,key and value */
public static final class LongObjectCursor<VType> {
/**
* The current key and value's index in the container this cursor belongs to. The meaning of
* this index is defined by the container (usually it will be an index in the underlying storage
* buffer).
*/
public int index;
/** The current key. */
public long key;
/** The current value. */
public VType value;
@Override
public String toString() {
return "[cursor, index: " + index + ", key: " + key + ", value: " + value + "]";
}
}
}

35
lucene/core/src/java/org/apache/lucene/util/hppc/ObjectCursor.java Normal file
View File

@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.util.hppc;
/** Forked from HPPC, holding int index and Object value */
public final class ObjectCursor<VType> {
/**
* The current value's index in the container this cursor belongs to. The meaning of this index is
* defined by the container (usually it will be an index in the underlying storage buffer).
*/
public int index;
/** The current value. */
public VType value;
@Override
public String toString() {
return "[cursor, index: " + index + ", value: " + value + "]";
}
}

47
lucene/core/src/test/org/apache/lucene/util/hppc/TestIntIntHashMap.java
View File

@ -48,59 +48,16 @@ public class TestIntIntHashMap extends LuceneTestCase {
protected int key9 = cast(9), k9 = key9;
/** Convert to target type from an integer used to test stuff. */
public int cast(Integer v) {
return v.intValue();
}
public int[] asArray(int... ints) {
int[] values = (new int[ints.length]);
for (int i = 0; i < ints.length; i++) values[i] = ints[i];
return values;
public int cast(int v) {
return v;
}
/** Create a new array of a given type and copy the arguments to this array. */
/* */
public final int[] newArray(int... elements) {
return newArray0(elements);
}
/* */
private final int[] newArray0(int... elements) {
return elements;
}
public int[] newArray(int v0) {
return this.newArray0(v0);
}
public int[] newArray(int v0, int v1) {
return this.newArray0(v0, v1);
}
public int[] newArray(int v0, int v1, int v2) {
return this.newArray0(v0, v1, v2);
}
public int[] newArray(int v0, int v1, int v2, int v3) {
return this.newArray0(v0, v1, v2, v3);
}
public int[] newArray(int v0, int v1, int v2, int v3, int v4, int v5, int v6) {
return this.newArray0(v0, v1, v2, v3, v4, v5, v6);
}
public int[] newArray(int v0, int v1, int v2, int v3, int v4, int v5) {
return this.newArray0(v0, v1, v2, v3, v4, v5);
}
public int[] newArray(int v0, int v1, int v2, int v3, int v4) {
return this.newArray0(v0, v1, v2, v3, v4);
}
public int[] newArray(int v0, int v1, int v2, int v3, int v4, int v5, int v6, int v7) {
return this.newArray0(v0, v1, v2, v3, v4, v5, v6, v7);
}
public static int randomIntBetween(int min, int max) {
return min + random().nextInt(max + 1 - min);
}

75
lucene/core/src/test/org/apache/lucene/util/hppc/TestIntObjectHashMap.java
View File

@ -49,59 +49,16 @@ public class TestIntObjectHashMap extends LuceneTestCase {
protected int key9 = cast(9), k9 = key9;
/** Convert to target type from an integer used to test stuff. */
public int cast(Integer v) {
return v.intValue();
}
public int[] asArray(int... ints) {
int[] values = (new int[ints.length]);
for (int i = 0; i < ints.length; i++) values[i] = ints[i];
return values;
public int cast(int v) {
return v;
}
/** Create a new array of a given type and copy the arguments to this array. */
/* */
public final int[] newArray(int... elements) {
return newArray0(elements);
}
/* */
private final int[] newArray0(int... elements) {
return elements;
}
public int[] newArray(int v0) {
return this.newArray0(v0);
}
public int[] newArray(int v0, int v1) {
return this.newArray0(v0, v1);
}
public int[] newArray(int v0, int v1, int v2) {
return this.newArray0(v0, v1, v2);
}
public int[] newArray(int v0, int v1, int v2, int v3) {
return this.newArray0(v0, v1, v2, v3);
}
public int[] newArray(int v0, int v1, int v2, int v3, int v4, int v5, int v6) {
return this.newArray0(v0, v1, v2, v3, v4, v5, v6);
}
public int[] newArray(int v0, int v1, int v2, int v3, int v4, int v5) {
return this.newArray0(v0, v1, v2, v3, v4, v5);
}
public int[] newArray(int v0, int v1, int v2, int v3, int v4) {
return this.newArray0(v0, v1, v2, v3, v4);
}
public int[] newArray(int v0, int v1, int v2, int v3, int v4, int v5, int v6, int v7) {
return this.newArray0(v0, v1, v2, v3, v4, v5, v6, v7);
}
public static int randomIntBetween(int min, int max) {
return min + random().nextInt(max + 1 - min);
}
@ -297,11 +254,26 @@ public class TestIntObjectHashMap extends LuceneTestCase {
}
/* */
@Test
public void testNullValue() {
map.put(key1, null);
assertTrue(map.containsKey(key1));
assertNull(map.get(key1));
}
@Test
public void testPutOverExistingKey() {
map.put(key1, value1);
assertEquals(value1, map.put(key1, value3));
assertEquals(value3, map.get(key1));
assertEquals(value3, map.put(key1, null));
assertTrue(map.containsKey(key1));
assertNull(map.get(key1));
assertNull(map.put(key1, value1));
assertEquals(value1, map.get(key1));
}
/* */
@ -381,6 +353,16 @@ public class TestIntObjectHashMap extends LuceneTestCase {
map.remove(empty);
assertEquals(null, map.get(empty));
map.put(empty, null);
assertEquals(1, map.size());
assertTrue(map.containsKey(empty));
assertNull(map.get(empty));
map.remove(empty);
assertEquals(0, map.size());
assertFalse(map.containsKey(empty));
assertNull(map.get(empty));
}
/* */
@ -577,7 +559,7 @@ public class TestIntObjectHashMap extends LuceneTestCase {
map.put(key3, value1);
int counted = 0;
for (IntObjectHashMap.ObjectCursor c : map.values()) {
for (ObjectCursor c : map.values()) {
assertEquals(map.values[c.index], c.value);
counted++;
}
@ -608,7 +590,6 @@ public class TestIntObjectHashMap extends LuceneTestCase {
@Test
public void testEqualsSubClass() {
class Sub extends IntObjectHashMap {}
;
IntObjectHashMap l1 = newInstance();
l1.put(k1, value0);

597
lucene/core/src/test/org/apache/lucene/util/hppc/TestLongIntHashMap.java Normal file
View File

@ -0,0 +1,597 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.util.hppc;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.junit.After;
import org.junit.Test;
/**
* Tests for {@link LongIntHashMap}.
*
* <p>Mostly forked and trimmed from com.carrotsearch.hppc.LongIntHashMapTest
*
* <p>github: https://github.com/carrotsearch/hppc release: 0.9.0
*/
public class TestLongIntHashMap extends LuceneTestCase {
/* Ready to use key values. */
protected long keyE = 0;
protected long key0 = cast(0), k0 = key0;
protected long key1 = cast(1), k1 = key1;
protected long key2 = cast(2), k2 = key2;
protected long key3 = cast(3), k3 = key3;
protected long key4 = cast(4), k4 = key4;
protected long key5 = cast(5), k5 = key5;
protected long key6 = cast(6), k6 = key6;
protected long key7 = cast(7), k7 = key7;
protected long key8 = cast(8), k8 = key8;
protected long key9 = cast(9), k9 = key9;
/** Convert to target type from an integer used to test stuff. */
public long cast(int v) {
return v;
}
/** Create a new array of a given type and copy the arguments to this array. */
/* */
public final long[] newArray(long... elements) {
return elements;
}
public static int randomIntBetween(int min, int max) {
return min + random().nextInt(max + 1 - min);
}
/** Check if the array's content is identical to a given sequence of elements. */
public static void assertSortedListEquals(long[] array, long... elements) {
assertEquals(elements.length, array.length);
Arrays.sort(array);
Arrays.sort(elements);
assertArrayEquals(elements, array);
}
/** Check if the array's content is identical to a given sequence of elements. */
public static void assertSortedListEquals(int[] array, int... elements) {
assertEquals(elements.length, array.length);
Arrays.sort(array);
Arrays.sort(elements);
assertArrayEquals(elements, array);
}
protected int value0 = vcast(0);
protected int value1 = vcast(1);
protected int value2 = vcast(2);
protected int value3 = vcast(3);
protected int value4 = vcast(4);
/** Per-test fresh initialized instance. */
public LongIntHashMap map = newInstance();
protected LongIntHashMap newInstance() {
return new LongIntHashMap();
}
@After
public void checkEmptySlotsUninitialized() {
if (map != null) {
int occupied = 0;
for (int i = 0; i <= map.mask; i++) {
if (((map.keys[i]) == 0)) {
} else {
occupied++;
}
}
assertEquals(occupied, map.assigned);
if (!map.hasEmptyKey) {}
}
}
/** Convert to target type from an integer used to test stuff. */
protected int vcast(int value) {
return value;
}
/** Create a new array of a given type and copy the arguments to this array. */
/* */
protected final int[] newvArray(int... elements) {
return elements;
}
private void assertSameMap(final LongIntHashMap c1, final LongIntHashMap c2) {
assertEquals(c1.size(), c2.size());
for (LongIntHashMap.LongIntCursor entry : c1) {
assertTrue(c2.containsKey(entry.key));
assertEquals(entry.value, c2.get(entry.key));
}
}
/* */
@Test
public void testEnsureCapacity() {
final AtomicInteger expands = new AtomicInteger();
LongIntHashMap map =
new LongIntHashMap(0) {
@Override
protected void allocateBuffers(int arraySize) {
super.allocateBuffers(arraySize);
expands.incrementAndGet();
}
};
// Add some elements.
final int max = rarely() ? 0 : randomIntBetween(0, 250);
for (int i = 0; i < max; i++) {
map.put(cast(i), value0);
}
final int additions = randomIntBetween(max, max + 5000);
map.ensureCapacity(additions + map.size());
final int before = expands.get();
for (int i = 0; i < additions; i++) {
map.put(cast(i), value0);
}
assertEquals(before, expands.get());
}
@Test
public void testCursorIndexIsValid() {
map.put(keyE, value1);
map.put(key1, value2);
map.put(key2, value3);
for (LongIntHashMap.LongIntCursor c : map) {
assertTrue(map.indexExists(c.index));
assertEquals(c.value, map.indexGet(c.index));
}
}
@Test
public void testIndexMethods() {
map.put(keyE, value1);
map.put(key1, value2);
assertTrue(map.indexOf(keyE) >= 0);
assertTrue(map.indexOf(key1) >= 0);
assertTrue(map.indexOf(key2) < 0);
assertTrue(map.indexExists(map.indexOf(keyE)));
assertTrue(map.indexExists(map.indexOf(key1)));
assertFalse(map.indexExists(map.indexOf(key2)));
assertEquals(value1, map.indexGet(map.indexOf(keyE)));
assertEquals(value2, map.indexGet(map.indexOf(key1)));
expectThrows(
AssertionError.class,
() -> {
map.indexGet(map.indexOf(key2));
fail();
});
assertEquals(value1, map.indexReplace(map.indexOf(keyE), value3));
assertEquals(value2, map.indexReplace(map.indexOf(key1), value4));
assertEquals(value3, map.indexGet(map.indexOf(keyE)));
assertEquals(value4, map.indexGet(map.indexOf(key1)));
map.indexInsert(map.indexOf(key2), key2, value1);
assertEquals(value1, map.indexGet(map.indexOf(key2)));
assertEquals(3, map.size());
assertEquals(value3, map.indexRemove(map.indexOf(keyE)));
assertEquals(2, map.size());
assertEquals(value1, map.indexRemove(map.indexOf(key2)));
assertEquals(1, map.size());
assertTrue(map.indexOf(keyE) < 0);
assertTrue(map.indexOf(key1) >= 0);
assertTrue(map.indexOf(key2) < 0);
}
/* */
@Test
public void testCloningConstructor() {
map.put(key1, value1);
map.put(key2, value2);
map.put(key3, value3);
assertSameMap(map, new LongIntHashMap(map));
}
/* */
@Test
public void testFromArrays() {
map.put(key1, value1);
map.put(key2, value2);
map.put(key3, value3);
LongIntHashMap map2 =
LongIntHashMap.from(newArray(key1, key2, key3), newvArray(value1, value2, value3));
assertSameMap(map, map2);
}
@Test
public void testGetOrDefault() {
map.put(key2, value2);
assertTrue(map.containsKey(key2));
map.put(key1, value1);
assertEquals(value1, map.getOrDefault(key1, value3));
assertEquals(value3, map.getOrDefault(key3, value3));
map.remove(key1);
assertEquals(value3, map.getOrDefault(key1, value3));
}
/* */
@Test
public void testPut() {
map.put(key1, value1);
assertTrue(map.containsKey(key1));
assertEquals(value1, map.get(key1));
}
/* */
@Test
public void testPutOverExistingKey() {
map.put(key1, value1);
assertEquals(value1, map.put(key1, value3));
assertEquals(value3, map.get(key1));
}
/* */
@Test
public void testPutWithExpansions() {
final int COUNT = 10000;
final Random rnd = new Random(random().nextLong());
final HashSet<Object> values = new HashSet<Object>();
for (int i = 0; i < COUNT; i++) {
final int v = rnd.nextInt();
final boolean hadKey = values.contains(cast(v));
values.add(cast(v));
assertEquals(hadKey, map.containsKey(cast(v)));
map.put(cast(v), vcast(v));
assertEquals(values.size(), map.size());
}
assertEquals(values.size(), map.size());
}
/* */
@Test
public void testPutAll() {
map.put(key1, value1);
map.put(key2, value1);
LongIntHashMap map2 = newInstance();
map2.put(key2, value2);
map2.put(keyE, value1);
// One new key (keyE).
assertEquals(1, map.putAll(map2));
// Assert the value under key2 has been replaced.
assertEquals(value2, map.get(key2));
// And key3 has been added.
assertEquals(value1, map.get(keyE));
assertEquals(3, map.size());
}
/* */
@Test
public void testPutIfAbsent() {
assertTrue(map.putIfAbsent(key1, value1));
assertFalse(map.putIfAbsent(key1, value2));
assertEquals(value1, map.get(key1));
}
@Test
public void testPutOrAdd() {
assertEquals(value1, map.putOrAdd(key1, value1, value2));
assertEquals(value3, map.putOrAdd(key1, value1, value2));
}
@Test
public void testAddTo() {
assertEquals(value1, map.addTo(key1, value1));
assertEquals(value3, map.addTo(key1, value2));
}
/* */
@Test
public void testRemove() {
map.put(key1, value1);
assertEquals(value1, map.remove(key1));
assertEquals(0, map.remove(key1));
assertEquals(0, map.size());
// These are internals, but perhaps worth asserting too.
assertEquals(0, map.assigned);
}
/* */
@Test
public void testEmptyKey() {
final int empty = 0;
map.put(empty, value1);
assertEquals(1, map.size());
assertEquals(false, map.isEmpty());
assertEquals(value1, map.get(empty));
assertEquals(value1, map.getOrDefault(empty, value2));
assertEquals(true, map.iterator().hasNext());
assertEquals(empty, map.iterator().next().key);
assertEquals(value1, map.iterator().next().value);
map.remove(empty);
assertEquals(0, map.get(empty));
}
/* */
@Test
public void testMapKeySet() {
map.put(key1, value3);
map.put(key2, value2);
map.put(key3, value1);
assertSortedListEquals(map.keys().toArray(), key1, key2, key3);
}
/* */
@Test
public void testMapKeySetIterator() {
map.put(key1, value3);
map.put(key2, value2);
map.put(key3, value1);
int counted = 0;
for (LongCursor c : map.keys()) {
assertEquals(map.keys[c.index], c.value);
counted++;
}
assertEquals(counted, map.size());
}
/* */
@Test
public void testClear() {
map.put(key1, value1);
map.put(key2, value1);
map.clear();
assertEquals(0, map.size());
// These are internals, but perhaps worth asserting too.
assertEquals(0, map.assigned);
// Check if the map behaves properly upon subsequent use.
testPutWithExpansions();
}
/* */
@Test
public void testRelease() {
map.put(key1, value1);
map.put(key2, value1);
map.release();
assertEquals(0, map.size());
// These are internals, but perhaps worth asserting too.
assertEquals(0, map.assigned);
// Check if the map behaves properly upon subsequent use.
testPutWithExpansions();
}
/* */
@Test
public void testIterable() {
map.put(key1, value1);
map.put(key2, value2);
map.put(key3, value3);
map.remove(key2);
int count = 0;
for (LongIntHashMap.LongIntCursor cursor : map) {
count++;
assertTrue(map.containsKey(cursor.key));
assertEquals(cursor.value, map.get(cursor.key));
assertEquals(cursor.value, map.values[cursor.index]);
assertEquals(cursor.key, map.keys[cursor.index]);
}
assertEquals(count, map.size());
map.clear();
assertFalse(map.iterator().hasNext());
}
/* */
@Test
public void testBug_HPPC73_FullCapacityGet() {
final AtomicInteger reallocations = new AtomicInteger();
final int elements = 0x7F;
map =
new LongIntHashMap(elements, 1f) {
@Override
protected double verifyLoadFactor(double loadFactor) {
// Skip load factor sanity range checking.
return loadFactor;
}
@Override
protected void allocateBuffers(int arraySize) {
super.allocateBuffers(arraySize);
reallocations.incrementAndGet();
}
};
int reallocationsBefore = reallocations.get();
assertEquals(reallocationsBefore, 1);
for (int i = 1; i <= elements; i++) {
map.put(cast(i), value1);
}
// Non-existent key.
long outOfSet = cast(elements + 1);
map.remove(outOfSet);
assertFalse(map.containsKey(outOfSet));
assertEquals(reallocationsBefore, reallocations.get());
// Should not expand because we're replacing an existing element.
map.put(k1, value2);
assertEquals(reallocationsBefore, reallocations.get());
// Remove from a full map.
map.remove(k1);
assertEquals(reallocationsBefore, reallocations.get());
map.put(k1, value2);
// Check expand on "last slot of a full map" condition.
map.put(outOfSet, value1);
assertEquals(reallocationsBefore + 1, reallocations.get());
}
@Test
public void testHashCodeEquals() {
LongIntHashMap l0 = newInstance();
assertEquals(0, l0.hashCode());
assertEquals(l0, newInstance());
LongIntHashMap l1 =
LongIntHashMap.from(newArray(key1, key2, key3), newvArray(value1, value2, value3));
LongIntHashMap l2 =
LongIntHashMap.from(newArray(key2, key1, key3), newvArray(value2, value1, value3));
LongIntHashMap l3 = LongIntHashMap.from(newArray(key1, key2), newvArray(value2, value1));
assertEquals(l1.hashCode(), l2.hashCode());
assertEquals(l1, l2);
assertFalse(l1.equals(l3));
assertFalse(l2.equals(l3));
}
@Test
public void testBug_HPPC37() {
LongIntHashMap l1 = LongIntHashMap.from(newArray(key1), newvArray(value1));
LongIntHashMap l2 = LongIntHashMap.from(newArray(key2), newvArray(value1));
assertFalse(l1.equals(l2));
assertFalse(l2.equals(l1));
}
/*
*
*/
@Test
public void testClone() {
this.map.put(key1, value1);
this.map.put(key2, value2);
this.map.put(key3, value3);
LongIntHashMap cloned = map.clone();
cloned.remove(key1);
assertSortedListEquals(map.keys().toArray(), key1, key2, key3);
assertSortedListEquals(cloned.keys().toArray(), key2, key3);
}
/* */
@Test
public void testMapValues() {
map.put(key1, value3);
map.put(key2, value2);
map.put(key3, value1);
assertSortedListEquals(map.values().toArray(), value1, value2, value3);
map.clear();
map.put(key1, value1);
map.put(key2, value2);
map.put(key3, value2);
assertSortedListEquals(map.values().toArray(), value1, value2, value2);
}
/* */
@Test
public void testMapValuesIterator() {
map.put(key1, value3);
map.put(key2, value2);
map.put(key3, value1);
int counted = 0;
for (IntCursor c : map.values()) {
assertEquals(map.values[c.index], c.value);
counted++;
}
assertEquals(counted, map.size());
}
/* */
@Test
public void testEqualsSameClass() {
LongIntHashMap l1 = newInstance();
l1.put(k1, value0);
l1.put(k2, value1);
l1.put(k3, value2);
LongIntHashMap l2 = new LongIntHashMap(l1);
l2.putAll(l1);
LongIntHashMap l3 = new LongIntHashMap(l2);
l3.putAll(l2);
l3.put(k4, value0);
assertEquals(l2, l1);
assertEquals(l2.hashCode(), l1.hashCode());
assertNotEquals(l1, l3);
}
/* */
@Test
public void testEqualsSubClass() {
class Sub extends LongIntHashMap {}
LongIntHashMap l1 = newInstance();
l1.put(k1, value0);
l1.put(k2, value1);
l1.put(k3, value2);
LongIntHashMap l2 = new Sub();
l2.putAll(l1);
l2.put(k4, value3);
LongIntHashMap l3 = new Sub();
l3.putAll(l2);
assertNotEquals(l1, l2);
assertEquals(l3.hashCode(), l2.hashCode());
assertEquals(l3, l2);
}
}

611
lucene/core/src/test/org/apache/lucene/util/hppc/TestLongObjectHashMap.java Normal file
View File

@ -0,0 +1,611 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.util.hppc;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.junit.After;
import org.junit.Test;
/**
* Tests for {@link LongObjectHashMap}.
*
* <p>Mostly forked and trimmed from com.carrotsearch.hppc.LongObjectHashMapTest
*
* <p>github: https://github.com/carrotsearch/hppc release: 0.9.0
*/
@SuppressWarnings({"rawtypes", "unchecked"})
public class TestLongObjectHashMap extends LuceneTestCase {
/* Ready to use key values. */
protected long keyE = 0;
protected long key0 = cast(0), k0 = key0;
protected long key1 = cast(1), k1 = key1;
protected long key2 = cast(2), k2 = key2;
protected long key3 = cast(3), k3 = key3;
protected long key4 = cast(4), k4 = key4;
protected long key5 = cast(5), k5 = key5;
protected long key6 = cast(6), k6 = key6;
protected long key7 = cast(7), k7 = key7;
protected long key8 = cast(8), k8 = key8;
protected long key9 = cast(9), k9 = key9;
/** Convert to target type from an integer used to test stuff. */
public long cast(int v) {
return v;
}
/** Create a new array of a given type and copy the arguments to this array. */
/* */
public final long[] newArray(long... elements) {
return elements;
}
public static int randomIntBetween(int min, int max) {
return min + random().nextInt(max + 1 - min);
}
/** Check if the array's content is identical to a given sequence of elements. */
public static void assertSortedListEquals(long[] array, long... elements) {
assertEquals(elements.length, array.length);
Arrays.sort(array);
Arrays.sort(elements);
assertArrayEquals(elements, array);
}
/** Check if the array's content is identical to a given sequence of elements. */
public static void assertSortedListEquals(Object[] array, Object... elements) {
assertEquals(elements.length, array.length);
Arrays.sort(array);
assertArrayEquals(elements, array);
}
protected int value0 = vcast(0);
protected int value1 = vcast(1);
protected int value2 = vcast(2);
protected int value3 = vcast(3);
protected int value4 = vcast(4);
/** Per-test fresh initialized instance. */
public LongObjectHashMap<Object> map = newInstance();
protected LongObjectHashMap newInstance() {
return new LongObjectHashMap();
}
@After
public void checkEmptySlotsUninitialized() {
if (map != null) {
int occupied = 0;
for (int i = 0; i <= map.mask; i++) {
if (((map.keys[i]) == 0)) {
} else {
occupied++;
}
}
assertEquals(occupied, map.assigned);
if (!map.hasEmptyKey) {}
}
}
/** Convert to target type from an integer used to test stuff. */
protected int vcast(int value) {
return value;
}
/** Create a new array of a given type and copy the arguments to this array. */
/* */
protected final Object[] newvArray(Object... elements) {
return elements;
}
private void assertSameMap(
final LongObjectHashMap<Object> c1, final LongObjectHashMap<Object> c2) {
assertEquals(c1.size(), c2.size());
for (LongObjectHashMap.LongObjectCursor entry : c1) {
assertTrue(c2.containsKey(entry.key));
assertEquals(entry.value, c2.get(entry.key));
}
}
/* */
@Test
public void testEnsureCapacity() {
final AtomicInteger expands = new AtomicInteger();
LongObjectHashMap map =
new LongObjectHashMap(0) {
@Override
protected void allocateBuffers(int arraySize) {
super.allocateBuffers(arraySize);
expands.incrementAndGet();
}
};
// Add some elements.
final int max = rarely() ? 0 : randomIntBetween(0, 250);
for (int i = 0; i < max; i++) {
map.put(cast(i), value0);
}
final int additions = randomIntBetween(max, max + 5000);
map.ensureCapacity(additions + map.size());
final int before = expands.get();
for (int i = 0; i < additions; i++) {
map.put(cast(i), value0);
}
assertEquals(before, expands.get());
}
@Test
public void testCursorIndexIsValid() {
map.put(keyE, value1);
map.put(key1, value2);
map.put(key2, value3);
for (LongObjectHashMap.LongObjectCursor c : map) {
assertTrue(map.indexExists(c.index));
assertEquals(c.value, map.indexGet(c.index));
}
}
@Test
public void testIndexMethods() {
map.put(keyE, value1);
map.put(key1, value2);
assertTrue(map.indexOf(keyE) >= 0);
assertTrue(map.indexOf(key1) >= 0);
assertTrue(map.indexOf(key2) < 0);
assertTrue(map.indexExists(map.indexOf(keyE)));
assertTrue(map.indexExists(map.indexOf(key1)));
assertFalse(map.indexExists(map.indexOf(key2)));
assertEquals(value1, map.indexGet(map.indexOf(keyE)));
assertEquals(value2, map.indexGet(map.indexOf(key1)));
expectThrows(
AssertionError.class,
() -> {
map.indexGet(map.indexOf(key2));
fail();
});
assertEquals(value1, map.indexReplace(map.indexOf(keyE), value3));
assertEquals(value2, map.indexReplace(map.indexOf(key1), value4));
assertEquals(value3, map.indexGet(map.indexOf(keyE)));
assertEquals(value4, map.indexGet(map.indexOf(key1)));
map.indexInsert(map.indexOf(key2), key2, value1);
assertEquals(value1, map.indexGet(map.indexOf(key2)));
assertEquals(3, map.size());
assertEquals(value3, map.indexRemove(map.indexOf(keyE)));
assertEquals(2, map.size());
assertEquals(value1, map.indexRemove(map.indexOf(key2)));
assertEquals(1, map.size());
assertTrue(map.indexOf(keyE) < 0);
assertTrue(map.indexOf(key1) >= 0);
assertTrue(map.indexOf(key2) < 0);
}
/* */
@Test
public void testCloningConstructor() {
map.put(key1, value1);
map.put(key2, value2);
map.put(key3, value3);
assertSameMap(map, new LongObjectHashMap(map));
}
/* */
@Test
public void testFromArrays() {
map.put(key1, value1);
map.put(key2, value2);
map.put(key3, value3);
LongObjectHashMap map2 =
LongObjectHashMap.from(newArray(key1, key2, key3), newvArray(value1, value2, value3));
assertSameMap(map, map2);
}
@Test
public void testGetOrDefault() {
map.put(key2, value2);
assertTrue(map.containsKey(key2));
map.put(key1, value1);
assertEquals(value1, map.getOrDefault(key1, value3));
assertEquals(value3, map.getOrDefault(key3, value3));
map.remove(key1);
assertEquals(value3, map.getOrDefault(key1, value3));
}
/* */
@Test
public void testPut() {
map.put(key1, value1);
assertTrue(map.containsKey(key1));
assertEquals(value1, map.get(key1));
}
/* */
@Test
public void testNullValue() {
map.put(key1, null);
assertTrue(map.containsKey(key1));
assertNull(map.get(key1));
}
@Test
public void testPutOverExistingKey() {
map.put(key1, value1);
assertEquals(value1, map.put(key1, value3));
assertEquals(value3, map.get(key1));
assertEquals(value3, map.put(key1, null));
assertTrue(map.containsKey(key1));
assertNull(map.get(key1));
assertNull(map.put(key1, value1));
assertEquals(value1, map.get(key1));
}
/* */
@Test
public void testPutWithExpansions() {
final int COUNT = 10000;
final Random rnd = new Random(random().nextLong());
final HashSet<Object> values = new HashSet<Object>();
for (int i = 0; i < COUNT; i++) {
final int v = rnd.nextInt();
final boolean hadKey = values.contains(cast(v));
values.add(cast(v));
assertEquals(hadKey, map.containsKey(cast(v)));
map.put(cast(v), vcast(v));
assertEquals(values.size(), map.size());
}
assertEquals(values.size(), map.size());
}
/* */
@Test
public void testPutAll() {
map.put(key1, value1);
map.put(key2, value1);
LongObjectHashMap map2 = newInstance();
map2.put(key2, value2);
map2.put(keyE, value1);
// One new key (keyE).
assertEquals(1, map.putAll(map2));
// Assert the value under key2 has been replaced.
assertEquals(value2, map.get(key2));
// And key3 has been added.
assertEquals(value1, map.get(keyE));
assertEquals(3, map.size());
}
/* */
@Test
public void testPutIfAbsent() {
assertTrue(map.putIfAbsent(key1, value1));
assertFalse(map.putIfAbsent(key1, value2));
assertEquals(value1, map.get(key1));
}
/* */
@Test
public void testRemove() {
map.put(key1, value1);
assertEquals(value1, map.remove(key1));
assertEquals(null, map.remove(key1));
assertEquals(0, map.size());
// These are internals, but perhaps worth asserting too.
assertEquals(0, map.assigned);
}
/* */
@Test
public void testEmptyKey() {
final int empty = 0;
map.put(empty, value1);
assertEquals(1, map.size());
assertEquals(false, map.isEmpty());
assertEquals(value1, map.get(empty));
assertEquals(value1, map.getOrDefault(empty, value2));
assertEquals(true, map.iterator().hasNext());
assertEquals(empty, map.iterator().next().key);
assertEquals(value1, map.iterator().next().value);
map.remove(empty);
assertEquals(null, map.get(empty));
map.put(empty, null);
assertEquals(1, map.size());
assertTrue(map.containsKey(empty));
assertNull(map.get(empty));
map.remove(empty);
assertEquals(0, map.size());
assertFalse(map.containsKey(empty));
assertNull(map.get(empty));
}
/* */
@Test
public void testMapKeySet() {
map.put(key1, value3);
map.put(key2, value2);
map.put(key3, value1);
assertSortedListEquals(map.keys().toArray(), key1, key2, key3);
}
/* */
@Test
public void testMapKeySetIterator() {
map.put(key1, value3);
map.put(key2, value2);
map.put(key3, value1);
int counted = 0;
for (LongCursor c : map.keys()) {
assertEquals(map.keys[c.index], c.value);
counted++;
}
assertEquals(counted, map.size());
}
/* */
@Test
public void testClear() {
map.put(key1, value1);
map.put(key2, value1);
map.clear();
assertEquals(0, map.size());
// These are internals, but perhaps worth asserting too.
assertEquals(0, map.assigned);
// Check if the map behaves properly upon subsequent use.
testPutWithExpansions();
}
/* */
@Test
public void testRelease() {
map.put(key1, value1);
map.put(key2, value1);
map.release();
assertEquals(0, map.size());
// These are internals, but perhaps worth asserting too.
assertEquals(0, map.assigned);
// Check if the map behaves properly upon subsequent use.
testPutWithExpansions();
}
/* */
@Test
public void testIterable() {
map.put(key1, value1);
map.put(key2, value2);
map.put(key3, value3);
map.remove(key2);
int count = 0;
for (LongObjectHashMap.LongObjectCursor cursor : map) {
count++;
assertTrue(map.containsKey(cursor.key));
assertEquals(cursor.value, map.get(cursor.key));
assertEquals(cursor.value, map.values[cursor.index]);
assertEquals(cursor.key, map.keys[cursor.index]);
}
assertEquals(count, map.size());
map.clear();
assertFalse(map.iterator().hasNext());
}
/* */
@Test
public void testBug_HPPC73_FullCapacityGet() {
final AtomicInteger reallocations = new AtomicInteger();
final int elements = 0x7F;
map =
new LongObjectHashMap(elements, 1f) {
@Override
protected double verifyLoadFactor(double loadFactor) {
// Skip load factor sanity range checking.
return loadFactor;
}
@Override
protected void allocateBuffers(int arraySize) {
super.allocateBuffers(arraySize);
reallocations.incrementAndGet();
}
};
int reallocationsBefore = reallocations.get();
assertEquals(reallocationsBefore, 1);
for (int i = 1; i <= elements; i++) {
map.put(cast(i), value1);
}
// Non-existent key.
long outOfSet = cast(elements + 1);
map.remove(outOfSet);
assertFalse(map.containsKey(outOfSet));
assertEquals(reallocationsBefore, reallocations.get());
// Should not expand because we're replacing an existing element.
map.put(k1, value2);
assertEquals(reallocationsBefore, reallocations.get());
// Remove from a full map.
map.remove(k1);
assertEquals(reallocationsBefore, reallocations.get());
map.put(k1, value2);
// Check expand on "last slot of a full map" condition.
map.put(outOfSet, value1);
assertEquals(reallocationsBefore + 1, reallocations.get());
}
@Test
public void testHashCodeEquals() {
LongObjectHashMap l0 = newInstance();
assertEquals(0, l0.hashCode());
assertEquals(l0, newInstance());
LongObjectHashMap l1 =
LongObjectHashMap.from(newArray(key1, key2, key3), newvArray(value1, value2, value3));
LongObjectHashMap l2 =
LongObjectHashMap.from(newArray(key2, key1, key3), newvArray(value2, value1, value3));
LongObjectHashMap l3 = LongObjectHashMap.from(newArray(key1, key2), newvArray(value2, value1));
assertEquals(l1.hashCode(), l2.hashCode());
assertEquals(l1, l2);
assertFalse(l1.equals(l3));
assertFalse(l2.equals(l3));
}
@Test
public void testBug_HPPC37() {
LongObjectHashMap l1 = LongObjectHashMap.from(newArray(key1), newvArray(value1));
LongObjectHashMap l2 = LongObjectHashMap.from(newArray(key2), newvArray(value1));
assertFalse(l1.equals(l2));
assertFalse(l2.equals(l1));
}
/*
*
*/
@Test
public void testClone() {
this.map.put(key1, value1);
this.map.put(key2, value2);
this.map.put(key3, value3);
LongObjectHashMap cloned = map.clone();
cloned.remove(key1);
assertSortedListEquals(map.keys().toArray(), key1, key2, key3);
assertSortedListEquals(cloned.keys().toArray(), key2, key3);
}
/* */
@Test
public void testMapValues() {
map.put(key1, value3);
map.put(key2, value2);
map.put(key3, value1);
assertSortedListEquals(map.values().toArray(), value1, value2, value3);
map.clear();
map.put(key1, value1);
map.put(key2, value2);
map.put(key3, value2);
assertSortedListEquals(map.values().toArray(), value1, value2, value2);
}
/* */
@Test
public void testMapValuesIterator() {
map.put(key1, value3);
map.put(key2, value2);
map.put(key3, value1);
int counted = 0;
for (ObjectCursor c : map.values()) {
assertEquals(map.values[c.index], c.value);
counted++;
}
assertEquals(counted, map.size());
}
/* */
@Test
public void testEqualsSameClass() {
LongObjectHashMap l1 = newInstance();
l1.put(k1, value0);
l1.put(k2, value1);
l1.put(k3, value2);
LongObjectHashMap l2 = new LongObjectHashMap(l1);
l2.putAll(l1);
LongObjectHashMap l3 = new LongObjectHashMap(l2);
l3.putAll(l2);
l3.put(k4, value0);
assertEquals(l2, l1);
assertEquals(l2.hashCode(), l1.hashCode());
assertNotEquals(l1, l3);
}
/* */
@Test
public void testEqualsSubClass() {
class Sub extends LongObjectHashMap {}
LongObjectHashMap l1 = newInstance();
l1.put(k1, value0);
l1.put(k2, value1);
l1.put(k3, value2);
LongObjectHashMap l2 = new Sub();
l2.putAll(l1);
l2.put(k4, value3);
LongObjectHashMap l3 = new Sub();
l3.putAll(l2);
assertNotEquals(l1, l2);
assertEquals(l3.hashCode(), l2.hashCode());
assertEquals(l3, l2);
}
}

28
lucene/facet/src/java/org/apache/lucene/facet/range/OverlappingLongRangeCounter.java
View File

@ -16,11 +16,11 @@
*/
package org.apache.lucene.facet.range;
import com.carrotsearch.hppc.LongArrayList;
import com.carrotsearch.hppc.LongIntHashMap;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import org.apache.lucene.util.FixedBitSet;
/**
@ -238,28 +238,28 @@ class OverlappingLongRangeCounter extends LongRangeCounter {
// track the start vs end case separately because if a
// given point is both, then it must be its own
// elementary interval:
Map<Long, Integer> endsMap = new HashMap<>();
LongIntHashMap endsMap = new LongIntHashMap();
endsMap.put(Long.MIN_VALUE, 1);
endsMap.put(Long.MAX_VALUE, 2);
for (LongRange range : ranges) {
Integer cur = endsMap.get(range.min);
if (cur == null) {
endsMap.put(range.min, 1);
int index = endsMap.indexOf(range.min);
if (index < 0) {
endsMap.indexInsert(index, range.min, 1);
} else {
endsMap.put(range.min, cur | 1);
endsMap.indexReplace(index, endsMap.indexGet(index) | 1);
}
cur = endsMap.get(range.max);
if (cur == null) {
endsMap.put(range.max, 2);
index = endsMap.indexOf(range.max);
if (index < 0) {
endsMap.indexInsert(index, range.max, 2);
} else {
endsMap.put(range.max, cur | 2);
endsMap.indexReplace(index, endsMap.indexGet(index) | 2);
}
}
List<Long> endsList = new ArrayList<>(endsMap.keySet());
Collections.sort(endsList);
LongArrayList endsList = new LongArrayList(endsMap.keys());
Arrays.sort(endsList.buffer, 0, endsList.size());
// Build elementaryIntervals (a 1D Venn diagram):
List<InclusiveRange> elementaryIntervals = new ArrayList<>();

2
lucene/join/build.gradle
View File

@ -21,5 +21,7 @@ description = 'Index-time and Query-time joins for normalized content'
dependencies {
moduleApi project(':lucene:core')
moduleImplementation 'com.carrotsearch:hppc'
moduleTestImplementation project(':lucene:test-framework')
}

2
lucene/join/src/java/module-info.java
View File

@ -16,8 +16,10 @@
*/
/** Index-time and Query-time joins for normalized content */
@SuppressWarnings({"requires-automatic"})
module org.apache.lucene.join {
requires org.apache.lucene.core;
requires com.carrotsearch.hppc;
exports org.apache.lucene.search.join;
}

13
lucene/join/src/java/org/apache/lucene/search/join/DiversifyingNearestChildrenKnnCollector.java
View File

@ -17,14 +17,13 @@
package org.apache.lucene.search.join;
import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.search.AbstractKnnCollector;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.hppc.IntIntHashMap;
/**
* This collects the nearest children vectors. Diversifying the results over the provided parent
@ -117,7 +116,7 @@ class DiversifyingNearestChildrenKnnCollector extends AbstractKnnCollector {
// Used to keep track of nodeId -> positionInHeap. This way when new scores are added for a
// node, the heap can be
// updated efficiently.
private final Map<Integer, Integer> nodeIdHeapIndex;
private final IntIntHashMap nodeIdHeapIndex;
private boolean closed = false;
public NodeIdCachingHeap(int maxSize) {
@ -130,8 +129,7 @@ class DiversifyingNearestChildrenKnnCollector extends AbstractKnnCollector {
// NOTE: we add +1 because all access to heap is 1-based not 0-based. heap[0] is unused.
heapSize = maxSize + 1;
this.maxSize = maxSize;
this.nodeIdHeapIndex =
new HashMap<>(maxSize < 2 ? maxSize + 1 : (int) (maxSize / 0.75 + 1.0));
this.nodeIdHeapIndex = new IntIntHashMap(maxSize);
this.heapNodes = new ParentChildScore[heapSize];
}
@ -179,8 +177,9 @@ class DiversifyingNearestChildrenKnnCollector extends AbstractKnnCollector {
if (closed) {
throw new IllegalStateException();
}
Integer previousNodeIndex = nodeIdHeapIndex.get(parentNode);
if (previousNodeIndex != null) {
int index = nodeIdHeapIndex.indexOf(parentNode);
if (index >= 0) {
int previousNodeIndex = nodeIdHeapIndex.indexGet(index);
if (heapNodes[previousNodeIndex].score < score) {
updateElement(previousNodeIndex, node, parentNode, score);
return true;

101
lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java
View File

@ -16,14 +16,16 @@
*/
package org.apache.lucene.search.join;
import com.carrotsearch.hppc.LongArrayList;
import com.carrotsearch.hppc.LongFloatHashMap;
import com.carrotsearch.hppc.LongHashSet;
import com.carrotsearch.hppc.LongIntHashMap;
import com.carrotsearch.hppc.cursors.LongCursor;
import com.carrotsearch.hppc.procedures.LongFloatProcedure;
import java.io.IOException;
import java.util.HashMap;
import java.util.Arrays;
import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
import java.util.TreeSet;
import java.util.function.BiConsumer;
import java.util.function.LongFunction;
import org.apache.lucene.document.DoublePoint;
import org.apache.lucene.document.FloatPoint;
import org.apache.lucene.document.IntPoint;
@ -147,46 +149,40 @@ public final class JoinUtil {
IndexSearcher fromSearcher,
ScoreMode scoreMode)
throws IOException {
TreeSet<Long> joinValues = new TreeSet<>();
Map<Long, Float> aggregatedScores = new HashMap<>();
Map<Long, Integer> occurrences = new HashMap<>();
LongHashSet joinValues = new LongHashSet();
LongFloatHashMap aggregatedScores = new LongFloatHashMap();
LongIntHashMap occurrences = new LongIntHashMap();
boolean needsScore = scoreMode != ScoreMode.None;
BiConsumer<Long, Float> scoreAggregator;
LongFloatProcedure scoreAggregator;
if (scoreMode == ScoreMode.Max) {
scoreAggregator =
(key, score) -> {
Float currentValue = aggregatedScores.putIfAbsent(key, score);
if (currentValue != null) {
aggregatedScores.put(key, Math.max(currentValue, score));
int index = aggregatedScores.indexOf(key);
if (index < 0) {
aggregatedScores.indexInsert(index, key, score);
} else {
float currentScore = aggregatedScores.indexGet(index);
aggregatedScores.indexReplace(index, Math.max(currentScore, score));
}
};
} else if (scoreMode == ScoreMode.Min) {
scoreAggregator =
(key, score) -> {
Float currentValue = aggregatedScores.putIfAbsent(key, score);
if (currentValue != null) {
aggregatedScores.put(key, Math.min(currentValue, score));
int index = aggregatedScores.indexOf(key);
if (index < 0) {
aggregatedScores.indexInsert(index, key, score);
} else {
float currentScore = aggregatedScores.indexGet(index);
aggregatedScores.indexReplace(index, Math.min(currentScore, score));
}
};
} else if (scoreMode == ScoreMode.Total) {
scoreAggregator =
(key, score) -> {
Float currentValue = aggregatedScores.putIfAbsent(key, score);
if (currentValue != null) {
aggregatedScores.put(key, currentValue + score);
}
};
scoreAggregator = aggregatedScores::addTo;
} else if (scoreMode == ScoreMode.Avg) {
scoreAggregator =
(key, score) -> {
Float currentSore = aggregatedScores.putIfAbsent(key, score);
if (currentSore != null) {
aggregatedScores.put(key, currentSore + score);
}
Integer currentOccurrence = occurrences.putIfAbsent(key, 1);
if (currentOccurrence != null) {
occurrences.put(key, ++currentOccurrence);
}
aggregatedScores.addTo(key, score);
occurrences.addTo(key, 1);
};
} else {
scoreAggregator =
@ -195,12 +191,12 @@ public final class JoinUtil {
};
}
LongFunction<Float> joinScorer;
LongFloatFunction joinScorer;
if (scoreMode == ScoreMode.Avg) {
joinScorer =
(joinValue) -> {
Float aggregatedScore = aggregatedScores.get(joinValue);
Integer occurrence = occurrences.get(joinValue);
float aggregatedScore = aggregatedScores.get(joinValue);
int occurrence = occurrences.get(joinValue);
return aggregatedScore / occurrence;
};
} else {
@ -222,7 +218,7 @@ public final class JoinUtil {
long value = sortedNumericDocValues.nextValue();
joinValues.add(value);
if (needsScore) {
scoreAggregator.accept(value, scorer.score());
scoreAggregator.apply(value, scorer.score());
}
}
}
@ -271,7 +267,7 @@ public final class JoinUtil {
}
joinValues.add(value);
if (needsScore) {
scoreAggregator.accept(value, scorer.score());
scoreAggregator.apply(value, scorer.score());
}
}
@ -296,7 +292,9 @@ public final class JoinUtil {
}
fromSearcher.search(fromQuery, collector);
Iterator<Long> iterator = joinValues.iterator();
LongArrayList joinValuesList = new LongArrayList(joinValues);
Arrays.sort(joinValuesList.buffer, 0, joinValuesList.size());
Iterator<LongCursor> iterator = joinValuesList.iterator();
final int bytesPerDim;
final BytesRef encoded = new BytesRef();
@ -308,10 +306,10 @@ public final class JoinUtil {
@Override
public BytesRef next() {
if (iterator.hasNext()) {
long value = iterator.next();
IntPoint.encodeDimension((int) value, encoded.bytes, 0);
LongCursor value = iterator.next();
IntPoint.encodeDimension((int) value.value, encoded.bytes, 0);
if (needsScore) {
score = joinScorer.apply(value);
score = joinScorer.apply(value.value);
}
return encoded;
} else {
@ -326,10 +324,10 @@ public final class JoinUtil {
@Override
public BytesRef next() {
if (iterator.hasNext()) {
long value = iterator.next();
LongPoint.encodeDimension(value, encoded.bytes, 0);
LongCursor value = iterator.next();
LongPoint.encodeDimension(value.value, encoded.bytes, 0);
if (needsScore) {
score = joinScorer.apply(value);
score = joinScorer.apply(value.value);
}
return encoded;
} else {
@ -344,10 +342,11 @@ public final class JoinUtil {
@Override
public BytesRef next() {
if (iterator.hasNext()) {
long value = iterator.next();
FloatPoint.encodeDimension(Float.intBitsToFloat((int) value), encoded.bytes, 0);
LongCursor value = iterator.next();
FloatPoint.encodeDimension(
Float.intBitsToFloat((int) value.value), encoded.bytes, 0);
if (needsScore) {
score = joinScorer.apply(value);
score = joinScorer.apply(value.value);
}
return encoded;
} else {
@ -362,10 +361,10 @@ public final class JoinUtil {
@Override
public BytesRef next() {
if (iterator.hasNext()) {
long value = iterator.next();
DoublePoint.encodeDimension(Double.longBitsToDouble(value), encoded.bytes, 0);
LongCursor value = iterator.next();
DoublePoint.encodeDimension(Double.longBitsToDouble(value.value), encoded.bytes, 0);
if (needsScore) {
score = joinScorer.apply(value);
score = joinScorer.apply(value.value);
}
return encoded;
} else {
@ -583,4 +582,10 @@ public final class JoinUtil {
max,
searcher.getTopReaderContext().id());
}
/** Similar to {@link java.util.function.LongFunction} for primitive argument and result. */
private interface LongFloatFunction {
float apply(long value);
}
}

7
lucene/misc/src/java/org/apache/lucene/misc/search/DiversifiedTopDocsCollector.java
View File

@ -19,8 +19,6 @@ package org.apache.lucene.misc.search;
import java.io.IOException;
import java.util.ArrayDeque;
import java.util.Deque;
import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.misc.search.DiversifiedTopDocsCollector.ScoreDocKey;
@ -32,6 +30,7 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopDocsCollector;
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.hppc.LongObjectHashMap;
/**
* A {@link TopDocsCollector} that controls diversity in results by ensuring no more than
@ -69,7 +68,7 @@ public abstract class DiversifiedTopDocsCollector extends TopDocsCollector<Score
ScoreDocKey spare;
private ScoreDocKeyQueue globalQueue;
private int numHits;
private Map<Long, ScoreDocKeyQueue> perKeyQueues;
private LongObjectHashMap<ScoreDocKeyQueue> perKeyQueues;
protected int maxNumPerKey;
private Deque<ScoreDocKeyQueue> sparePerKeyQueues = new ArrayDeque<>();
@ -77,7 +76,7 @@ public abstract class DiversifiedTopDocsCollector extends TopDocsCollector<Score
super(new ScoreDocKeyQueue(numHits));
// Need to access pq.lessThan() which is protected so have to cast here...
this.globalQueue = (ScoreDocKeyQueue) pq;
perKeyQueues = new HashMap<Long, ScoreDocKeyQueue>();
perKeyQueues = new LongObjectHashMap<>();
this.numHits = numHits;
this.maxNumPerKey = maxHitsPerKey;
}

28
lucene/sandbox/src/java/org/apache/lucene/sandbox/search/TermAutomatonQuery.java
View File

@ -54,6 +54,7 @@ import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.Operations;
import org.apache.lucene.util.automaton.Transition;
import org.apache.lucene.util.hppc.IntObjectHashMap;
// TODO
// - compare perf to PhraseQuery exact and sloppy
@ -86,7 +87,7 @@ public class TermAutomatonQuery extends Query implements Accountable {
private final Automaton.Builder builder;
Automaton det;
private final Map<BytesRef, Integer> termToID = new HashMap<>();
private final Map<Integer, BytesRef> idToTerm = new HashMap<>();
private final IntObjectHashMap<BytesRef> idToTerm = new IntObjectHashMap<>();
private int anyTermID = -1;
public TermAutomatonQuery(String field) {
@ -209,7 +210,7 @@ public class TermAutomatonQuery extends Query implements Accountable {
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost)
throws IOException {
Map<Integer, TermStates> termStates = new HashMap<>();
IntObjectHashMap<TermStates> termStates = new IntObjectHashMap<>();
for (Map.Entry<BytesRef, Integer> ent : termToID.entrySet()) {
if (ent.getKey() != null) {
@ -360,14 +361,14 @@ public class TermAutomatonQuery extends Query implements Accountable {
final class TermAutomatonWeight extends Weight {
final Automaton automaton;
private final Map<Integer, TermStates> termStates;
private final IntObjectHashMap<TermStates> termStates;
private final Similarity.SimScorer stats;
private final Similarity similarity;
public TermAutomatonWeight(
Automaton automaton,
IndexSearcher searcher,
Map<Integer, TermStates> termStates,
IntObjectHashMap<TermStates> termStates,
float boost)
throws IOException {
super(TermAutomatonQuery.this);
@ -375,14 +376,13 @@ public class TermAutomatonQuery extends Query implements Accountable {
this.termStates = termStates;
this.similarity = searcher.getSimilarity();
List<TermStatistics> allTermStats = new ArrayList<>();
for (Map.Entry<Integer, BytesRef> ent : idToTerm.entrySet()) {
Integer termID = ent.getKey();
if (ent.getValue() != null) {
TermStates ts = termStates.get(termID);
for (IntObjectHashMap.IntObjectCursor<BytesRef> ent : idToTerm) {
if (ent.value != null) {
TermStates ts = termStates.get(ent.key);
if (ts.docFreq() > 0) {
allTermStats.add(
searcher.termStatistics(
new Term(field, ent.getValue()), ts.docFreq(), ts.totalTermFreq()));
new Term(field, ent.value), ts.docFreq(), ts.totalTermFreq()));
}
}
}
@ -410,18 +410,18 @@ public class TermAutomatonQuery extends Query implements Accountable {
EnumAndScorer[] enums = new EnumAndScorer[idToTerm.size()];
boolean any = false;
for (Map.Entry<Integer, TermStates> ent : termStates.entrySet()) {
TermStates termStates = ent.getValue();
for (IntObjectHashMap.IntObjectCursor<TermStates> ent : termStates) {
TermStates termStates = ent.value;
assert termStates.wasBuiltFor(ReaderUtil.getTopLevelContext(context))
: "The top-reader used to create Weight is not the same as the current reader's top-reader ("
+ ReaderUtil.getTopLevelContext(context);
BytesRef term = idToTerm.get(ent.getKey());
BytesRef term = idToTerm.get(ent.key);
TermState state = termStates.get(context);
if (state != null) {
TermsEnum termsEnum = context.reader().terms(field).iterator();
termsEnum.seekExact(term, state);
enums[ent.getKey()] =
new EnumAndScorer(ent.getKey(), termsEnum.postings(null, PostingsEnum.POSITIONS));
enums[ent.key] =
new EnumAndScorer(ent.key, termsEnum.postings(null, PostingsEnum.POSITIONS));
any = true;
}
}

2
lucene/spatial-extras/build.gradle
View File

@ -30,6 +30,8 @@ dependencies {
moduleApi 'org.locationtech.spatial4j:spatial4j'
moduleApi 'io.sgr:s2-geometry-library-java'
moduleImplementation 'com.carrotsearch:hppc'
moduleTestImplementation project(':lucene:test-framework')
moduleTestImplementation project(':lucene:spatial-test-fixtures')
moduleTestImplementation 'org.locationtech.jts:jts-core'

1
lucene/spatial-extras/src/java/module-info.java
View File

@ -20,6 +20,7 @@
module org.apache.lucene.spatial_extras {
requires spatial4j;
requires s2.geometry.library.java;
requires com.carrotsearch.hppc;
requires org.apache.lucene.core;
requires org.apache.lucene.spatial3d;

15
lucene/spatial-extras/src/java/org/apache/lucene/spatial/util/CachingDoubleValueSource.java
View File

@ -16,13 +16,13 @@
*/
package org.apache.lucene.spatial.util;
import com.carrotsearch.hppc.IntDoubleHashMap;
import java.io.IOException;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.DoubleValues;
import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.util.hppc.IntObjectHashMap;
/**
* Caches the doubleVal of another value source in a HashMap so that it is computed only once.
@ -32,11 +32,11 @@ import org.apache.lucene.util.hppc.IntObjectHashMap;
public class CachingDoubleValueSource extends DoubleValuesSource {
final DoubleValuesSource source;
final IntObjectHashMap<Double> cache;
final IntDoubleHashMap cache;
public CachingDoubleValueSource(DoubleValuesSource source) {
this.source = source;
cache = new IntObjectHashMap<>();
cache = new IntDoubleHashMap();
}
@Override
@ -53,11 +53,14 @@ public class CachingDoubleValueSource extends DoubleValuesSource {
@Override
public double doubleValue() throws IOException {
double v;
int key = base + doc;
Double v = cache.get(key);
if (v == null) {
int index = cache.indexOf(key);
if (index < 0) {
v = vals.doubleValue();
cache.put(key, v);
cache.indexInsert(index, key, v);
} else {
v = cache.indexGet(index);
}
return v;
}