Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5757: move RamUsageEstimator reflector to test-framework 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1602515 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2014-06-13 20:41:17 +00:00
parent f319ebd10c
commit 1ea4ad0b03
22 changed files with 536 additions and 538 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/CHANGES.txt
View File

@ -223,6 +223,9 @@ API Changes
* LUCENE-5695: DocIdSet implements Accountable. (Adrien Grand)
* LUCENE-5757: Moved RamUsageEstimator's reflection-based processing to RamUsageTester
in the test-framework module. (Robert Muir)
Optimizations
* LUCENE-5603: hunspell stemmer more efficiently strips prefixes

19
lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java
View File

@ -24,9 +24,8 @@ import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import org.apache.lucene.analysis.hunspell.Dictionary;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.RamUsageTester;
import org.junit.Ignore;
/**
@ -167,14 +166,14 @@ public class TestAllDictionaries extends LuceneTestCase {
try (InputStream dictionary = zip.getInputStream(dicEntry);
InputStream affix = zip.getInputStream(affEntry)) {
Dictionary dic = new Dictionary(affix, dictionary);
System.out.println(tests[i] + "\t" + RamUsageEstimator.humanSizeOf(dic) + "\t(" +
"words=" + RamUsageEstimator.humanSizeOf(dic.words) + ", " +
"flags=" + RamUsageEstimator.humanSizeOf(dic.flagLookup) + ", " +
"strips=" + RamUsageEstimator.humanSizeOf(dic.stripData) + ", " +
"conditions=" + RamUsageEstimator.humanSizeOf(dic.patterns) + ", " +
"affixData=" + RamUsageEstimator.humanSizeOf(dic.affixData) + ", " +
"prefixes=" + RamUsageEstimator.humanSizeOf(dic.prefixes) + ", " +
"suffixes=" + RamUsageEstimator.humanSizeOf(dic.suffixes) + ")");
System.out.println(tests[i] + "\t" + RamUsageTester.humanSizeOf(dic) + "\t(" +
"words=" + RamUsageTester.humanSizeOf(dic.words) + ", " +
"flags=" + RamUsageTester.humanSizeOf(dic.flagLookup) + ", " +
"strips=" + RamUsageTester.humanSizeOf(dic.stripData) + ", " +
"conditions=" + RamUsageTester.humanSizeOf(dic.patterns) + ", " +
"affixData=" + RamUsageTester.humanSizeOf(dic.affixData) + ", " +
"prefixes=" + RamUsageTester.humanSizeOf(dic.prefixes) + ", " +
"suffixes=" + RamUsageTester.humanSizeOf(dic.suffixes) + ")");
}
}
}

19
lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries2.java
View File

@ -24,9 +24,8 @@ import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import org.apache.lucene.analysis.hunspell.Dictionary;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.RamUsageTester;
import org.junit.Ignore;
/**
@ -183,14 +182,14 @@ public class TestAllDictionaries2 extends LuceneTestCase {
try (InputStream dictionary = zip.getInputStream(dicEntry);
InputStream affix = zip.getInputStream(affEntry)) {
Dictionary dic = new Dictionary(affix, dictionary);
System.out.println(tests[i] + "\t" + RamUsageEstimator.humanSizeOf(dic) + "\t(" +
"words=" + RamUsageEstimator.humanSizeOf(dic.words) + ", " +
"flags=" + RamUsageEstimator.humanSizeOf(dic.flagLookup) + ", " +
"strips=" + RamUsageEstimator.humanSizeOf(dic.stripData) + ", " +
"conditions=" + RamUsageEstimator.humanSizeOf(dic.patterns) + ", " +
"affixData=" + RamUsageEstimator.humanSizeOf(dic.affixData) + ", " +
"prefixes=" + RamUsageEstimator.humanSizeOf(dic.prefixes) + ", " +
"suffixes=" + RamUsageEstimator.humanSizeOf(dic.suffixes) + ")");
System.out.println(tests[i] + "\t" + RamUsageTester.humanSizeOf(dic) + "\t(" +
"words=" + RamUsageTester.humanSizeOf(dic.words) + ", " +
"flags=" + RamUsageTester.humanSizeOf(dic.flagLookup) + ", " +
"strips=" + RamUsageTester.humanSizeOf(dic.stripData) + ", " +
"conditions=" + RamUsageTester.humanSizeOf(dic.patterns) + ", " +
"affixData=" + RamUsageTester.humanSizeOf(dic.affixData) + ", " +
"prefixes=" + RamUsageTester.humanSizeOf(dic.prefixes) + ", " +
"suffixes=" + RamUsageTester.humanSizeOf(dic.suffixes) + ")");
}
}
}

9
lucene/common-build.xml
View File

@ -2214,7 +2214,6 @@ ${ant.project.name}.test.dependencies=${test.classpath.list}
<property name="forbidden-base-excludes" value=""/>
<property name="forbidden-tests-excludes" value=""/>
<property name="forbidden-sysout-excludes" value=""/>
<property name="forbidden-rue-excludes" value=""/>
<target name="-install-forbidden-apis" unless="forbidden-apis.loaded" depends="ivy-availability-check,ivy-configure">
<ivy:cachepath organisation="de.thetaphi" module="forbiddenapis" revision="1.5.1"
@ -2255,13 +2254,7 @@ ${ant.project.name}.test.dependencies=${test.classpath.list}
</target>
<!-- applies to only source code -->
<target name="-check-forbidden-core" depends="-init-forbidden-apis,compile-core,-check-forbidden-sysout,-check-forbidden-rue" />
<target name="-check-forbidden-rue" depends="-init-forbidden-apis,compile-core">
<forbidden-apis signaturesFile="${common.dir}/tools/forbiddenApis/rue.txt" classpathref="forbidden-apis.allclasses.classpath">
<fileset dir="${build.dir}/classes/java" excludes="${forbidden-rue-excludes}"/>
</forbidden-apis>
</target>
<target name="-check-forbidden-core" depends="-init-forbidden-apis,compile-core,-check-forbidden-sysout" />
<target name="-check-forbidden-sysout" depends="-init-forbidden-apis,compile-core">
<forbidden-apis bundledSignatures="jdk-system-out" classpathref="forbidden-apis.allclasses.classpath">

389
lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java
View File

@ -27,7 +27,6 @@ import java.util.*;
/**
* Estimates the size (memory representation) of Java objects.
*
* @see #sizeOf(Object)
* @see #shallowSizeOf(Object)
* @see #shallowSizeOfInstance(Class)
*
@ -235,18 +234,6 @@ public final class RamUsageEstimator {
Constants.JAVA_VENDOR + ", " + Constants.JAVA_VERSION + "]";
}
/**
* Cached information about a given class.
*/
private static final class ClassCache {
public final long alignedShallowInstanceSize;
public final Field[] referenceFields;
public ClassCache(long alignedShallowInstanceSize, Field[] referenceFields) {
this.alignedShallowInstanceSize = alignedShallowInstanceSize;
this.referenceFields = referenceFields;
}
}
// Object with just one field to determine the object header size by getting the offset of the dummy field:
@SuppressWarnings("unused")
@ -320,19 +307,6 @@ public final class RamUsageEstimator {
return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_DOUBLE * arr.length);
}
/**
* Estimates the RAM usage by the given object. It will
* walk the object tree and sum up all referenced objects.
*
* <p><b>Resource Usage:</b> This method internally uses a set of
* every object seen during traversals so it does allocate memory
* (it isn't side-effect free). After the method exits, this memory
* should be GCed.</p>
*/
public static long sizeOf(Object obj) {
return measureObjectSize(obj);
}
/**
* Estimates a "shallow" memory usage of the given object. For arrays, this will be the
* memory taken by array storage (no subreferences will be followed). For objects, this
@ -395,119 +369,6 @@ public final class RamUsageEstimator {
return alignObjectSize(size);
}
/*
* Non-recursive version of object descend. This consumes more memory than recursive in-depth
* traversal but prevents stack overflows on long chains of objects
* or complex graphs (a max. recursion depth on my machine was ~5000 objects linked in a chain
* so not too much).
*/
private static long measureObjectSize(Object root) {
// Objects seen so far.
final IdentityHashSet<Object> seen = new IdentityHashSet<>();
// Class cache with reference Field and precalculated shallow size.
final IdentityHashMap<Class<?>, ClassCache> classCache = new IdentityHashMap<>();
// Stack of objects pending traversal. Recursion caused stack overflows.
final ArrayList<Object> stack = new ArrayList<>();
stack.add(root);
long totalSize = 0;
while (!stack.isEmpty()) {
final Object ob = stack.remove(stack.size() - 1);
if (ob == null || seen.contains(ob)) {
continue;
}
seen.add(ob);
final Class<?> obClazz = ob.getClass();
assert obClazz != null : "jvm bug detected (Object.getClass() == null). please report this to your vendor";
if (obClazz.isArray()) {
/*
* Consider an array, possibly of primitive types. Push any of its references to
* the processing stack and accumulate this array's shallow size.
*/
long size = NUM_BYTES_ARRAY_HEADER;
final int len = Array.getLength(ob);
if (len > 0) {
Class<?> componentClazz = obClazz.getComponentType();
if (componentClazz.isPrimitive()) {
size += (long) len * primitiveSizes.get(componentClazz);
} else {
size += (long) NUM_BYTES_OBJECT_REF * len;
// Push refs for traversal later.
for (int i = len; --i >= 0 ;) {
final Object o = Array.get(ob, i);
if (o != null && !seen.contains(o)) {
stack.add(o);
}
}
}
}
totalSize += alignObjectSize(size);
} else {
/*
* Consider an object. Push any references it has to the processing stack
* and accumulate this object's shallow size.
*/
try {
ClassCache cachedInfo = classCache.get(obClazz);
if (cachedInfo == null) {
classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
}
for (Field f : cachedInfo.referenceFields) {
// Fast path to eliminate redundancies.
final Object o = f.get(ob);
if (o != null && !seen.contains(o)) {
stack.add(o);
}
}
totalSize += cachedInfo.alignedShallowInstanceSize;
} catch (IllegalAccessException e) {
// this should never happen as we enabled setAccessible().
throw new RuntimeException("Reflective field access failed?", e);
}
}
}
// Help the GC (?).
seen.clear();
stack.clear();
classCache.clear();
return totalSize;
}
/**
* Create a cached information about shallow size and reference fields for
* a given class.
*/
private static ClassCache createCacheEntry(final Class<?> clazz) {
ClassCache cachedInfo;
long shallowInstanceSize = NUM_BYTES_OBJECT_HEADER;
final ArrayList<Field> referenceFields = new ArrayList<>(32);
for (Class<?> c = clazz; c != null; c = c.getSuperclass()) {
final Field[] fields = c.getDeclaredFields();
for (final Field f : fields) {
if (!Modifier.isStatic(f.getModifiers())) {
shallowInstanceSize = adjustForField(shallowInstanceSize, f);
if (!f.getType().isPrimitive()) {
f.setAccessible(true);
referenceFields.add(f);
}
}
}
}
cachedInfo = new ClassCache(
alignObjectSize(shallowInstanceSize),
referenceFields.toArray(new Field[referenceFields.size()]));
return cachedInfo;
}
/**
* This method returns the maximum representation size of an object. <code>sizeSoFar</code>
* is the object's size measured so far. <code>f</code> is the field being probed.
@ -515,7 +376,7 @@ public final class RamUsageEstimator {
* <p>The returned offset will be the maximum of whatever was measured so far and
* <code>f</code> field's offset and representation size (unaligned).
*/
private static long adjustForField(long sizeSoFar, final Field f) {
static long adjustForField(long sizeSoFar, final Field f) {
final Class<?> type = f.getType();
final int fsize = type.isPrimitive() ? primitiveSizes.get(type) : NUM_BYTES_OBJECT_REF;
if (objectFieldOffsetMethod != null) {
@ -577,252 +438,4 @@ public final class RamUsageEstimator {
return bytes + " bytes";
}
}
/**
* Return a human-readable size of a given object.
* @see #sizeOf(Object)
* @see #humanReadableUnits(long)
*/
public static String humanSizeOf(Object object) {
return humanReadableUnits(sizeOf(object));
}
/**
* An identity hash set implemented using open addressing. No null keys are allowed.
*
* TODO: If this is useful outside this class, make it public - needs some work
*/
static final class IdentityHashSet<KType> implements Iterable<KType> {
/**
* Default load factor.
*/
public final static float DEFAULT_LOAD_FACTOR = 0.75f;
/**
* Minimum capacity for the set.
*/
public final static int MIN_CAPACITY = 4;
/**
* All of set entries. Always of power of two length.
*/
public Object[] keys;
/**
* Cached number of assigned slots.
*/
public int assigned;
/**
* The load factor for this set (fraction of allocated or deleted slots before
* the buffers must be rehashed or reallocated).
*/
public final float loadFactor;
/**
* Cached capacity threshold at which we must resize the buffers.
*/
private int resizeThreshold;
/**
* Creates a hash set with the default capacity of 16.
* load factor of {@value #DEFAULT_LOAD_FACTOR}. `
*/
public IdentityHashSet() {
this(16, DEFAULT_LOAD_FACTOR);
}
/**
* Creates a hash set with the given capacity, load factor of
* {@value #DEFAULT_LOAD_FACTOR}.
*/
public IdentityHashSet(int initialCapacity) {
this(initialCapacity, DEFAULT_LOAD_FACTOR);
}
/**
* Creates a hash set with the given capacity and load factor.
*/
public IdentityHashSet(int initialCapacity, float loadFactor) {
initialCapacity = Math.max(MIN_CAPACITY, initialCapacity);
assert initialCapacity > 0 : "Initial capacity must be between (0, "
+ Integer.MAX_VALUE + "].";
assert loadFactor > 0 && loadFactor < 1 : "Load factor must be between (0, 1).";
this.loadFactor = loadFactor;
allocateBuffers(roundCapacity(initialCapacity));
}
/**
* Adds a reference to the set. Null keys are not allowed.
*/
public boolean add(KType e) {
assert e != null : "Null keys not allowed.";
if (assigned >= resizeThreshold) expandAndRehash();
final int mask = keys.length - 1;
int slot = rehash(e) & mask;
Object existing;
while ((existing = keys[slot]) != null) {
if (e == existing) {
return false; // already found.
}
slot = (slot + 1) & mask;
}
assigned++;
keys[slot] = e;
return true;
}
/**
* Checks if the set contains a given ref.
*/
public boolean contains(KType e) {
final int mask = keys.length - 1;
int slot = rehash(e) & mask;
Object existing;
while ((existing = keys[slot]) != null) {
if (e == existing) {
return true;
}
slot = (slot + 1) & mask;
}
return false;
}
/** Rehash via MurmurHash.
*
* <p>The implementation is based on the
* finalization step from Austin Appleby's
* <code>MurmurHash3</code>.
*
* @see "http://sites.google.com/site/murmurhash/"
*/
private static int rehash(Object o) {
int k = System.identityHashCode(o);
k ^= k >>> 16;
k *= 0x85ebca6b;
k ^= k >>> 13;
k *= 0xc2b2ae35;
k ^= k >>> 16;
return k;
}
/**
* Expand the internal storage buffers (capacity) or rehash current keys and
* values if there are a lot of deleted slots.
*/
private void expandAndRehash() {
final Object[] oldKeys = this.keys;
assert assigned >= resizeThreshold;
allocateBuffers(nextCapacity(keys.length));
/*
* Rehash all assigned slots from the old hash table.
*/
final int mask = keys.length - 1;
for (int i = 0; i < oldKeys.length; i++) {
final Object key = oldKeys[i];
if (key != null) {
int slot = rehash(key) & mask;
while (keys[slot] != null) {
slot = (slot + 1) & mask;
}
keys[slot] = key;
}
}
Arrays.fill(oldKeys, null);
}
/**
* Allocate internal buffers for a given capacity.
*
* @param capacity
* New capacity (must be a power of two).
*/
private void allocateBuffers(int capacity) {
this.keys = new Object[capacity];
this.resizeThreshold = (int) (capacity * DEFAULT_LOAD_FACTOR);
}
/**
* Return the next possible capacity, counting from the current buffers' size.
*/
protected int nextCapacity(int current) {
assert current > 0 && Long.bitCount(current) == 1 : "Capacity must be a power of two.";
assert ((current << 1) > 0) : "Maximum capacity exceeded ("
+ (0x80000000 >>> 1) + ").";
if (current < MIN_CAPACITY / 2) current = MIN_CAPACITY / 2;
return current << 1;
}
/**
* Round the capacity to the next allowed value.
*/
protected int roundCapacity(int requestedCapacity) {
// Maximum positive integer that is a power of two.
if (requestedCapacity > (0x80000000 >>> 1)) return (0x80000000 >>> 1);
int capacity = MIN_CAPACITY;
while (capacity < requestedCapacity) {
capacity <<= 1;
}
return capacity;
}
public void clear() {
assigned = 0;
Arrays.fill(keys, null);
}
public int size() {
return assigned;
}
public boolean isEmpty() {
return size() == 0;
}
@Override
public Iterator<KType> iterator() {
return new Iterator<KType>() {
int pos = -1;
Object nextElement = fetchNext();
@Override
public boolean hasNext() {
return nextElement != null;
}
@SuppressWarnings("unchecked")
@Override
public KType next() {
Object r = this.nextElement;
if (r == null) {
throw new NoSuchElementException();
}
this.nextElement = fetchNext();
return (KType) r;
}
private Object fetchNext() {
pos++;
while (pos < keys.length && keys[pos] == null) {
pos++;
}
return (pos >= keys.length ? null : keys[pos]);
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
};
}
}
}

2
lucene/core/src/test/org/apache/lucene/util/StressRamUsageEstimator.java
View File

@ -58,7 +58,7 @@ public class StressRamUsageEstimator extends LuceneTestCase {
while (true) {
// Check the current memory consumption and provide the estimate.
long jvmUsed = memoryMXBean.getHeapMemoryUsage().getUsed();
long estimated = RamUsageEstimator.sizeOf(first);
long estimated = RamUsageTester.sizeOf(first);
System.out.println(String.format(Locale.ROOT, "%10d, %10d",
jvmUsed, estimated));

2
lucene/core/src/test/org/apache/lucene/util/TestIdentityHashSet.java
View File

@ -28,7 +28,7 @@ public class TestIdentityHashSet extends LuceneTestCase {
Random rnd = random();
Set<Object> jdk = Collections.newSetFromMap(
new IdentityHashMap<Object,Boolean>());
RamUsageEstimator.IdentityHashSet<Object> us = new RamUsageEstimator.IdentityHashSet<>();
RamUsageTester.IdentityHashSet<Object> us = new RamUsageTester.IdentityHashSet<>();
int max = 100000;
int threshold = 256;

3
lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java
View File

@ -18,11 +18,10 @@ package org.apache.lucene.util;
*/
import static org.apache.lucene.util.RamUsageEstimator.*;
import static org.apache.lucene.util.RamUsageTester.sizeOf;
import java.util.Random;
import com.carrotsearch.randomizedtesting.generators.RandomStrings;
public class TestRamUsageEstimator extends LuceneTestCase {
public void testSanity() {
assertTrue(sizeOf(new String("test string")) > shallowSizeOfInstance(String.class));

2
lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimatorOnWildAnimals.java
View File

@ -40,7 +40,7 @@ public class TestRamUsageEstimatorOnWildAnimals extends LuceneTestCase {
for (int i = 0; i < mid; i++) {
last = (last.next = new ListElement());
}
RamUsageEstimator.sizeOf(first); // cause SOE or pass.
RamUsageTester.sizeOf(first); // cause SOE or pass.
lower = mid;
} catch (StackOverflowError e) {
upper = mid;

11
lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
View File

@ -38,6 +38,7 @@ import org.apache.lucene.util.LongsRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.RamUsageTester;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.packed.PackedInts.Reader;
import org.junit.Ignore;
@ -146,7 +147,7 @@ public class TestPackedInts extends LuceneTestCase {
}
in.close();
final long expectedBytesUsed = RamUsageEstimator.sizeOf(r);
final long expectedBytesUsed = RamUsageTester.sizeOf(r);
final long computedBytesUsed = r.ramBytesUsed();
assertEquals(r.getClass() + "expected " + expectedBytesUsed + ", got: " + computedBytesUsed,
expectedBytesUsed, computedBytesUsed);
@ -687,7 +688,7 @@ public class TestPackedInts extends LuceneTestCase {
assertEquals(10, wrt.get(7));
assertEquals(99, wrt.get(valueCount - 10));
assertEquals(1 << 10, wrt.get(valueCount - 1));
assertEquals(RamUsageEstimator.sizeOf(wrt), wrt.ramBytesUsed());
assertEquals(RamUsageTester.sizeOf(wrt), wrt.ramBytesUsed());
}
public void testPagedGrowableWriter() {
@ -716,7 +717,7 @@ public class TestPackedInts extends LuceneTestCase {
}
// test ramBytesUsed
assertEquals(RamUsageEstimator.sizeOf(writer), writer.ramBytesUsed(), 8);
assertEquals(RamUsageTester.sizeOf(writer), writer.ramBytesUsed(), 8);
// test copy
PagedGrowableWriter copy = writer.resize(TestUtil.nextLong(random(), writer.size() / 2, writer.size() * 3 / 2));
@ -764,7 +765,7 @@ public class TestPackedInts extends LuceneTestCase {
}
// test ramBytesUsed
assertEquals(RamUsageEstimator.sizeOf(writer) - RamUsageEstimator.sizeOf(writer.format), writer.ramBytesUsed());
assertEquals(RamUsageTester.sizeOf(writer) - RamUsageTester.sizeOf(writer.format), writer.ramBytesUsed());
// test copy
PagedMutable copy = writer.resize(TestUtil.nextLong(random(), writer.size() / 2, writer.size() * 3 / 2));
@ -1048,7 +1049,7 @@ public class TestPackedInts extends LuceneTestCase {
}
}
final long expectedBytesUsed = RamUsageEstimator.sizeOf(buf);
final long expectedBytesUsed = RamUsageTester.sizeOf(buf);
final long computedBytesUsed = buf.ramBytesUsed();
assertEquals(expectedBytesUsed, computedBytesUsed);
}

4
lucene/memory/build.xml
View File

@ -19,10 +19,6 @@
<project name="memory" default="default">
<property name="forbidden-rue-excludes" value="
org/apache/lucene/index/memory/MemoryIndex.class
"/>
<description>
Single-document in-memory index implementation
</description>

13
lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
View File

@ -33,7 +33,6 @@ import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
@ -51,7 +50,6 @@ import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
@ -574,15 +572,6 @@ public class MemoryIndex {
*/
}
}
/**
* Returns a reasonable approximation of the main memory [bytes] consumed by
* this instance. Useful for smart memory sensititive caches/pools.
* @return the main memory consumption
*/
public long getMemorySize() {
return RamUsageEstimator.sizeOf(this);
}
/** sorts into ascending order (on demand), reusing memory along the way */
private void sortFields() {
@ -655,7 +644,6 @@ public class MemoryIndex {
result.append("\tterms=" + info.terms.size());
result.append(", positions=" + numPositions);
result.append(", memory=" + RamUsageEstimator.humanReadableUnits(RamUsageEstimator.sizeOf(info)));
result.append("\n");
sumPositions += numPositions;
sumTerms += info.terms.size();
@ -664,7 +652,6 @@ public class MemoryIndex {
result.append("\nfields=" + sortedFields.length);
result.append(", terms=" + sumTerms);
result.append(", positions=" + sumPositions);
result.append(", memory=" + RamUsageEstimator.humanReadableUnits(getMemorySize()));
return result.toString();
}

8
lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
View File

@ -161,14 +161,6 @@ public class MemoryIndexTest extends BaseTokenStreamTestCase {
memory.addField("foo", fooField.toString(), analyzer);
memory.addField("term", termField.toString(), analyzer);
if (VERBOSE) {
System.out.println("Random MemoryIndex:\n" + memory.toString());
System.out.println("Same index as RAMDirectory: " +
RamUsageEstimator.humanReadableUnits(RamUsageEstimator.sizeOf(ramdir)));
System.out.println();
} else {
assertTrue(memory.getMemorySize() > 0L);
}
AtomicReader reader = (AtomicReader) memory.createSearcher().getIndexReader();
DirectoryReader competitor = DirectoryReader.open(ramdir);
duellReaders(competitor, reader);

4
lucene/misc/build.xml
View File

@ -32,10 +32,6 @@
org/apache/lucene/misc/IndexMergeTool.class
"/>
<property name="forbidden-rue-excludes" value="
org/apache/lucene/uninverting/FieldCache$CacheEntry.class
"/>
<import file="../module-build.xml"/>
<target name="install-cpptasks" unless="cpptasks.uptodate" depends="ivy-availability-check,ivy-fail,ivy-configure">

5
lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java
View File

@ -33,6 +33,7 @@ import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.PagedBytes;
@ -110,7 +111,7 @@ import org.apache.lucene.util.StringHelper;
*
*/
public class DocTermOrds {
public class DocTermOrds implements Accountable {
// Term ords are shifted by this, internally, to reserve
// values 0 (end term) and 1 (index is a pointer into byte array)
@ -167,7 +168,7 @@ public class DocTermOrds {
protected DocsEnum docsEnum;
/** Returns total bytes used. */
public long ramUsedInBytes() {
public long ramBytesUsed() {
// can cache the mem size since it shouldn't change
if (memsz!=0) return memsz;
long sz = 8*8 + 32; // local fields

32
lucene/misc/src/java/org/apache/lucene/uninverting/FieldCache.java
View File

@ -34,6 +34,7 @@ import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
@ -54,8 +55,14 @@ interface FieldCache {
/**
* Placeholder indicating creation of this cache is currently in-progress.
*/
public static final class CreationPlaceholder {
Object value;
public static final class CreationPlaceholder implements Accountable {
Accountable value;
@Override
public long ramBytesUsed() {
// don't call on the in-progress value, might make things angry.
return RamUsageEstimator.NUM_BYTES_OBJECT_REF;
}
}
/**
@ -270,13 +277,12 @@ interface FieldCache {
private final String fieldName;
private final Class<?> cacheType;
private final Object custom;
private final Object value;
private String size;
private final Accountable value;
public CacheEntry(Object readerKey, String fieldName,
Class<?> cacheType,
Object custom,
Object value) {
Accountable value) {
this.readerKey = readerKey;
this.fieldName = fieldName;
this.cacheType = cacheType;
@ -304,21 +310,13 @@ interface FieldCache {
return value;
}
/**
* Computes (and stores) the estimated size of the cache Value
* @see #getEstimatedSize
*/
public void estimateSize() {
long bytesUsed = RamUsageEstimator.sizeOf(getValue());
size = RamUsageEstimator.humanReadableUnits(bytesUsed);
}
/**
* The most recently estimated size of the value, null unless
* estimateSize has been called.
*/
public String getEstimatedSize() {
return size;
long bytesUsed = value == null ? 0L : value.ramBytesUsed();
return RamUsageEstimator.humanReadableUnits(bytesUsed);
}
@Override
@ -331,9 +329,7 @@ interface FieldCache {
b.append(System.identityHashCode(getValue()));
String s = getEstimatedSize();
if(null != s) {
b.append(" (size =~ ").append(s).append(')');
}
b.append(" (size =~ ").append(s).append(')');
return b.toString();
}

86
lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheImpl.java
View File

@ -38,10 +38,12 @@ import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.packed.GrowableWriter;
import org.apache.lucene.util.packed.MonotonicAppendingLongBuffer;
import org.apache.lucene.util.packed.PackedInts;
@ -87,11 +89,11 @@ class FieldCacheImpl implements FieldCache {
final Cache cache = cacheEntry.getValue();
final Class<?> cacheType = cacheEntry.getKey();
synchronized(cache.readerCache) {
for (final Map.Entry<Object,Map<CacheKey, Object>> readerCacheEntry : cache.readerCache.entrySet()) {
for (final Map.Entry<Object,Map<CacheKey, Accountable>> readerCacheEntry : cache.readerCache.entrySet()) {
final Object readerKey = readerCacheEntry.getKey();
if (readerKey == null) continue;
final Map<CacheKey, Object> innerCache = readerCacheEntry.getValue();
for (final Map.Entry<CacheKey, Object> mapEntry : innerCache.entrySet()) {
final Map<CacheKey, Accountable> innerCache = readerCacheEntry.getValue();
for (final Map.Entry<CacheKey, Accountable> mapEntry : innerCache.entrySet()) {
CacheKey entry = mapEntry.getKey();
result.add(new CacheEntry(readerKey, entry.field,
cacheType, entry.custom,
@ -124,9 +126,9 @@ class FieldCacheImpl implements FieldCache {
final FieldCacheImpl wrapper;
final Map<Object,Map<CacheKey,Object>> readerCache = new WeakHashMap<>();
final Map<Object,Map<CacheKey,Accountable>> readerCache = new WeakHashMap<>();
protected abstract Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField)
protected abstract Accountable createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField)
throws IOException;
/** Remove this reader from the cache, if present. */
@ -138,10 +140,10 @@ class FieldCacheImpl implements FieldCache {
/** Sets the key to the value for the provided reader;
* if the key is already set then this doesn't change it. */
public void put(AtomicReader reader, CacheKey key, Object value) {
public void put(AtomicReader reader, CacheKey key, Accountable value) {
final Object readerKey = reader.getCoreCacheKey();
synchronized (readerCache) {
Map<CacheKey,Object> innerCache = readerCache.get(readerKey);
Map<CacheKey,Accountable> innerCache = readerCache.get(readerKey);
if (innerCache == null) {
// First time this reader is using FieldCache
innerCache = new HashMap<>();
@ -158,8 +160,8 @@ class FieldCacheImpl implements FieldCache {
}
public Object get(AtomicReader reader, CacheKey key, boolean setDocsWithField) throws IOException {
Map<CacheKey,Object> innerCache;
Object value;
Map<CacheKey,Accountable> innerCache;
Accountable value;
final Object readerKey = reader.getCoreCacheKey();
synchronized (readerCache) {
innerCache = readerCache.get(readerKey);
@ -324,7 +326,7 @@ class FieldCacheImpl implements FieldCache {
} else {
bits = docsWithField;
}
caches.get(DocsWithFieldCache.class).put(reader, new CacheKey(field, null), bits);
caches.get(DocsWithFieldCache.class).put(reader, new CacheKey(field, null), new BitsEntry(bits));
}
private static class HoldsOneThing<T> {
@ -358,7 +360,26 @@ class FieldCacheImpl implements FieldCache {
} else if (!fieldInfo.isIndexed()) {
return new Bits.MatchNoBits(reader.maxDoc());
}
return (Bits) caches.get(DocsWithFieldCache.class).get(reader, new CacheKey(field, null), false);
BitsEntry bitsEntry = (BitsEntry) caches.get(DocsWithFieldCache.class).get(reader, new CacheKey(field, null), false);
return bitsEntry.bits;
}
static class BitsEntry implements Accountable {
final Bits bits;
BitsEntry(Bits bits) {
this.bits = bits;
}
@Override
public long ramBytesUsed() {
long base = RamUsageEstimator.NUM_BYTES_OBJECT_REF;
if (bits instanceof Bits.MatchAllBits || bits instanceof Bits.MatchNoBits) {
return base;
} else {
return base + (bits.length() >>> 3);
}
}
}
static final class DocsWithFieldCache extends Cache {
@ -367,7 +388,7 @@ class FieldCacheImpl implements FieldCache {
}
@Override
protected Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField /* ignored */)
protected BitsEntry createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField /* ignored */)
throws IOException {
final String field = key.field;
final int maxDoc = reader.maxDoc();
@ -380,7 +401,7 @@ class FieldCacheImpl implements FieldCache {
assert termsDocCount <= maxDoc;
if (termsDocCount == maxDoc) {
// Fast case: all docs have this field:
return new Bits.MatchAllBits(maxDoc);
return new BitsEntry(new Bits.MatchAllBits(maxDoc));
}
final TermsEnum termsEnum = terms.iterator(null);
DocsEnum docs = null;
@ -406,15 +427,15 @@ class FieldCacheImpl implements FieldCache {
}
}
if (res == null) {
return new Bits.MatchNoBits(maxDoc);
return new BitsEntry(new Bits.MatchNoBits(maxDoc));
}
final int numSet = res.cardinality();
if (numSet >= maxDoc) {
// The cardinality of the BitSet is maxDoc if all documents have a value.
assert numSet == maxDoc;
return new Bits.MatchAllBits(maxDoc);
return new BitsEntry(new Bits.MatchAllBits(maxDoc));
}
return res;
return new BitsEntry(res);
}
}
@ -441,7 +462,7 @@ class FieldCacheImpl implements FieldCache {
}
}
static class LongsFromArray extends NumericDocValues {
static class LongsFromArray extends NumericDocValues implements Accountable {
private final PackedInts.Reader values;
private final long minValue;
@ -454,6 +475,11 @@ class FieldCacheImpl implements FieldCache {
public long get(int docID) {
return minValue + values.get(docID);
}
@Override
public long ramBytesUsed() {
return values.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.NUM_BYTES_LONG;
}
}
static final class LongCache extends Cache {
@ -462,7 +488,7 @@ class FieldCacheImpl implements FieldCache {
}
@Override
protected Object createValue(final AtomicReader reader, CacheKey key, boolean setDocsWithField)
protected Accountable createValue(final AtomicReader reader, CacheKey key, boolean setDocsWithField)
throws IOException {
final Parser parser = (Parser) key.custom;
@ -523,7 +549,7 @@ class FieldCacheImpl implements FieldCache {
}
}
public static class SortedDocValuesImpl {
public static class SortedDocValuesImpl implements Accountable {
private final PagedBytes.Reader bytes;
private final MonotonicAppendingLongBuffer termOrdToBytesOffset;
private final PackedInts.Reader docToTermOrd;
@ -563,6 +589,15 @@ class FieldCacheImpl implements FieldCache {
}
};
}
@Override
public long ramBytesUsed() {
return bytes.ramBytesUsed() +
termOrdToBytesOffset.ramBytesUsed() +
docToTermOrd.ramBytesUsed() +
3*RamUsageEstimator.NUM_BYTES_OBJECT_REF +
RamUsageEstimator.NUM_BYTES_INT;
}
}
public SortedDocValues getTermsIndex(AtomicReader reader, String field) throws IOException {
@ -597,7 +632,7 @@ class FieldCacheImpl implements FieldCache {
}
@Override
protected Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField /* ignored */)
protected Accountable createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField /* ignored */)
throws IOException {
final int maxDoc = reader.maxDoc();
@ -679,7 +714,7 @@ class FieldCacheImpl implements FieldCache {
}
}
private static class BinaryDocValuesImpl {
private static class BinaryDocValuesImpl implements Accountable {
private final PagedBytes.Reader bytes;
private final PackedInts.Reader docToOffset;
@ -703,6 +738,11 @@ class FieldCacheImpl implements FieldCache {
}
};
}
@Override
public long ramBytesUsed() {
return bytes.ramBytesUsed() + docToOffset.ramBytesUsed() + 2*RamUsageEstimator.NUM_BYTES_OBJECT_REF;
}
}
// TODO: this if DocTermsIndex was already created, we
@ -742,7 +782,7 @@ class FieldCacheImpl implements FieldCache {
}
@Override
protected Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField)
protected Accountable createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField)
throws IOException {
// TODO: would be nice to first check if DocTermsIndex
@ -880,7 +920,7 @@ class FieldCacheImpl implements FieldCache {
}
@Override
protected Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField /* ignored */)
protected Accountable createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField /* ignored */)
throws IOException {
BytesRef prefix = (BytesRef) key.custom;
return new DocTermOrds(reader, null, key.field, prefix);

20
lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheSanityChecker.java
View File

@ -27,7 +27,6 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.uninverting.FieldCache.CacheEntry;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.MapOfSets;
/**
@ -54,20 +53,10 @@ import org.apache.lucene.util.MapOfSets;
*/
final class FieldCacheSanityChecker {
private boolean estimateRam;
public FieldCacheSanityChecker() {
/* NOOP */
}
/**
* If set, estimate size for all CacheEntry objects will be calculateed.
*/
public void setRamUsageEstimator(boolean flag) {
estimateRam = flag;
}
/**
* Quick and dirty convenience method
* @see #check
@ -83,7 +72,6 @@ final class FieldCacheSanityChecker {
*/
public static Insanity[] checkSanity(CacheEntry... cacheEntries) {
FieldCacheSanityChecker sanityChecker = new FieldCacheSanityChecker();
sanityChecker.setRamUsageEstimator(true);
return sanityChecker.check(cacheEntries);
}
@ -99,12 +87,6 @@ final class FieldCacheSanityChecker {
if (null == cacheEntries || 0 == cacheEntries.length)
return new Insanity[0];
if (estimateRam) {
for (int i = 0; i < cacheEntries.length; i++) {
cacheEntries[i].estimateSize();
}
}
// the indirect mapping lets MapOfSet dedup identical valIds for us
//
// maps the (valId) identityhashCode of cache values to
@ -125,7 +107,7 @@ final class FieldCacheSanityChecker {
// It's OK to have dup entries, where one is eg
// float[] and the other is the Bits (from
// getDocWithField())
if (val instanceof Bits) {
if (val instanceof FieldCacheImpl.BitsEntry) {
continue;
}

3
lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
View File

@ -59,6 +59,7 @@ import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.RamUsageTester;
import org.apache.lucene.util.TestUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@ -1566,7 +1567,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest
while (bytesIndexed < bytesToIndex) {
Document doc = docs.nextDoc();
w.addDocument(doc);
bytesIndexed += RamUsageEstimator.sizeOf(doc);
bytesIndexed += RamUsageTester.sizeOf(doc);
}
IndexReader r = w.getReader();

419
lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java Normal file
View File

@ -0,0 +1,419 @@
package org.apache.lucene.util;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.lang.reflect.Array;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.NoSuchElementException;
/** Crawls object graph to collect RAM usage for testing */
public final class RamUsageTester {
/**
* Estimates the RAM usage by the given object. It will
* walk the object tree and sum up all referenced objects.
*
* <p><b>Resource Usage:</b> This method internally uses a set of
* every object seen during traversals so it does allocate memory
* (it isn't side-effect free). After the method exits, this memory
* should be GCed.</p>
*/
public static long sizeOf(Object obj) {
return measureObjectSize(obj);
}
/**
* Return a human-readable size of a given object.
* @see #sizeOf(Object)
* @see RamUsageEstimator#humanReadableUnits(long)
*/
public static String humanSizeOf(Object object) {
return RamUsageEstimator.humanReadableUnits(sizeOf(object));
}
/*
* Non-recursive version of object descend. This consumes more memory than recursive in-depth
* traversal but prevents stack overflows on long chains of objects
* or complex graphs (a max. recursion depth on my machine was ~5000 objects linked in a chain
* so not too much).
*/
private static long measureObjectSize(Object root) {
// Objects seen so far.
final IdentityHashSet<Object> seen = new IdentityHashSet<>();
// Class cache with reference Field and precalculated shallow size.
final IdentityHashMap<Class<?>, ClassCache> classCache = new IdentityHashMap<>();
// Stack of objects pending traversal. Recursion caused stack overflows.
final ArrayList<Object> stack = new ArrayList<>();
stack.add(root);
long totalSize = 0;
while (!stack.isEmpty()) {
final Object ob = stack.remove(stack.size() - 1);
if (ob == null || seen.contains(ob)) {
continue;
}
seen.add(ob);
final Class<?> obClazz = ob.getClass();
assert obClazz != null : "jvm bug detected (Object.getClass() == null). please report this to your vendor";
if (obClazz.isArray()) {
/*
* Consider an array, possibly of primitive types. Push any of its references to
* the processing stack and accumulate this array's shallow size.
*/
long size = RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
final int len = Array.getLength(ob);
if (len > 0) {
Class<?> componentClazz = obClazz.getComponentType();
if (componentClazz.isPrimitive()) {
size += (long) len * RamUsageEstimator.shallowSizeOfInstance(componentClazz);
} else {
size += (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF * len;
// Push refs for traversal later.
for (int i = len; --i >= 0 ;) {
final Object o = Array.get(ob, i);
if (o != null && !seen.contains(o)) {
stack.add(o);
}
}
}
}
totalSize += RamUsageEstimator.alignObjectSize(size);
} else {
/*
* Consider an object. Push any references it has to the processing stack
* and accumulate this object's shallow size.
*/
try {
ClassCache cachedInfo = classCache.get(obClazz);
if (cachedInfo == null) {
classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
}
for (Field f : cachedInfo.referenceFields) {
// Fast path to eliminate redundancies.
final Object o = f.get(ob);
if (o != null && !seen.contains(o)) {
stack.add(o);
}
}
totalSize += cachedInfo.alignedShallowInstanceSize;
} catch (IllegalAccessException e) {
// this should never happen as we enabled setAccessible().
throw new RuntimeException("Reflective field access failed?", e);
}
}
}
// Help the GC (?).
seen.clear();
stack.clear();
classCache.clear();
return totalSize;
}
/**
* Cached information about a given class.
*/
private static final class ClassCache {
public final long alignedShallowInstanceSize;
public final Field[] referenceFields;
public ClassCache(long alignedShallowInstanceSize, Field[] referenceFields) {
this.alignedShallowInstanceSize = alignedShallowInstanceSize;
this.referenceFields = referenceFields;
}
}
/**
* Create a cached information about shallow size and reference fields for
* a given class.
*/
private static ClassCache createCacheEntry(final Class<?> clazz) {
ClassCache cachedInfo;
long shallowInstanceSize = RamUsageEstimator.NUM_BYTES_OBJECT_HEADER;
final ArrayList<Field> referenceFields = new ArrayList<>(32);
for (Class<?> c = clazz; c != null; c = c.getSuperclass()) {
final Field[] fields = c.getDeclaredFields();
for (final Field f : fields) {
if (!Modifier.isStatic(f.getModifiers())) {
shallowInstanceSize = RamUsageEstimator.adjustForField(shallowInstanceSize, f);
if (!f.getType().isPrimitive()) {
f.setAccessible(true);
referenceFields.add(f);
}
}
}
}
cachedInfo = new ClassCache(
RamUsageEstimator.alignObjectSize(shallowInstanceSize),
referenceFields.toArray(new Field[referenceFields.size()]));
return cachedInfo;
}
/**
* An identity hash set implemented using open addressing. No null keys are allowed.
*
* TODO: If this is useful outside this class, make it public - needs some work
*/
static final class IdentityHashSet<KType> implements Iterable<KType> {
/**
* Default load factor.
*/
public final static float DEFAULT_LOAD_FACTOR = 0.75f;
/**
* Minimum capacity for the set.
*/
public final static int MIN_CAPACITY = 4;
/**
* All of set entries. Always of power of two length.
*/
public Object[] keys;
/**
* Cached number of assigned slots.
*/
public int assigned;
/**
* The load factor for this set (fraction of allocated or deleted slots before
* the buffers must be rehashed or reallocated).
*/
public final float loadFactor;
/**
* Cached capacity threshold at which we must resize the buffers.
*/
private int resizeThreshold;
/**
* Creates a hash set with the default capacity of 16.
* load factor of {@value #DEFAULT_LOAD_FACTOR}. `
*/
public IdentityHashSet() {
this(16, DEFAULT_LOAD_FACTOR);
}
/**
* Creates a hash set with the given capacity, load factor of
* {@value #DEFAULT_LOAD_FACTOR}.
*/
public IdentityHashSet(int initialCapacity) {
this(initialCapacity, DEFAULT_LOAD_FACTOR);
}
/**
* Creates a hash set with the given capacity and load factor.
*/
public IdentityHashSet(int initialCapacity, float loadFactor) {
initialCapacity = Math.max(MIN_CAPACITY, initialCapacity);
assert initialCapacity > 0 : "Initial capacity must be between (0, "
+ Integer.MAX_VALUE + "].";
assert loadFactor > 0 && loadFactor < 1 : "Load factor must be between (0, 1).";
this.loadFactor = loadFactor;
allocateBuffers(roundCapacity(initialCapacity));
}
/**
* Adds a reference to the set. Null keys are not allowed.
*/
public boolean add(KType e) {
assert e != null : "Null keys not allowed.";
if (assigned >= resizeThreshold) expandAndRehash();
final int mask = keys.length - 1;
int slot = rehash(e) & mask;
Object existing;
while ((existing = keys[slot]) != null) {
if (e == existing) {
return false; // already found.
}
slot = (slot + 1) & mask;
}
assigned++;
keys[slot] = e;
return true;
}
/**
* Checks if the set contains a given ref.
*/
public boolean contains(KType e) {
final int mask = keys.length - 1;
int slot = rehash(e) & mask;
Object existing;
while ((existing = keys[slot]) != null) {
if (e == existing) {
return true;
}
slot = (slot + 1) & mask;
}
return false;
}
/** Rehash via MurmurHash.
*
* <p>The implementation is based on the
* finalization step from Austin Appleby's
* <code>MurmurHash3</code>.
*
* @see "http://sites.google.com/site/murmurhash/"
*/
private static int rehash(Object o) {
int k = System.identityHashCode(o);
k ^= k >>> 16;
k *= 0x85ebca6b;
k ^= k >>> 13;
k *= 0xc2b2ae35;
k ^= k >>> 16;
return k;
}
/**
* Expand the internal storage buffers (capacity) or rehash current keys and
* values if there are a lot of deleted slots.
*/
private void expandAndRehash() {
final Object[] oldKeys = this.keys;
assert assigned >= resizeThreshold;
allocateBuffers(nextCapacity(keys.length));
/*
* Rehash all assigned slots from the old hash table.
*/
final int mask = keys.length - 1;
for (int i = 0; i < oldKeys.length; i++) {
final Object key = oldKeys[i];
if (key != null) {
int slot = rehash(key) & mask;
while (keys[slot] != null) {
slot = (slot + 1) & mask;
}
keys[slot] = key;
}
}
Arrays.fill(oldKeys, null);
}
/**
* Allocate internal buffers for a given capacity.
*
* @param capacity
* New capacity (must be a power of two).
*/
private void allocateBuffers(int capacity) {
this.keys = new Object[capacity];
this.resizeThreshold = (int) (capacity * DEFAULT_LOAD_FACTOR);
}
/**
* Return the next possible capacity, counting from the current buffers' size.
*/
protected int nextCapacity(int current) {
assert current > 0 && Long.bitCount(current) == 1 : "Capacity must be a power of two.";
assert ((current << 1) > 0) : "Maximum capacity exceeded ("
+ (0x80000000 >>> 1) + ").";
if (current < MIN_CAPACITY / 2) current = MIN_CAPACITY / 2;
return current << 1;
}
/**
* Round the capacity to the next allowed value.
*/
protected int roundCapacity(int requestedCapacity) {
// Maximum positive integer that is a power of two.
if (requestedCapacity > (0x80000000 >>> 1)) return (0x80000000 >>> 1);
int capacity = MIN_CAPACITY;
while (capacity < requestedCapacity) {
capacity <<= 1;
}
return capacity;
}
public void clear() {
assigned = 0;
Arrays.fill(keys, null);
}
public int size() {
return assigned;
}
public boolean isEmpty() {
return size() == 0;
}
@Override
public Iterator<KType> iterator() {
return new Iterator<KType>() {
int pos = -1;
Object nextElement = fetchNext();
@Override
public boolean hasNext() {
return nextElement != null;
}
@SuppressWarnings("unchecked")
@Override
public KType next() {
Object r = this.nextElement;
if (r == null) {
throw new NoSuchElementException();
}
this.nextElement = fetchNext();
return (KType) r;
}
private Object fetchNext() {
pos++;
while (pos < keys.length && keys[pos] == null) {
pos++;
}
return (pos >= keys.length ? null : keys[pos]);
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
};
}
}
}

19
lucene/tools/forbiddenApis/rue.txt
View File

@ -1,19 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
@defaultMessage This method is useful for testing but is slow at runtime
org.apache.lucene.util.RamUsageEstimator#sizeOf(java.lang.Object)
org.apache.lucene.util.RamUsageEstimator#humanSizeOf(java.lang.Object)

2
solr/core/src/java/org/apache/solr/request/UnInvertedField.java
View File

@ -156,7 +156,7 @@ public class UnInvertedField extends DocTermOrds {
public long memSize() {
// can cache the mem size since it shouldn't change
if (memsz!=0) return memsz;
long sz = super.ramUsedInBytes();
long sz = super.ramBytesUsed();
sz += 8*8 + 32; // local fields
sz += bigTerms.size() * 64;
for (TopTerm tt : bigTerms.values()) {