Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-10182: Be specific about which sizeOf() is called; rename RamUsageTester.sizeOf to ramUsed (#386) 

Co-authored-by: Stefan Vodita <voditas@amazon.com>
This commit is contained in:
Stefan Vodita 2021-10-19 01:13:32 +03:00 committed by GitHub
parent a93dfe93c9
commit d9e3d99ec9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 62 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lucene/CHANGES.txt
View File

@ -167,6 +167,11 @@ API Changes
Improvements
---------------------
* LUCENE-10182: TestRamUsageEstimator used RamUsageTester.sizeOf throughout, making some of the
tests trivial. Now, it compares results from RamUsageEstimator with those from RamUsageTester.
To prevent this error in the future, RamUsageTester.sizeOf was renamed to ramUsed.
(Uwe Schindler, Dawid Weiss, Stefan Vodita)
* LUCENE-10129: RamUsageEstimator overloads the shallowSizeOf method for primitive arrays
to avoid falling back on shallowSizeOf(Object), which could lead to performance traps.
(Robert Muir, Uwe Schindler, Stefan Vodita)

4
lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java
View File

@ -159,8 +159,8 @@ public class TestAllDictionaries extends LuceneTestCase {
(Path aff) -> {
try {
Dictionary dic = loadDictionary(aff);
totalMemory.addAndGet(RamUsageTester.sizeOf(dic));
totalWords.addAndGet(RamUsageTester.sizeOf(dic.words));
totalMemory.addAndGet(RamUsageTester.ramUsed(dic));
totalWords.addAndGet(RamUsageTester.ramUsed(dic.words));
System.out.println(aff + "\t" + memoryUsageSummary(dic));
} catch (Throwable e) {
failures.add(aff);

2
lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/packed/TestLegacyPackedInts.java
View File

@ -85,7 +85,7 @@ public class TestLegacyPackedInts extends LuceneTestCase {
r.get(i));
}
in.close();
final long expectedBytesUsed = RamUsageTester.sizeOf(r);
final long expectedBytesUsed = RamUsageTester.ramUsed(r);
final long computedBytesUsed = r.ramBytesUsed();
assertEquals(
r.getClass() + "expected " + expectedBytesUsed + ", got: " + computedBytesUsed,

4
lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java
View File

@ -93,12 +93,12 @@ public class TestOrdinalMap extends LuceneTestCase {
SortedDocValues sdv = MultiDocValues.getSortedValues(r, "sdv");
if (sdv instanceof MultiDocValues.MultiSortedDocValues) {
OrdinalMap map = ((MultiDocValues.MultiSortedDocValues) sdv).mapping;
assertEquals(RamUsageTester.sizeOf(map, ORDINAL_MAP_ACCUMULATOR), map.ramBytesUsed());
assertEquals(RamUsageTester.ramUsed(map, ORDINAL_MAP_ACCUMULATOR), map.ramBytesUsed());
}
SortedSetDocValues ssdv = MultiDocValues.getSortedSetValues(r, "ssdv");
if (ssdv instanceof MultiDocValues.MultiSortedSetDocValues) {
OrdinalMap map = ((MultiDocValues.MultiSortedSetDocValues) ssdv).mapping;
assertEquals(RamUsageTester.sizeOf(map, ORDINAL_MAP_ACCUMULATOR), map.ramBytesUsed());
assertEquals(RamUsageTester.ramUsed(map, ORDINAL_MAP_ACCUMULATOR), map.ramBytesUsed());
}
iw.close();
r.close();

4
lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
View File

@ -418,7 +418,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
}
}
queryCache.assertConsistent();
assertEquals(RamUsageTester.sizeOf(queryCache, acc), queryCache.ramBytesUsed());
assertEquals(RamUsageTester.ramUsed(queryCache, acc), queryCache.ramBytesUsed());
}
w.close();
@ -523,7 +523,7 @@ public class TestLRUQueryCache extends LuceneTestCase {
}
assertTrue(queryCache.getCacheCount() > 0);
final long actualRamBytesUsed = RamUsageTester.sizeOf(queryCache, acc);
final long actualRamBytesUsed = RamUsageTester.ramUsed(queryCache, acc);
final long expectedRamBytesUsed = queryCache.ramBytesUsed();
// error < 30%
assertEquals(actualRamBytesUsed, expectedRamBytesUsed, 30 * actualRamBytesUsed / 100);

2
lucene/core/src/test/org/apache/lucene/search/TestTermInSetQuery.java
View File

@ -183,7 +183,7 @@ public class TestTermInSetQuery extends LuceneTestCase {
terms.add(newBytesRef(RandomStrings.randomUnicodeOfLength(random(), 10)));
}
TermInSetQuery query = new TermInSetQuery("f", terms);
final long actualRamBytesUsed = RamUsageTester.sizeOf(query);
final long actualRamBytesUsed = RamUsageTester.ramUsed(query);
final long expectedRamBytesUsed = query.ramBytesUsed();
// error margin within 5%
assertEquals(expectedRamBytesUsed, actualRamBytesUsed, actualRamBytesUsed / 20);

2
lucene/core/src/test/org/apache/lucene/util/TestFrequencyTrackingRingBuffer.java
View File

@ -72,6 +72,6 @@ public class TestFrequencyTrackingRingBuffer extends LuceneTestCase {
for (int i = 0; i < 10000; ++i) {
buffer.add(random().nextInt());
}
assertEquals(RamUsageTester.sizeOf(buffer), buffer.ramBytesUsed());
assertEquals(RamUsageTester.ramUsed(buffer), buffer.ramBytesUsed());
}
}

6
lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java
View File

@ -218,9 +218,9 @@ public class TestPagedBytes extends LuceneTestCase {
BytesRef bytes = newBytesRef(TestUtil.randomSimpleString(random(), 10));
pointer = b.copyUsingLengthPrefix(bytes);
}
assertEquals(RamUsageTester.sizeOf(b), b.ramBytesUsed());
assertEquals(RamUsageTester.ramUsed(b), b.ramBytesUsed());
final PagedBytes.Reader reader = b.freeze(random().nextBoolean());
assertEquals(RamUsageTester.sizeOf(b), b.ramBytesUsed());
assertEquals(RamUsageTester.sizeOf(reader), reader.ramBytesUsed());
assertEquals(RamUsageTester.ramUsed(b), b.ramBytesUsed());
assertEquals(RamUsageTester.ramUsed(reader), reader.ramBytesUsed());
}
}

72
lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java
View File

@ -16,18 +16,8 @@
*/
package org.apache.lucene.util;
import static org.apache.lucene.util.RamUsageEstimator.COMPRESSED_REFS_ENABLED;
import static org.apache.lucene.util.RamUsageEstimator.HOTSPOT_BEAN_CLASS;
import static org.apache.lucene.util.RamUsageEstimator.JVM_IS_HOTSPOT_64BIT;
import static org.apache.lucene.util.RamUsageEstimator.LONG_SIZE;
import static org.apache.lucene.util.RamUsageEstimator.MANAGEMENT_FACTORY_CLASS;
import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_ALIGNMENT;
import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_HEADER;
import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOf;
import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance;
import static org.apache.lucene.util.RamUsageTester.sizeOf;
import static org.apache.lucene.util.RamUsageEstimator.*;
import static org.apache.lucene.util.RamUsageTester.ramUsed;
import java.util.ArrayList;
import java.util.Arrays;
@ -47,73 +37,73 @@ public class TestRamUsageEstimator extends LuceneTestCase {
static final String[] strings = new String[] {"test string", "hollow", "catchmaster"};
public void testSanity() {
assertTrue(sizeOf("test string") > shallowSizeOfInstance(String.class));
assertTrue(ramUsed("test string") > shallowSizeOfInstance(String.class));
Holder holder = new Holder();
holder.holder = new Holder("string2", 5000L);
assertTrue(sizeOf(holder) > shallowSizeOfInstance(Holder.class));
assertTrue(sizeOf(holder) > sizeOf(holder.holder));
assertTrue(ramUsed(holder) > shallowSizeOfInstance(Holder.class));
assertTrue(ramUsed(holder) > ramUsed(holder.holder));
assertTrue(shallowSizeOfInstance(HolderSubclass.class) >= shallowSizeOfInstance(Holder.class));
assertTrue(shallowSizeOfInstance(Holder.class) == shallowSizeOfInstance(HolderSubclass2.class));
assertTrue(sizeOf(strings) > shallowSizeOf(strings));
assertTrue(ramUsed(strings) > shallowSizeOf(strings));
}
public void testStaticOverloads() {
Random rnd = random();
{
byte[] array = new byte[rnd.nextInt(1024)];
assertEquals(sizeOf(array), sizeOf((Object) array));
assertEquals(shallowSizeOf(array), sizeOf((Object) array));
assertEquals(sizeOf(array), ramUsed(array));
assertEquals(shallowSizeOf(array), ramUsed(array));
}
{
boolean[] array = new boolean[rnd.nextInt(1024)];
assertEquals(sizeOf(array), sizeOf((Object) array));
assertEquals(shallowSizeOf(array), sizeOf((Object) array));
assertEquals(sizeOf(array), ramUsed(array));
assertEquals(shallowSizeOf(array), ramUsed(array));
}
{
char[] array = new char[rnd.nextInt(1024)];
assertEquals(sizeOf(array), sizeOf((Object) array));
assertEquals(shallowSizeOf(array), sizeOf((Object) array));
assertEquals(sizeOf(array), ramUsed(array));
assertEquals(shallowSizeOf(array), ramUsed(array));
}
{
short[] array = new short[rnd.nextInt(1024)];
assertEquals(sizeOf(array), sizeOf((Object) array));
assertEquals(shallowSizeOf(array), sizeOf((Object) array));
assertEquals(sizeOf(array), ramUsed(array));
assertEquals(shallowSizeOf(array), ramUsed(array));
}
{
int[] array = new int[rnd.nextInt(1024)];
assertEquals(sizeOf(array), sizeOf((Object) array));
assertEquals(shallowSizeOf(array), sizeOf((Object) array));
assertEquals(sizeOf(array), ramUsed(array));
assertEquals(shallowSizeOf(array), ramUsed(array));
}
{
float[] array = new float[rnd.nextInt(1024)];
assertEquals(sizeOf(array), sizeOf((Object) array));
assertEquals(shallowSizeOf(array), sizeOf((Object) array));
assertEquals(sizeOf(array), ramUsed(array));
assertEquals(shallowSizeOf(array), ramUsed(array));
}
{
long[] array = new long[rnd.nextInt(1024)];
assertEquals(sizeOf(array), sizeOf((Object) array));
assertEquals(shallowSizeOf(array), sizeOf((Object) array));
assertEquals(sizeOf(array), ramUsed(array));
assertEquals(shallowSizeOf(array), ramUsed(array));
}
{
double[] array = new double[rnd.nextInt(1024)];
assertEquals(sizeOf(array), sizeOf((Object) array));
assertEquals(shallowSizeOf(array), sizeOf((Object) array));
assertEquals(sizeOf(array), ramUsed(array));
assertEquals(shallowSizeOf(array), ramUsed(array));
}
}
public void testStrings() {
long actual = sizeOf(strings);
long estimated = RamUsageEstimator.sizeOf(strings);
long actual = ramUsed(strings);
long estimated = sizeOf(strings);
assertEquals(actual, estimated);
}
@ -123,8 +113,8 @@ public class TestRamUsageEstimator extends LuceneTestCase {
bytes.add(new BytesRef("foo bar " + i));
bytes.add(new BytesRef("baz bam " + i));
}
long actual = sizeOf(bytes);
long estimated = RamUsageEstimator.sizeOf(bytes);
long actual = ramUsed(bytes);
long estimated = sizeOf(bytes);
assertEquals((double) actual, (double) estimated, (double) actual * 0.1);
}
@ -137,13 +127,13 @@ public class TestRamUsageEstimator extends LuceneTestCase {
map.put("complex " + i, new Term("foo " + i, "bar " + i));
}
double errorFactor = COMPRESSED_REFS_ENABLED ? 0.2 : 0.3;
long actual = sizeOf(map);
long actual = ramUsed(map);
long estimated = RamUsageEstimator.sizeOfObject(map);
assertEquals((double) actual, (double) estimated, (double) actual * errorFactor);
// test recursion
map.put("self", map);
actual = sizeOf(map);
actual = ramUsed(map);
estimated = RamUsageEstimator.sizeOfObject(map);
assertEquals((double) actual, (double) estimated, (double) actual * errorFactor);
}
@ -155,13 +145,13 @@ public class TestRamUsageEstimator extends LuceneTestCase {
for (int i = 0; i < 100; i++) {
list.add(new Term("foo " + i, "term " + i));
}
long actual = sizeOf(list);
long actual = ramUsed(list);
long estimated = RamUsageEstimator.sizeOfObject(list);
assertEquals((double) actual, (double) estimated, (double) actual * 0.1);
// test recursion
list.add(list);
actual = sizeOf(list);
actual = ramUsed(list);
estimated = RamUsageEstimator.sizeOfObject(list);
assertEquals((double) actual, (double) estimated, (double) actual * 0.1);
}
@ -180,7 +170,7 @@ public class TestRamUsageEstimator extends LuceneTestCase {
BooleanClause.Occur.MUST_NOT)
.add(dismax, BooleanClause.Occur.MUST)
.build();
long actual = sizeOf(bq);
long actual = ramUsed(bq);
long estimated = RamUsageEstimator.sizeOfObject(bq);
// sizeOfObject uses much lower default size estimate than we normally use
// but the query-specific default is so large that the comparison becomes meaningless.

10
lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
View File

@ -667,7 +667,7 @@ public class TestPackedInts extends LuceneTestCase {
assertEquals(10, wrt.get(7));
assertEquals(99, wrt.get(valueCount - 10));
assertEquals(1 << 10, wrt.get(valueCount - 1));
assertEquals(RamUsageTester.sizeOf(wrt), wrt.ramBytesUsed());
assertEquals(RamUsageTester.ramUsed(wrt), wrt.ramBytesUsed());
}
public void testPagedGrowableWriter() {
@ -703,7 +703,7 @@ public class TestPackedInts extends LuceneTestCase {
}
// test ramBytesUsed
assertEquals(RamUsageTester.sizeOf(writer), writer.ramBytesUsed(), 8);
assertEquals(RamUsageTester.ramUsed(writer), writer.ramBytesUsed(), 8);
// test copy
PagedGrowableWriter copy =
@ -756,7 +756,7 @@ public class TestPackedInts extends LuceneTestCase {
// test ramBytesUsed
assertEquals(
RamUsageTester.sizeOf(writer) - RamUsageTester.sizeOf(writer.format),
RamUsageTester.ramUsed(writer) - RamUsageTester.ramUsed(writer.format),
writer.ramBytesUsed());
// test copy
@ -1017,7 +1017,7 @@ public class TestPackedInts extends LuceneTestCase {
for (int i = 0; i < arr.length; ++i) {
buf.add(arr[i]);
if (rarely() && !TEST_NIGHTLY) {
final long expectedBytesUsed = RamUsageTester.sizeOf(buf);
final long expectedBytesUsed = RamUsageTester.ramUsed(buf);
final long computedBytesUsed = buf.ramBytesUsed();
assertEquals(expectedBytesUsed, computedBytesUsed);
}
@ -1044,7 +1044,7 @@ public class TestPackedInts extends LuceneTestCase {
}
assertFalse(it.hasNext());
final long expectedBytesUsed = RamUsageTester.sizeOf(values);
final long expectedBytesUsed = RamUsageTester.ramUsed(values);
final long computedBytesUsed = values.ramBytesUsed();
assertEquals(expectedBytesUsed, computedBytesUsed);
}

2
lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
View File

@ -594,7 +594,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest
Document justBodyDoc = new Document();
justBodyDoc.add(doc.getField("body"));
w.addDocument(justBodyDoc);
bytesIndexed += RamUsageTester.sizeOf(justBodyDoc);
bytesIndexed += RamUsageTester.ramUsed(justBodyDoc);
}
IndexReader r = w.getReader();

4
lucene/test-framework/src/java/org/apache/lucene/util/BaseDocIdSetTestCase.java
View File

@ -190,9 +190,9 @@ public abstract class BaseDocIdSetTestCase<T extends DocIdSet> extends LuceneTes
Dummy dummy = new Dummy();
dummy.o1 = copyOf(new BitSet(length), length);
dummy.o2 = set;
long bytes1 = RamUsageTester.sizeOf(dummy);
long bytes1 = RamUsageTester.ramUsed(dummy);
dummy.o2 = null;
long bytes2 = RamUsageTester.sizeOf(dummy);
long bytes2 = RamUsageTester.ramUsed(dummy);
return bytes1 - bytes2;
}
}

10
lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
View File

@ -82,23 +82,23 @@ public final class RamUsageTester {
* traversals so it does allocate memory (it isn't side-effect free). After the method exits, this
* memory should be GCed.
*/
public static long sizeOf(Object obj, Accumulator accumulator) {
public static long ramUsed(Object obj, Accumulator accumulator) {
return measureObjectSize(obj, accumulator);
}
/** Same as calling <code>sizeOf(obj, DEFAULT_FILTER)</code>. */
public static long sizeOf(Object obj) {
return sizeOf(obj, new Accumulator());
public static long ramUsed(Object obj) {
return ramUsed(obj, new Accumulator());
}
/**
* Return a human-readable size of a given object.
*
* @see #sizeOf(Object)
* @see #ramUsed(Object)
* @see RamUsageEstimator#humanReadableUnits(long)
*/
public static String humanSizeOf(Object object) {
return RamUsageEstimator.humanReadableUnits(sizeOf(object));
return RamUsageEstimator.humanReadableUnits(ramUsed(object));
}
/*

2
lucene/test-framework/src/test/org/apache/lucene/util/TestRamUsageTesterOnWildAnimals.java
View File

@ -37,7 +37,7 @@ public class TestRamUsageTesterOnWildAnimals extends LuceneTestCase {
for (int i = 0; i < mid; i++) {
last = (last.next = new ListElement());
}
RamUsageTester.sizeOf(first); // cause SOE or pass.
RamUsageTester.ramUsed(first); // cause SOE or pass.
lower = mid;
} catch (
@SuppressWarnings("unused")