Apache/lucene
1
0
Fork 0
mirror of https://github.com/apache/lucene.git synced 2025-02-28 21:39:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

LUCENE-8642: RamUsageTester.sizeOf ignores arrays and collections if --illegal-access=deny.

Browse Source
This commit is contained in:
Dawid Weiss 2019-01-18 11:55:53 +01:00
parent 75a7827bf4
commit efef89adc6
3 changed files with 122 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java
View File

@ -173,7 +173,6 @@ public class TermInSetQueryTest extends LuceneTestCase {
QueryUtils.checkEqual(query1, query2);
}
@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-8641")
public void testRamBytesUsed() {
List<BytesRef> terms = new ArrayList<>();
final int numTerms = 10000 + random().nextInt(1000);

27
lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java
View File

@ -27,6 +27,7 @@ import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
@ -289,12 +290,26 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
new SimpleMergedSegmentWarmer(InfoStream.NO_OUTPUT).warm(reader);
}
final long actualBytes = RamUsageTester.sizeOf(reader2, new Accumulator(reader2)) - RamUsageTester.sizeOf(reader1, new Accumulator(reader1));
final long expectedBytes = ((SegmentReader) reader2).ramBytesUsed() - ((SegmentReader) reader1).ramBytesUsed();
final long absoluteError = actualBytes - expectedBytes;
final double relativeError = (double) absoluteError / actualBytes;
final String message = "Actual RAM usage " + actualBytes + ", but got " + expectedBytes + ", " + 100*relativeError + "% error";
assertTrue(message, Math.abs(relativeError) < 0.20d || Math.abs(absoluteError) < 1000);
long act1 = RamUsageTester.sizeOf(reader2, new Accumulator(reader2));
long act2 = RamUsageTester.sizeOf(reader1, new Accumulator(reader1));
final long measuredBytes = act1 - act2;
long reported1 = ((SegmentReader) reader2).ramBytesUsed();
long reported2 = ((SegmentReader) reader1).ramBytesUsed();
final long reportedBytes = reported1 - reported2;
final long absoluteError = Math.abs(measuredBytes - reportedBytes);
final double relativeError = (double) absoluteError / measuredBytes;
final String message = String.format(Locale.ROOT,
"RamUsageTester reports %d bytes but ramBytesUsed() returned %d (%.1f error). " +
" [Measured: %d, %d. Reported: %d, %d]",
measuredBytes,
reportedBytes,
(100 * relativeError),
act1, act2,
reported1, reported2);
assertTrue(message, relativeError < 0.20d || absoluteError < 1000);
reader1.close();
reader2.close();

182
lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
View File

@ -32,8 +32,10 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.function.Predicate;
import java.util.function.ToLongFunction;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -112,80 +114,18 @@ public final class RamUsageTester {
}
seen.add(ob);
final long obSize;
final Class<?> obClazz = ob.getClass();
assert obClazz != null : "jvm bug detected (Object.getClass() == null). please report this to your vendor";
if (obClazz.isArray()) {
/*
* Consider an array, possibly of primitive types. Push any of its references to
* the processing stack and accumulate this array's shallow size.
*/
final long shallowSize = RamUsageEstimator.shallowSizeOf(ob);
final int len = Array.getLength(ob);
final List<Object> values;
Class<?> componentClazz = obClazz.getComponentType();
if (componentClazz.isPrimitive()) {
values = Collections.emptyList();
} else {
values = new AbstractList<Object>() {
@Override
public Object get(int index) {
return Array.get(ob, index);
}
@Override
public int size() {
return len;
}
};
}
totalSize += accumulator.accumulateArray(ob, shallowSize, values, stack);
obSize = handleArray(accumulator, stack, ob, obClazz);
} else {
/*
* Consider an object. Push any references it has to the processing stack
* and accumulate this object's shallow size.
*/
try {
ClassCache cachedInfo = classCache.get(obClazz);
if (cachedInfo == null) {
classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
}
boolean needsReflection = true;
if (Constants.JRE_IS_MINIMUM_JAVA9 && obClazz.getName().startsWith("java.")) {
// Java 9: Best guess for some known types, as we cannot precisely look into runtime classes:
final ToLongFunction<Object> func = SIMPLE_TYPES.get(obClazz);
if (func != null) { // some simple type like String where the size is easy to get from public properties
totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize + func.applyAsLong(ob),
Collections.emptyMap(), stack);
needsReflection = false;
} else if (ob instanceof Iterable) {
final List<Object> values = StreamSupport.stream(((Iterable<?>) ob).spliterator(), false)
.collect(Collectors.toList());
totalSize += accumulator.accumulateArray(ob, cachedInfo.alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
needsReflection = false;
} else if (ob instanceof Map) {
final List<Object> values = ((Map<?,?>) ob).entrySet().stream()
.flatMap(e -> Stream.of(e.getKey(), e.getValue()))
.collect(Collectors.toList());
totalSize += accumulator.accumulateArray(ob, cachedInfo.alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
totalSize += RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
needsReflection = false;
}
}
if (needsReflection) {
final Map<Field, Object> fieldValues = new HashMap<>();
for (Field f : cachedInfo.referenceFields) {
fieldValues.put(f, f.get(ob));
}
totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize, fieldValues, stack);
}
} catch (IllegalAccessException e) {
// this should never happen as we enabled setAccessible().
throw new RuntimeException("Reflective field access failed?", e);
}
obSize = handleOther(accumulator, classCache, stack, ob, obClazz);
}
totalSize += obSize;
// Dump size of each object for comparisons across JVMs and flags.
// System.out.println(" += " + obClazz + " | " + obSize);
}
// Help the GC (?).
@ -195,7 +135,91 @@ public final class RamUsageTester {
return totalSize;
}
private static long handleOther(Accumulator accumulator, IdentityHashMap<Class<?>, ClassCache> classCache, ArrayList<Object> stack, Object ob, Class<?> obClazz) {
/*
* Consider an object. Push any references it has to the processing stack
* and accumulate this object's shallow size.
*/
try {
if (Constants.JRE_IS_MINIMUM_JAVA9) {
long alignedShallowInstanceSize = RamUsageEstimator.shallowSizeOf(ob);
Predicate<Class<?>> isJavaModule = (clazz) -> {
return clazz.getName().startsWith("java.");
};
// Java 9: Best guess for some known types, as we cannot precisely look into runtime classes:
final ToLongFunction<Object> func = SIMPLE_TYPES.get(obClazz);
if (func != null) { // some simple type like String where the size is easy to get from public properties
return accumulator.accumulateObject(ob, alignedShallowInstanceSize + func.applyAsLong(ob),
Collections.emptyMap(), stack);
} else if (ob instanceof Enum) {
return alignedShallowInstanceSize;
} else if (ob instanceof ByteBuffer) {
// Approximate ByteBuffers with their underlying storage (ignores field overhead).
return byteArraySize(((ByteBuffer) ob).capacity());
} else if (isJavaModule.test(obClazz) && ob instanceof Map) {
final List<Object> values = ((Map<?,?>) ob).entrySet().stream()
.flatMap(e -> Stream.of(e.getKey(), e.getValue()))
.collect(Collectors.toList());
return accumulator.accumulateArray(ob, alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack)
+ RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
} else if (isJavaModule.test(obClazz) && ob instanceof Iterable) {
final List<Object> values = StreamSupport.stream(((Iterable<?>) ob).spliterator(), false)
.collect(Collectors.toList());
return accumulator.accumulateArray(ob, alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack)
+ RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
} else {
// Fallback to reflective access.
}
}
ClassCache cachedInfo = classCache.get(obClazz);
if (cachedInfo == null) {
classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
}
final Map<Field, Object> fieldValues = new HashMap<>();
for (Field f : cachedInfo.referenceFields) {
fieldValues.put(f, f.get(ob));
}
return accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize, fieldValues, stack);
} catch (IllegalAccessException e) {
// this should never happen as we enabled setAccessible().
throw new RuntimeException("Reflective field access failed?", e);
}
}
private static long handleArray(Accumulator accumulator, ArrayList<Object> stack, Object ob, Class<?> obClazz) {
/*
* Consider an array, possibly of primitive types. Push any of its references to
* the processing stack and accumulate this array's shallow size.
*/
final long shallowSize = RamUsageEstimator.shallowSizeOf(ob);
final int len = Array.getLength(ob);
final List<Object> values;
Class<?> componentClazz = obClazz.getComponentType();
if (componentClazz.isPrimitive()) {
values = Collections.emptyList();
} else {
values = new AbstractList<Object>() {
@Override
public Object get(int index) {
return Array.get(ob, index);
}
@Override
public int size() {
return len;
}
};
}
return accumulator.accumulateArray(ob, shallowSize, values, stack);
}
/**
* This map contains a function to calculate sizes of some "simple types" like String just from their public properties.
* This is needed for Java 9, which does not allow to look into runtime class fields.
@ -212,8 +236,6 @@ public final class RamUsageTester {
a(StringBuffer.class, v -> charArraySize(v.capacity()));
// Types with large buffers:
a(ByteArrayOutputStream.class, v -> byteArraySize(v.size()));
// Approximate ByteBuffers with their underling storage (ignores field overhead).
a(ByteBuffer.class, v -> byteArraySize(v.capacity()));
// For File and Path, we just take the length of String representation as approximation:
a(File.class, v -> charArraySize(v.toString().length()));
a(Path.class, v -> charArraySize(v.toString().length()));
@ -227,10 +249,6 @@ public final class RamUsageTester {
private long charArraySize(int len) {
return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long)Character.BYTES * len);
}
private long byteArraySize(int len) {
return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + len);
}
});
/**
@ -271,11 +289,10 @@ public final class RamUsageTester {
f.setAccessible(true);
referenceFields.add(f);
} catch (RuntimeException re) {
if ("java.lang.reflect.InaccessibleObjectException".equals(re.getClass().getName())) {
// LUCENE-7595: this is Java 9, which prevents access to fields in foreign modules
} else {
throw re;
}
throw new RuntimeException(String.format(Locale.ROOT,
"Can't access field '%s' of class '%s' for RAM estimation.",
f.getName(),
clazz.getName()), re);
}
}
}
@ -289,4 +306,7 @@ public final class RamUsageTester {
});
}
private static long byteArraySize(int len) {
return RamUsageEstimator.alignObjectSize((long) RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + len);
}
}