Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-8642, LUCENE-8641: correct RamUsageTester.sizeOf's handling of ByteBuffers. Throw exceptions on denied reflection to catch problems early. This affects tests only.

This commit is contained in:
Dawid Weiss 2019-01-17 12:23:30 +01:00
parent 9926b45597
commit a16f0833ed
2 changed files with 26 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java
View File

@ -173,7 +173,6 @@ public class TermInSetQueryTest extends LuceneTestCase {
QueryUtils.checkEqual(query1, query2);
}
@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-8641")
public void testRamBytesUsed() {
List<BytesRef> terms = new ArrayList<>();
final int numTerms = 10000 + random().nextInt(1000);

48
lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
View File

@ -32,6 +32,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.function.ToLongFunction;
@ -147,34 +148,41 @@ public final class RamUsageTester {
* and accumulate this object's shallow size.
*/
try {
ClassCache cachedInfo = classCache.get(obClazz);
if (cachedInfo == null) {
classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
}
boolean needsReflection = true;
if (Constants.JRE_IS_MINIMUM_JAVA9 && obClazz.getName().startsWith("java.")) {
long alignedShallowInstanceSize = RamUsageEstimator.shallowSizeOf(ob);
// Java 9: Best guess for some known types, as we cannot precisely look into runtime classes:
final ToLongFunction<Object> func = SIMPLE_TYPES.get(obClazz);
if (func != null) { // some simple type like String where the size is easy to get from public properties
totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize + func.applyAsLong(ob),
totalSize += accumulator.accumulateObject(ob, alignedShallowInstanceSize + func.applyAsLong(ob),
Collections.emptyMap(), stack);
needsReflection = false;
} else if (ob instanceof Iterable) {
final List<Object> values = StreamSupport.stream(((Iterable<?>) ob).spliterator(), false)
.collect(Collectors.toList());
totalSize += accumulator.accumulateArray(ob, cachedInfo.alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
} else if (ob instanceof ByteBuffer) {
// Approximate ByteBuffers with their underlying storage (ignores field overhead).
totalSize += byteArraySize(((ByteBuffer) ob).capacity());
needsReflection = false;
} else if (ob instanceof Map) {
final List<Object> values = ((Map<?,?>) ob).entrySet().stream()
.flatMap(e -> Stream.of(e.getKey(), e.getValue()))
.collect(Collectors.toList());
totalSize += accumulator.accumulateArray(ob, cachedInfo.alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
totalSize += accumulator.accumulateArray(ob, alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
totalSize += RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
needsReflection = false;
} else if (ob instanceof Iterable) {
final List<Object> values = StreamSupport.stream(((Iterable<?>) ob).spliterator(), false)
.collect(Collectors.toList());
totalSize += accumulator.accumulateArray(ob, alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
needsReflection = false;
}
}
if (needsReflection) {
ClassCache cachedInfo = classCache.get(obClazz);
if (cachedInfo == null) {
classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
}
final Map<Field, Object> fieldValues = new HashMap<>();
for (Field f : cachedInfo.referenceFields) {
fieldValues.put(f, f.get(ob));
@ -212,8 +220,6 @@ public final class RamUsageTester {
a(StringBuffer.class, v -> charArraySize(v.capacity()));
// Types with large buffers:
a(ByteArrayOutputStream.class, v -> byteArraySize(v.size()));
// Approximate ByteBuffers with their underling storage (ignores field overhead).
a(ByteBuffer.class, v -> byteArraySize(v.capacity()));
// For File and Path, we just take the length of String representation as approximation:
a(File.class, v -> charArraySize(v.toString().length()));
a(Path.class, v -> charArraySize(v.toString().length()));
@ -227,10 +233,6 @@ public final class RamUsageTester {
private long charArraySize(int len) {
return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long)Character.BYTES * len);
}
private long byteArraySize(int len) {
return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + len);
}
});
/**
@ -271,11 +273,10 @@ public final class RamUsageTester {
f.setAccessible(true);
referenceFields.add(f);
} catch (RuntimeException re) {
if ("java.lang.reflect.InaccessibleObjectException".equals(re.getClass().getName())) {
// LUCENE-7595: this is Java 9, which prevents access to fields in foreign modules
} else {
throw re;
}
throw new RuntimeException(String.format(Locale.ROOT,
"Can't access field %s of class %s for RAM estimation.",
f.toString(),
c.getName()), re);
}
}
}
@ -289,4 +290,7 @@ public final class RamUsageTester {
});
}
private static long byteArraySize(int len) {
return RamUsageEstimator.alignObjectSize((long) RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + len);
}
}