Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-7595: Improve RAMUsageTester in test-framework to estimate memory usage of runtime classes and work with Java 9 EA (b148+). Disable static field heap usage checker in LuceneTestCase

This commit is contained in:
Uwe Schindler 2016-12-29 01:56:23 +01:00
parent 262049fc8f
commit f29d2b5668
4 changed files with 139 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lucene/CHANGES.txt
View File

@ -208,6 +208,10 @@ Other
* LUCENE-7599: Simplify TestRandomChains using Java's built-in Predicate and
Function interfaces. (Ahmet Arslan via Adrien Grand)
* LUCENE-7595: Improve RAMUsageTester in test-framework to estimate memory usage of
runtime classes and work with Java 9 EA (b148+). Disable static field heap usage
checker in LuceneTestCase. (Uwe Schindler, Dawid Weiss)
Build
* LUCENE-7387: fix defaultCodec in build.xml to account for the line ending (hossman)

3
lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
View File

@ -265,6 +265,8 @@ public class TestLRUQueryCache extends LuceneTestCase {
// This test makes sure that by making the same assumptions as LRUQueryCache, RAMUsageTester
// computes the same memory usage.
public void testRamBytesUsedAgreesWithRamUsageTester() throws IOException {
assumeFalse("LUCENE-7595: RamUsageTester does not work exact in Java 9 (estimations for maps and lists)", Constants.JRE_IS_MINIMUM_JAVA9);
final LRUQueryCache queryCache = new LRUQueryCache(1 + random().nextInt(5), 1 + random().nextInt(10000), context -> random().nextBoolean());
// an accumulator that only sums up memory usage of referenced filters and doc id sets
final RamUsageTester.Accumulator acc = new RamUsageTester.Accumulator() {
@ -379,7 +381,6 @@ public class TestLRUQueryCache extends LuceneTestCase {
// by the cache itself, not cache entries, and we want to make sure that
// memory usage is not grossly underestimated.
public void testRamBytesUsedConstantEntryOverhead() throws IOException {
LuceneTestCase.assumeFalse("RamUsageTester does not fully work on Java 9", Constants.JRE_IS_MINIMUM_JAVA9);
final LRUQueryCache queryCache = new LRUQueryCache(1000000, 10000000, context -> true);
final RamUsageTester.Accumulator acc = new RamUsageTester.Accumulator() {

90
lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
View File

@ -598,51 +598,55 @@ public abstract class LuceneTestCase extends Assert {
* other.
*/
@ClassRule
public static TestRule classRules = RuleChain
.outerRule(new TestRuleIgnoreTestSuites())
.around(ignoreAfterMaxFailures)
.around(suiteFailureMarker = new TestRuleMarkFailure())
.around(new TestRuleAssertionsRequired())
.around(new TestRuleLimitSysouts(suiteFailureMarker))
.around(tempFilesCleanupRule = new TestRuleTemporaryFilesCleanup(suiteFailureMarker))
.around(new StaticFieldsInvariantRule(STATIC_LEAK_THRESHOLD, true) {
@Override
protected boolean accept(java.lang.reflect.Field field) {
// Don't count known classes that consume memory once.
if (STATIC_LEAK_IGNORED_TYPES.contains(field.getType().getName())) {
return false;
public static TestRule classRules;
static {
RuleChain r = RuleChain.outerRule(new TestRuleIgnoreTestSuites())
.around(ignoreAfterMaxFailures)
.around(suiteFailureMarker = new TestRuleMarkFailure())
.around(new TestRuleAssertionsRequired())
.around(new TestRuleLimitSysouts(suiteFailureMarker))
.around(tempFilesCleanupRule = new TestRuleTemporaryFilesCleanup(suiteFailureMarker));
// TODO LUCENE-7595: Java 9 does not allow to look into runtime classes, so we have to fix the RAM usage checker!
if (!Constants.JRE_IS_MINIMUM_JAVA9) {
r = r.around(new StaticFieldsInvariantRule(STATIC_LEAK_THRESHOLD, true) {
@Override
protected boolean accept(java.lang.reflect.Field field) {
// Don't count known classes that consume memory once.
if (STATIC_LEAK_IGNORED_TYPES.contains(field.getType().getName())) {
return false;
}
// Don't count references from ourselves, we're top-level.
if (field.getDeclaringClass() == LuceneTestCase.class) {
return false;
}
return super.accept(field);
}
// Don't count references from ourselves, we're top-level.
if (field.getDeclaringClass() == LuceneTestCase.class) {
return false;
});
}
classRules = r.around(new NoClassHooksShadowingRule())
.around(new NoInstanceHooksOverridesRule() {
@Override
protected boolean verify(Method key) {
String name = key.getName();
return !(name.equals("setUp") || name.equals("tearDown"));
}
return super.accept(field);
}
})
.around(new NoClassHooksShadowingRule())
.around(new NoInstanceHooksOverridesRule() {
@Override
protected boolean verify(Method key) {
String name = key.getName();
return !(name.equals("setUp") || name.equals("tearDown"));
}
})
.around(classNameRule = new TestRuleStoreClassName())
.around(new TestRuleRestoreSystemProperties(
// Enlist all properties to which we have write access (security manager);
// these should be restored to previous state, no matter what the outcome of the test.
// We reset the default locale and timezone; these properties change as a side-effect
"user.language",
"user.timezone",
// TODO: these should, ideally, be moved to Solr's base class.
"solr.directoryFactory",
"solr.solr.home",
"solr.data.dir"
))
.around(classEnvRule = new TestRuleSetupAndRestoreClassEnv());
})
.around(classNameRule = new TestRuleStoreClassName())
.around(new TestRuleRestoreSystemProperties(
// Enlist all properties to which we have write access (security manager);
// these should be restored to previous state, no matter what the outcome of the test.
// We reset the default locale and timezone; these properties change as a side-effect
"user.language",
"user.timezone",
// TODO: these should, ideally, be moved to Solr's base class.
"solr.directoryFactory",
"solr.solr.home",
"solr.data.dir"
))
.around(classEnvRule = new TestRuleSetupAndRestoreClassEnv());
}
// -----------------------------------------------------------------
// Test level rules.

102
lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
View File

@ -16,9 +16,12 @@
*/
package org.apache.lucene.util;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.lang.reflect.Array;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.nio.file.Path;
import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.AbstractList;
@ -30,6 +33,10 @@ import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.ToLongFunction;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
/** Crawls object graph to collect RAM usage for testing */
public final class RamUsageTester {
@ -40,9 +47,7 @@ public final class RamUsageTester {
/** Accumulate transitive references for the provided fields of the given
* object into <code>queue</code> and return the shallow size of this object. */
public long accumulateObject(Object o, long shallowSize, Map<Field, Object> fieldValues, Collection<Object> queue) {
for (Object value : fieldValues.values()) {
queue.add(value);
}
queue.addAll(fieldValues.values());
return shallowSize;
}
@ -130,10 +135,10 @@ public final class RamUsageTester {
@Override
public int size() {
return len;
}
};
}
}
};
}
totalSize += accumulator.accumulateArray(ob, shallowSize, values, stack);
} else {
/*
@ -145,13 +150,36 @@ public final class RamUsageTester {
if (cachedInfo == null) {
classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
}
Map<Field, Object> fieldValues = new HashMap<>();
for (Field f : cachedInfo.referenceFields) {
fieldValues.put(f, f.get(ob));
boolean needsReflection = true;
if (Constants.JRE_IS_MINIMUM_JAVA9) {
// Java 9: Best guess for some known types, as we cannot precisely look into runtime classes:
final ToLongFunction<Object> func = SIMPLE_TYPES.get(obClazz);
if (func != null) { // some simple type like String where the size is easy to get from public properties
totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize + func.applyAsLong(ob),
Collections.emptyMap(), stack);
needsReflection = false;
} else if (ob instanceof Iterable) {
final List<Object> values = StreamSupport.stream(((Iterable<?>) ob).spliterator(), false)
.collect(Collectors.toList());
totalSize += accumulator.accumulateArray(ob, cachedInfo.alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
needsReflection = false;
} else if (ob instanceof Map) {
final List<Object> values = ((Map<?,?>) ob).entrySet().stream()
.flatMap(e -> Stream.of(e.getKey(), e.getValue()))
.collect(Collectors.toList());
totalSize += accumulator.accumulateArray(ob, cachedInfo.alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
totalSize += RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
needsReflection = false;
}
}
if (needsReflection) {
final Map<Field, Object> fieldValues = new HashMap<>();
for (Field f : cachedInfo.referenceFields) {
fieldValues.put(f, f.get(ob));
}
totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize, fieldValues, stack);
}
totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize, fieldValues, stack);
} catch (IllegalAccessException e) {
// this should never happen as we enabled setAccessible().
throw new RuntimeException("Reflective field access failed?", e);
@ -167,7 +195,41 @@ public final class RamUsageTester {
return totalSize;
}
/**
* This map contains a function to calculate sizes of some "simple types" like String just from their public properties.
* This is needed for Java 9, which does not allow to look into runtime class fields.
*/
@SuppressWarnings("serial")
private static final Map<Class<?>, ToLongFunction<Object>> SIMPLE_TYPES = Collections.unmodifiableMap(new IdentityHashMap<Class<?>, ToLongFunction<Object>>() {
{ init(); }
@SuppressForbidden(reason = "We measure some forbidden classes")
private void init() {
// String types:
a(String.class, v -> charArraySize(v.length())); // may not be correct with Java 9's compact strings!
a(StringBuilder.class, v -> charArraySize(v.capacity()));
a(StringBuffer.class, v -> charArraySize(v.capacity()));
// Types with large buffers:
a(ByteArrayOutputStream.class, v -> byteArraySize(v.size()));
// For File and Path, we just take the length of String representation as approximation:
a(File.class, v -> charArraySize(v.toString().length()));
a(Path.class, v -> charArraySize(v.toString().length()));
}
@SuppressWarnings("unchecked")
private <T> void a(Class<T> clazz, ToLongFunction<T> func) {
put(clazz, (ToLongFunction<Object>) func);
}
private long charArraySize(int len) {
return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long)Character.BYTES * len);
}
private long byteArraySize(int len) {
return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + len);
}
});
/**
* Cached information about a given class.
*/
@ -202,8 +264,16 @@ public final class RamUsageTester {
shallowInstanceSize = RamUsageEstimator.adjustForField(shallowInstanceSize, f);
if (!f.getType().isPrimitive()) {
f.setAccessible(true);
referenceFields.add(f);
try {
f.setAccessible(true);
referenceFields.add(f);
} catch (RuntimeException re) {
if ("java.lang.reflect.InaccessibleObjectException".equals(re.getClass().getName())) {
// LUCENE-7595: this is Java 9, which prevents access to fields in foreign modules
} else {
throw re;
}
}
}
}
}