diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 255867d68c3..7a118f1d8c2 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -208,6 +208,10 @@ Other * LUCENE-7599: Simplify TestRandomChains using Java's built-in Predicate and Function interfaces. (Ahmet Arslan via Adrien Grand) +* LUCENE-7595: Improve RAMUsageTester in test-framework to estimate memory usage of + runtime classes and work with Java 9 EA (b148+). Disable static field heap usage + checker in LuceneTestCase. (Uwe Schindler, Dawid Weiss) + Build * LUCENE-7387: fix defaultCodec in build.xml to account for the line ending (hossman) diff --git a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java index 87382f98387..9ebacf7292a 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java @@ -265,6 +265,8 @@ public class TestLRUQueryCache extends LuceneTestCase { // This test makes sure that by making the same assumptions as LRUQueryCache, RAMUsageTester // computes the same memory usage. public void testRamBytesUsedAgreesWithRamUsageTester() throws IOException { + assumeFalse("LUCENE-7595: RamUsageTester does not work exact in Java 9 (estimations for maps and lists)", Constants.JRE_IS_MINIMUM_JAVA9); + final LRUQueryCache queryCache = new LRUQueryCache(1 + random().nextInt(5), 1 + random().nextInt(10000), context -> random().nextBoolean()); // an accumulator that only sums up memory usage of referenced filters and doc id sets final RamUsageTester.Accumulator acc = new RamUsageTester.Accumulator() { @@ -379,7 +381,6 @@ public class TestLRUQueryCache extends LuceneTestCase { // by the cache itself, not cache entries, and we want to make sure that // memory usage is not grossly underestimated. public void testRamBytesUsedConstantEntryOverhead() throws IOException { - LuceneTestCase.assumeFalse("RamUsageTester does not fully work on Java 9", Constants.JRE_IS_MINIMUM_JAVA9); final LRUQueryCache queryCache = new LRUQueryCache(1000000, 10000000, context -> true); final RamUsageTester.Accumulator acc = new RamUsageTester.Accumulator() { diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java index 1848c4e7964..50139a099de 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java @@ -598,51 +598,55 @@ public abstract class LuceneTestCase extends Assert { * other. */ @ClassRule - public static TestRule classRules = RuleChain - .outerRule(new TestRuleIgnoreTestSuites()) - .around(ignoreAfterMaxFailures) - .around(suiteFailureMarker = new TestRuleMarkFailure()) - .around(new TestRuleAssertionsRequired()) - .around(new TestRuleLimitSysouts(suiteFailureMarker)) - .around(tempFilesCleanupRule = new TestRuleTemporaryFilesCleanup(suiteFailureMarker)) - .around(new StaticFieldsInvariantRule(STATIC_LEAK_THRESHOLD, true) { - @Override - protected boolean accept(java.lang.reflect.Field field) { - // Don't count known classes that consume memory once. - if (STATIC_LEAK_IGNORED_TYPES.contains(field.getType().getName())) { - return false; + public static TestRule classRules; + static { + RuleChain r = RuleChain.outerRule(new TestRuleIgnoreTestSuites()) + .around(ignoreAfterMaxFailures) + .around(suiteFailureMarker = new TestRuleMarkFailure()) + .around(new TestRuleAssertionsRequired()) + .around(new TestRuleLimitSysouts(suiteFailureMarker)) + .around(tempFilesCleanupRule = new TestRuleTemporaryFilesCleanup(suiteFailureMarker)); + // TODO LUCENE-7595: Java 9 does not allow to look into runtime classes, so we have to fix the RAM usage checker! + if (!Constants.JRE_IS_MINIMUM_JAVA9) { + r = r.around(new StaticFieldsInvariantRule(STATIC_LEAK_THRESHOLD, true) { + @Override + protected boolean accept(java.lang.reflect.Field field) { + // Don't count known classes that consume memory once. + if (STATIC_LEAK_IGNORED_TYPES.contains(field.getType().getName())) { + return false; + } + // Don't count references from ourselves, we're top-level. + if (field.getDeclaringClass() == LuceneTestCase.class) { + return false; + } + return super.accept(field); } - // Don't count references from ourselves, we're top-level. - if (field.getDeclaringClass() == LuceneTestCase.class) { - return false; + }); + } + classRules = r.around(new NoClassHooksShadowingRule()) + .around(new NoInstanceHooksOverridesRule() { + @Override + protected boolean verify(Method key) { + String name = key.getName(); + return !(name.equals("setUp") || name.equals("tearDown")); } - return super.accept(field); - } - }) - .around(new NoClassHooksShadowingRule()) - .around(new NoInstanceHooksOverridesRule() { - @Override - protected boolean verify(Method key) { - String name = key.getName(); - return !(name.equals("setUp") || name.equals("tearDown")); - } - }) - .around(classNameRule = new TestRuleStoreClassName()) - .around(new TestRuleRestoreSystemProperties( - // Enlist all properties to which we have write access (security manager); - // these should be restored to previous state, no matter what the outcome of the test. - - // We reset the default locale and timezone; these properties change as a side-effect - "user.language", - "user.timezone", - - // TODO: these should, ideally, be moved to Solr's base class. - "solr.directoryFactory", - "solr.solr.home", - "solr.data.dir" - )) - .around(classEnvRule = new TestRuleSetupAndRestoreClassEnv()); - + }) + .around(classNameRule = new TestRuleStoreClassName()) + .around(new TestRuleRestoreSystemProperties( + // Enlist all properties to which we have write access (security manager); + // these should be restored to previous state, no matter what the outcome of the test. + + // We reset the default locale and timezone; these properties change as a side-effect + "user.language", + "user.timezone", + + // TODO: these should, ideally, be moved to Solr's base class. + "solr.directoryFactory", + "solr.solr.home", + "solr.data.dir" + )) + .around(classEnvRule = new TestRuleSetupAndRestoreClassEnv()); + } // ----------------------------------------------------------------- // Test level rules. diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java b/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java index 985052654c1..daf81a96b35 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java @@ -16,9 +16,12 @@ */ package org.apache.lucene.util; +import java.io.ByteArrayOutputStream; +import java.io.File; import java.lang.reflect.Array; import java.lang.reflect.Field; import java.lang.reflect.Modifier; +import java.nio.file.Path; import java.security.AccessController; import java.security.PrivilegedAction; import java.util.AbstractList; @@ -30,6 +33,10 @@ import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.ToLongFunction; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; /** Crawls object graph to collect RAM usage for testing */ public final class RamUsageTester { @@ -40,9 +47,7 @@ public final class RamUsageTester { /** Accumulate transitive references for the provided fields of the given * object into queue and return the shallow size of this object. */ public long accumulateObject(Object o, long shallowSize, Map fieldValues, Collection queue) { - for (Object value : fieldValues.values()) { - queue.add(value); - } + queue.addAll(fieldValues.values()); return shallowSize; } @@ -130,10 +135,10 @@ public final class RamUsageTester { @Override public int size() { return len; - } - - }; - } + } + + }; + } totalSize += accumulator.accumulateArray(ob, shallowSize, values, stack); } else { /* @@ -145,13 +150,36 @@ public final class RamUsageTester { if (cachedInfo == null) { classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz)); } - - Map fieldValues = new HashMap<>(); - for (Field f : cachedInfo.referenceFields) { - fieldValues.put(f, f.get(ob)); + + boolean needsReflection = true; + if (Constants.JRE_IS_MINIMUM_JAVA9) { + // Java 9: Best guess for some known types, as we cannot precisely look into runtime classes: + final ToLongFunction func = SIMPLE_TYPES.get(obClazz); + if (func != null) { // some simple type like String where the size is easy to get from public properties + totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize + func.applyAsLong(ob), + Collections.emptyMap(), stack); + needsReflection = false; + } else if (ob instanceof Iterable) { + final List values = StreamSupport.stream(((Iterable) ob).spliterator(), false) + .collect(Collectors.toList()); + totalSize += accumulator.accumulateArray(ob, cachedInfo.alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack); + needsReflection = false; + } else if (ob instanceof Map) { + final List values = ((Map) ob).entrySet().stream() + .flatMap(e -> Stream.of(e.getKey(), e.getValue())) + .collect(Collectors.toList()); + totalSize += accumulator.accumulateArray(ob, cachedInfo.alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack); + totalSize += RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; + needsReflection = false; + } + } + if (needsReflection) { + final Map fieldValues = new HashMap<>(); + for (Field f : cachedInfo.referenceFields) { + fieldValues.put(f, f.get(ob)); + } + totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize, fieldValues, stack); } - - totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize, fieldValues, stack); } catch (IllegalAccessException e) { // this should never happen as we enabled setAccessible(). throw new RuntimeException("Reflective field access failed?", e); @@ -167,7 +195,41 @@ public final class RamUsageTester { return totalSize; } - + /** + * This map contains a function to calculate sizes of some "simple types" like String just from their public properties. + * This is needed for Java 9, which does not allow to look into runtime class fields. + */ + @SuppressWarnings("serial") + private static final Map, ToLongFunction> SIMPLE_TYPES = Collections.unmodifiableMap(new IdentityHashMap, ToLongFunction>() { + { init(); } + + @SuppressForbidden(reason = "We measure some forbidden classes") + private void init() { + // String types: + a(String.class, v -> charArraySize(v.length())); // may not be correct with Java 9's compact strings! + a(StringBuilder.class, v -> charArraySize(v.capacity())); + a(StringBuffer.class, v -> charArraySize(v.capacity())); + // Types with large buffers: + a(ByteArrayOutputStream.class, v -> byteArraySize(v.size())); + // For File and Path, we just take the length of String representation as approximation: + a(File.class, v -> charArraySize(v.toString().length())); + a(Path.class, v -> charArraySize(v.toString().length())); + } + + @SuppressWarnings("unchecked") + private void a(Class clazz, ToLongFunction func) { + put(clazz, (ToLongFunction) func); + } + + private long charArraySize(int len) { + return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long)Character.BYTES * len); + } + + private long byteArraySize(int len) { + return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + len); + } + }); + /** * Cached information about a given class. */ @@ -202,8 +264,16 @@ public final class RamUsageTester { shallowInstanceSize = RamUsageEstimator.adjustForField(shallowInstanceSize, f); if (!f.getType().isPrimitive()) { - f.setAccessible(true); - referenceFields.add(f); + try { + f.setAccessible(true); + referenceFields.add(f); + } catch (RuntimeException re) { + if ("java.lang.reflect.InaccessibleObjectException".equals(re.getClass().getName())) { + // LUCENE-7595: this is Java 9, which prevents access to fields in foreign modules + } else { + throw re; + } + } } } }