LUCENE-8642: RamUsageTester.sizeOf ignores arrays and collections if --illegal-access=deny.

lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java

@@ -173,7 +173,6 @@ public class TermInSetQueryTest extends LuceneTestCase {
     QueryUtils.checkEqual(query1, query2);
   }
 
-  @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-8641")
   public void testRamBytesUsed() {
     List<BytesRef> terms = new ArrayList<>();
     final int numTerms = 10000 + random().nextInt(1000);

lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java

@@ -27,6 +27,7 @@ import java.util.HashSet;
 import java.util.IdentityHashMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
@@ -289,12 +290,26 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {
       new SimpleMergedSegmentWarmer(InfoStream.NO_OUTPUT).warm(reader);
     }
 
-    final long actualBytes = RamUsageTester.sizeOf(reader2, new Accumulator(reader2)) - RamUsageTester.sizeOf(reader1, new Accumulator(reader1));
+    long act1 = RamUsageTester.sizeOf(reader2, new Accumulator(reader2));
-    final long expectedBytes = ((SegmentReader) reader2).ramBytesUsed() - ((SegmentReader) reader1).ramBytesUsed();
+    long act2 = RamUsageTester.sizeOf(reader1, new Accumulator(reader1));
-    final long absoluteError = actualBytes - expectedBytes;
+    final long measuredBytes = act1 - act2;
-    final double relativeError = (double) absoluteError / actualBytes;
+
-    final String message = "Actual RAM usage " + actualBytes + ", but got " + expectedBytes + ", " + 100*relativeError + "% error";
+    long reported1 = ((SegmentReader) reader2).ramBytesUsed();
-    assertTrue(message, Math.abs(relativeError) < 0.20d || Math.abs(absoluteError) < 1000);
+    long reported2 = ((SegmentReader) reader1).ramBytesUsed();
+    final long reportedBytes = reported1 - reported2;
+
+    final long absoluteError = Math.abs(measuredBytes - reportedBytes);
+    final double relativeError = (double) absoluteError / measuredBytes;
+    final String message = String.format(Locale.ROOT,
+        "RamUsageTester reports %d bytes but ramBytesUsed() returned %d (%.1f error). " +
+        " [Measured: %d, %d. Reported: %d, %d]",
+        measuredBytes,
+        reportedBytes,
+        (100 * relativeError),
+        act1, act2,
+        reported1, reported2);
+
+    assertTrue(message, relativeError < 0.20d || absoluteError < 1000);
 
     reader1.close();
     reader2.close();

lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java

@@ -32,8 +32,10 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.IdentityHashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Predicate;
 import java.util.function.ToLongFunction;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -112,80 +114,18 @@ public final class RamUsageTester {
       }
       seen.add(ob);
 
+      final long obSize;
       final Class<?> obClazz = ob.getClass();
       assert obClazz != null : "jvm bug detected (Object.getClass() == null). please report this to your vendor";
       if (obClazz.isArray()) {
-        /*
+        obSize = handleArray(accumulator, stack, ob, obClazz);
-         * Consider an array, possibly of primitive types. Push any of its references to
-         * the processing stack and accumulate this array's shallow size. 
-         */
-        final long shallowSize = RamUsageEstimator.shallowSizeOf(ob);
-        final int len = Array.getLength(ob);
-        final List<Object> values;
-        Class<?> componentClazz = obClazz.getComponentType();
-        if (componentClazz.isPrimitive()) {
-          values = Collections.emptyList();
-        } else {
-          values = new AbstractList<Object>() {
-
-            @Override
-            public Object get(int index) {
-              return Array.get(ob, index);
-            }
-
-            @Override
-            public int size() {
-              return len;
-            }
-            
-          };
-        }
-        totalSize += accumulator.accumulateArray(ob, shallowSize, values, stack);
       } else {
-        /*
+        obSize = handleOther(accumulator, classCache, stack, ob, obClazz);
-         * Consider an object. Push any references it has to the processing stack
-         * and accumulate this object's shallow size. 
-         */
-        try {
-          ClassCache cachedInfo = classCache.get(obClazz);
-          if (cachedInfo == null) {
-            classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
-          }
-          
-          boolean needsReflection = true;
-          if (Constants.JRE_IS_MINIMUM_JAVA9 && obClazz.getName().startsWith("java.")) {
-            // Java 9: Best guess for some known types, as we cannot precisely look into runtime classes:
-            final ToLongFunction<Object> func = SIMPLE_TYPES.get(obClazz);
-            if (func != null) { // some simple type like String where the size is easy to get from public properties
-              totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize + func.applyAsLong(ob), 
-                  Collections.emptyMap(), stack);
-              needsReflection = false;
-            } else if (ob instanceof Iterable) {
-              final List<Object> values = StreamSupport.stream(((Iterable<?>) ob).spliterator(), false)
-                  .collect(Collectors.toList());
-              totalSize += accumulator.accumulateArray(ob, cachedInfo.alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
-              needsReflection = false;
-            }  else if (ob instanceof Map) {
-              final List<Object> values = ((Map<?,?>) ob).entrySet().stream()
-                  .flatMap(e -> Stream.of(e.getKey(), e.getValue()))
-                  .collect(Collectors.toList());
-              totalSize += accumulator.accumulateArray(ob, cachedInfo.alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
-              totalSize += RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
-              needsReflection = false;
-            }
-          }
-          if (needsReflection) {
-            final Map<Field, Object> fieldValues = new HashMap<>();
-            for (Field f : cachedInfo.referenceFields) {
-              fieldValues.put(f, f.get(ob));
-            }
-            totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize, fieldValues, stack);
-          }
-        } catch (IllegalAccessException e) {
-          // this should never happen as we enabled setAccessible().
-          throw new RuntimeException("Reflective field access failed?", e);
-        }
       }
+
+      totalSize += obSize;
+      // Dump size of each object for comparisons across JVMs and flags.
+      // System.out.println("  += " + obClazz + " | " + obSize);
     }
 
     // Help the GC (?).
@@ -195,7 +135,91 @@ public final class RamUsageTester {
 
     return totalSize;
   }
-  
+
+  private static long handleOther(Accumulator accumulator, IdentityHashMap<Class<?>, ClassCache> classCache, ArrayList<Object> stack, Object ob, Class<?> obClazz) {
+    /*
+     * Consider an object. Push any references it has to the processing stack
+     * and accumulate this object's shallow size.
+     */
+    try {
+      if (Constants.JRE_IS_MINIMUM_JAVA9) {
+        long alignedShallowInstanceSize = RamUsageEstimator.shallowSizeOf(ob);
+
+        Predicate<Class<?>> isJavaModule = (clazz) -> {
+          return clazz.getName().startsWith("java.");
+        };
+
+        // Java 9: Best guess for some known types, as we cannot precisely look into runtime classes:
+        final ToLongFunction<Object> func = SIMPLE_TYPES.get(obClazz);
+        if (func != null) { // some simple type like String where the size is easy to get from public properties
+          return accumulator.accumulateObject(ob, alignedShallowInstanceSize + func.applyAsLong(ob),
+              Collections.emptyMap(), stack);
+        } else if (ob instanceof Enum) {
+          return alignedShallowInstanceSize;
+        } else if (ob instanceof ByteBuffer) {
+          // Approximate ByteBuffers with their underlying storage (ignores field overhead).
+          return byteArraySize(((ByteBuffer) ob).capacity());
+        }  else if (isJavaModule.test(obClazz) && ob instanceof Map) {
+          final List<Object> values = ((Map<?,?>) ob).entrySet().stream()
+              .flatMap(e -> Stream.of(e.getKey(), e.getValue()))
+              .collect(Collectors.toList());
+          return accumulator.accumulateArray(ob, alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack)
+              + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
+        } else if (isJavaModule.test(obClazz) && ob instanceof Iterable) {
+          final List<Object> values = StreamSupport.stream(((Iterable<?>) ob).spliterator(), false)
+            .collect(Collectors.toList());
+          return accumulator.accumulateArray(ob, alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack)
+              + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
+        } else {
+          // Fallback to reflective access.
+        }
+      }
+
+      ClassCache cachedInfo = classCache.get(obClazz);
+      if (cachedInfo == null) {
+        classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
+      }
+
+      final Map<Field, Object> fieldValues = new HashMap<>();
+      for (Field f : cachedInfo.referenceFields) {
+        fieldValues.put(f, f.get(ob));
+      }
+      return accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize, fieldValues, stack);
+    } catch (IllegalAccessException e) {
+      // this should never happen as we enabled setAccessible().
+      throw new RuntimeException("Reflective field access failed?", e);
+    }
+  }
+
+  private static long handleArray(Accumulator accumulator, ArrayList<Object> stack, Object ob, Class<?> obClazz) {
+    /*
+     * Consider an array, possibly of primitive types. Push any of its references to
+     * the processing stack and accumulate this array's shallow size.
+     */
+    final long shallowSize = RamUsageEstimator.shallowSizeOf(ob);
+    final int len = Array.getLength(ob);
+    final List<Object> values;
+    Class<?> componentClazz = obClazz.getComponentType();
+    if (componentClazz.isPrimitive()) {
+      values = Collections.emptyList();
+    } else {
+      values = new AbstractList<Object>() {
+
+        @Override
+        public Object get(int index) {
+          return Array.get(ob, index);
+        }
+
+        @Override
+        public int size() {
+          return len;
+        }
+
+      };
+    }
+    return accumulator.accumulateArray(ob, shallowSize, values, stack);
+  }
+
   /**
    * This map contains a function to calculate sizes of some "simple types" like String just from their public properties.
    * This is needed for Java 9, which does not allow to look into runtime class fields.
@@ -212,8 +236,6 @@ public final class RamUsageTester {
       a(StringBuffer.class, v -> charArraySize(v.capacity()));
       // Types with large buffers:
       a(ByteArrayOutputStream.class, v -> byteArraySize(v.size()));
-      // Approximate ByteBuffers with their underling storage (ignores field overhead).
-      a(ByteBuffer.class, v -> byteArraySize(v.capacity()));
       // For File and Path, we just take the length of String representation as approximation:
       a(File.class, v -> charArraySize(v.toString().length()));
       a(Path.class, v -> charArraySize(v.toString().length()));
@@ -227,10 +249,6 @@ public final class RamUsageTester {
     private long charArraySize(int len) {
       return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long)Character.BYTES * len);
     }
-    
-    private long byteArraySize(int len) {
-      return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + len);
-    }
   });
   
   /**
@@ -271,11 +289,10 @@ public final class RamUsageTester {
                 f.setAccessible(true);
                 referenceFields.add(f);
               } catch (RuntimeException re) {
-                if ("java.lang.reflect.InaccessibleObjectException".equals(re.getClass().getName())) {
+                throw new RuntimeException(String.format(Locale.ROOT,
-                  // LUCENE-7595: this is Java 9, which prevents access to fields in foreign modules
+                    "Can't access field '%s' of class '%s' for RAM estimation.",
-                } else {
+                    f.getName(),
-                  throw re;
+                    clazz.getName()), re);
-                }
               }
             }
           }
@@ -289,4 +306,7 @@ public final class RamUsageTester {
     });
   }
 
+  private static long byteArraySize(int len) {
+    return RamUsageEstimator.alignObjectSize((long) RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + len);
+  }
 }