diff --git a/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java b/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java
index aba8f237342..caabfefac44 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java
@@ -173,7 +173,6 @@ public class TermInSetQueryTest extends LuceneTestCase {
     QueryUtils.checkEqual(query1, query2);
   }
 
-  @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-8641")
   public void testRamBytesUsed() {
     List<BytesRef> terms = new ArrayList<>();
     final int numTerms = 10000 + random().nextInt(1000);
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java b/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
index 9b7b666d001..fb8889b0ba9 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
@@ -32,6 +32,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.IdentityHashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.ToLongFunction;
@@ -147,34 +148,41 @@ public final class RamUsageTester {
          * and accumulate this object's shallow size. 
          */
         try {
-          ClassCache cachedInfo = classCache.get(obClazz);
-          if (cachedInfo == null) {
-            classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
-          }
-          
           boolean needsReflection = true;
           if (Constants.JRE_IS_MINIMUM_JAVA9 && obClazz.getName().startsWith("java.")) {
+            long alignedShallowInstanceSize = RamUsageEstimator.shallowSizeOf(ob);
+
             // Java 9: Best guess for some known types, as we cannot precisely look into runtime classes:
             final ToLongFunction<Object> func = SIMPLE_TYPES.get(obClazz);
             if (func != null) { // some simple type like String where the size is easy to get from public properties
-              totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize + func.applyAsLong(ob), 
+              totalSize += accumulator.accumulateObject(ob, alignedShallowInstanceSize + func.applyAsLong(ob),
                   Collections.emptyMap(), stack);
               needsReflection = false;
-            } else if (ob instanceof Iterable) {
-              final List<Object> values = StreamSupport.stream(((Iterable<?>) ob).spliterator(), false)
-                  .collect(Collectors.toList());
-              totalSize += accumulator.accumulateArray(ob, cachedInfo.alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
+            } else if (ob instanceof ByteBuffer) {
+              // Approximate ByteBuffers with their underlying storage (ignores field overhead).
+              totalSize += byteArraySize(((ByteBuffer) ob).capacity());
               needsReflection = false;
             }  else if (ob instanceof Map) {
               final List<Object> values = ((Map<?,?>) ob).entrySet().stream()
                   .flatMap(e -> Stream.of(e.getKey(), e.getValue()))
                   .collect(Collectors.toList());
-              totalSize += accumulator.accumulateArray(ob, cachedInfo.alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
+              totalSize += accumulator.accumulateArray(ob, alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
               totalSize += RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
               needsReflection = false;
+            } else if (ob instanceof Iterable) {
+            final List<Object> values = StreamSupport.stream(((Iterable<?>) ob).spliterator(), false)
+                .collect(Collectors.toList());
+              totalSize += accumulator.accumulateArray(ob, alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
+              needsReflection = false;
             }
           }
+
           if (needsReflection) {
+            ClassCache cachedInfo = classCache.get(obClazz);
+            if (cachedInfo == null) {
+              classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
+            }
+
             final Map<Field, Object> fieldValues = new HashMap<>();
             for (Field f : cachedInfo.referenceFields) {
               fieldValues.put(f, f.get(ob));
@@ -212,8 +220,6 @@ public final class RamUsageTester {
       a(StringBuffer.class, v -> charArraySize(v.capacity()));
       // Types with large buffers:
       a(ByteArrayOutputStream.class, v -> byteArraySize(v.size()));
-      // Approximate ByteBuffers with their underling storage (ignores field overhead).
-      a(ByteBuffer.class, v -> byteArraySize(v.capacity()));
       // For File and Path, we just take the length of String representation as approximation:
       a(File.class, v -> charArraySize(v.toString().length()));
       a(Path.class, v -> charArraySize(v.toString().length()));
@@ -227,10 +233,6 @@ public final class RamUsageTester {
     private long charArraySize(int len) {
       return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long)Character.BYTES * len);
     }
-    
-    private long byteArraySize(int len) {
-      return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + len);
-    }
   });
   
   /**
@@ -271,11 +273,10 @@ public final class RamUsageTester {
                 f.setAccessible(true);
                 referenceFields.add(f);
               } catch (RuntimeException re) {
-                if ("java.lang.reflect.InaccessibleObjectException".equals(re.getClass().getName())) {
-                  // LUCENE-7595: this is Java 9, which prevents access to fields in foreign modules
-                } else {
-                  throw re;
-                }
+                throw new RuntimeException(String.format(Locale.ROOT,
+                    "Can't access field %s of class %s for RAM estimation.",
+                    f.toString(),
+                    c.getName()), re);
               }
             }
           }
@@ -289,4 +290,7 @@ public final class RamUsageTester {
     });
   }
 
+  private static long byteArraySize(int len) {
+    return RamUsageEstimator.alignObjectSize((long) RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + len);
+  }
 }