LUCENE-5463: RUE.(human)sizeOf(Object) is now a forbidden API.

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1571384 13f79535-47bb-0310-9956-ffa450edef68

lucene/CHANGES.txt

@@ -88,6 +88,11 @@ Test Framework
 
 * LUCENE-5449: Rename _TestUtil and _TestHelper to remove the leading _.
 
+Build
+
+* LUCENE-5463: RamUsageEstimator.(human)sizeOf(Object) is now a forbidden API.
+  (Adrien Grand, Robert Muir)
+
 ======================= Lucene 4.7.0 =======================
 
 New Features

lucene/common-build.xml

@@ -2192,6 +2192,7 @@ ${ant.project.name}.test.dependencies=${test.classpath.list}
   <property name="forbidden-base-excludes" value=""/>
   <property name="forbidden-tests-excludes" value=""/>
   <property name="forbidden-sysout-excludes" value=""/>
+  <property name="forbidden-rue-excludes" value=""/>
   
   <target name="-install-forbidden-apis" unless="forbidden-apis.loaded" depends="ivy-availability-check,ivy-configure">
     <ivy:cachepath organisation="de.thetaphi" module="forbiddenapis" revision="1.4"
@@ -2211,9 +2212,10 @@ ${ant.project.name}.test.dependencies=${test.classpath.list}
     </path>
   </target>  
 
-  <target name="check-forbidden-apis" depends="-check-forbidden-base,-check-forbidden-tests,-check-forbidden-sysout" description="Check forbidden API calls in compiled class files"/>
+  <target name="check-forbidden-apis" depends="-check-forbidden-all,-check-forbidden-tests,-check-forbidden-sources" description="Check forbidden API calls in compiled class files"/>
   
-  <target name="-check-forbidden-base" depends="-init-forbidden-apis,compile-core,compile-test">
+  <!-- applies to both source and test code -->
+  <target name="-check-forbidden-all" depends="-init-forbidden-apis,compile-core,compile-test">
     <forbidden-apis internalRuntimeForbidden="true" classpathref="forbidden-apis.allclasses.classpath">
       <bundledSignatures name="jdk-unsafe-${javac.target}"/>
       <bundledSignatures name="jdk-deprecated-${javac.target}"/>
@@ -2223,12 +2225,22 @@ ${ant.project.name}.test.dependencies=${test.classpath.list}
     </forbidden-apis>
   </target>
   
+  <!-- applies to only test code -->
   <target name="-check-forbidden-tests" depends="-init-forbidden-apis,compile-test">
     <forbidden-apis signaturesFile="${common.dir}/tools/forbiddenApis/tests.txt" classpathref="forbidden-apis.allclasses.classpath"> 
       <fileset dir="${build.dir}/classes/test" excludes="${forbidden-tests-excludes}"/>
     </forbidden-apis>
   </target>
  
+  <!-- applies to only source code -->
+  <target name="-check-forbidden-sources" depends="-init-forbidden-apis,compile-core,-check-forbidden-sysout,-check-forbidden-rue" />
+
+  <target name="-check-forbidden-rue" depends="-init-forbidden-apis,compile-core">
+    <forbidden-apis signaturesFile="${common.dir}/tools/forbiddenApis/rue.txt" classpathref="forbidden-apis.allclasses.classpath">
+      <fileset dir="${build.dir}/classes/java" excludes="${forbidden-rue-excludes}"/>
+    </forbidden-apis>
+  </target>
+
   <target name="-check-forbidden-sysout" depends="-init-forbidden-apis,compile-core">
     <forbidden-apis bundledSignatures="jdk-system-out" classpathref="forbidden-apis.allclasses.classpath">
       <fileset dir="${build.dir}/classes/java" excludes="${forbidden-sysout-excludes}"/>

lucene/core/build.xml

@@ -30,6 +30,12 @@
     org/apache/lucene/util/PrintStreamInfoStream.class
   "/>
 
+  <property name="forbidden-rue-excludes" value="
+    org/apache/lucene/search/FieldCache$CacheEntry.class
+    org/apache/lucene/util/RamUsageEstimator.class
+    org/apache/lucene/search/CachingWrapperFilter.class
+  "/>
+
   <import file="../common-build.xml"/>
 
   <property name="moman.commit-hash" value="5c5c2a1e4dea" />

lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java

@@ -46,6 +46,15 @@ import org.apache.lucene.util.BytesRef;
  */
 public class FilterAtomicReader extends AtomicReader {
 
+  /** Get the wrapped instance by <code>reader</code> as long as this reader is
+   *  an intance of {@link FilterAtomicReader}.  */
+  public static AtomicReader unwrap(AtomicReader reader) {
+    while (reader instanceof FilterAtomicReader) {
+      reader = ((FilterAtomicReader) reader).in;
+    }
+    return reader;
+  }
+
   /** Base class for filtering {@link Fields}
    *  implementations. */
   public static class FilterFields extends Fields {

lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java

@@ -320,6 +320,11 @@ public final class RamUsageEstimator {
     return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_DOUBLE * arr.length);
   }
 
+  /** Returns the size in bytes of the String object. */
+  public static long sizeOf(String s) {
+    return shallowSizeOf(s) + alignObjectSize(NUM_BYTES_ARRAY_HEADER + NUM_BYTES_CHAR * s.length());
+  }
+
   /** 
    * Estimates the RAM usage by the given object. It will
    * walk the object tree and sum up all referenced objects.

lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java

@@ -21,6 +21,8 @@ import static org.apache.lucene.util.RamUsageEstimator.*;
 
 import java.util.Random;
 
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
 public class TestRamUsageEstimator extends LuceneTestCase {
   public void testSanity() {
     assertTrue(sizeOf(new String("test string")) > shallowSizeOfInstance(String.class));
@@ -84,6 +86,10 @@ public class TestRamUsageEstimator extends LuceneTestCase {
       double[] array = new double[rnd.nextInt(1024)];
       assertEquals(sizeOf(array), sizeOf((Object) array));
     }
+    {
+      String s = RandomStrings.randomUnicodeOfCodepointLength(random(), random().nextInt(10));
+      assertEquals(sizeOf(s), sizeOf((Object) s));
+    }
   }
   
   public void testReferenceSize() {

lucene/facet/src/java/org/apache/lucene/facet/taxonomy/CachedOrdinalsReader.java

@@ -136,13 +136,21 @@ public class CachedOrdinalsReader extends OrdinalsReader {
         this.ordinals = ords;
       }
     }
+
+    public long ramBytesUsed() {
+      long mem = RamUsageEstimator.shallowSizeOf(this) + RamUsageEstimator.sizeOf(offsets);
+      if (offsets != ordinals) {
+        mem += RamUsageEstimator.sizeOf(ordinals);
+      }
+      return mem;
+    }
   }
 
   /** How many bytes is this cache using? */
   public synchronized long ramBytesUsed() {
     long bytes = 0;
     for(CachedOrds ords : ordsCache.values()) {
-      bytes += RamUsageEstimator.sizeOf(ords);
+      bytes += ords.ramBytesUsed();
     }
 
     return bytes;

lucene/memory/build.xml

@@ -19,6 +19,10 @@
 
 <project name="memory" default="default">
 
+  <property name="forbidden-rue-excludes" value="
+    org/apache/lucene/index/memory/MemoryIndex.class
+  "/>
+
   <description>
     Single-document in-memory index implementation
   </description>

lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java

@@ -44,11 +44,13 @@ import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FilterAtomicReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.MultiDocValues;
 import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SegmentReader;
 import org.apache.lucene.index.SlowCompositeReaderWrapper;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.sorter.Sorter;
@@ -640,7 +642,16 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
 
   @Override
   public long sizeInBytes() {
-    return RamUsageEstimator.sizeOf(this);
+    long mem = RamUsageEstimator.shallowSizeOf(this);
+    if (searcher != null) {
+      for (AtomicReaderContext context : searcher.getIndexReader().leaves()) {
+        AtomicReader reader = FilterAtomicReader.unwrap(context.reader());
+        if (reader instanceof SegmentReader) {
+          mem += ((SegmentReader) context.reader()).ramBytesUsed();
+        }
+      }
+    }
+    return mem;
   }
 
   @Override

lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java

@@ -291,7 +291,15 @@ public class FSTCompletionLookup extends Lookup {
 
   @Override
   public long sizeInBytes() {
-    return RamUsageEstimator.sizeOf(this);
+    long mem = RamUsageEstimator.shallowSizeOf(this) + RamUsageEstimator.shallowSizeOf(normalCompletion) + RamUsageEstimator.shallowSizeOf(higherWeightsCompletion);
+    if (normalCompletion != null) {
+      mem += normalCompletion.getFST().sizeInBytes();
+    }
+    if (higherWeightsCompletion != null && (normalCompletion == null || normalCompletion.getFST() != higherWeightsCompletion.getFST())) {
+      // the fst should be shared between the 2 completion instances, don't count it twice
+      mem += higherWeightsCompletion.getFST().sizeInBytes();
+    }
+    return mem;
   }
 
   @Override

lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java

@@ -198,7 +198,7 @@ public class JaspellLookup extends Lookup {
   /** Returns byte size of the underlying TST. */
   @Override
   public long sizeInBytes() {
-    return RamUsageEstimator.sizeOf(trie);
+    return trie.sizeInBytes();
   }
   
   @Override

lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java

@@ -40,6 +40,7 @@ import java.util.Vector;
 import java.util.zip.GZIPInputStream;
 
 import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
 
 /**
  * Implementation of a Ternary Search Trie, a data structure for storing
@@ -75,7 +76,7 @@ public class JaspellTernarySearchTrie {
     protected Object data;
 
     /** The relative nodes. */
-    protected TSTNode[] relatives = new TSTNode[4];
+    protected final TSTNode[] relatives = new TSTNode[4];
 
     /** The char used in the split. */
     protected char splitchar;
@@ -92,6 +93,17 @@ public class JaspellTernarySearchTrie {
       this.splitchar = splitchar;
       relatives[PARENT] = parent;
     }
+
+    public long sizeInBytes() {
+      long mem = RamUsageEstimator.shallowSizeOf(this) + RamUsageEstimator.shallowSizeOf(relatives);
+      for (TSTNode node : relatives) {
+        if (node != null) {
+          mem += node.sizeInBytes();
+        }
+      }
+      return mem;
+    }
+
   }
 
   /**
@@ -873,4 +885,14 @@ public class JaspellTernarySearchTrie {
             sortKeysNumReturnValues, sortKeysResult);
   }
 
+  /** Return an approximate memory usage for this trie. */
+  public long sizeInBytes() {
+    long mem = RamUsageEstimator.shallowSizeOf(this);
+    final TSTNode root = getRoot();
+    if (root != null) {
+      mem += root.sizeInBytes();
+    }
+    return mem;
+  }
+
 }

lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java

@@ -216,7 +216,11 @@ public class TSTLookup extends Lookup {
   /** Returns byte size of the underlying TST */
   @Override
   public long sizeInBytes() {
-    return RamUsageEstimator.sizeOf(autocomplete);
+    long mem = RamUsageEstimator.shallowSizeOf(this);
+    if (root != null) {
+      mem += root.sizeInBytes();
+    }
+    return mem;
   }
   
   @Override

lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TernaryTreeNode.java

@@ -1,5 +1,7 @@
 package org.apache.lucene.search.suggest.tst;
 
+import org.apache.lucene.util.RamUsageEstimator;
+
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -42,4 +44,22 @@ public class TernaryTreeNode {
    */
   String token;
   Object val;
+
+  long sizeInBytes() {
+    long mem = RamUsageEstimator.shallowSizeOf(this);
+    if (loKid != null) {
+      mem += loKid.sizeInBytes();
+    }
+    if (eqKid != null) {
+      mem += eqKid.sizeInBytes();
+    }
+    if (hiKid != null) {
+      mem += hiKid.sizeInBytes();
+    }
+    if (token != null) {
+      mem += RamUsageEstimator.sizeOf(token);
+    }
+    mem += RamUsageEstimator.shallowSizeOf(val);
+    return mem;
+  }
 }

lucene/test-framework/build.xml

@@ -51,6 +51,7 @@
     </forbidden-apis>
   </target>
   <target name="-check-forbidden-sysout"/>
+  <target name="-check-forbidden-rue"/>
 
   <target name="javadocs-core" depends="javadocs"/>
   <target name="javadocs" depends="init,javadocs-lucene-core,javadocs-lucene-codecs">

lucene/tools/build.xml

@@ -39,6 +39,7 @@
   <!-- redefine the forbidden apis to be no-ops -->
   <target name="-check-forbidden-tests"/>
   <target name="-check-forbidden-sysout"/>
+  <target name="-check-forbidden-rue"/>
 
   <!--  
       Specialize compile-core to not depend on clover, to exclude a 

lucene/tools/forbiddenApis/rue.txt

@@ -0,0 +1,19 @@
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+@defaultMessage slow
+
+org.apache.lucene.util.RamUsageEstimator#sizeOf(java.lang.Object)
+org.apache.lucene.util.RamUsageEstimator#humanSizeOf(java.lang.Object)