From 858718d17ad96109b4f0191df425f038c5b389ab Mon Sep 17 00:00:00 2001 From: Yonik Seeley Date: Thu, 9 Apr 2015 16:52:16 +0000 Subject: [PATCH] SOLR-7371: don't cache size for now and tolerate small alignment errors git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1672421 13f79535-47bb-0310-9956-ffa450edef68 --- .../src/java/org/apache/solr/search/BitDocSet.java | 8 +++++--- .../core/src/java/org/apache/solr/search/DocSlice.java | 8 +++----- .../src/java/org/apache/solr/search/HashDocSet.java | 10 ++-------- .../java/org/apache/solr/search/SortedIntDocSet.java | 9 +++------ 4 files changed, 13 insertions(+), 22 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/search/BitDocSet.java b/solr/core/src/java/org/apache/solr/search/BitDocSet.java index dd0b6ffabac..df9ca8f168a 100644 --- a/solr/core/src/java/org/apache/solr/search/BitDocSet.java +++ b/solr/core/src/java/org/apache/solr/search/BitDocSet.java @@ -40,7 +40,9 @@ import org.apache.lucene.util.RamUsageEstimator; * @since solr 0.9 */ public class BitDocSet extends DocSetBase { - private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BitDocSet.class); + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(BitDocSet.class) + + RamUsageEstimator.shallowSizeOfInstance(FixedBitSet.class) + + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; // for the array object inside the FixedBitSet. long[] array won't change alignment, so no need to calculate it. final FixedBitSet bits; int size; // number of docs in the set (cached for perf) @@ -198,7 +200,7 @@ public class BitDocSet extends DocSetBase { if (other instanceof BitDocSet) { // if we don't know our current size, this is faster than // size + other.size - intersection_size - return (int) FixedBitSet.unionCount(this.bits, ((BitDocSet)other).bits); + return (int) FixedBitSet.unionCount(this.bits, ((BitDocSet) other).bits); } else { // they had better not call us back! return other.unionSize(this); @@ -364,7 +366,7 @@ public class BitDocSet extends DocSetBase { @Override public long ramBytesUsed() { - return BASE_RAM_BYTES_USED + bits.ramBytesUsed(); + return BASE_RAM_BYTES_USED + ((long)bits.getBits().length << 3); } @Override diff --git a/solr/core/src/java/org/apache/solr/search/DocSlice.java b/solr/core/src/java/org/apache/solr/search/DocSlice.java index 6ab2d2e8d07..92a7e159550 100644 --- a/solr/core/src/java/org/apache/solr/search/DocSlice.java +++ b/solr/core/src/java/org/apache/solr/search/DocSlice.java @@ -31,7 +31,7 @@ import org.apache.lucene.util.RamUsageEstimator; * @since solr 0.9 */ public class DocSlice extends DocSetBase implements DocList { - private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DocSlice.class); + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(DocSlice.class) + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; final int offset; // starting position of the docs (zero based) final int len; // number of positions used in arrays @@ -41,8 +41,6 @@ public class DocSlice extends DocSetBase implements DocList { final int matches; final float maxScore; - final long ramBytesUsed; - /** * Primary constructor for a DocSlice instance. * @@ -59,7 +57,6 @@ public class DocSlice extends DocSetBase implements DocList { this.scores=scores; this.matches=matches; this.maxScore=maxScore; - this.ramBytesUsed = BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(docs) + (scores != null ? RamUsageEstimator.sizeOf(scores) : 0); } @Override @@ -178,9 +175,10 @@ public class DocSlice extends DocSetBase implements DocList { return null; } + /** WARNING: this can over-estimate real memory use since backing arrays are shared with other DocSlice instances */ @Override public long ramBytesUsed() { - return ramBytesUsed; + return BASE_RAM_BYTES_USED + ((long)docs.length << 2) + (scores == null ? 0 : ((long)scores.length<<2)+RamUsageEstimator.NUM_BYTES_ARRAY_HEADER); } @Override diff --git a/solr/core/src/java/org/apache/solr/search/HashDocSet.java b/solr/core/src/java/org/apache/solr/search/HashDocSet.java index 892bee6eadc..bcab43d66e3 100644 --- a/solr/core/src/java/org/apache/solr/search/HashDocSet.java +++ b/solr/core/src/java/org/apache/solr/search/HashDocSet.java @@ -35,7 +35,7 @@ import org.apache.lucene.util.RamUsageEstimator; * @since solr 0.9 */ public final class HashDocSet extends DocSetBase { - private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(HashDocSet.class); + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(HashDocSet.class) + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; /** Default load factor to use for HashDocSets. We keep track of the inverse * since multiplication is so much faster than division. The default @@ -52,16 +52,12 @@ public final class HashDocSet extends DocSetBase { private final static int EMPTY=-1; private final int[] table; private final int size; - private final int mask; - private final long ramBytesUsed; - public HashDocSet(HashDocSet set) { this.table = set.table.clone(); this.size = set.size; this.mask = set.mask; - this.ramBytesUsed = BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(table); } /** Create a HashDocSet from a list of *unique* ids */ @@ -89,8 +85,6 @@ public final class HashDocSet extends DocSetBase { } size = len; - - ramBytesUsed = BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(table); } void put(int doc) { @@ -307,7 +301,7 @@ public final class HashDocSet extends DocSetBase { @Override public long ramBytesUsed() { - return ramBytesUsed; + return BASE_RAM_BYTES_USED + (table.length<<2); } @Override diff --git a/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java b/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java index 33f58728f87..7b1349def58 100644 --- a/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java +++ b/solr/core/src/java/org/apache/solr/search/SortedIntDocSet.java @@ -35,18 +35,15 @@ import org.apache.lucene.util.RamUsageEstimator; * SortedIntDocSet represents a sorted set of Lucene Document Ids. */ public class SortedIntDocSet extends DocSetBase { - private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(SortedIntDocSet.class); + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(SortedIntDocSet.class) + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; protected final int[] docs; - protected final long ramBytesUsed; /** * @param docs Sorted list of ids */ public SortedIntDocSet(int[] docs) { this.docs = docs; - this.ramBytesUsed = BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(docs); - // if (firstNonSorted(docs,0,docs.length)>=0) throw new RuntimeException("NON SORTED DOCS!!!"); } /** @@ -246,7 +243,7 @@ public class SortedIntDocSet extends DocSetBase { // if b is 8 times bigger than a, use the modified binary search. if ((b.length>>3) >= a.length) { - return intersectionSize(a,b); + return intersectionSize(a, b); } // if they are close in size, just do a linear walk of both. @@ -791,7 +788,7 @@ public class SortedIntDocSet extends DocSetBase { @Override public long ramBytesUsed() { - return ramBytesUsed; + return BASE_RAM_BYTES_USED + (docs.length << 2); } @Override