From e4ad92203f2e479ed617a1e710fe05025c7700c2 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 29 Jul 2013 00:02:42 +0200 Subject: [PATCH] remove RamUsage and use Lucene one --- .../org/elasticsearch/common/RamUsage.java | 53 ------------------- .../common/util/BigDoubleArrayList.java | 14 +++-- .../common/util/BigFloatArrayList.java | 14 +++-- .../common/util/BigIntArray.java | 14 +++-- .../id/simple/SimpleIdReaderTypeCache.java | 16 +++--- .../fielddata/ordinals/DocIdOrdinals.java | 4 +- .../fielddata/ordinals/MultiOrdinals.java | 12 +++-- .../ordinals/SinglePackedOrdinals.java | 6 +-- .../plain/DoubleArrayAtomicFieldData.java | 8 +-- .../plain/DoubleArrayIndexFieldData.java | 10 ++-- .../plain/FloatArrayAtomicFieldData.java | 8 +-- .../plain/FloatArrayIndexFieldData.java | 8 +-- .../GeoPointDoubleArrayAtomicFieldData.java | 10 ++-- .../plain/PackedArrayAtomicFieldData.java | 6 +-- 14 files changed, 69 insertions(+), 114 deletions(-) delete mode 100644 src/main/java/org/elasticsearch/common/RamUsage.java diff --git a/src/main/java/org/elasticsearch/common/RamUsage.java b/src/main/java/org/elasticsearch/common/RamUsage.java deleted file mode 100644 index 97bafa4afe5..00000000000 --- a/src/main/java/org/elasticsearch/common/RamUsage.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common; - -/** - * - */ -public class RamUsage { - - private static final String OS_ARCH = System.getProperty("os.arch"); - private static final boolean JRE_IS_64BIT; - - static { - String x = System.getProperty("sun.arch.data.model"); - if (x != null) { - JRE_IS_64BIT = x.indexOf("64") != -1; - } else { - if (OS_ARCH != null && OS_ARCH.indexOf("64") != -1) { - JRE_IS_64BIT = true; - } else { - JRE_IS_64BIT = false; - } - } - } - - public final static int NUM_BYTES_SHORT = 2; - public final static int NUM_BYTES_INT = 4; - public final static int NUM_BYTES_LONG = 8; - public final static int NUM_BYTES_FLOAT = 4; - public final static int NUM_BYTES_DOUBLE = 8; - public final static int NUM_BYTES_CHAR = 2; - public final static int NUM_BYTES_OBJECT_HEADER = 8; - public final static int NUM_BYTES_OBJECT_REF = JRE_IS_64BIT ? 8 : 4; - public final static int NUM_BYTES_ARRAY_HEADER = NUM_BYTES_OBJECT_HEADER + NUM_BYTES_INT + NUM_BYTES_OBJECT_REF; - -} diff --git a/src/main/java/org/elasticsearch/common/util/BigDoubleArrayList.java b/src/main/java/org/elasticsearch/common/util/BigDoubleArrayList.java index 6f213588597..50344d3e4e5 100644 --- a/src/main/java/org/elasticsearch/common/util/BigDoubleArrayList.java +++ b/src/main/java/org/elasticsearch/common/util/BigDoubleArrayList.java @@ -20,15 +20,19 @@ package org.elasticsearch.common.util; import org.apache.lucene.util.ArrayUtil; -import org.elasticsearch.common.RamUsage; +import org.apache.lucene.util.RamUsageEstimator; import java.util.Arrays; -/** Float array abstraction able to support more than 2B values. This implementation slices data into fixed-sized blocks of - * configurable length. */ +/** + * Float array abstraction able to support more than 2B values. This implementation slices data into fixed-sized blocks of + * configurable length. + */ public final class BigDoubleArrayList extends AbstractBigArray { - /** Default page size, 16KB of memory per page. */ + /** + * Default page size, 16KB of memory per page. + */ private static final int DEFAULT_PAGE_SIZE = 1 << 11; private double[][] pages; @@ -69,7 +73,7 @@ public final class BigDoubleArrayList extends AbstractBigArray { @Override protected int numBytesPerElement() { - return RamUsage.NUM_BYTES_DOUBLE; + return RamUsageEstimator.NUM_BYTES_DOUBLE; } } diff --git a/src/main/java/org/elasticsearch/common/util/BigFloatArrayList.java b/src/main/java/org/elasticsearch/common/util/BigFloatArrayList.java index 15ac6dd0a9c..b3bb3dcb5f5 100644 --- a/src/main/java/org/elasticsearch/common/util/BigFloatArrayList.java +++ b/src/main/java/org/elasticsearch/common/util/BigFloatArrayList.java @@ -20,13 +20,17 @@ package org.elasticsearch.common.util; import org.apache.lucene.util.ArrayUtil; -import org.elasticsearch.common.RamUsage; +import org.apache.lucene.util.RamUsageEstimator; -/** Float array abstraction able to support more than 2B values. This implementation slices data into fixed-sized blocks of - * configurable length. */ +/** + * Float array abstraction able to support more than 2B values. This implementation slices data into fixed-sized blocks of + * configurable length. + */ public final class BigFloatArrayList extends AbstractBigArray { - /** Default page size, 16KB of memory per page. */ + /** + * Default page size, 16KB of memory per page. + */ private static final int DEFAULT_PAGE_SIZE = 1 << 12; private float[][] pages; @@ -64,7 +68,7 @@ public final class BigFloatArrayList extends AbstractBigArray { @Override protected int numBytesPerElement() { - return RamUsage.NUM_BYTES_FLOAT; + return RamUsageEstimator.NUM_BYTES_FLOAT; } } diff --git a/src/main/java/org/elasticsearch/common/util/BigIntArray.java b/src/main/java/org/elasticsearch/common/util/BigIntArray.java index 669940bd273..2e181491c9b 100644 --- a/src/main/java/org/elasticsearch/common/util/BigIntArray.java +++ b/src/main/java/org/elasticsearch/common/util/BigIntArray.java @@ -19,13 +19,17 @@ package org.elasticsearch.common.util; -import org.elasticsearch.common.RamUsage; +import org.apache.lucene.util.RamUsageEstimator; -/** Int array abstraction able to support more than 2B values. This implementation slices data into fixed-sized blocks of - * configurable length. */ +/** + * Int array abstraction able to support more than 2B values. This implementation slices data into fixed-sized blocks of + * configurable length. + */ public final class BigIntArray extends AbstractBigArray implements IntArray { - /** Default page size, 16KB of memory per page. */ + /** + * Default page size, 16KB of memory per page. + */ public static final int DEFAULT_PAGE_SIZE = 1 << 12; private int[][] pages; @@ -63,7 +67,7 @@ public final class BigIntArray extends AbstractBigArray implements IntArray { @Override protected int numBytesPerElement() { - return RamUsage.NUM_BYTES_INT; + return RamUsageEstimator.NUM_BYTES_INT; } } diff --git a/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdReaderTypeCache.java b/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdReaderTypeCache.java index 083e79f3b51..c31c043f049 100644 --- a/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdReaderTypeCache.java +++ b/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdReaderTypeCache.java @@ -20,7 +20,7 @@ package org.elasticsearch.index.cache.id.simple; import gnu.trove.impl.hash.TObjectHash; -import org.elasticsearch.common.RamUsage; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.bytes.HashedBytesArray; import org.elasticsearch.common.trove.ExtTObjectIntHasMap; import org.elasticsearch.index.cache.id.IdReaderTypeCache; @@ -86,26 +86,26 @@ public class SimpleIdReaderTypeCache implements IdReaderTypeCache { long sizeInBytes = 0; // Ignore type field // sizeInBytes += ((type.length() * RamUsage.NUM_BYTES_CHAR) + (3 * RamUsage.NUM_BYTES_INT)) + RamUsage.NUM_BYTES_OBJECT_HEADER; - sizeInBytes += RamUsage.NUM_BYTES_ARRAY_HEADER + (idToDoc._valuesSize() * RamUsage.NUM_BYTES_INT); + sizeInBytes += RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (idToDoc._valuesSize() * RamUsageEstimator.NUM_BYTES_INT); for (Object o : idToDoc._set) { if (o == TObjectHash.FREE || o == TObjectHash.REMOVED) { - sizeInBytes += RamUsage.NUM_BYTES_OBJECT_REF; + sizeInBytes += RamUsageEstimator.NUM_BYTES_OBJECT_REF; } else { HashedBytesArray bytesArray = (HashedBytesArray) o; - sizeInBytes += RamUsage.NUM_BYTES_OBJECT_HEADER + (bytesArray.length() + RamUsage.NUM_BYTES_INT); + sizeInBytes += RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + (bytesArray.length() + RamUsageEstimator.NUM_BYTES_INT); } } // The docIdToId array contains references to idToDoc for this segment or other segments, so we can use OBJECT_REF - sizeInBytes += RamUsage.NUM_BYTES_ARRAY_HEADER + (RamUsage.NUM_BYTES_OBJECT_REF * docIdToId.length); + sizeInBytes += RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (RamUsageEstimator.NUM_BYTES_OBJECT_REF * docIdToId.length); for (HashedBytesArray bytesArray : parentIdsValues) { if (bytesArray == null) { - sizeInBytes += RamUsage.NUM_BYTES_OBJECT_REF; + sizeInBytes += RamUsageEstimator.NUM_BYTES_OBJECT_REF; } else { - sizeInBytes += RamUsage.NUM_BYTES_OBJECT_HEADER + (bytesArray.length() + RamUsage.NUM_BYTES_INT); + sizeInBytes += RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + (bytesArray.length() + RamUsageEstimator.NUM_BYTES_INT); } } - sizeInBytes += RamUsage.NUM_BYTES_ARRAY_HEADER + (RamUsage.NUM_BYTES_INT * parentIdsOrdinals.length); + sizeInBytes += RamUsageEstimator.sizeOf(parentIdsOrdinals); return sizeInBytes; } diff --git a/src/main/java/org/elasticsearch/index/fielddata/ordinals/DocIdOrdinals.java b/src/main/java/org/elasticsearch/index/fielddata/ordinals/DocIdOrdinals.java index 53c4b078bb6..238172cbace 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/ordinals/DocIdOrdinals.java +++ b/src/main/java/org/elasticsearch/index/fielddata/ordinals/DocIdOrdinals.java @@ -20,7 +20,7 @@ package org.elasticsearch.index.fielddata.ordinals; import org.apache.lucene.util.LongsRef; -import org.elasticsearch.common.RamUsage; +import org.apache.lucene.util.RamUsageEstimator; /** * Ordinals that effectively are single valued and map "one to one" to the @@ -50,7 +50,7 @@ public class DocIdOrdinals implements Ordinals { @Override public long getMemorySizeInBytes() { - return RamUsage.NUM_BYTES_OBJECT_REF; + return RamUsageEstimator.NUM_BYTES_OBJECT_REF; } @Override diff --git a/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java b/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java index f242abf3b79..541e7308a49 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java +++ b/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java @@ -21,19 +21,23 @@ package org.elasticsearch.index.fielddata.ordinals; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.LongsRef; +import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.packed.AppendingLongBuffer; import org.apache.lucene.util.packed.MonotonicAppendingLongBuffer; import org.apache.lucene.util.packed.PackedInts; -import org.elasticsearch.common.RamUsage; import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs.Iter; -/** {@link Ordinals} implementation which is efficient at storing field data ordinals for multi-valued or sparse fields. */ +/** + * {@link Ordinals} implementation which is efficient at storing field data ordinals for multi-valued or sparse fields. + */ public class MultiOrdinals implements Ordinals { private static final int OFFSETS_PAGE_SIZE = 1024; private static final int OFFSET_INIT_PAGE_COUNT = 16; - /** Return true if this impl is going to be smaller than {@link SinglePackedOrdinals} by at least 20%. */ + /** + * Return true if this impl is going to be smaller than {@link SinglePackedOrdinals} by at least 20%. + */ public static boolean significantlySmallerThanSinglePackedOrdinals(int maxDoc, int numDocsWithValue, long numOrds) { final int bitsPerOrd = PackedInts.bitsRequired(numOrds); // Compute the worst-case number of bits per value for offsets in the worst case, eg. if no docs have a value at the @@ -168,7 +172,7 @@ public class MultiOrdinals implements Ordinals { final long endOffset = endOffsets.get(docId); final int numValues = (int) (endOffset - startOffset); if (longsScratch.length < numValues) { - longsScratch.longs = new long[ArrayUtil.oversize(numValues, RamUsage.NUM_BYTES_LONG)]; + longsScratch.longs = new long[ArrayUtil.oversize(numValues, RamUsageEstimator.NUM_BYTES_LONG)]; } for (int i = 0; i < numValues; ++i) { longsScratch.longs[i] = 1L + ords.get(startOffset + i); diff --git a/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java b/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java index 8647699ddcd..7128ddb7562 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java +++ b/src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java @@ -20,8 +20,8 @@ package org.elasticsearch.index.fielddata.ordinals; import org.apache.lucene.util.LongsRef; +import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.packed.PackedInts; -import org.elasticsearch.common.RamUsage; /** */ @@ -60,7 +60,7 @@ public class SinglePackedOrdinals implements Ordinals { @Override public long getMemorySizeInBytes() { if (size == -1) { - size = RamUsage.NUM_BYTES_OBJECT_REF + reader.ramBytesUsed(); + size = RamUsageEstimator.NUM_BYTES_OBJECT_REF + reader.ramBytesUsed(); } return size; } @@ -136,7 +136,7 @@ public class SinglePackedOrdinals implements Ordinals { @Override public LongsRef getOrds(int docId) { final long ordinal = reader.get(docId); - if (ordinal == 0) { + if (ordinal == 0) { longsScratch.length = 0; } else { longsScratch.offset = 0; diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java index 6b0949d5136..7a1c3f8ca8d 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java +++ b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java @@ -20,7 +20,7 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.util.FixedBitSet; -import org.elasticsearch.common.RamUsage; +import org.apache.lucene.util.RamUsageEstimator; import org.elasticsearch.common.util.BigDoubleArrayList; import org.elasticsearch.index.fielddata.*; import org.elasticsearch.index.fielddata.ordinals.Ordinals; @@ -115,7 +115,7 @@ public abstract class DoubleArrayAtomicFieldData extends AtomicNumericFieldData @Override public long getMemorySizeInBytes() { if (size == -1) { - size = RamUsage.NUM_BYTES_INT/*size*/ + RamUsage.NUM_BYTES_INT/*numDocs*/ + values.sizeInBytes() + ordinals.getMemorySizeInBytes(); + size = RamUsageEstimator.NUM_BYTES_INT/*size*/ + RamUsageEstimator.NUM_BYTES_INT/*numDocs*/ + values.sizeInBytes() + ordinals.getMemorySizeInBytes(); } return size; } @@ -191,7 +191,7 @@ public abstract class DoubleArrayAtomicFieldData extends AtomicNumericFieldData @Override public long getMemorySizeInBytes() { if (size == -1) { - size = RamUsage.NUM_BYTES_ARRAY_HEADER + values.sizeInBytes() + (set.getBits().length * RamUsage.NUM_BYTES_LONG); + size = RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + values.sizeInBytes() + RamUsageEstimator.sizeOf(set.getBits()); } return size; } @@ -281,7 +281,7 @@ public abstract class DoubleArrayAtomicFieldData extends AtomicNumericFieldData @Override public long getMemorySizeInBytes() { if (size == -1) { - size = RamUsage.NUM_BYTES_ARRAY_HEADER + values.sizeInBytes(); + size = RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + values.sizeInBytes(); } return size; } diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java index dbd5cf030a9..fe072298c03 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java +++ b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java @@ -22,13 +22,9 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.Terms; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.BytesRefIterator; -import org.apache.lucene.util.FixedBitSet; -import org.apache.lucene.util.NumericUtils; +import org.apache.lucene.util.*; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.RamUsage; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigDoubleArrayList; import org.elasticsearch.index.Index; @@ -108,8 +104,8 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData