From fde32cc599cdc763efaea7c77742619d1cac9078 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 8 Dec 2014 14:58:32 -0800 Subject: [PATCH] Stats: Add more fine grained memory stats from Lucene segment reader. This is a start to exposing memory stats improvements from Lucene 5.0. This adds the following categories of Lucene index pieces to index stats: * Terms * Stored fields * Term Vectors * Norms * Doc values --- .../index/engine/SegmentsStats.java | 120 ++++++++++++++++-- .../index/engine/internal/InternalEngine.java | 22 +++- .../indices/stats/IndicesStatsTests.java | 84 ++++++++++++ 3 files changed, 213 insertions(+), 13 deletions(-) create mode 100644 src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java diff --git a/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java b/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java index bee0a315d3e..229750e840e 100644 --- a/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java +++ b/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.engine; -import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; @@ -30,26 +29,46 @@ import org.elasticsearch.common.xcontent.XContentBuilderString; import java.io.IOException; -/** - * - */ public class SegmentsStats implements Streamable, ToXContent { private long count; private long memoryInBytes; + private long termsMemoryInBytes; + private long storedFieldsMemoryInBytes; + private long termVectorsMemoryInBytes; + private long normsMemoryInBytes; + private long docValuesMemoryInBytes; private long indexWriterMemoryInBytes; private long indexWriterMaxMemoryInBytes; private long versionMapMemoryInBytes; private long bitsetMemoryInBytes; - public SegmentsStats() { - - } + public SegmentsStats() {} public void add(long count, long memoryInBytes) { this.count += count; this.memoryInBytes += memoryInBytes; } + + public void addTermsMemoryInBytes(long termsMemoryInBytes) { + this.termsMemoryInBytes += termsMemoryInBytes; + } + + public void addStoredFieldsMemoryInBytes(long storedFieldsMemoryInBytes) { + this.storedFieldsMemoryInBytes += storedFieldsMemoryInBytes; + } + + public void addTermVectorsMemoryInBytes(long termVectorsMemoryInBytes) { + this.termVectorsMemoryInBytes += termVectorsMemoryInBytes; + } + + public void addNormsMemoryInBytes(long normsMemoryInBytes) { + this.normsMemoryInBytes += normsMemoryInBytes; + } + + public void addDocValuesMemoryInBytes(long docValuesMemoryInBytes) { + this.docValuesMemoryInBytes += docValuesMemoryInBytes; + } public void addIndexWriterMemoryInBytes(long indexWriterMemoryInBytes) { this.indexWriterMemoryInBytes += indexWriterMemoryInBytes; @@ -72,6 +91,11 @@ public class SegmentsStats implements Streamable, ToXContent { return; } add(mergeStats.count, mergeStats.memoryInBytes); + addTermsMemoryInBytes(mergeStats.termsMemoryInBytes); + addStoredFieldsMemoryInBytes(mergeStats.storedFieldsMemoryInBytes); + addTermVectorsMemoryInBytes(mergeStats.termVectorsMemoryInBytes); + addNormsMemoryInBytes(mergeStats.normsMemoryInBytes); + addDocValuesMemoryInBytes(mergeStats.docValuesMemoryInBytes); addIndexWriterMemoryInBytes(mergeStats.indexWriterMemoryInBytes); addIndexWriterMaxMemoryInBytes(mergeStats.indexWriterMaxMemoryInBytes); addVersionMapMemoryInBytes(mergeStats.versionMapMemoryInBytes); @@ -79,7 +103,7 @@ public class SegmentsStats implements Streamable, ToXContent { } /** - * The the segments count. + * The number of segments. */ public long getCount() { return this.count; @@ -96,6 +120,61 @@ public class SegmentsStats implements Streamable, ToXContent { return new ByteSizeValue(memoryInBytes); } + /** + * Estimation of the terms dictionary memory usage by a segment. + */ + public long getTermsMemoryInBytes() { + return this.termsMemoryInBytes; + } + + public ByteSizeValue getTermsMemory() { + return new ByteSizeValue(termsMemoryInBytes); + } + + /** + * Estimation of the stored fields memory usage by a segment. + */ + public long getStoredFieldsMemoryInBytes() { + return this.storedFieldsMemoryInBytes; + } + + public ByteSizeValue getStoredFieldsMemory() { + return new ByteSizeValue(storedFieldsMemoryInBytes); + } + + /** + * Estimation of the term vectors memory usage by a segment. + */ + public long getTermVectorsMemoryInBytes() { + return this.termVectorsMemoryInBytes; + } + + public ByteSizeValue getTermVectorsMemory() { + return new ByteSizeValue(termVectorsMemoryInBytes); + } + + /** + * Estimation of the norms memory usage by a segment. + */ + public long getNormsMemoryInBytes() { + return this.normsMemoryInBytes; + } + + public ByteSizeValue getNormsMemory() { + return new ByteSizeValue(normsMemoryInBytes); + } + + /** + * Estimation of the doc values memory usage by a segment. + */ + public long getDocValuesMemoryInBytes() { + return this.docValuesMemoryInBytes; + } + + public ByteSizeValue getDocValuesMemory() { + return new ByteSizeValue(docValuesMemoryInBytes); + } + /** * Estimation of the memory usage by index writer */ @@ -151,6 +230,11 @@ public class SegmentsStats implements Streamable, ToXContent { builder.startObject(Fields.SEGMENTS); builder.field(Fields.COUNT, count); builder.byteSizeField(Fields.MEMORY_IN_BYTES, Fields.MEMORY, memoryInBytes); + builder.byteSizeField(Fields.TERMS_MEMORY_IN_BYTES, Fields.TERMS_MEMORY, termsMemoryInBytes); + builder.byteSizeField(Fields.STORED_FIELDS_MEMORY_IN_BYTES, Fields.STORED_FIELDS_MEMORY, storedFieldsMemoryInBytes); + builder.byteSizeField(Fields.TERM_VECTORS_MEMORY_IN_BYTES, Fields.TERM_VECTORS_MEMORY, termVectorsMemoryInBytes); + builder.byteSizeField(Fields.NORMS_MEMORY_IN_BYTES, Fields.NORMS_MEMORY, normsMemoryInBytes); + builder.byteSizeField(Fields.DOC_VALUES_MEMORY_IN_BYTES, Fields.DOC_VALUES_MEMORY, docValuesMemoryInBytes); builder.byteSizeField(Fields.INDEX_WRITER_MEMORY_IN_BYTES, Fields.INDEX_WRITER_MEMORY, indexWriterMemoryInBytes); builder.byteSizeField(Fields.INDEX_WRITER_MAX_MEMORY_IN_BYTES, Fields.INDEX_WRITER_MAX_MEMORY, indexWriterMaxMemoryInBytes); builder.byteSizeField(Fields.VERSION_MAP_MEMORY_IN_BYTES, Fields.VERSION_MAP_MEMORY, versionMapMemoryInBytes); @@ -164,6 +248,16 @@ public class SegmentsStats implements Streamable, ToXContent { static final XContentBuilderString COUNT = new XContentBuilderString("count"); static final XContentBuilderString MEMORY = new XContentBuilderString("memory"); static final XContentBuilderString MEMORY_IN_BYTES = new XContentBuilderString("memory_in_bytes"); + static final XContentBuilderString TERMS_MEMORY = new XContentBuilderString("terms_memory"); + static final XContentBuilderString TERMS_MEMORY_IN_BYTES = new XContentBuilderString("terms_memory_in_bytes"); + static final XContentBuilderString STORED_FIELDS_MEMORY = new XContentBuilderString("stored_fields_memory"); + static final XContentBuilderString STORED_FIELDS_MEMORY_IN_BYTES = new XContentBuilderString("stored_fields_memory_in_bytes"); + static final XContentBuilderString TERM_VECTORS_MEMORY = new XContentBuilderString("term_vectors_memory"); + static final XContentBuilderString TERM_VECTORS_MEMORY_IN_BYTES = new XContentBuilderString("term_vectors_memory_in_bytes"); + static final XContentBuilderString NORMS_MEMORY = new XContentBuilderString("norms_memory"); + static final XContentBuilderString NORMS_MEMORY_IN_BYTES = new XContentBuilderString("norms_memory_in_bytes"); + static final XContentBuilderString DOC_VALUES_MEMORY = new XContentBuilderString("doc_values_memory"); + static final XContentBuilderString DOC_VALUES_MEMORY_IN_BYTES = new XContentBuilderString("doc_values_memory_in_bytes"); static final XContentBuilderString INDEX_WRITER_MEMORY = new XContentBuilderString("index_writer_memory"); static final XContentBuilderString INDEX_WRITER_MEMORY_IN_BYTES = new XContentBuilderString("index_writer_memory_in_bytes"); static final XContentBuilderString INDEX_WRITER_MAX_MEMORY = new XContentBuilderString("index_writer_max_memory"); @@ -178,6 +272,11 @@ public class SegmentsStats implements Streamable, ToXContent { public void readFrom(StreamInput in) throws IOException { count = in.readVLong(); memoryInBytes = in.readLong(); + termsMemoryInBytes = in.readLong(); + storedFieldsMemoryInBytes = in.readLong(); + termVectorsMemoryInBytes = in.readLong(); + normsMemoryInBytes = in.readLong(); + docValuesMemoryInBytes = in.readLong(); indexWriterMemoryInBytes = in.readLong(); versionMapMemoryInBytes = in.readLong(); indexWriterMaxMemoryInBytes = in.readLong(); @@ -188,6 +287,11 @@ public class SegmentsStats implements Streamable, ToXContent { public void writeTo(StreamOutput out) throws IOException { out.writeVLong(count); out.writeLong(memoryInBytes); + out.writeLong(termsMemoryInBytes); + out.writeLong(storedFieldsMemoryInBytes); + out.writeLong(termVectorsMemoryInBytes); + out.writeLong(normsMemoryInBytes); + out.writeLong(docValuesMemoryInBytes); out.writeLong(indexWriterMemoryInBytes); out.writeLong(versionMapMemoryInBytes); out.writeLong(indexWriterMaxMemoryInBytes); diff --git a/src/main/java/org/elasticsearch/index/engine/internal/InternalEngine.java b/src/main/java/org/elasticsearch/index/engine/internal/InternalEngine.java index 5cb6f8a1a47..df40a63237b 100644 --- a/src/main/java/org/elasticsearch/index/engine/internal/InternalEngine.java +++ b/src/main/java/org/elasticsearch/index/engine/internal/InternalEngine.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.*; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; @@ -1213,9 +1214,12 @@ public class InternalEngine implements Engine { } return t; } - - private static long getReaderRamBytesUsed(LeafReaderContext reader) { - return segmentReader(reader.reader()).ramBytesUsed(); + + private long guardedRamBytesUsed(Accountable a) { + if (a == null) { + return 0; + } + return a.ramBytesUsed(); } @Override @@ -1225,7 +1229,13 @@ public class InternalEngine implements Engine { try (final Searcher searcher = acquireSearcher("segments_stats")) { SegmentsStats stats = new SegmentsStats(); for (LeafReaderContext reader : searcher.reader().leaves()) { - stats.add(1, getReaderRamBytesUsed(reader)); + final SegmentReader segmentReader = segmentReader(reader.reader()); + stats.add(1, segmentReader.ramBytesUsed()); + stats.addTermsMemoryInBytes(guardedRamBytesUsed(segmentReader.fields())); + stats.addStoredFieldsMemoryInBytes(guardedRamBytesUsed(segmentReader.getFieldsReader())); + stats.addTermVectorsMemoryInBytes(guardedRamBytesUsed(segmentReader.getTermVectorsReader())); + stats.addNormsMemoryInBytes(guardedRamBytesUsed(segmentReader.getNormsReader())); + stats.addDocValuesMemoryInBytes(guardedRamBytesUsed(segmentReader.getDocValuesReader())); } stats.addVersionMapMemoryInBytes(versionMap.ramBytesUsed()); stats.addIndexWriterMemoryInBytes(indexWriter.ramBytesUsed()); @@ -1258,7 +1268,9 @@ public class InternalEngine implements Engine { } catch (IOException e) { logger.trace("failed to get size for [{}]", e, info.info.name); } - segment.memoryInBytes = getReaderRamBytesUsed(reader); + final SegmentReader segmentReader = segmentReader(reader.reader()); + segment.memoryInBytes = segmentReader.ramBytesUsed(); + // TODO: add more fine grained mem stats values to per segment info here segments.put(info.info.name, segment); } } finally { diff --git a/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java b/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java new file mode 100644 index 00000000000..4e1f25fb1bb --- /dev/null +++ b/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java @@ -0,0 +1,84 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.stats; + +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.engine.SegmentsStats; +import org.elasticsearch.test.ElasticsearchSingleNodeTest; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; + +import static org.hamcrest.Matchers.*; + +public class IndicesStatsTests extends ElasticsearchSingleNodeTest { + + public void testSegmentStatsEmptyIndex() { + createIndex("test"); + IndicesStatsResponse rsp = client().admin().indices().prepareStats("test").get(); + SegmentsStats stats = rsp.getTotal().getSegments(); + assertEquals(0, stats.getTermsMemoryInBytes()); + assertEquals(0, stats.getStoredFieldsMemoryInBytes()); + assertEquals(0, stats.getTermVectorsMemoryInBytes()); + assertEquals(0, stats.getNormsMemoryInBytes()); + assertEquals(0, stats.getDocValuesMemoryInBytes()); + } + + public void testSegmentStats() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("doc") + .startObject("properties") + .startObject("foo") + .field("type", "string") + .field("index", "not_analyzed") + .field("doc_values", true) + .field("store", true) + .field("term_vector", "with_positions_offsets_payloads") + .endObject() + .endObject() + .endObject() + .endObject(); + assertAcked(client().admin().indices().prepareCreate("test").addMapping("doc", mapping)); + ensureGreen("test"); + client().prepareIndex("test", "doc", "1").setSource("foo", "bar").get(); + client().admin().indices().prepareRefresh("test").get(); + + IndicesStatsResponse rsp = client().admin().indices().prepareStats("test").get(); + SegmentsStats stats = rsp.getIndex("test").getTotal().getSegments(); + assertThat(stats.getTermsMemoryInBytes(), greaterThan(0l)); + assertThat(stats.getStoredFieldsMemoryInBytes(), greaterThan(0l)); + assertThat(stats.getTermVectorsMemoryInBytes(), greaterThan(0l)); + assertThat(stats.getNormsMemoryInBytes(), greaterThan(0l)); + assertThat(stats.getDocValuesMemoryInBytes(), greaterThan(0l)); + + // now check multiple segments stats are merged together + client().prepareIndex("test", "doc", "2").setSource("foo", "bar").get(); + client().admin().indices().prepareRefresh("test").get(); + + rsp = client().admin().indices().prepareStats("test").get(); + SegmentsStats stats2 = rsp.getIndex("test").getTotal().getSegments(); + assertThat(stats2.getTermsMemoryInBytes(), greaterThan(stats.getTermsMemoryInBytes())); + assertThat(stats2.getStoredFieldsMemoryInBytes(), greaterThan(stats.getStoredFieldsMemoryInBytes())); + assertThat(stats2.getTermVectorsMemoryInBytes(), greaterThan(stats.getTermVectorsMemoryInBytes())); + assertThat(stats2.getNormsMemoryInBytes(), greaterThan(stats.getNormsMemoryInBytes())); + assertThat(stats2.getDocValuesMemoryInBytes(), greaterThan(stats.getDocValuesMemoryInBytes())); + } +}