honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-17 10:25:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

add filter_cache_size to node stats

Browse Source
This commit is contained in:
kimchy 2010-12-28 16:57:55 +02:00
parent bb4017769e
commit d8aef57baa
14 changed files with 120 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/AllDocSet.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.common.lucene.docset;
import org.apache.lucene.search.DocIdSetIterator;
import org.elasticsearch.common.RamUsage;
import java.io.IOException;
@ -44,6 +45,10 @@ public class AllDocSet extends DocSet {
return doc < maxDoc;
}
@Override public long sizeInBytes() {
return RamUsage.NUM_BYTES_INT;
}
@Override public DocIdSetIterator iterator() throws IOException {
return new AllDocIdSetIterator(maxDoc);
}

8
modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/AndDocSet.java
View File

@ -55,6 +55,14 @@ public class AndDocSet extends DocSet {
// return true;
}
@Override public long sizeInBytes() {
long sizeInBytes = 0;
for (DocSet set : sets) {
sizeInBytes += set.sizeInBytes();
}
return sizeInBytes;
}
@Override public DocIdSetIterator iterator() throws IOException {
return new AndDocIdSetIterator();
}

6
modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/DocSet.java
View File

@ -41,7 +41,13 @@ public abstract class DocSet extends DocIdSet {
@Override public boolean isCacheable() {
return true;
}
@Override public long sizeInBytes() {
return 0;
}
};
public abstract boolean get(int doc) throws IOException;
public abstract long sizeInBytes();
}

4
modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/GetDocSet.java
View File

@ -34,6 +34,10 @@ public abstract class GetDocSet extends DocSet {
this.maxDoc = maxDoc;
}
@Override public long sizeInBytes() {
return 0;
}
@Override public DocIdSetIterator iterator() throws IOException {
return new DocIdSetIterator() {
private int doc = -1;

4
modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/NotDocSet.java
View File

@ -44,6 +44,10 @@ public class NotDocSet extends GetDocSet {
return !set.get(doc);
}
@Override public long sizeInBytes() {
return set.sizeInBytes();
}
// This seems like overhead compared to testing with get and iterating over docs
// @Override public DocIdSetIterator iterator() throws IOException {
// return new NotDocIdSetIterator();

5
modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/OpenBitDocSet.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.common.lucene.docset;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.OpenBitSetDISI;
import org.elasticsearch.common.RamUsage;
import java.io.IOException;
@ -59,4 +60,8 @@ public class OpenBitDocSet extends DocSet {
@Override public DocIdSetIterator iterator() throws IOException {
return set.iterator();
}
@Override public long sizeInBytes() {
return set.getBits().length * RamUsage.NUM_BYTES_LONG + RamUsage.NUM_BYTES_ARRAY_HEADER + RamUsage.NUM_BYTES_INT /* wlen */;
}
}

28
modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/OrDocSet.java
View File

@ -37,22 +37,30 @@ public class OrDocSet extends DocSet {
}
@Override public boolean get(int doc) throws IOException {
// not cacheable, the reason is that by default, when constructing the filter, it is not cacheable,
// so if someone wants it to be cacheable, we might as well construct a cached version of the result
for (DocSet s : sets) {
if (s.get(doc)) return true;
}
return false;
// for (DocSet s : sets) {
// if (s.get(doc)) return true;
// }
// return false;
}
@Override public boolean isCacheable() {
// not cacheable, the reason is that by default, when constructing the filter, it is not cacheable,
// so if someone wants it to be cacheable, we might as well construct a cached version of the result
return false;
// for (DocSet set : sets) {
// if (!set.isCacheable()) {
// return false;
// }
// }
// return true;
}
@Override public long sizeInBytes() {
long sizeInBytes = 0;
for (DocSet set : sets) {
if (!set.isCacheable()) {
return false;
}
sizeInBytes += set.sizeInBytes();
}
return true;
return sizeInBytes;
}
@Override public DocIdSetIterator iterator() throws IOException {

2
modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/FilterCache.java vendored
View File

@ -45,4 +45,6 @@ public interface FilterCache extends IndexComponent, CloseableComponent {
* Clears unreferenced filters.
*/
void clearUnreferenced();
long sizeInBytes();
}

4
modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/none/NoneFilterCache.java vendored
View File

@ -69,4 +69,8 @@ public class NoneFilterCache extends AbstractIndexComponent implements FilterCac
@Override public void clearUnreferenced() {
// nothing to do here
}
@Override public long sizeInBytes() {
return 0;
}
}

10
modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java vendored
View File

@ -88,6 +88,16 @@ public abstract class AbstractConcurrentMapFilterCache extends AbstractIndexComp
// }
}
@Override public long sizeInBytes() {
long sizeInBytes = 0;
for (ConcurrentMap<Filter, DocSet> map : cache.values()) {
for (DocSet docSet : map.values()) {
sizeInBytes += docSet.sizeInBytes();
}
}
return sizeInBytes;
}
@Override public Filter cache(Filter filterToCache) {
if (isCached(filterToCache)) {
return filterToCache;

15
modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractDoubleConcurrentMapFilterCache.java vendored
View File

@ -79,6 +79,21 @@ public abstract class AbstractDoubleConcurrentMapFilterCache extends AbstractInd
@Override public void clearUnreferenced() {
}
@Override public long sizeInBytes() {
long sizeInBytes = 0;
for (ConcurrentMap<Filter, DocSet> map : cache.values()) {
for (DocSet docSet : map.values()) {
sizeInBytes += docSet.sizeInBytes();
}
}
for (ConcurrentMap<Filter, DocSet> map : weakCache.values()) {
for (DocSet docSet : map.values()) {
sizeInBytes += docSet.sizeInBytes();
}
}
return sizeInBytes;
}
@Override public Filter cache(Filter filterToCache) {
if (isCached(filterToCache)) {
return filterToCache;

4
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/ScriptFilterParser.java
View File

@ -164,6 +164,10 @@ public class ScriptFilterParser extends AbstractIndexComponent implements XConte
this.searchScript = searchScript;
}
@Override public long sizeInBytes() {
return 0;
}
@Override public boolean isCacheable() {
// not cacheable for several reasons:
// 1. The script service is shared and holds the current reader executing against, and it

38
modules/elasticsearch/src/main/java/org/elasticsearch/indices/IndicesStats.java
View File

@ -25,6 +25,7 @@ import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentBuilderString;
import java.io.IOException;
import java.io.Serializable;
@ -40,12 +41,15 @@ public class IndicesStats implements Streamable, Serializable, ToXContent {
private ByteSizeValue fieldCacheSize;
private ByteSizeValue filterCacheSize;
IndicesStats() {
}
public IndicesStats(ByteSizeValue storeSize, ByteSizeValue fieldCacheSize) {
public IndicesStats(ByteSizeValue storeSize, ByteSizeValue fieldCacheSize, ByteSizeValue filterCacheSize) {
this.storeSize = storeSize;
this.fieldCacheSize = fieldCacheSize;
this.filterCacheSize = filterCacheSize;
}
/**
@ -70,6 +74,14 @@ public class IndicesStats implements Streamable, Serializable, ToXContent {
return this.fieldCacheSize;
}
public ByteSizeValue filterCacheSize() {
return this.filterCacheSize;
}
public ByteSizeValue getFilterCacheSize() {
return this.filterCacheSize;
}
public static IndicesStats readIndicesStats(StreamInput in) throws IOException {
IndicesStats stats = new IndicesStats();
stats.readFrom(in);
@ -79,19 +91,33 @@ public class IndicesStats implements Streamable, Serializable, ToXContent {
@Override public void readFrom(StreamInput in) throws IOException {
storeSize = ByteSizeValue.readBytesSizeValue(in);
fieldCacheSize = ByteSizeValue.readBytesSizeValue(in);
filterCacheSize = ByteSizeValue.readBytesSizeValue(in);
}
@Override public void writeTo(StreamOutput out) throws IOException {
storeSize.writeTo(out);
fieldCacheSize.writeTo(out);
filterCacheSize.writeTo(out);
}
@Override public void toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject("indices");
builder.field("store_size", storeSize.toString());
builder.field("store_size_in_bytes", storeSize.bytes());
builder.field("field_cache_size", fieldCacheSize.toString());
builder.field("field_cache_size_in_bytes", fieldCacheSize.bytes());
builder.startObject(Fields.INDICES);
builder.field(Fields.STORE_SIZE, storeSize.toString());
builder.field(Fields.STORE_SIZE_IN_BYTES, storeSize.bytes());
builder.field(Fields.FIELD_CACHE_SIZE, fieldCacheSize.toString());
builder.field(Fields.FIELD_CACHE_SIZE_IN_BYTES, fieldCacheSize.bytes());
builder.field(Fields.FILTER_CACHE_SIZE, filterCacheSize.toString());
builder.field(Fields.FILTER_CACHE_SIZE_IN_BYTES, filterCacheSize.bytes());
builder.endObject();
}
static final class Fields {
static final XContentBuilderString INDICES = new XContentBuilderString("indices");
static final XContentBuilderString STORE_SIZE = new XContentBuilderString("store_size");
static final XContentBuilderString STORE_SIZE_IN_BYTES = new XContentBuilderString("store_size_in_bytes");
static final XContentBuilderString FIELD_CACHE_SIZE = new XContentBuilderString("field_cache_size");
static final XContentBuilderString FIELD_CACHE_SIZE_IN_BYTES = new XContentBuilderString("field_cache_size_in_bytes");
static final XContentBuilderString FILTER_CACHE_SIZE = new XContentBuilderString("filter_cache_size");
static final XContentBuilderString FILTER_CACHE_SIZE_IN_BYTES = new XContentBuilderString("filter_cache_size_in_bytes");
}
}

4
modules/elasticsearch/src/main/java/org/elasticsearch/indices/InternalIndicesService.java
View File

@ -157,6 +157,7 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
@Override public IndicesStats stats() {
long storeTotalSize = 0;
long fieldCacheTotalSize = 0;
long filterCacheTotalSize = 0;
for (IndexService indexService : indices.values()) {
for (IndexShard indexShard : indexService) {
try {
@ -166,8 +167,9 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
}
}
fieldCacheTotalSize += indexService.cache().fieldData().sizeInBytes();
filterCacheTotalSize += indexService.cache().filter().sizeInBytes();
}
return new IndicesStats(new ByteSizeValue(storeTotalSize), new ByteSizeValue(fieldCacheTotalSize));
return new IndicesStats(new ByteSizeValue(storeTotalSize), new ByteSizeValue(fieldCacheTotalSize), new ByteSizeValue(filterCacheTotalSize));
}
/**