honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Stats: add segments.index_writer_max_memory to see index writer's max RAM usage before buffered documents must be written to a new segment 

Closes #7438

Closes #7440
This commit is contained in:
mikemccand 2014-08-25 14:43:09 -04:00
parent 74287865b2
commit 783a9cbb18
7 changed files with 42 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/reference/cat/nodes.asciidoc
View File

@ -180,7 +180,9 @@ operations |9
|`segments.memory` |`sm`, `segmentsMemory` |No |Memory used by
segments |1.4kb
|`segments.index_writer_memory` |`siwm`, `segmentsIndexWriterMemory` |No
|Memory used by index writer |1.2kb
|Memory used by index writer |18mb
|`segments.index_writer_max_memory` |`siwmx`, `segmentsIndexWriterMaxMemory` |No
|Maximum memory index writer may use before it must write buffered documents to a new segment |32mb
|`segments.version_map_memory` |`svmm`, `segmentsVersionMapMemory` |No
|Memory used by version map |1.0kb
|=======================================================================

28
src/main/java/org/elasticsearch/index/engine/SegmentsStats.java
View File

@ -38,6 +38,7 @@ public class SegmentsStats implements Streamable, ToXContent {
private long count;
private long memoryInBytes;
private long indexWriterMemoryInBytes;
private long indexWriterMaxMemoryInBytes;
private long versionMapMemoryInBytes;
public SegmentsStats() {
@ -53,6 +54,10 @@ public class SegmentsStats implements Streamable, ToXContent {
this.indexWriterMemoryInBytes += indexWriterMemoryInBytes;
}
public void addIndexWriterMaxMemoryInBytes(long indexWriterMaxMemoryInBytes) {
this.indexWriterMaxMemoryInBytes += indexWriterMaxMemoryInBytes;
}
public void addVersionMapMemoryInBytes(long versionMapMemoryInBytes) {
this.versionMapMemoryInBytes += versionMapMemoryInBytes;
}
@ -63,6 +68,7 @@ public class SegmentsStats implements Streamable, ToXContent {
}
add(mergeStats.count, mergeStats.memoryInBytes);
addIndexWriterMemoryInBytes(mergeStats.indexWriterMemoryInBytes);
addIndexWriterMaxMemoryInBytes(mergeStats.indexWriterMaxMemoryInBytes);
addVersionMapMemoryInBytes(mergeStats.versionMapMemoryInBytes);
}
@ -95,6 +101,17 @@ public class SegmentsStats implements Streamable, ToXContent {
return new ByteSizeValue(indexWriterMemoryInBytes);
}
/**
* Maximum memory index writer may use before it must write buffered documents to a new segment.
*/
public long getIndexWriterMaxMemoryInBytes() {
return this.indexWriterMaxMemoryInBytes;
}
public ByteSizeValue getIndexWriterMaxMemory() {
return new ByteSizeValue(indexWriterMaxMemoryInBytes);
}
/**
* Estimation of the memory usage by version map
*/
@ -118,6 +135,7 @@ public class SegmentsStats implements Streamable, ToXContent {
builder.field(Fields.COUNT, count);
builder.byteSizeField(Fields.MEMORY_IN_BYTES, Fields.MEMORY, memoryInBytes);
builder.byteSizeField(Fields.INDEX_WRITER_MEMORY_IN_BYTES, Fields.INDEX_WRITER_MEMORY, indexWriterMemoryInBytes);
builder.byteSizeField(Fields.INDEX_WRITER_MAX_MEMORY_IN_BYTES, Fields.INDEX_WRITER_MAX_MEMORY, indexWriterMaxMemoryInBytes);
builder.byteSizeField(Fields.VERSION_MAP_MEMORY_IN_BYTES, Fields.VERSION_MAP_MEMORY, versionMapMemoryInBytes);
builder.endObject();
return builder;
@ -130,6 +148,8 @@ public class SegmentsStats implements Streamable, ToXContent {
static final XContentBuilderString MEMORY_IN_BYTES = new XContentBuilderString("memory_in_bytes");
static final XContentBuilderString INDEX_WRITER_MEMORY = new XContentBuilderString("index_writer_memory");
static final XContentBuilderString INDEX_WRITER_MEMORY_IN_BYTES = new XContentBuilderString("index_writer_memory_in_bytes");
static final XContentBuilderString INDEX_WRITER_MAX_MEMORY = new XContentBuilderString("index_writer_max_memory");
static final XContentBuilderString INDEX_WRITER_MAX_MEMORY_IN_BYTES = new XContentBuilderString("index_writer_max_memory_in_bytes");
static final XContentBuilderString VERSION_MAP_MEMORY = new XContentBuilderString("version_map_memory");
static final XContentBuilderString VERSION_MAP_MEMORY_IN_BYTES = new XContentBuilderString("version_map_memory_in_bytes");
}
@ -142,6 +162,9 @@ public class SegmentsStats implements Streamable, ToXContent {
indexWriterMemoryInBytes = in.readLong();
versionMapMemoryInBytes = in.readLong();
}
if (in.getVersion().onOrAfter(Version.V_1_4_0)) {
indexWriterMaxMemoryInBytes = in.readLong();
}
}
@Override
@ -152,5 +175,8 @@ public class SegmentsStats implements Streamable, ToXContent {
out.writeLong(indexWriterMemoryInBytes);
out.writeLong(versionMapMemoryInBytes);
}
if (out.getVersion().onOrAfter(Version.V_1_4_0)) {
out.writeLong(indexWriterMaxMemoryInBytes);
}
}
}
}

1
src/main/java/org/elasticsearch/index/engine/internal/InternalEngine.java
View File

@ -1149,6 +1149,7 @@ public class InternalEngine extends AbstractIndexShardComponent implements Engin
}
stats.addVersionMapMemoryInBytes(versionMap.ramBytesUsed());
stats.addIndexWriterMemoryInBytes(indexWriter.ramBytesUsed());
stats.addIndexWriterMaxMemoryInBytes((long) (indexWriter.getConfig().getRAMBufferSizeMB()*1024*1024));
return stats;
} finally {
searcher.close();

6
src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
View File

@ -255,6 +255,9 @@ public class RestIndicesAction extends AbstractCatAction {
table.addCell("segments.index_writer_memory", "sibling:pri;alias:siwm,segmentsIndexWriterMemory;default:false;text-align:right;desc:memory used by index writer");
table.addCell("pri.segments.index_writer_memory", "default:false;text-align:right;desc:memory used by index writer");
table.addCell("segments.index_writer_max_memory", "sibling:pri;alias:siwmx,segmentsIndexWriterMaxMemory;default:false;text-align:right;desc:maximum memory index writer may use before it must write buffered documents to a new segment");
table.addCell("pri.segments.index_writer_max_memory", "default:false;text-align:right;desc:maximum memory index writer may use before it must write buffered documents to a new segment");
table.addCell("segments.version_map_memory", "sibling:pri;alias:svmm,segmentsVersionMapMemory;default:false;text-align:right;desc:memory used by version map");
table.addCell("pri.segments.version_map_memory", "default:false;text-align:right;desc:memory used by version map");
@ -446,6 +449,9 @@ public class RestIndicesAction extends AbstractCatAction {
table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getIndexWriterMemory());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getIndexWriterMemory());
table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getIndexWriterMaxMemory());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getIndexWriterMaxMemory());
table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getVersionMapMemory());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getVersionMapMemory());

2
src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
View File

@ -175,6 +175,7 @@ public class RestNodesAction extends AbstractCatAction {
table.addCell("segments.count", "alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments");
table.addCell("segments.memory", "alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments");
table.addCell("segments.index_writer_memory", "alias:siwm,segmentsIndexWriterMemory;default:false;text-align:right;desc:memory used by index writer");
table.addCell("segments.index_writer_max_memory", "alias:siwmx,segmentsIndexWriterMaxMemory;default:false;text-align:right;desc:maximum memory index writer may use before it must write buffered documents to a new segment");
table.addCell("segments.version_map_memory", "alias:svmm,segmentsVersionMapMemory;default:false;text-align:right;desc:memory used by version map");
table.addCell("suggest.current", "alias:suc,suggestCurrent;default:false;text-align:right;desc:number of current suggest ops");
@ -284,6 +285,7 @@ public class RestNodesAction extends AbstractCatAction {
table.addCell(stats == null ? null : stats.getIndices().getSegments().getCount());
table.addCell(stats == null ? null : stats.getIndices().getSegments().getMemory());
table.addCell(stats == null ? null : stats.getIndices().getSegments().getIndexWriterMemory());
table.addCell(stats == null ? null : stats.getIndices().getSegments().getIndexWriterMaxMemory());
table.addCell(stats == null ? null : stats.getIndices().getSegments().getVersionMapMemory());
table.addCell(stats == null ? null : stats.getIndices().getSuggest().getCurrent());

2
src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
View File

@ -146,6 +146,7 @@ public class RestShardsAction extends AbstractCatAction {
table.addCell("segments.count", "alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments");
table.addCell("segments.memory", "alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments");
table.addCell("segments.index_writer_memory", "alias:siwm,segmentsIndexWriterMemory;default:false;text-align:right;desc:memory used by index writer");
table.addCell("segments.index_writer_max_memory", "alias:siwmx,segmentsIndexWriterMaxMemory;default:false;text-align:right;desc:maximum memory index writer may use before it must write buffered documents to a new segment");
table.addCell("segments.version_map_memory", "alias:svmm,segmentsVersionMapMemory;default:false;text-align:right;desc:memory used by version map");
table.addCell("warmer.current", "alias:wc,warmerCurrent;default:false;text-align:right;desc:current warmer ops");
@ -245,6 +246,7 @@ public class RestShardsAction extends AbstractCatAction {
table.addCell(shardStats == null ? null : shardStats.getSegments().getCount());
table.addCell(shardStats == null ? null : shardStats.getSegments().getMemory());
table.addCell(shardStats == null ? null : shardStats.getSegments().getIndexWriterMemory());
table.addCell(shardStats == null ? null : shardStats.getSegments().getIndexWriterMaxMemory());
table.addCell(shardStats == null ? null : shardStats.getSegments().getVersionMapMemory());
table.addCell(shardStats == null ? null : shardStats.getWarmer().current());

1
src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java
View File

@ -434,6 +434,7 @@ public class IndexStatsTests extends ElasticsearchIntegrationTest {
IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get();
assertThat(stats.getTotal().getSegments().getIndexWriterMemoryInBytes(), greaterThan(0l));
assertThat(stats.getTotal().getSegments().getIndexWriterMaxMemoryInBytes(), greaterThan(0l));
assertThat(stats.getTotal().getSegments().getVersionMapMemoryInBytes(), greaterThan(0l));
client().admin().indices().prepareFlush().get();