honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #20255 from javanna/enhancement/cluster_stats_available_memory 

Add mem section back to cluster stats
This commit is contained in:
Luca Cavanna 2016-09-02 10:19:51 +02:00 committed by GitHub
parent 825b80f2a6 186a5d74b8
commit faa03ad9fa
12 changed files with 237 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java
View File

@ -212,7 +212,7 @@ public class NodeStats extends BaseNodeResponse implements ToXContent {
indices = NodeIndicesStats.readIndicesStats(in);
}
if (in.readBoolean()) {
os = OsStats.readOsStats(in);
os = new OsStats(in);
}
if (in.readBoolean()) {
process = ProcessStats.readProcessStats(in);

26
core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java
View File

@ -87,7 +87,7 @@ public class ClusterStatsNodes implements ToXContent {
}
}
this.counts = new Counts(nodeInfos);
this.os = new OsStats(nodeInfos);
this.os = new OsStats(nodeInfos, nodeStats);
this.process = new ProcessStats(nodeStats);
this.jvm = new JvmStats(nodeInfos, nodeStats);
this.networkTypes = new NetworkTypes(nodeInfos);
@ -226,11 +226,12 @@ public class ClusterStatsNodes implements ToXContent {
final int availableProcessors;
final int allocatedProcessors;
final ObjectIntHashMap<String> names;
final org.elasticsearch.monitor.os.OsStats.Mem mem;
/**
* Build the stats from information about each node.
*/
private OsStats(List<NodeInfo> nodeInfos) {
private OsStats(List<NodeInfo> nodeInfos, List<NodeStats> nodeStatsList) {
this.names = new ObjectIntHashMap<>();
int availableProcessors = 0;
int allocatedProcessors = 0;
@ -244,6 +245,22 @@ public class ClusterStatsNodes implements ToXContent {
}
this.availableProcessors = availableProcessors;
this.allocatedProcessors = allocatedProcessors;
long totalMemory = 0;
long freeMemory = 0;
for (NodeStats nodeStats : nodeStatsList) {
if (nodeStats.getOs() != null) {
long total = nodeStats.getOs().getMem().getTotal().bytes();
if (total > 0) {
totalMemory += total;
}
long free = nodeStats.getOs().getMem().getFree().bytes();
if (free > 0) {
freeMemory += free;
}
}
}
this.mem = new org.elasticsearch.monitor.os.OsStats.Mem(totalMemory, freeMemory);
}
public int getAvailableProcessors() {
@ -254,6 +271,10 @@ public class ClusterStatsNodes implements ToXContent {
return allocatedProcessors;
}
public org.elasticsearch.monitor.os.OsStats.Mem getMem() {
return mem;
}
static final class Fields {
static final String AVAILABLE_PROCESSORS = "available_processors";
static final String ALLOCATED_PROCESSORS = "allocated_processors";
@ -274,6 +295,7 @@ public class ClusterStatsNodes implements ToXContent {
builder.endObject();
}
builder.endArray();
mem.toXContent(builder, params);
return builder;
}
}

2
core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
View File

@ -92,7 +92,7 @@ public class TransportClusterStatsAction extends TransportNodesAction<ClusterSta
@Override
protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeRequest) {
NodeInfo nodeInfo = nodeService.info(true, true, false, true, false, true, false, true, false, false);
NodeStats nodeStats = nodeService.stats(CommonStatsFlags.NONE, false, true, true, false, true, false, false, false, false, false, false);
NodeStats nodeStats = nodeService.stats(CommonStatsFlags.NONE, true, true, true, false, true, false, false, false, false, false, false);
List<ShardStats> shardsStats = new ArrayList<>();
for (IndexService indexService : indicesService) {
for (IndexShard indexShard : indexService) {

21
core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
View File

@ -173,23 +173,10 @@ public class OsProbe {
}
public OsStats osStats() {
OsStats stats = new OsStats();
stats.timestamp = System.currentTimeMillis();
stats.cpu = new OsStats.Cpu();
stats.cpu.percent = getSystemCpuPercent();
stats.cpu.loadAverage = getSystemLoadAverage();
OsStats.Mem mem = new OsStats.Mem();
mem.total = getTotalPhysicalMemorySize();
mem.free = getFreePhysicalMemorySize();
stats.mem = mem;
OsStats.Swap swap = new OsStats.Swap();
swap.total = getTotalSwapSpaceSize();
swap.free = getFreeSwapSpaceSize();
stats.swap = swap;
return stats;
OsStats.Cpu cpu = new OsStats.Cpu(getSystemCpuPercent(), getSystemLoadAverage());
OsStats.Mem mem = new OsStats.Mem(getTotalPhysicalMemorySize(), getFreePhysicalMemorySize());
OsStats.Swap swap = new OsStats.Swap(getTotalSwapSpaceSize(), getFreeSwapSpaceSize());
return new OsStats(System.currentTimeMillis(), cpu, mem , swap);
}
/**

247
core/src/main/java/org/elasticsearch/monitor/os/OsStats.java
View File

@ -21,28 +21,42 @@ package org.elasticsearch.monitor.os;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.Arrays;
import java.util.Objects;
/**
*
*/
public class OsStats implements Streamable, ToXContent {
public class OsStats implements Writeable, ToXContent {
long timestamp;
private final long timestamp;
private final Cpu cpu;
private final Mem mem;
private final Swap swap;
Cpu cpu = null;
public OsStats(long timestamp, Cpu cpu, Mem mem, Swap swap) {
this.timestamp = timestamp;
this.cpu = Objects.requireNonNull(cpu, "cpu must not be null");
this.mem = Objects.requireNonNull(mem, "mem must not be null");;
this.swap = Objects.requireNonNull(swap, "swap must not be null");;
}
Mem mem = null;
public OsStats(StreamInput in) throws IOException {
this.timestamp = in.readVLong();
this.cpu = new Cpu(in);
this.mem = new Mem(in);
this.swap = new Swap(in);
}
Swap swap = null;
OsStats() {
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVLong(timestamp);
cpu.writeTo(out);
mem.writeTo(out);
swap.writeTo(out);
}
public long getTimestamp() {
@ -65,9 +79,9 @@ public class OsStats implements Streamable, ToXContent {
static final String CPU = "cpu";
static final String PERCENT = "percent";
static final String LOAD_AVERAGE = "load_average";
static final String LOAD_AVERAGE_1M = new String("1m");
static final String LOAD_AVERAGE_5M = new String("5m");
static final String LOAD_AVERAGE_15M = new String("15m");
static final String LOAD_AVERAGE_1M = "1m";
static final String LOAD_AVERAGE_5M = "5m";
static final String LOAD_AVERAGE_15M = "15m";
static final String MEM = "mem";
static final String SWAP = "swap";
@ -86,105 +100,29 @@ public class OsStats implements Streamable, ToXContent {
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(Fields.OS);
builder.field(Fields.TIMESTAMP, getTimestamp());
if (cpu != null) {
builder.startObject(Fields.CPU);
builder.field(Fields.PERCENT, cpu.getPercent());
if (cpu.getLoadAverage() != null && Arrays.stream(cpu.getLoadAverage()).anyMatch(load -> load != -1)) {
builder.startObject(Fields.LOAD_AVERAGE);
if (cpu.getLoadAverage()[0] != -1) {
builder.field(Fields.LOAD_AVERAGE_1M, cpu.getLoadAverage()[0]);
}
if (cpu.getLoadAverage()[1] != -1) {
builder.field(Fields.LOAD_AVERAGE_5M, cpu.getLoadAverage()[1]);
}
if (cpu.getLoadAverage()[2] != -1) {
builder.field(Fields.LOAD_AVERAGE_15M, cpu.getLoadAverage()[2]);
}
builder.endObject();
}
builder.endObject();
}
if (mem != null) {
builder.startObject(Fields.MEM);
builder.byteSizeField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, mem.getTotal());
builder.byteSizeField(Fields.FREE_IN_BYTES, Fields.FREE, mem.getFree());
builder.byteSizeField(Fields.USED_IN_BYTES, Fields.USED, mem.getUsed());
builder.field(Fields.FREE_PERCENT, mem.getFreePercent());
builder.field(Fields.USED_PERCENT, mem.getUsedPercent());
builder.endObject();
}
if (swap != null) {
builder.startObject(Fields.SWAP);
builder.byteSizeField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, swap.getTotal());
builder.byteSizeField(Fields.FREE_IN_BYTES, Fields.FREE, swap.getFree());
builder.byteSizeField(Fields.USED_IN_BYTES, Fields.USED, swap.getUsed());
builder.endObject();
}
cpu.toXContent(builder, params);
mem.toXContent(builder, params);
swap.toXContent(builder, params);
builder.endObject();
return builder;
}
public static OsStats readOsStats(StreamInput in) throws IOException {
OsStats stats = new OsStats();
stats.readFrom(in);
return stats;
}
public static class Cpu implements Writeable, ToXContent {
@Override
public void readFrom(StreamInput in) throws IOException {
timestamp = in.readVLong();
cpu = in.readOptionalStreamable(Cpu::new);
if (in.readBoolean()) {
mem = Mem.readMem(in);
}
if (in.readBoolean()) {
swap = Swap.readSwap(in);
}
}
private final short percent;
private final double[] loadAverage;
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVLong(timestamp);
out.writeOptionalStreamable(cpu);
if (mem == null) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
mem.writeTo(out);
}
if (swap == null) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
swap.writeTo(out);
}
}
public static class Cpu implements Streamable {
short percent = -1;
double[] loadAverage = null;
Cpu() {}
public static Cpu readCpu(StreamInput in) throws IOException {
Cpu cpu = new Cpu();
cpu.readFrom(in);
return cpu;
public Cpu(short systemCpuPercent, double[] systemLoadAverage) {
this.percent = systemCpuPercent;
this.loadAverage = systemLoadAverage;
}
@Override
public void readFrom(StreamInput in) throws IOException {
percent = in.readShort();
public Cpu(StreamInput in) throws IOException {
this.percent = in.readShort();
if (in.readBoolean()) {
loadAverage = in.readDoubleArray();
this.loadAverage = in.readDoubleArray();
} else {
loadAverage = null;
this.loadAverage = null;
}
}
@ -206,12 +144,49 @@ public class OsStats implements Streamable, ToXContent {
public double[] getLoadAverage() {
return loadAverage;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(Fields.CPU);
builder.field(Fields.PERCENT, getPercent());
if (getLoadAverage() != null && Arrays.stream(getLoadAverage()).anyMatch(load -> load != -1)) {
builder.startObject(Fields.LOAD_AVERAGE);
if (getLoadAverage()[0] != -1) {
builder.field(Fields.LOAD_AVERAGE_1M, getLoadAverage()[0]);
}
if (getLoadAverage()[1] != -1) {
builder.field(Fields.LOAD_AVERAGE_5M, getLoadAverage()[1]);
}
if (getLoadAverage()[2] != -1) {
builder.field(Fields.LOAD_AVERAGE_15M, getLoadAverage()[2]);
}
builder.endObject();
}
builder.endObject();
return builder;
}
}
public static class Swap implements Streamable {
public static class Swap implements Writeable, ToXContent {
long total = -1;
long free = -1;
private final long total;
private final long free;
public Swap(long total, long free) {
this.total = total;
this.free = free;
}
public Swap(StreamInput in) throws IOException {
this.total = in.readLong();
this.free = in.readLong();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeLong(total);
out.writeLong(free);
}
public ByteSizeValue getFree() {
return new ByteSizeValue(free);
@ -225,40 +200,30 @@ public class OsStats implements Streamable, ToXContent {
return new ByteSizeValue(total);
}
public static Swap readSwap(StreamInput in) throws IOException {
Swap swap = new Swap();
swap.readFrom(in);
return swap;
}
@Override
public void readFrom(StreamInput in) throws IOException {
total = in.readLong();
free = in.readLong();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeLong(total);
out.writeLong(free);
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(Fields.SWAP);
builder.byteSizeField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, getTotal());
builder.byteSizeField(Fields.FREE_IN_BYTES, Fields.FREE, getFree());
builder.byteSizeField(Fields.USED_IN_BYTES, Fields.USED, getUsed());
builder.endObject();
return builder;
}
}
public static class Mem implements Streamable {
public static class Mem implements Writeable, ToXContent {
long total = -1;
long free = -1;
private final long total;
private final long free;
public static Mem readMem(StreamInput in) throws IOException {
Mem mem = new Mem();
mem.readFrom(in);
return mem;
public Mem(long total, long free) {
this.total = total;
this.free = free;
}
@Override
public void readFrom(StreamInput in) throws IOException {
total = in.readLong();
free = in.readLong();
public Mem(StreamInput in) throws IOException {
this.total = in.readLong();
this.free = in.readLong();
}
@Override
@ -276,7 +241,7 @@ public class OsStats implements Streamable, ToXContent {
}
public short getUsedPercent() {
return calculatePercentage(getUsed().bytes(), getTotal().bytes());
return calculatePercentage(getUsed().bytes(), total);
}
public ByteSizeValue getFree() {
@ -284,11 +249,23 @@ public class OsStats implements Streamable, ToXContent {
}
public short getFreePercent() {
return calculatePercentage(getFree().bytes(), getTotal().bytes());
return calculatePercentage(free, total);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(Fields.MEM);
builder.byteSizeField(Fields.TOTAL_IN_BYTES, Fields.TOTAL, getTotal());
builder.byteSizeField(Fields.FREE_IN_BYTES, Fields.FREE, getFree());
builder.byteSizeField(Fields.USED_IN_BYTES, Fields.USED, getUsed());
builder.field(Fields.FREE_PERCENT, getFreePercent());
builder.field(Fields.USED_PERCENT, getUsedPercent());
builder.endObject();
return builder;
}
}
private static short calculatePercentage(long used, long max) {
public static short calculatePercentage(long used, long max) {
return max <= 0 ? 0 : (short) (Math.round((100d * used) / max));
}
}

17
core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java
View File

@ -21,12 +21,15 @@ package org.elasticsearch.action.admin.cluster.stats;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
import org.elasticsearch.client.Requests;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.monitor.os.OsStats;
import org.elasticsearch.node.Node;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@ -181,6 +184,20 @@ public class ClusterStatsIT extends ESIntegTestCase {
assertThat(msg, response.nodesStats.getProcess().getMinOpenFileDescriptors(), Matchers.greaterThanOrEqualTo(-1L));
assertThat(msg, response.nodesStats.getProcess().getMaxOpenFileDescriptors(), Matchers.greaterThanOrEqualTo(-1L));
NodesStatsResponse nodesStatsResponse = client().admin().cluster().prepareNodesStats().setOs(true).get();
long total = 0;
long free = 0;
long used = 0;
for (NodeStats nodeStats : nodesStatsResponse.getNodes()) {
total += nodeStats.getOs().getMem().getTotal().bytes();
free += nodeStats.getOs().getMem().getFree().bytes();
used += nodeStats.getOs().getMem().getUsed().bytes();
}
assertEquals(msg, free, response.nodesStats.getOs().getMem().getFree().bytes());
assertEquals(msg, total, response.nodesStats.getOs().getMem().getTotal().bytes());
assertEquals(msg, used, response.nodesStats.getOs().getMem().getUsed().bytes());
assertEquals(msg, OsStats.calculatePercentage(used, total), response.nodesStats.getOs().getMem().getUsedPercent());
assertEquals(msg, OsStats.calculatePercentage(free, total), response.nodesStats.getOs().getMem().getFreePercent());
}
public void testAllocatedProcessors() throws Exception {

2
core/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java
View File

@ -49,7 +49,7 @@ public class OsProbeTests extends ESTestCase {
assertNotNull(stats);
assertThat(stats.getTimestamp(), greaterThan(0L));
assertThat(stats.getCpu().getPercent(), anyOf(equalTo((short) -1), is(both(greaterThanOrEqualTo((short) 0)).and(lessThanOrEqualTo((short) 100)))));
double[] loadAverage = stats.getCpu().loadAverage;
double[] loadAverage = stats.getCpu().getLoadAverage();
if (loadAverage != null) {
assertThat(loadAverage.length, equalTo(3));
}

55
core/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java Normal file
View File

@ -0,0 +1,55 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.monitor.os;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
public class OsStatsTests extends ESTestCase {
public void testSerialization() throws IOException {
int numLoadAverages = randomIntBetween(1, 5);
double loadAverages[] = new double[numLoadAverages];
for (int i = 0; i < loadAverages.length; i++) {
loadAverages[i] = randomDouble();
}
OsStats.Cpu cpu = new OsStats.Cpu(randomShort(), loadAverages);
OsStats.Mem mem = new OsStats.Mem(randomLong(), randomLong());
OsStats.Swap swap = new OsStats.Swap(randomLong(), randomLong());
OsStats osStats = new OsStats(System.currentTimeMillis(), cpu, mem, swap);
try (BytesStreamOutput out = new BytesStreamOutput()) {
osStats.writeTo(out);
try (StreamInput in = out.bytes().streamInput()) {
OsStats deserializedOsStats = new OsStats(in);
assertEquals(osStats.getTimestamp(), deserializedOsStats.getTimestamp());
assertEquals(osStats.getCpu().getPercent(), deserializedOsStats.getCpu().getPercent());
assertArrayEquals(osStats.getCpu().getLoadAverage(), deserializedOsStats.getCpu().getLoadAverage(), 0);
assertEquals(osStats.getMem().getFree(), deserializedOsStats.getMem().getFree());
assertEquals(osStats.getMem().getTotal(), deserializedOsStats.getMem().getTotal());
assertEquals(osStats.getSwap().getFree(), deserializedOsStats.getSwap().getFree());
assertEquals(osStats.getSwap().getTotal(), deserializedOsStats.getSwap().getTotal());
}
}
}
}

12
docs/reference/cluster/stats.asciidoc
View File

@ -116,7 +116,17 @@ Will return, for example:
"name": "Mac OS X",
"count": 1
}
]
],
"mem" : {
"total" : "16gb",
"total_in_bytes" : 17179869184,
"free" : "78.1mb",
"free_in_bytes" : 81960960,
"used" : "15.9gb",
"used_in_bytes" : 17097908224,
"free_percent" : 0,
"used_percent" : 100
}
},
"process": {
"cpu": {

4
docs/reference/migration/migrate_5_0/java.asciidoc
View File

@ -349,7 +349,9 @@ The `setQuery(BytesReference)` method have been removed in favor of using `setQu
==== ClusterStatsResponse
Removed the `getMemoryAvailable` method from `OsStats`, which could be previously accessed calling
`clusterStatsResponse.getNodesStats().getOs().getMemoryAvailable()`.
`clusterStatsResponse.getNodesStats().getOs().getMemoryAvailable()`. It is now replaced with
`clusterStatsResponse.getNodesStats().getOs().getMem()` which exposes `getTotal()`, `getFree()`,
`getUsed()`, `getFreePercent()` and `getUsedPercent()`.
==== setRefresh(boolean) has been removed

6
docs/reference/migration/migrate_5_0/rest.asciidoc
View File

@ -29,9 +29,9 @@ document exists in an index. The old endpoint will keep working until 6.0.
==== Removed `mem` section from `/_cluster/stats` response
The `mem` section contained only one value, the total memory available
throughout all nodes in the cluster. The section was removed as it didn't
prove useful.
The `mem` section contained only the `total` value, which was actually the
memory available throughout all nodes in the cluster. The section contains now
`total`, `free`, `used`, `used_percent` and `free_percent`.
==== Revised node roles aggregate returned by `/_cluster/stats`

5
rest-api-spec/src/main/resources/rest-api-spec/test/cluster.stats/10_basic.yaml
View File

@ -19,6 +19,11 @@
- gte: { nodes.count.ingest: 0}
- gte: { nodes.count.coordinating_only: 0}
- is_true: nodes.os
- is_true: nodes.os.mem.total_in_bytes
- is_true: nodes.os.mem.free_in_bytes
- is_true: nodes.os.mem.used_in_bytes
- is_true: nodes.os.mem.free_percent
- is_true: nodes.os.mem.used_percent
- is_true: nodes.process
- is_true: nodes.jvm
- is_true: nodes.fs