honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support for Heap after GC stats (#1265) 

* Support for Heap after GC stats

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

* Addressing code review comments

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

* Using the right version 2.0.0 (instead of 1.2.0) for the change

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
This commit is contained in:
Andriy Redko 2021-09-28 14:40:00 -04:00 committed by GitHub
parent 457c1cd6ec
commit 80388a8a29
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 118 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
server/src/main/java/org/opensearch/monitor/jvm/JvmStats.java
View File

@ -32,6 +32,7 @@
package org.opensearch.monitor.jvm;
import org.opensearch.Version;
import org.opensearch.common.io.stream.StreamInput;
import org.opensearch.common.io.stream.StreamOutput;
import org.opensearch.common.io.stream.Writeable;
@ -85,6 +86,11 @@ public class JvmStats implements Writeable, ToXContentFragment {
try {
MemoryUsage usage = memoryPoolMXBean.getUsage();
MemoryUsage peakUsage = memoryPoolMXBean.getPeakUsage();
MemoryUsage collectionUsage = memoryPoolMXBean.getCollectionUsage();
// For some pools, the collection usage may not be supported and the method returns null.
if (collectionUsage == null) {
collectionUsage = new MemoryUsage(0, 0, 0, 0);
}
String name = GcNames.getByMemoryPoolName(memoryPoolMXBean.getName(), null);
if (name == null) { // if we can't resolve it, its not interesting.... (Per Gen, Code Cache)
continue;
@ -93,8 +99,13 @@ public class JvmStats implements Writeable, ToXContentFragment {
usage.getUsed() < 0 ? 0 : usage.getUsed(),
usage.getMax() < 0 ? 0 : usage.getMax(),
peakUsage.getUsed() < 0 ? 0 : peakUsage.getUsed(),
peakUsage.getMax() < 0 ? 0 : peakUsage.getMax()
));
peakUsage.getMax() < 0 ? 0 : peakUsage.getMax(),
new MemoryPoolGcStats(
collectionUsage.getUsed() < 0 ? 0 : collectionUsage.getUsed(),
collectionUsage.getMax() < 0 ? 0 : collectionUsage.getMax()
)
));
} catch (final Exception ignored) {
}
@ -223,6 +234,12 @@ public class JvmStats implements Writeable, ToXContentFragment {
builder.humanReadableField(Fields.PEAK_USED_IN_BYTES, Fields.PEAK_USED, new ByteSizeValue(pool.peakUsed));
builder.humanReadableField(Fields.PEAK_MAX_IN_BYTES, Fields.PEAK_MAX, new ByteSizeValue(pool.peakMax));
builder.startObject(Fields.LAST_GC_STATS);
builder.humanReadableField(Fields.USED_IN_BYTES, Fields.USED, new ByteSizeValue(pool.getLastGcStats().used));
builder.humanReadableField(Fields.MAX_IN_BYTES, Fields.MAX, new ByteSizeValue(pool.getLastGcStats().max));
builder.field(Fields.USAGE_PERCENT, pool.getLastGcStats().getUsagePercent());
builder.endObject();
builder.endObject();
}
builder.endObject();
@ -299,7 +316,9 @@ public class JvmStats implements Writeable, ToXContentFragment {
static final String PEAK_USED_IN_BYTES = "peak_used_in_bytes";
static final String PEAK_MAX = "peak_max";
static final String PEAK_MAX_IN_BYTES = "peak_max_in_bytes";
static final String USAGE_PERCENT = "usage_percent";
static final String LAST_GC_STATS = "last_gc_stats";
static final String THREADS = "threads";
static final String COUNT = "count";
static final String PEAK_COUNT = "peak_count";
@ -414,6 +433,48 @@ public class JvmStats implements Writeable, ToXContentFragment {
return peakCount;
}
}
/**
* Stores the memory usage after the Java virtual machine
* most recently expended effort in recycling unused objects
* in particular memory pool.
*/
public static class MemoryPoolGcStats implements Writeable {
private final long used;
private final long max;
public MemoryPoolGcStats(long used, long max) {
this.used = used;
this.max = max;
}
public MemoryPoolGcStats(StreamInput in) throws IOException {
used = in.readVLong();
max = in.readVLong();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVLong(used);
out.writeVLong(max);
}
public ByteSizeValue getUsed() {
return new ByteSizeValue(used);
}
public ByteSizeValue getMax() {
return new ByteSizeValue(max);
}
public short getUsagePercent() {
if (max == 0) {
return -1;
}
return (short) (used * 100 / max);
}
}
public static class MemoryPool implements Writeable {
@ -422,13 +483,15 @@ public class JvmStats implements Writeable, ToXContentFragment {
private final long max;
private final long peakUsed;
private final long peakMax;
private final MemoryPoolGcStats lastGcStats;
public MemoryPool(String name, long used, long max, long peakUsed, long peakMax) {
public MemoryPool(String name, long used, long max, long peakUsed, long peakMax, MemoryPoolGcStats lastGcStats) {
this.name = name;
this.used = used;
this.max = max;
this.peakUsed = peakUsed;
this.peakMax = peakMax;
this.lastGcStats = lastGcStats;
}
public MemoryPool(StreamInput in) throws IOException {
@ -437,6 +500,11 @@ public class JvmStats implements Writeable, ToXContentFragment {
max = in.readVLong();
peakUsed = in.readVLong();
peakMax = in.readVLong();
if (in.getVersion().onOrAfter(Version.V_2_0_0)) {
lastGcStats = new MemoryPoolGcStats(in);
} else {
lastGcStats = new MemoryPoolGcStats(0, 0);
}
}
@Override
@ -446,6 +514,9 @@ public class JvmStats implements Writeable, ToXContentFragment {
out.writeVLong(max);
out.writeVLong(peakUsed);
out.writeVLong(peakMax);
if (out.getVersion().onOrAfter(Version.V_2_0_0)) {
lastGcStats.writeTo(out);
}
}
public String getName() {
@ -467,6 +538,13 @@ public class JvmStats implements Writeable, ToXContentFragment {
public ByteSizeValue getPeakMax() {
return new ByteSizeValue(peakMax);
}
/**
* Returns the heap usage after last garbage collection cycle
*/
public MemoryPoolGcStats getLastGcStats() {
return lastGcStats;
}
}
public static class Mem implements Writeable, Iterable<MemoryPool> {

38
server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
View File

@ -54,7 +54,6 @@ import org.opensearch.test.OpenSearchTestCase;
import org.opensearch.test.VersionUtils;
import org.opensearch.threadpool.ThreadPoolStats;
import org.opensearch.transport.TransportStats;
import org.opensearch.action.admin.cluster.node.stats.NodeStats;
import java.io.IOException;
import java.util.ArrayList;
@ -63,9 +62,15 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import static java.util.Collections.emptyMap;
import static java.util.Collections.emptySet;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.not;
import static org.hamcrest.collection.IsEmptyCollection.empty;
public class NodeStatsTests extends OpenSearchTestCase {
public void testSerialization() throws IOException {
@ -149,6 +154,34 @@ public class NodeStatsTests extends OpenSearchTestCase {
assertEquals(mem.getNonHeapCommitted(), deserializedMem.getNonHeapCommitted());
assertEquals(mem.getNonHeapUsed(), deserializedMem.getNonHeapUsed());
assertEquals(mem.getHeapMax(), deserializedMem.getHeapMax());
final Map<String, JvmStats.MemoryPool> pools = StreamSupport
.stream(mem.spliterator(), false)
.collect(Collectors.toMap(JvmStats.MemoryPool::getName, Function.identity()));
final Map<String, JvmStats.MemoryPool> deserializedPools = StreamSupport
.stream(deserializedMem.spliterator(), false)
.collect(Collectors.toMap(JvmStats.MemoryPool::getName, Function.identity()));
assertThat(pools.keySet(), not(empty()));
assertThat(deserializedPools.keySet(), not(empty()));
for (final Map.Entry<String, JvmStats.MemoryPool> entry: pools.entrySet()) {
assertThat(deserializedPools.containsKey(entry.getKey()), is(true));
assertEquals(entry.getValue().getName(), deserializedPools.get(entry.getKey()).getName());
assertEquals(entry.getValue().getMax(), deserializedPools.get(entry.getKey()).getMax());
assertEquals(entry.getValue().getPeakMax(), deserializedPools.get(entry.getKey()).getPeakMax());
assertEquals(entry.getValue().getPeakUsed(), deserializedPools.get(entry.getKey()).getPeakUsed());
assertEquals(entry.getValue().getUsed(), deserializedPools.get(entry.getKey()).getUsed());
assertEquals(entry.getValue().getLastGcStats().getUsed(),
deserializedPools.get(entry.getKey()).getLastGcStats().getUsed());
assertEquals(entry.getValue().getLastGcStats().getMax(),
deserializedPools.get(entry.getKey()).getLastGcStats().getMax());
assertEquals(entry.getValue().getLastGcStats().getUsagePercent(),
deserializedPools.get(entry.getKey()).getLastGcStats().getUsagePercent());
}
JvmStats.Classes classes = jvm.getClasses();
assertEquals(classes.getLoadedClassCount(), deserializedJvm.getClasses().getLoadedClassCount());
assertEquals(classes.getTotalLoadedClassCount(), deserializedJvm.getClasses().getTotalLoadedClassCount());
@ -397,7 +430,8 @@ public class NodeStatsTests extends OpenSearchTestCase {
List<JvmStats.MemoryPool> memoryPools = new ArrayList<>(numMemoryPools);
for (int i = 0; i < numMemoryPools; i++) {
memoryPools.add(new JvmStats.MemoryPool(randomAlphaOfLengthBetween(3, 10), randomNonNegativeLong(),
randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(),
new JvmStats.MemoryPoolGcStats(randomNonNegativeLong(), randomNonNegativeLong())));
}
JvmStats.Threads threads = new JvmStats.Threads(randomIntBetween(1, 1000), randomIntBetween(1, 1000));
int numGarbageCollectors = randomIntBetween(0, 10);