honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Core: ignore known idle threads by default in /_nodes/hot_threads 

Add a new ignore_idle_threads boolean option (default true) to
/_nodes/hot_threads, to filter out threads in known idle places like
waiting on a socket select or on pulling the next task from an empty
queue.

Closes #8985

Closes #8908
This commit is contained in:
Michael McCandless 2014-12-17 11:59:31 -05:00 committed by mikemccand
parent f1da788211
commit 242e631e95
8 changed files with 122 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/reference/cluster/nodes-hot-threads.asciidoc
View File

@ -14,3 +14,5 @@ threads. Parameters allowed are:
Defaults to 500ms.
`type`:: The type to sample, defaults to cpu, but supports wait and
block to see hot threads that are in wait or block state.
`ignore_idle_threads`:: If true, known idle threads (e.g. waiting in a socket select, or to
get a task from an empty queue) are filtered out. Defaults to true.

4
rest-api-spec/api/nodes.hot_threads.json
View File

@ -24,6 +24,10 @@
"type" : "number",
"description" : "Specify the number of threads to provide information for (default: 3)"
},
"ignore_idle_threads": {
"type" : "boolean",
"description" : "Don't show threads that are in known-idle places, such as waiting on a socket select or pulling from an empty task queue (default: true)"
},
"type": {
"type" : "enum",
"options" : ["cpu", "wait", "block"],

20
src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.action.admin.cluster.node.hotthreads;
import org.elasticsearch.Version;
import org.elasticsearch.action.support.nodes.NodesOperationRequest;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
@ -35,6 +36,7 @@ public class NodesHotThreadsRequest extends NodesOperationRequest<NodesHotThread
String type = "cpu";
TimeValue interval = new TimeValue(500, TimeUnit.MILLISECONDS);
int snapshots = 10;
boolean ignoreIdleThreads = true;
/**
* Get hot threads from nodes based on the nodes ids specified. If none are passed, hot
@ -53,6 +55,15 @@ public class NodesHotThreadsRequest extends NodesOperationRequest<NodesHotThread
return this;
}
public boolean ignoreIdleThreads() {
return this.ignoreIdleThreads;
}
public NodesHotThreadsRequest ignoreIdleThreads(boolean ignoreIdleThreads) {
this.ignoreIdleThreads = ignoreIdleThreads;
return this;
}
public NodesHotThreadsRequest type(String type) {
this.type = type;
return this;
@ -84,6 +95,12 @@ public class NodesHotThreadsRequest extends NodesOperationRequest<NodesHotThread
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
threads = in.readInt();
if (in.getVersion().before(Version.V_1_5_0)) {
// Pre-1.5.0 did not filter hot threads, so we shouldn't:
ignoreIdleThreads = false;
} else {
ignoreIdleThreads = in.readBoolean();
}
type = in.readString();
interval = TimeValue.readTimeValue(in);
snapshots = in.readInt();
@ -93,6 +110,9 @@ public class NodesHotThreadsRequest extends NodesOperationRequest<NodesHotThread
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeInt(threads);
if (out.getVersion().onOrAfter(Version.V_1_5_0)) {
out.writeBoolean(ignoreIdleThreads);
}
out.writeString(type);
interval.writeTo(out);
out.writeInt(snapshots);

5
src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestBuilder.java
View File

@ -37,6 +37,11 @@ public class NodesHotThreadsRequestBuilder extends NodesOperationRequestBuilder<
return this;
}
public NodesHotThreadsRequestBuilder setIgnoreIdleThreads(boolean ignoreIdleThreads) {
request.ignoreIdleThreads(ignoreIdleThreads);
return this;
}
public NodesHotThreadsRequestBuilder setType(String type) {
request.type(type);
return this;

5
src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
View File

@ -92,7 +92,8 @@ public class TransportNodesHotThreadsAction extends TransportNodesOperationActio
.busiestThreads(request.request.threads)
.type(request.request.type)
.interval(request.request.interval)
.threadElementsSnapshotCount(request.request.snapshots);
.threadElementsSnapshotCount(request.request.snapshots)
.ignoreIdleThreads(request.request.ignoreIdleThreads);
try {
return new NodeHotThreads(clusterService.localNode(), hotThreads.detect());
} catch (Exception e) {
@ -130,4 +131,4 @@ public class TransportNodesHotThreadsAction extends TransportNodesOperationActio
request.writeTo(out);
}
}
}
}

66
src/main/java/org/elasticsearch/monitor/jvm/HotThreads.java
View File

@ -40,6 +40,7 @@ public class HotThreads {
private TimeValue threadElementsSnapshotDelay = new TimeValue(10);
private int threadElementsSnapshotCount = 10;
private String type = "cpu";
private boolean ignoreIdleThreads = true;
public HotThreads interval(TimeValue interval) {
this.interval = interval;
@ -51,6 +52,11 @@ public class HotThreads {
return this;
}
public HotThreads ignoreIdleThreads(boolean ignoreIdleThreads) {
this.ignoreIdleThreads = ignoreIdleThreads;
return this;
}
public HotThreads threadElementsSnapshotDelay(TimeValue threadElementsSnapshotDelay) {
this.threadElementsSnapshotDelay = threadElementsSnapshotDelay;
return this;
@ -76,6 +82,44 @@ public class HotThreads {
}
}
private static boolean isIdleThread(ThreadInfo threadInfo) {
String threadName = threadInfo.getThreadName();
// NOTE: these are likely JVM dependent
if (threadName.equals("Signal Dispatcher") ||
threadName.equals("Finalizer") ||
threadName.equals("Reference Handler")) {
return true;
}
for (StackTraceElement frame : threadInfo.getStackTrace()) {
String className = frame.getClassName();
String methodName = frame.getMethodName();
if (className.equals("java.util.concurrent.ThreadPoolExecutor") &&
methodName.equals("getTask")) {
return true;
}
if (className.equals("sun.nio.ch.SelectorImpl") &&
methodName.equals("select")) {
return true;
}
if (className.equals("org.elasticsearch.threadpool.ThreadPool$EstimatedTimeThread") &&
methodName.equals("run")) {
return true;
}
if (className.equals("org.elasticsearch.indices.ttl.IndicesTTLService$Notifier") &&
methodName.equals("await")) {
return true;
}
if (className.equals("java.util.concurrent.LinkedTransferQueue") &&
methodName.equals("poll")) {
return true;
}
}
return false;
}
private String innerDetect() throws Exception {
StringBuilder sb = new StringBuilder();
ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
@ -168,18 +212,18 @@ public class HotThreads {
time = hotties.get(t).blockedTime;
}
String threadName = null;
if (allInfos[0][t] == null) {
for (ThreadInfo[] info : allInfos) {
if (info != null && info[t] != null) {
threadName = info[t].getThreadName();
break;
for (ThreadInfo[] info : allInfos) {
if (info != null && info[t] != null) {
if (ignoreIdleThreads && isIdleThread(info[t])) {
info[t] = null;
continue;
}
threadName = info[t].getThreadName();
break;
}
if (threadName == null) {
continue; // thread is not alive yet or died before the first snapshot - ignore it!
}
} else {
threadName = allInfos[0][t].getThreadName();
}
if (threadName == null) {
continue; // thread is not alive yet or died before the first snapshot - ignore it!
}
double percent = (((double) time) / interval.nanos()) * 100;
sb.append(String.format(Locale.ROOT, "%n%4.1f%% (%s out of %s) %s usage by thread '%s'%n", percent, TimeValue.timeValueNanos(time), interval, type, threadName));
@ -277,4 +321,4 @@ public class HotThreads {
this.info = info;
}
}
}
}

1
src/main/java/org/elasticsearch/rest/action/admin/cluster/node/hotthreads/RestNodesHotThreadsAction.java
View File

@ -54,6 +54,7 @@ public class RestNodesHotThreadsAction extends BaseRestHandler {
String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId"));
NodesHotThreadsRequest nodesHotThreadsRequest = new NodesHotThreadsRequest(nodesIds);
nodesHotThreadsRequest.threads(request.paramAsInt("threads", nodesHotThreadsRequest.threads()));
nodesHotThreadsRequest.ignoreIdleThreads(request.paramAsBoolean("ignore_idle_threads", nodesHotThreadsRequest.ignoreIdleThreads()));
nodesHotThreadsRequest.type(request.param("type", nodesHotThreadsRequest.type()));
nodesHotThreadsRequest.interval(TimeValue.parseTimeValue(request.param("interval"), nodesHotThreadsRequest.interval()));
nodesHotThreadsRequest.snapshots(request.paramAsInt("snapshots", nodesHotThreadsRequest.snapshots()));

32
src/test/java/org/elasticsearch/action/admin/HotThreadsTest.java
View File

@ -40,6 +40,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitC
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.notNullValue;
import static org.hamcrest.Matchers.lessThan;
/**
*/
@ -63,6 +64,7 @@ public class HotThreadsTest extends ElasticsearchIntegrationTest {
if (randomBoolean()) {
nodesHotThreadsRequestBuilder.setThreads(rarely() ? randomIntBetween(500, 5000) : randomIntBetween(1, 500));
}
nodesHotThreadsRequestBuilder.setIgnoreIdleThreads(randomBoolean());
if (randomBoolean()) {
switch (randomIntBetween(0, 2)) {
case 2:
@ -131,4 +133,34 @@ public class HotThreadsTest extends ElasticsearchIntegrationTest {
assertThat(hasErrors.get(), is(false));
}
}
public void testIgnoreIdleThreads() throws ExecutionException, InterruptedException {
// First time, don't ignore idle threads:
NodesHotThreadsRequestBuilder builder = client().admin().cluster().prepareNodesHotThreads();
builder.setIgnoreIdleThreads(false);
builder.setThreads(Integer.MAX_VALUE);
NodesHotThreadsResponse response = builder.execute().get();
int totSizeAll = 0;
for (NodeHotThreads node : response.getNodesMap().values()) {
totSizeAll += node.getHotThreads().length();
}
// Second time, do ignore idle threads:
builder = client().admin().cluster().prepareNodesHotThreads();
builder.setThreads(Integer.MAX_VALUE);
// Make sure default is true:
assertEquals(true, builder.request().ignoreIdleThreads());
response = builder.execute().get();
int totSizeIgnoreIdle = 0;
for (NodeHotThreads node : response.getNodesMap().values()) {
totSizeIgnoreIdle += node.getHotThreads().length();
}
// The filtered stacks should be smaller than unfiltered ones:
assertThat(totSizeIgnoreIdle, lessThan(totSizeAll));
}
}