honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[Monitoring] Add Logstash monitoring feature 

This PR adds UI for visualizing Logstash internal metrics with contributions from @ph, @jsvd and @pickypg

Changes include:

* x-pack component for Logstash which installs a monitoring exporter plugin to Logstash core. This periodically ships
  monitoring data to the monitoring bulk API endpoint.
* Adds xpack.monitoring.* setting to logstash.yml when x-pack is installed.
* UI changes to graph Logstash monitoring data. Logstash processes are called nodes. They send separate monitoring info
  which gets aggregated and displayed at cluster level and also at individual node level.
* Adds gradle build support for Logstash x-pack which can be controled via `xpack.logstash.build=false`

Fixes elastic/elasticsearch#4169

Original commit: elastic/x-pack-elasticsearch@f58ef406c1
This commit is contained in:
Chris Earle 2016-08-03 16:06:38 -04:00 committed by Suyog Rao
parent 37a29c0387
commit 2bce702f62
9 changed files with 271 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.asciidoc
View File

@ -51,7 +51,7 @@ gradle clean assemble
gradle clean install
-----
- If you don't work on the UI side of x-plugins, you can force gradle to skip building kibana by adding
`xpack.kibana.build=false` to your `~/.gradle/gradle.properties`. Alternatively you add `-Pxpack.kibana.build=false`
- If you don't work on the UI/Logstash side of x-plugins, you can force gradle to skip building kibana and/or Logstash by adding
`xpack.kibana.build=false`/`xpack.logstash.build=false` to your `~/.gradle/gradle.properties`. Alternatively you add `-Pxpack.kibana.build=false` or `-Pxpack.logstash.build=false`
on the command line if you only want to do this on individual builds (or `-Pxpack.kibana.build=true` if you need to
override having added this to your `gradle.properties`).

3
build.gradle
View File

@ -32,10 +32,13 @@ subprojects {
task bundlePack(type: Zip) {
onlyIf { project('kibana').bundlePlugin.enabled }
onlyIf { project('logstash').bundlePlugin.enabled }
dependsOn 'elasticsearch:bundlePlugin'
dependsOn 'kibana:bundlePlugin'
dependsOn 'logstash:bundlePlugin'
from { zipTree(project('elasticsearch').bundlePlugin.outputs.files.singleFile) }
from { zipTree(project('kibana').bundlePlugin.outputs.files.singleFile) }
from { zipTree(project('logstash').bundlePlugin.outputs.files.singleFile) }
destinationDir file('build/distributions')
baseName = 'x-pack'
version = VersionProperties.elasticsearch

5
elasticsearch/src/main/java/org/elasticsearch/xpack/monitoring/MonitoredSystem.java
View File

@ -10,7 +10,8 @@ import java.util.Locale;
public enum MonitoredSystem {
ES("es"),
KIBANA("kibana");
KIBANA("kibana"),
LOGSTASH("logstash");
private final String system;
@ -28,6 +29,8 @@ public enum MonitoredSystem {
return ES;
case "kibana":
return KIBANA;
case "logstash":
return LOGSTASH;
default:
throw new IllegalArgumentException("Unknown monitoring system [" + system + "]");
}

1
elasticsearch/src/main/java/org/elasticsearch/xpack/monitoring/resolver/ResolversRegistry.java
View File

@ -50,6 +50,7 @@ public class ResolversRegistry implements Iterable<MonitoringIndexNameResolver>
// register resolvers for monitored systems
registerMonitoredSystem(MonitoredSystem.KIBANA, settings);
registerMonitoredSystem(MonitoredSystem.LOGSTASH, settings);
}
/**

3
elasticsearch/src/main/resources/monitoring-data.json
View File

@ -18,6 +18,9 @@
},
"node": {
"enabled": false
},
"logstash": {
"enabled": false
}
}
}

211
elasticsearch/src/main/resources/monitoring-logstash.json Normal file
View File

@ -0,0 +1,211 @@
{
"template": ".monitoring-logstash-${monitoring.template.version}-*",
"settings": {
"index.number_of_shards": 1,
"index.number_of_replicas": 1,
"index.codec": "best_compression"
},
"mappings": {
"_default_": {
"_all": {
"enabled": false
},
"properties": {
"cluster_uuid": {
"type": "keyword"
},
"timestamp": {
"type": "date",
"format": "date_time"
},
"source_node": {
"properties": {
"uuid": {
"type": "keyword"
},
"host": {
"type": "keyword"
},
"transport_address": {
"type": "keyword"
},
"ip": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"attributes": {
"dynamic": true,
"properties": {
"data": {
"type": "boolean"
},
"master": {
"type": "boolean"
},
"client": {
"type": "boolean"
}
}
}
}
}
}
},
"logstash_stats": {
"properties": {
"logstash_stats": {
"properties": {
"logstash": {
"properties": {
"uuid": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"host": {
"type": "keyword"
},
"http_address": {
"type": "keyword"
},
"version": {
"type": "keyword"
},
"snapshot": {
"type": "boolean"
},
"status": {
"type": "keyword"
},
"pipeline": {
"properties": {
"workers": {
"type": "short"
},
"batch_size": {
"type": "long"
}
}
}
}
},
"events": {
"properties": {
"filtered": {
"type": "long"
},
"in": {
"type": "long"
},
"out": {
"type": "long"
},
"duration_in_millis": {
"type": "long"
}
}
},
"timestamp": {
"type": "date"
},
"jvm": {
"properties": {
"uptime_in_millis": {
"type": "long"
},
"gc": {
"properties": {
"collectors": {
"properties": {
"old": {
"properties": {
"collection_count": {
"type": "long"
},
"collection_time_in_millis": {
"type": "long"
}
}
},
"young": {
"properties": {
"collection_count": {
"type": "long"
},
"collection_time_in_millis": {
"type": "long"
}
}
}
}
}
}
},
"mem": {
"properties": {
"heap_max_in_bytes": {
"type": "long"
},
"heap_used_in_bytes": {
"type": "long"
},
"heap_used_percent": {
"type": "long"
}
}
}
}
},
"os": {
"properties": {
"load": {
"properties": {
"1m": {
"type": "half_float"
},
"5m": {
"type": "half_float"
},
"15m": {
"type": "half_float"
}
}
}
}
},
"process": {
"properties": {
"cpu": {
"properties": {
"percent": {
"type": "long"
}
}
},
"max_file_descriptors": {
"type": "long"
},
"open_file_descriptors": {
"type": "long"
}
}
},
"reloads": {
"properties": {
"failures": {
"type": "long"
},
"successes": {
"type": "long"
}
}
}
}
}
}
}
}
}

45
elasticsearch/src/test/java/org/elasticsearch/xpack/monitoring/MonitoredSystemTests.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.monitoring;
import org.elasticsearch.test.ESTestCase;
import java.util.Locale;
import static org.hamcrest.Matchers.containsString;
/**
* Tests {@link MonitoredSystem}.
*/
public class MonitoredSystemTests extends ESTestCase {
public void testGetSystem() {
// everything is just lowercased...
for (final MonitoredSystem system : MonitoredSystem.values()) {
assertEquals(system.name().toLowerCase(Locale.ROOT), system.getSystem());
}
}
public void testFromSystem() {
for (final MonitoredSystem system : MonitoredSystem.values()) {
final String lowercased = system.name().toLowerCase(Locale.ROOT);
assertSame(system, MonitoredSystem.fromSystem(system.name()));
assertSame(system, MonitoredSystem.fromSystem(lowercased));
}
}
public void testFromUnknownSystem() {
final String unknownSystem = randomAsciiOfLengthBetween(3, 4);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
MonitoredSystem.fromSystem(unknownSystem);
});
assertThat(e.getMessage(), containsString(unknownSystem));
}
}

2
elasticsearch/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java
View File

@ -37,7 +37,7 @@ import static org.mockito.Mockito.when;
*/
public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTestCase {
private final int EXPECTED_TEMPLATES = 3;
private final int EXPECTED_TEMPLATES = 4;
private final RestClient client = mock(RestClient.class);
private final Response versionResponse = mock(Response.class);

2
elasticsearch/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterTests.java
View File

@ -300,7 +300,7 @@ public class HttpExporterTests extends ESTestCase {
// expected number of resources
assertThat(multiResource.getResources().size(), equalTo(version + templates.size() + pipelines.size() + bwc.size()));
assertThat(version, equalTo(1));
assertThat(templates, hasSize(3));
assertThat(templates, hasSize(4));
assertThat(pipelines, hasSize(useIngest ? 1 : 0));
assertThat(bwc, hasSize(1));