[Monitoring] Add Logstash monitoring feature

This PR adds UI for visualizing Logstash internal metrics with contributions from @ph, @jsvd and @pickypg

Changes include:

* x-pack component for Logstash which installs a monitoring exporter plugin to Logstash core. This periodically ships
  monitoring data to the monitoring bulk API endpoint.
* Adds xpack.monitoring.* setting to logstash.yml when x-pack is installed.
* UI changes to graph Logstash monitoring data. Logstash processes are called nodes. They send separate monitoring info
  which gets aggregated and displayed at cluster level and also at individual node level.
* Adds gradle build support for Logstash x-pack which can be controled via `xpack.logstash.build=false`

Fixes elastic/elasticsearch#4169

Original commit: elastic/x-pack-elasticsearch@f58ef406c1

README.asciidoc

@@ -51,7 +51,7 @@ gradle clean assemble
 gradle clean install
 -----
 
-- If you don't work on the UI side of x-plugins, you can force gradle to skip building kibana by adding
+- If you don't work on the UI/Logstash side of x-plugins, you can force gradle to skip building kibana and/or Logstash by adding
-  `xpack.kibana.build=false` to your `~/.gradle/gradle.properties`. Alternatively you add `-Pxpack.kibana.build=false`
+  `xpack.kibana.build=false`/`xpack.logstash.build=false` to your `~/.gradle/gradle.properties`. Alternatively you add `-Pxpack.kibana.build=false` or `-Pxpack.logstash.build=false`
   on the command line if you only want to do this on individual builds (or `-Pxpack.kibana.build=true` if you need to
   override having added this to your `gradle.properties`).

build.gradle

@@ -32,10 +32,13 @@ subprojects {
 
 task bundlePack(type: Zip) {
   onlyIf { project('kibana').bundlePlugin.enabled }
+  onlyIf { project('logstash').bundlePlugin.enabled }
   dependsOn 'elasticsearch:bundlePlugin'
   dependsOn 'kibana:bundlePlugin'
+  dependsOn 'logstash:bundlePlugin'
   from { zipTree(project('elasticsearch').bundlePlugin.outputs.files.singleFile) }
   from { zipTree(project('kibana').bundlePlugin.outputs.files.singleFile) }
+  from { zipTree(project('logstash').bundlePlugin.outputs.files.singleFile) }
   destinationDir file('build/distributions')
   baseName = 'x-pack'
   version = VersionProperties.elasticsearch

elasticsearch/src/main/java/org/elasticsearch/xpack/monitoring/MonitoredSystem.java

@@ -10,7 +10,8 @@ import java.util.Locale;
 public enum MonitoredSystem {
 
     ES("es"),
-    KIBANA("kibana");
+    KIBANA("kibana"),
+    LOGSTASH("logstash");
 
     private final String system;
 
@@ -28,6 +29,8 @@ public enum MonitoredSystem {
                 return ES;
             case "kibana":
                 return KIBANA;
+            case "logstash":
+                return LOGSTASH;
             default:
                 throw new IllegalArgumentException("Unknown monitoring system [" + system + "]");
         }

elasticsearch/src/main/java/org/elasticsearch/xpack/monitoring/resolver/ResolversRegistry.java

@@ -50,6 +50,7 @@ public class ResolversRegistry implements Iterable<MonitoringIndexNameResolver>
 
         // register resolvers for monitored systems
         registerMonitoredSystem(MonitoredSystem.KIBANA, settings);
+        registerMonitoredSystem(MonitoredSystem.LOGSTASH, settings);
     }
 
     /**

elasticsearch/src/main/resources/monitoring-data.json

@@ -18,6 +18,9 @@
     },
     "node": {
       "enabled": false
+    },
+    "logstash": {
+      "enabled": false
     }
   }
 }

elasticsearch/src/main/resources/monitoring-logstash.json

@@ -0,0 +1,211 @@
+{
+  "template": ".monitoring-logstash-${monitoring.template.version}-*",
+  "settings": {
+    "index.number_of_shards": 1,
+    "index.number_of_replicas": 1,
+    "index.codec": "best_compression"
+  },
+  "mappings": {
+    "_default_": {
+      "_all": {
+        "enabled": false
+      },
+      "properties": {
+        "cluster_uuid": {
+          "type": "keyword"
+        },
+        "timestamp": {
+          "type": "date",
+          "format": "date_time"
+        },
+        "source_node": {
+          "properties": {
+            "uuid": {
+              "type": "keyword"
+            },
+            "host": {
+              "type": "keyword"
+            },
+            "transport_address": {
+              "type": "keyword"
+            },
+            "ip": {
+              "type": "keyword"
+            },
+            "name": {
+              "type": "keyword"
+            },
+            "attributes": {
+              "dynamic": true,
+              "properties": {
+                "data": {
+                  "type": "boolean"
+                },
+                "master": {
+                  "type": "boolean"
+                },
+                "client": {
+                  "type": "boolean"
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "logstash_stats": {
+      "properties": {
+        "logstash_stats": {
+          "properties": {
+            "logstash": {
+              "properties": {
+                "uuid": {
+                  "type": "keyword"
+                },
+                "name": {
+                  "type": "keyword"
+                },
+                "host": {
+                  "type": "keyword"
+                },
+                "http_address": {
+                  "type": "keyword"
+                },
+                "version": {
+                  "type": "keyword"
+                },
+                "snapshot": {
+                  "type": "boolean"
+                },
+                "status": {
+                  "type": "keyword"
+                },
+                "pipeline": {
+                  "properties": {
+                    "workers": {
+                      "type": "short"
+                    },
+                    "batch_size": {
+                      "type": "long"
+                    }
+                  }
+                }
+              }
+            },
+            "events": {
+              "properties": {
+                "filtered": {
+                  "type": "long"
+                },
+                "in": {
+                  "type": "long"
+                },
+                "out": {
+                  "type": "long"
+                },
+                "duration_in_millis": {
+                  "type": "long"
+                }
+              }
+            },
+            "timestamp": {
+              "type": "date"
+            },
+            "jvm": {
+              "properties": {
+                "uptime_in_millis": {
+                  "type": "long"
+                },
+                "gc": {
+                  "properties": {
+                    "collectors": {
+                      "properties": {
+                        "old": {
+                          "properties": {
+                            "collection_count": {
+                              "type": "long"
+                            },
+                            "collection_time_in_millis": {
+                              "type": "long"
+                            }
+                          }
+                        },
+                        "young": {
+                          "properties": {
+                            "collection_count": {
+                              "type": "long"
+                            },
+                            "collection_time_in_millis": {
+                              "type": "long"
+                            }
+                          }
+                        }
+                      }
+                    }
+                  }
+                },
+                "mem": {
+                  "properties": {
+                    "heap_max_in_bytes": {
+                      "type": "long"
+                    },
+                    "heap_used_in_bytes": {
+                      "type": "long"
+                    },
+                    "heap_used_percent": {
+                      "type": "long"
+                    }
+                  }
+                }
+              }
+            },
+            "os": {
+              "properties": {
+                "load": {
+                  "properties": {
+                    "1m": {
+                      "type": "half_float"
+                    },
+                    "5m": {
+                      "type": "half_float"
+                    },
+                    "15m": {
+                      "type": "half_float"
+                    }
+                  }
+                }
+              }
+            },
+            "process": {
+              "properties": {
+                "cpu": {
+                  "properties": {
+                    "percent": {
+                      "type": "long"
+                    }
+                  }
+                },
+                "max_file_descriptors": {
+                  "type": "long"
+                },
+                "open_file_descriptors": {
+                  "type": "long"
+                }
+              }
+            },
+            "reloads": {
+              "properties": {
+                "failures": {
+                  "type": "long"
+                },
+                "successes": {
+                  "type": "long"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

elasticsearch/src/test/java/org/elasticsearch/xpack/monitoring/MonitoredSystemTests.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.monitoring;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Locale;
+
+import static org.hamcrest.Matchers.containsString;
+
+/**
+ * Tests {@link MonitoredSystem}.
+ */
+public class MonitoredSystemTests extends ESTestCase {
+
+    public void testGetSystem() {
+        // everything is just lowercased...
+        for (final MonitoredSystem system : MonitoredSystem.values()) {
+            assertEquals(system.name().toLowerCase(Locale.ROOT), system.getSystem());
+        }
+    }
+
+    public void testFromSystem() {
+        for (final MonitoredSystem system : MonitoredSystem.values()) {
+            final String lowercased = system.name().toLowerCase(Locale.ROOT);
+
+            assertSame(system, MonitoredSystem.fromSystem(system.name()));
+            assertSame(system, MonitoredSystem.fromSystem(lowercased));
+        }
+    }
+
+    public void testFromUnknownSystem() {
+        final String unknownSystem = randomAsciiOfLengthBetween(3, 4);
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
+            MonitoredSystem.fromSystem(unknownSystem);
+        });
+
+        assertThat(e.getMessage(), containsString(unknownSystem));
+    }
+
+}

elasticsearch/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java

@@ -37,7 +37,7 @@ import static org.mockito.Mockito.when;
  */
 public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTestCase {
 
-    private final int EXPECTED_TEMPLATES = 3;
+    private final int EXPECTED_TEMPLATES = 4;
 
     private final RestClient client = mock(RestClient.class);
     private final Response versionResponse = mock(Response.class);

elasticsearch/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterTests.java

@@ -300,7 +300,7 @@ public class HttpExporterTests extends ESTestCase {
         // expected number of resources
         assertThat(multiResource.getResources().size(), equalTo(version + templates.size() + pipelines.size() + bwc.size()));
         assertThat(version, equalTo(1));
-        assertThat(templates, hasSize(3));
+        assertThat(templates, hasSize(4));
         assertThat(pipelines, hasSize(useIngest ? 1 : 0));
         assertThat(bwc, hasSize(1));