[Logstash] New mapping for pipeline viewer (elastic/x-pack-elasticsearch#573)

Add `logstash-states` + other mapping fields for logstash pipeline viewer
This is part of https://github.com/elastic/x-pack-logstash/pull/25

Original commit: elastic/x-pack-elasticsearch@1940d79d13

plugin/src/main/java/org/elasticsearch/xpack/monitoring/MonitoredSystem.java

@@ -12,7 +12,8 @@ public enum MonitoredSystem {
     ES("es"),
     KIBANA("kibana"),
     LOGSTASH("logstash"),
-    BEATS("beats");
+    BEATS("beats"),
+    LOGSTASH_STATES("logstash-states");
 
     private final String system;
 
@@ -25,17 +26,23 @@ public enum MonitoredSystem {
     }
 
     public static MonitoredSystem fromSystem(String system) {
-        switch (system.toLowerCase(Locale.ROOT)) {
+        switch (transformSystemName(system)) {
             case "es":
                 return ES;
             case "kibana":
                 return KIBANA;
             case "logstash":
                 return LOGSTASH;
+            case "logstash-states":
+                return LOGSTASH_STATES;
             case "beats":
                 return BEATS;
             default:
                 throw new IllegalArgumentException("Unknown monitoring system [" + system + "]");
         }
     }
+
+    public static String transformSystemName(String systemName) {
+        return systemName.toLowerCase(Locale.ROOT).replace("_", "-");
+    }
 }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/ResolversRegistry.java

@@ -49,6 +49,7 @@ public class ResolversRegistry implements Iterable<MonitoringIndexNameResolver>
         // register resolvers for monitored systems
         registerMonitoredSystem(MonitoredSystem.KIBANA, settings);
         registerMonitoredSystem(MonitoredSystem.LOGSTASH, settings);
+        registerMonitoredSystem(MonitoredSystem.LOGSTASH_STATES, settings);
         registerMonitoredSystem(MonitoredSystem.BEATS, settings);
     }
 

plugin/src/main/resources/monitoring-logstash-states.json

@@ -0,0 +1,98 @@
+{
+  "index_patterns": ".monitoring-logstash-states-${monitoring.template.version}-*",
+  "version": 6000002,
+  "settings": {
+    "index.number_of_shards": 1,
+    "index.number_of_replicas": 1,
+    "index.codec": "best_compression"
+  },
+  "mappings": {
+    "doc": {
+      "dynamic": "false",
+      "properties": {
+        "cluster_uuid": {
+          "type": "keyword"
+        },
+        "timestamp": {
+          "type": "date",
+          "format": "date_time"
+        },
+        "type": {
+          "type": "keyword"
+        },
+        "source_node": {
+          "properties": {
+            "uuid": {
+              "type": "keyword"
+            },
+            "host": {
+              "type": "keyword"
+            },
+            "transport_address": {
+              "type": "keyword"
+            },
+            "ip": {
+              "type": "keyword"
+            },
+            "name": {
+              "type": "keyword"
+            }
+          }
+        },
+        "logstash_state": {
+          "properties": {
+            "timestamp": {
+              "type": "date"
+            },
+            "uuid": {
+              "type": "keyword"
+            },
+            "name": {
+              "type": "keyword"
+            },
+            "host": {
+              "type": "keyword"
+            },
+            "http_address": {
+              "type": "keyword"
+            },
+            "version": {
+              "type": "keyword"
+            },
+            "snapshot": {
+              "type": "boolean"
+            },
+            "status": {
+              "type": "keyword"
+            },
+            "pipeline": {
+              "properties": {
+                "name": {
+                  "type": "keyword"
+                },
+                "hash": {
+                  "type": "keyword"
+                },
+                "workers": {
+                  "type": "short"
+                },
+                "batch_size": {
+                  "type": "integer"
+                },
+                "format": {
+                  "type": "keyword"
+                },
+                "version": {
+                  "type": "keyword"
+                },
+                "representation": {
+                  "enabled": false
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

plugin/src/main/resources/monitoring-logstash.json

@@ -54,6 +54,7 @@
     "logstash_stats": {
       "properties": {
         "logstash_stats": {
+          "type": "object",
           "properties": {
             "logstash": {
               "properties": {
@@ -244,10 +245,86 @@
                 "events_count": {
                   "type": "long"
                 },
-                "type": {
+                "type": {"type": "keyword"}
+              }
+            },
+            "pipelines": {
+              "type": "nested",
+              "properties": {
+                "name": {
                   "type": "keyword"
+                },
+                "hash": {
+                  "type": "keyword"
+                },
+                "events": {
+                  "properties": {
+                    "input": {
+                      "type": "long"
+                    },
+                    "filter": {
+                      "type": "long"
+                    },
+                    "output": {
+                      "type": "long"
                     }
                   }
+                },
+                "queue": {
+                  "properties": {
+                    "events_count": {
+                      "type": "long"
+                    },
+                    "type": {"type": "keyword"}
+                  }
+                },
+                "components": {
+                  "type": "nested",
+                  "properties": {
+                    "id": {"type": "keyword"},
+                    "long_stat": {
+                      "type": "nested",
+                      "properties": {
+                        "name": {"type": "keyword"},
+                        "value": {"type": "long"},
+                        "metric_type": { "type": "keyword" }
+                      }
+                    },
+                    "double_stat": {
+                      "type": "nested",
+                      "properties": {
+                        "name": {"type": "keyword"},
+                        "value": {"type": "double"},
+                        "metric_type": { "type": "keyword" }
+                      }
+                    },
+                    "string_stat": {
+                      "type": "nested",
+                      "properties": {
+                        "name": {"type": "keyword"},
+                        "value": {"type": "keyword"},
+                        "metric_type": { "type": "keyword" }
+                      }
+                    }
+                  }
+                },
+                "reloads": {
+                  "properties": {
+                    "failures": {
+                      "type": "long"
+                    },
+                    "successes": {
+                      "type": "long"
+                    }
+                  }
+                }
+              }
+            },
+            "workers": {
+              "type": "short"
+            },
+            "batch_size": {
+              "type": "integer"
             }
           }
         }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/MonitoredSystemTests.java

@@ -19,16 +19,14 @@ public class MonitoredSystemTests extends ESTestCase {
     public void testGetSystem() {
         // everything is just lowercased...
         for (final MonitoredSystem system : MonitoredSystem.values()) {
-            assertEquals(system.name().toLowerCase(Locale.ROOT), system.getSystem());
+            assertEquals(MonitoredSystem.transformSystemName(system.name()), system.getSystem());
         }
     }
 
     public void testFromSystem() {
         for (final MonitoredSystem system : MonitoredSystem.values()) {
-            final String lowercased = system.name().toLowerCase(Locale.ROOT);
-
             assertSame(system, MonitoredSystem.fromSystem(system.name()));
-            assertSame(system, MonitoredSystem.fromSystem(lowercased));
+            assertSame(system, MonitoredSystem.fromSystem(MonitoredSystem.transformSystemName(system.name())));
         }
     }
 
@@ -42,4 +40,9 @@ public class MonitoredSystemTests extends ESTestCase {
         assertThat(e.getMessage(), containsString(unknownSystem));
     }
 
+    public void testTransformSystemName() {
+        final String systemName = "MiXED_System_Name";
+        assertEquals(MonitoredSystem.transformSystemName(systemName), "mixed-system-name");
+    }
+
 }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java

@@ -54,7 +54,7 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
      * kibana, logstash, beats
      */
     private final int EXPECTED_TYPES = MonitoringTemplateUtils.NEW_DATA_TYPES.length;
-    private final int EXPECTED_TEMPLATES = 6;
+    private final int EXPECTED_TEMPLATES = 7;
     private final int EXPECTED_WATCHES = 4;
 
     private final RestClient client = mock(RestClient.class);

plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterTests.java

@@ -357,7 +357,7 @@ public class HttpExporterTests extends ESTestCase {
                    equalTo(version + typeMappings.size() + templates.size() + pipelines.size() + watcherCheck.size() + bwc.size()));
         assertThat(version, equalTo(1));
         assertThat(typeMappings, hasSize(MonitoringTemplateUtils.NEW_DATA_TYPES.length));
-        assertThat(templates, hasSize(6));
+        assertThat(templates, hasSize(7));
         assertThat(pipelines, hasSize(useIngest ? 1 : 0));
         assertThat(watcherCheck, hasSize(clusterAlertManagement ? 1 : 0));
         assertThat(watches, hasSize(clusterAlertManagement ? ClusterAlertsUtil.WATCH_IDS.length : 0));