Monitoring: Manage multiple index templates

Original commit: elastic/x-pack-elasticsearch@fef9dcc5d1

elasticsearch/x-pack/build.gradle

@@ -71,8 +71,6 @@ compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-
 
 ext.expansions = [
   'project.version': version,
-  // Used in marvel index templates
-  'monitoring.template.version': '1',
   // Used in watcher index templates
   'watcher.template.version': '1',
 ]

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MarvelSettings.java

@@ -13,7 +13,6 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
 import org.elasticsearch.xpack.XPackPlugin;
 
 import java.util.Collections;
@@ -23,7 +22,6 @@ import java.util.function.Function;
 import static org.elasticsearch.common.settings.Setting.Property;
 import static org.elasticsearch.common.settings.Setting.boolSetting;
 import static org.elasticsearch.common.settings.Setting.groupSetting;
-import static org.elasticsearch.common.settings.Setting.intSetting;
 import static org.elasticsearch.common.settings.Setting.listSetting;
 import static org.elasticsearch.common.settings.Setting.timeSetting;
 
@@ -121,13 +119,6 @@ public class MarvelSettings extends AbstractComponent {
                         HISTORY_DURATION_MINIMUM,         // minimum value
                         Property.Dynamic, Property.NodeScope);
 
-    /**
-     * The index setting that holds the template version
-     */
-    public static final Setting<Integer> INDEX_TEMPLATE_VERSION =
-            intSetting("index.xpack.monitoring.template.version", MarvelTemplateUtils.TEMPLATE_VERSION,
-                    Property.Dynamic, Property.IndexScope);
-
     /**
      * Settings/Options per configured exporter
      */
@@ -147,7 +138,6 @@ public class MarvelSettings extends AbstractComponent {
         module.registerSetting(HISTORY_DURATION);
         module.registerSetting(EXPORTERS_SETTINGS);
         module.registerSetting(ENABLED);
-        module.registerSetting(INDEX_TEMPLATE_VERSION);
 
         module.registerSettingsFilter("xpack.monitoring.agent.exporters.*.auth.*");
         module.registerSettingsFilter("xpack.monitoring.agent.exporters.*.ssl.*");

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtils.java

@@ -5,103 +5,80 @@
  */
 package org.elasticsearch.marvel.agent.exporter;
 
-import org.elasticsearch.common.Strings;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.compress.NotXContentException;
 import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.xcontent.XContentHelper;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.Properties;
+import java.util.Locale;
+import java.util.regex.Pattern;
 
 public final class MarvelTemplateUtils {
 
-    static final String INDEX_TEMPLATE_FILE         = "/monitoring-es.json";
+    private static final String TEMPLATE_FILE = "/monitoring-%s.json";
-    static final String INDEX_TEMPLATE_NAME_PREFIX  = ".monitoring-es-";
+    private static final String TEMPLATE_VERSION_PROPERTY = Pattern.quote("${monitoring.template.version}");
 
-    static final String DATA_TEMPLATE_FILE          = "/monitoring-data.json";
+    /** Current version of es and data templates **/
-    static final String DATA_TEMPLATE_NAME_PREFIX   = ".monitoring-data-";
+    public static final Integer TEMPLATE_VERSION = 2;
-
-    static final String PROPERTIES_FILE             = "/monitoring.properties";
-    static final String TEMPLATE_VERSION_PROPERTY   = "template.version";
-
-    public static final Integer TEMPLATE_VERSION    = loadTemplateVersion();
 
     private MarvelTemplateUtils() {
     }
 
     /**
-     * Loads the default template for the timestamped indices
+     * Loads a built-in template and returns its source.
      */
-    public static byte[] loadTimestampedIndexTemplate() {
+    public static String loadTemplate(String id, Integer version) {
+        String resource = String.format(Locale.ROOT, TEMPLATE_FILE, id);
         try {
-            return load(INDEX_TEMPLATE_FILE);
+            BytesReference source = load(resource);
-        } catch (IOException e) {
+            validate(source);
-            throw new IllegalStateException("unable to load monitoring template", e);
+
+            return filter(source, version);
+        } catch (Exception e) {
+            throw new IllegalArgumentException("Unable to load monitoring template [" + resource + "]", e);
         }
     }
 
     /**
-     * Loads the default template for the data index
+     * Loads a resource from the classpath and returns it as a {@link BytesReference}
      */
-    public static byte[] loadDataIndexTemplate() {
+    static BytesReference load(String name) throws IOException {
-        try {
-            return load(DATA_TEMPLATE_FILE);
-        } catch (IOException e) {
-            throw new IllegalStateException("unable to load monitoring data template", e);
-        }
-    }
-
-    /**
-     * Loads a resource with a given name and returns it as a byte array.
-     */
-    static byte[] load(String name) throws IOException {
         try (InputStream is = MarvelTemplateUtils.class.getResourceAsStream(name)) {
-            ByteArrayOutputStream out = new ByteArrayOutputStream();
+            try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
                 Streams.copy(is, out);
-            return out.toByteArray();
+                return new BytesArray(out.toByteArray());
+            }
         }
     }
 
     /**
-     * Loads the current version of templates
+     * Parses and validates that the source is not empty.
-     *
-     * When executing tests in Intellij, the properties file might not be
-     * resolved: try running 'gradle processResources' first.
      */
-    static Integer loadTemplateVersion() {
+    static void validate(BytesReference source) {
-        try (InputStream is = MarvelTemplateUtils.class.getResourceAsStream(PROPERTIES_FILE)) {
+        if (source == null) {
-            Properties properties = new Properties();
+            throw new ElasticsearchParseException("Monitoring template must not be null");
-            properties.load(is);
-            String version = properties.getProperty(TEMPLATE_VERSION_PROPERTY);
-            if (Strings.hasLength(version)) {
-                return Integer.parseInt(version);
         }
-            throw new IllegalArgumentException("no monitoring template version found");
+
-        } catch (NumberFormatException e) {
+        try {
-            throw new IllegalArgumentException("failed to parse monitoring template version");
+            XContentHelper.convertToMap(source, false).v2();
-        } catch (IOException e) {
+        } catch (NotXContentException e) {
-            throw new IllegalArgumentException("failed to load monitoring template version");
+            throw new ElasticsearchParseException("Monitoring template must not be empty");
+        } catch (Exception e) {
+            throw new ElasticsearchParseException("Invalid monitoring template", e);
         }
     }
 
-    public static String indexTemplateName() {
+    /**
-        return indexTemplateName(TEMPLATE_VERSION);
+     * Filters the source: replaces any template version property with the version number
-    }
+     */
-
+    static String filter(BytesReference source, Integer version) {
-    public static String indexTemplateName(Integer version) {
+        return Pattern.compile(TEMPLATE_VERSION_PROPERTY)
-        return templateName(INDEX_TEMPLATE_NAME_PREFIX, version);
+                .matcher(source.toUtf8())
-    }
+                .replaceAll(String.valueOf(version));
-
-    public static String dataTemplateName() {
-        return dataTemplateName(TEMPLATE_VERSION);
-    }
-
-    public static String dataTemplateName(Integer version) {
-        return templateName(DATA_TEMPLATE_NAME_PREFIX, version);
-    }
-
-    static String templateName(String prefix, Integer version) {
-        assert version != null && version >= 0 : "version must be not null and greater or equal to zero";
-        return prefix + String.valueOf(version);
     }
 }

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporter.java

@@ -25,7 +25,6 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.marvel.agent.exporter.ExportBulk;
 import org.elasticsearch.marvel.agent.exporter.ExportException;
 import org.elasticsearch.marvel.agent.exporter.Exporter;
-import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
 import org.elasticsearch.marvel.agent.exporter.MonitoringDoc;
 import org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolver;
 import org.elasticsearch.marvel.agent.resolver.ResolversRegistry;
@@ -47,6 +46,7 @@ import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.AccessController;
@@ -55,6 +55,8 @@ import java.security.PrivilegedAction;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 
 /**
  *
@@ -107,11 +109,6 @@ public class HttpExporter extends Exporter {
     volatile boolean checkedAndUploadedIndexTemplate = false;
     volatile boolean supportedClusterVersion = false;
 
-    /**
-     * Version number of built-in templates
-     **/
-    private final Integer templateVersion;
-
     boolean keepAlive;
     final ConnectionKeepAliveWorker keepAliveWorker;
     Thread keepAliveThread;
@@ -145,14 +142,16 @@ public class HttpExporter extends Exporter {
         sslSocketFactory = createSSLSocketFactory(config.settings().getAsSettings(SSL_SETTING));
         hostnameVerification = config.settings().getAsBoolean(SSL_HOSTNAME_VERIFICATION_SETTING, true);
 
-        // Loads the current version number of built-in templates
-        templateVersion = MarvelTemplateUtils.TEMPLATE_VERSION;
-        if (templateVersion == null) {
-            throw new IllegalStateException("unable to find built-in template version");
-        }
         resolvers = new ResolversRegistry(config.settings());
-        logger.debug("initialized with hosts [{}], index prefix [{}], template version [{}]",
+        // Checks that required templates are loaded
-                Strings.arrayToCommaDelimitedString(hosts), MonitoringIndexNameResolver.PREFIX, templateVersion);
+        for (MonitoringIndexNameResolver resolver : resolvers) {
+            if (resolver.template() == null) {
+                throw new IllegalStateException("unable to find built-in template " + resolver.templateName());
+            }
+        }
+
+        logger.debug("initialized with hosts [{}], index prefix [{}]",
+                Strings.arrayToCommaDelimitedString(hosts), MonitoringIndexNameResolver.PREFIX);
     }
 
     ResolversRegistry getResolvers() {
@@ -424,24 +423,18 @@ public class HttpExporter extends Exporter {
      * @return true if template exists or was uploaded successfully.
      */
     private boolean checkAndUploadIndexTemplate(final String host) {
-        String templateName = MarvelTemplateUtils.indexTemplateName(templateVersion);
+        // List of distinct templates
-        boolean templateInstalled = hasTemplate(templateName, host);
+        Map<String, String> templates = StreamSupport.stream(new ResolversRegistry(Settings.EMPTY).spliterator(), false)
+                .collect(Collectors.toMap(MonitoringIndexNameResolver::templateName, MonitoringIndexNameResolver::template, (a, b) -> a));
 
-        // Works like LocalExporter on master:
+        for (Map.Entry<String, String> template : templates.entrySet()) {
-        // Install the index template for timestamped indices first, so that other nodes can ship data
+            if (hasTemplate(template.getKey(), host) == false) {
-        if (!templateInstalled) {
+                logger.debug("template [{}] not found", template.getKey());
-            logger.debug("could not find existing monitoring template, installing a new one");
+                if (!putTemplate(host, template.getKey(), template.getValue())) {
-            if (!putTemplate(host, templateName, MarvelTemplateUtils.loadTimestampedIndexTemplate())) {
                     return false;
                 }
-        }
+            } else {
-
+                logger.debug("template [{}] found", template.getKey());
-        // Install the index template for data index
-        templateName = MarvelTemplateUtils.dataTemplateName(templateVersion);
-        if (!hasTemplate(templateName, host)) {
-            logger.debug("could not find existing monitoring template for data index, installing a new one");
-            if (!putTemplate(host, templateName, MarvelTemplateUtils.loadDataIndexTemplate())) {
-                return false;
             }
         }
         return true;
@@ -481,7 +474,7 @@ public class HttpExporter extends Exporter {
         return false;
     }
 
-    boolean putTemplate(String host, String template, byte[] source) {
+    boolean putTemplate(String host, String template, String source) {
         logger.debug("installing template [{}]", template);
         HttpURLConnection connection = null;
         try {
@@ -492,13 +485,13 @@ public class HttpExporter extends Exporter {
             }
 
             // Uploads the template and closes the outputstream
-            Streams.copy(source, connection.getOutputStream());
+            Streams.copy(source.getBytes(StandardCharsets.UTF_8), connection.getOutputStream());
             if (connection.getResponseCode() != 200 && connection.getResponseCode() != 201) {
                 logConnectionError("error adding the monitoring template [" + template + "] to [" + host + "]", connection);
                 return false;
             }
 
-            logger.info("monitoring template [{}] updated to version [{}]", template, templateVersion);
+            logger.info("monitoring template [{}] updated ", template);
             return true;
         } catch (IOException e) {
             logger.error("failed to update monitoring template [{}] on host [{}]:\n{}", template, host, e.getMessage());

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporter.java

@@ -21,11 +21,11 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.marvel.agent.exporter.ExportBulk;
 import org.elasticsearch.marvel.agent.exporter.Exporter;
-import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
 import org.elasticsearch.marvel.agent.exporter.MonitoringDoc;
 import org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolver;
 import org.elasticsearch.marvel.agent.resolver.ResolversRegistry;
@@ -35,6 +35,7 @@ import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
@@ -56,23 +57,21 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
 
     private final AtomicReference<State> state = new AtomicReference<>(State.INITIALIZED);
 
-    /** Version number of built-in templates **/
-    private final Integer templateVersion;
-
     public LocalExporter(Exporter.Config config, MonitoringClientProxy client,
                          ClusterService clusterService, CleanerService cleanerService) {
         super(TYPE, config);
         this.client = client;
         this.clusterService = clusterService;
         this.cleanerService = cleanerService;
+        this.resolvers = new ResolversRegistry(config.settings());
 
-        // Loads the current version number of built-in templates
+        // Checks that required templates are loaded
-        templateVersion = MarvelTemplateUtils.TEMPLATE_VERSION;
+        for (MonitoringIndexNameResolver resolver : resolvers) {
-        if (templateVersion == null) {
+            if (resolver.template() == null) {
-            throw new IllegalStateException("unable to find built-in template version");
+                throw new IllegalStateException("unable to find built-in template " + resolver.templateName());
+            }
         }
 
-        resolvers = new ResolversRegistry(config.settings());
         clusterService.add(this);
         cleanerService.add(this);
     }
@@ -117,21 +116,23 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
             return null;
         }
 
-        String templateName = MarvelTemplateUtils.indexTemplateName(templateVersion);
+        // List of distinct templates
-        boolean templateInstalled = hasTemplate(templateName, clusterState);
+        Map<String, String> templates = StreamSupport.stream(new ResolversRegistry(Settings.EMPTY).spliterator(), false)
+                .collect(Collectors.toMap(MonitoringIndexNameResolver::templateName, MonitoringIndexNameResolver::template, (a, b) -> a));
 
-        // if this is not the master, we'll just look to see if the monitoring timestamped template is already
+
-        // installed and if so, if it has a compatible version. If it is (installed and compatible)
+        // if this is not the master, we'll just look to see if the monitoring templates are installed.
-        // we'll be able to start this exporter. Otherwise, we'll just wait for a new cluster state.
+        // If they all are, we'll be able to start this exporter. Otherwise, we'll just wait for a new cluster state.
         if (clusterService.state().nodes().isLocalNodeElectedMaster() == false) {
-            // We only need to check the index template for timestamped indices
+            for (String template : templates.keySet()) {
-            if (templateInstalled == false) {
+                if (hasTemplate(template, clusterState) == false) {
-                // the template for timestamped indices is not yet installed in the given cluster state, we'll wait.
+                    // the required template is not yet installed in the given cluster state, we'll wait.
-                logger.debug("monitoring index template does not exist, so service cannot start");
+                    logger.debug("monitoring index template [{}] does not exist, so service cannot start", template);
                     return null;
                 }
+            }
 
-            logger.debug("monitoring index template found, service can start");
+            logger.debug("monitoring index templates are installed, service can start");
 
         } else {
 
@@ -143,24 +144,18 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
                 return null;
             }
 
-            // Install the index template for timestamped indices first, so that other nodes can ship data
+            // Check that each required template exist, installing it if needed
-            if (templateInstalled == false) {
+            for (Map.Entry<String, String> template : templates.entrySet()) {
-                logger.debug("could not find existing monitoring template for timestamped indices, installing a new one");
+                if (hasTemplate(template.getKey(), clusterState) == false) {
-                putTemplate(templateName, MarvelTemplateUtils.loadTimestampedIndexTemplate());
+                    logger.debug("template [{}] not found", template.getKey());
-                // we'll get that template on the next cluster state update
+                    putTemplate(template.getKey(), template.getValue());
                     return null;
+                } else {
+                    logger.debug("template [{}] found", template.getKey());
+                }
             }
 
-            // Install the index template for data index
+            logger.debug("monitoring index templates are installed on master node, service can start");
-            templateName = MarvelTemplateUtils.dataTemplateName(templateVersion);
-            if (hasTemplate(templateName, clusterState) == false) {
-                logger.debug("could not find existing monitoring template for data index, installing a new one");
-                putTemplate(templateName, MarvelTemplateUtils.loadDataIndexTemplate());
-                // we'll get that template on the next cluster state update
-                return null;
-            }
-
-            logger.debug("monitoring index template found on master node, service can start");
         }
 
         if (state.compareAndSet(State.INITIALIZED, State.RUNNING)) {
@@ -197,7 +192,7 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
         return templates.size() > 0;
     }
 
-    void putTemplate(String template, byte[] source) {
+    void putTemplate(String template, String source) {
         logger.debug("installing template [{}]",template);
 
         PutIndexTemplateRequest request = new PutIndexTemplateRequest(template).source(source);

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/MonitoringIndexNameResolver.java

@@ -16,6 +16,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentBuilderString;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.marvel.MonitoredSystem;
+import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
 import org.elasticsearch.marvel.agent.exporter.MonitoringDoc;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
@@ -23,6 +24,7 @@ import org.joda.time.format.DateTimeFormat;
 import org.joda.time.format.DateTimeFormatter;
 
 import java.io.IOException;
+import java.util.Locale;
 import java.util.function.Function;
 
 /**
@@ -100,6 +102,16 @@ public abstract class MonitoringIndexNameResolver<T extends MonitoringDoc> {
         }
     }
 
+    /**
+     * @return the template name that is required by the resolver.
+     */
+    public abstract String templateName();
+
+    /**
+     * @return the template source required by the resolver
+     */
+    public abstract String template();
+
     int getVersion() {
         return version;
     }
@@ -145,6 +157,16 @@ public abstract class MonitoringIndexNameResolver<T extends MonitoringDoc> {
         public String indexPattern() {
             return index;
         }
+
+        @Override
+        public String templateName() {
+            return String.format(Locale.ROOT, "%s-%s-%d", PREFIX, DATA, getVersion());
+        }
+
+        @Override
+        public String template() {
+            return MarvelTemplateUtils.loadTemplate(DATA, getVersion());
+        }
     }
 
     /**
@@ -188,6 +210,16 @@ public abstract class MonitoringIndexNameResolver<T extends MonitoringDoc> {
             return null;
         }
 
+        @Override
+        public String templateName() {
+            return String.format(Locale.ROOT, "%s-%s-%d", PREFIX, getId(), getVersion());
+        }
+
+        @Override
+        public String template() {
+            return MarvelTemplateUtils.loadTemplate(getId(), getVersion());
+        }
+
         String getId() {
             return system.getSystem();
         }

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/resolver/ResolversRegistry.java

@@ -43,14 +43,17 @@ import static org.elasticsearch.marvel.MonitoredSystem.ES;
 
 public class ResolversRegistry implements Iterable<MonitoringIndexNameResolver> {
 
+    private static final int ES_TEMPLATE_VERSION = MarvelTemplateUtils.TEMPLATE_VERSION;
+    private static final int KIBANA_TEMPLATE_VERSION = MarvelTemplateUtils.TEMPLATE_VERSION;
+
     private final List<Registration> registrations = new ArrayList<>();
 
     public ResolversRegistry(Settings settings) {
         // register built-in defaults resolvers
-        registerBuiltIn(ES, MarvelTemplateUtils.TEMPLATE_VERSION, settings);
+        registerBuiltIn(ES, ES_TEMPLATE_VERSION, settings);
 
-        // register resolvers for external applications
+        // register resolvers for monitored systems
-        registerKibana(settings);
+        registerMonitoredSystem(MonitoredSystem.KIBANA, KIBANA_TEMPLATE_VERSION, settings);
     }
 
     /**
@@ -70,11 +73,11 @@ public class ResolversRegistry implements Iterable<MonitoringIndexNameResolver>
     }
 
     /**
-     * Registers resolvers for Kibana
+     * Registers resolvers for monitored systems
      */
-    private void registerKibana(Settings settings) {
+    private void registerMonitoredSystem(MonitoredSystem id, int version, Settings settings) {
-        final MonitoringBulkResolver kibana =  new MonitoringBulkResolver(MonitoredSystem.KIBANA, 0, settings);
+        final MonitoringBulkResolver resolver =  new MonitoringBulkResolver(id, version, settings);
-        registrations.add(resolveByClassSystemVersion(MonitoringBulkDoc.class, MonitoredSystem.KIBANA, Version.CURRENT, kibana));
+        registrations.add(resolveByClassSystemVersion(MonitoringBulkDoc.class, id, Version.CURRENT, resolver));
     }
 
     /**

elasticsearch/x-pack/marvel/src/main/resources/monitoring-kibana.json

@@ -0,0 +1,57 @@
+{
+  "template": ".monitoring-kibana-${monitoring.template.version}-*",
+  "settings": {
+    "index.number_of_shards": 1,
+    "index.number_of_replicas": 1,
+    "index.codec": "best_compression"
+  },
+  "mappings": {
+    "_default_": {
+      "_all": {
+        "enabled": false
+      },
+      "properties": {
+        "cluster_uuid": {
+          "type": "keyword"
+        },
+        "timestamp": {
+          "type": "date",
+          "format": "date_time"
+        },
+        "source_node": {
+          "properties": {
+            "uuid": {
+              "type": "keyword"
+            },
+            "host": {
+              "type": "keyword"
+            },
+            "transport_address": {
+              "type": "keyword"
+            },
+            "ip": {
+              "type": "keyword"
+            },
+            "name": {
+              "type": "keyword"
+            },
+            "attributes": {
+              "dynamic": true,
+              "properties": {
+                "data": {
+                  "type": "boolean"
+                },
+                "master": {
+                  "type": "boolean"
+                },
+                "client": {
+                  "type": "boolean"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

elasticsearch/x-pack/marvel/src/main/resources/monitoring.properties

@@ -1 +0,0 @@
-template.version=${monitoring.template.version}

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/AbstractExporterTemplateTestCase.java

@@ -19,8 +19,6 @@ import java.io.IOException;
 import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName;
-import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName;
 import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
 import static org.hamcrest.Matchers.notNullValue;
 
@@ -58,8 +56,9 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
         doExporting();
 
         logger.debug("--> templates does not exist: it should have been created in the current version");
-        assertTemplateExist(indexTemplateName());
+        for (String template : monitoringTemplates().keySet()) {
-        assertTemplateExist(dataTemplateName());
+            assertTemplateExist(template);
+        }
 
         doExporting();
 
@@ -82,8 +81,9 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
         assertTemplateExist(indexTemplateName(version));
 
         logger.debug("--> existing templates are old: new templates should be created");
-        assertTemplateExist(indexTemplateName());
+        for (String template : monitoringTemplates().keySet()) {
-        assertTemplateExist(dataTemplateName());
+            assertTemplateExist(template);
+        }
 
         doExporting();
 
@@ -101,12 +101,13 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
         doExporting();
 
         logger.debug("--> existing templates are up to date");
-        assertTemplateExist(indexTemplateName());
+        for (String template : monitoringTemplates().keySet()) {
-        assertTemplateExist(dataTemplateName());
+            assertTemplateExist(template);
+        }
 
         logger.debug("--> existing templates has the same version: they should not be changed");
-        assertTemplateNotUpdated(indexTemplateName());
+        assertTemplateNotUpdated(indexTemplateName(currentVersion));
-        assertTemplateNotUpdated(dataTemplateName());
+        assertTemplateNotUpdated(dataTemplateName(currentVersion));
 
         doExporting();
 
@@ -135,8 +136,9 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
         doExporting();
 
         logger.debug("--> templates should exist in current version");
-        assertTemplateExist(indexTemplateName());
+        for (String template : monitoringTemplates().keySet()) {
-        assertTemplateExist(dataTemplateName());
+            assertTemplateExist(template);
+        }
 
         if (previousIndexTemplateExist) {
             logger.debug("--> index template should exist in version [{}]", previousIndexTemplateVersion);
@@ -167,6 +169,16 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
         exporters.export(collector.collect());
     }
 
+    private String dataTemplateName(Integer version) {
+        MockDataIndexNameResolver resolver = new MockDataIndexNameResolver(version);
+        return resolver.templateName();
+    }
+
+    private String indexTemplateName(Integer version) {
+        MockTimestampedIndexNameResolver resolver = new MockTimestampedIndexNameResolver(MonitoredSystem.ES, version, exporterSettings());
+        return resolver.templateName();
+    }
+
     private String currentDataIndexName() {
         MockDataIndexNameResolver resolver = new MockDataIndexNameResolver(currentVersion);
         return resolver.index(null);
@@ -188,7 +200,6 @@ public abstract class AbstractExporterTemplateTestCase extends MarvelIntegTestCa
                                 .startObject("settings")
                                     .field("index.number_of_shards", 1)
                                     .field("index.number_of_replicas", 1)
-                                    .field(MarvelSettings.INDEX_TEMPLATE_VERSION.getKey(), String.valueOf(version))
                                 .endObject()
                                 .startObject("mappings")
                                     .startObject("_default_")

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/MarvelTemplateUtilsTests.java

@@ -5,52 +5,88 @@
  */
 package org.elasticsearch.marvel.agent.exporter;
 
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.test.ESTestCase;
+import org.junit.Rule;
+import org.junit.rules.ExpectedException;
 
 import java.io.IOException;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.notNullValue;
 
 public class MarvelTemplateUtilsTests extends ESTestCase {
 
-    public void testLoadTimestampedIndexTemplate() {
+    private static final String TEST_TEMPLATE = "/monitoring-test.json";
-        byte[] template = MarvelTemplateUtils.loadTimestampedIndexTemplate();
-        assertNotNull(template);
-        assertThat(template.length, greaterThan(0));
-    }
 
-    public void testLoadDataIndexTemplate() {
+    @Rule
-        byte[] template = MarvelTemplateUtils.loadDataIndexTemplate();
+    public ExpectedException expectedException = ExpectedException.none();
-        assertNotNull(template);
+
-        assertThat(template.length, greaterThan(0));
+    public void testLoadTemplate() throws IOException {
+        final int version = randomIntBetween(0, 10_000);
+        String source = MarvelTemplateUtils.loadTemplate("test", version);
+
+        assertThat(source, notNullValue());
+        assertThat(source.length(), greaterThan(0));
+        assertThat(source, equalTo("{\n" +
+                "  \"template\": \".monitoring-data-" + version + "\",\n" +
+                "  \"mappings\": {\n" +
+                "    \"type_1\": {\n" +
+                "      \"_meta\": {\n" +
+                "        \"template.version\": \"" + version + "\"\n" +
+                "      }\n" +
+                "    }\n" +
+                "  }\n" +
+                "}"));
     }
 
     public void testLoad() throws IOException {
-        String resource = randomFrom(MarvelTemplateUtils.INDEX_TEMPLATE_FILE, MarvelTemplateUtils.DATA_TEMPLATE_FILE);
+        BytesReference source = MarvelTemplateUtils.load(TEST_TEMPLATE);
-        byte[] template = MarvelTemplateUtils.load(resource);
+        assertThat(source, notNullValue());
-        assertNotNull(template);
+        assertThat(source.length(), greaterThan(0));
-        assertThat(template.length, greaterThan(0));
     }
 
-    public void testLoadTemplateVersion() {
+    public void testValidateNullSource() {
-        Integer version = MarvelTemplateUtils.loadTemplateVersion();
+        expectedException.expect(ElasticsearchParseException.class);
-        assertNotNull(version);
+        expectedException.expectMessage("Monitoring template must not be null");
-        assertThat(version, greaterThan(0));
+        MarvelTemplateUtils.validate(null);
-        assertThat(version, equalTo(MarvelTemplateUtils.TEMPLATE_VERSION));
     }
 
-    public void testIndexTemplateName() {
+    public void testValidateEmptySource() {
-        assertThat(MarvelTemplateUtils.indexTemplateName(),
+        expectedException.expect(ElasticsearchParseException.class);
-                equalTo(MarvelTemplateUtils.INDEX_TEMPLATE_NAME_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION));
+        expectedException.expectMessage("Monitoring template must not be empty");
-        int version = randomIntBetween(1, 100);
+        MarvelTemplateUtils.validate(new BytesArray(""));
-        assertThat(MarvelTemplateUtils.indexTemplateName(version), equalTo(".monitoring-es-" + version));
     }
 
-    public void testDataTemplateName() {
+    public void testValidateInvalidSource() {
-        assertThat(MarvelTemplateUtils.dataTemplateName(),
+        expectedException.expect(ElasticsearchParseException.class);
-                equalTo(MarvelTemplateUtils.DATA_TEMPLATE_NAME_PREFIX + MarvelTemplateUtils.TEMPLATE_VERSION));
+        expectedException.expectMessage("Invalid monitoring template");
-        int version = randomIntBetween(1, 100);
+        MarvelTemplateUtils.validate(new BytesArray("{\"foo\": \"bar"));
-        assertThat(MarvelTemplateUtils.dataTemplateName(version), equalTo(".monitoring-data-" + version));
+    }
+
+    public void testValidate() {
+        try {
+            MarvelTemplateUtils.validate(MarvelTemplateUtils.load(TEST_TEMPLATE));
+        } catch (Exception e) {
+            fail("failed to validate test template: " + e.getMessage());
+        }
+    }
+
+    public void testFilter() {
+        assertThat(MarvelTemplateUtils.filter(new BytesArray("${monitoring.template.version}"), 0), equalTo("0"));
+        assertThat(MarvelTemplateUtils.filter(new BytesArray("{\"template\": \"test-${monitoring.template.version}\"}"), 1),
+                equalTo("{\"template\": \"test-1\"}"));
+        assertThat(MarvelTemplateUtils.filter(new BytesArray("{\"template\": \"${monitoring.template.version}-test\"}"), 2),
+                equalTo("{\"template\": \"2-test\"}"));
+        assertThat(MarvelTemplateUtils.filter(new BytesArray("{\"template\": \"test-${monitoring.template.version}-test\"}"), 3),
+                equalTo("{\"template\": \"test-3-test\"}"));
+
+        final int version = randomIntBetween(0, 100);
+        assertThat(MarvelTemplateUtils.filter(new BytesArray("{\"foo-${monitoring.template.version}\": " +
+                        "\"bar-${monitoring.template.version}\"}"), version),
+                equalTo("{\"foo-" + version + "\": \"bar-" + version + "\"}"));
     }
 }

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTests.java

@@ -31,6 +31,7 @@ import org.elasticsearch.marvel.agent.collector.indices.IndexRecoveryMonitoringD
 import org.elasticsearch.marvel.agent.exporter.Exporters;
 import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
 import org.elasticsearch.marvel.agent.exporter.MonitoringDoc;
+import org.elasticsearch.marvel.agent.resolver.bulk.MonitoringBulkResolver;
 import org.elasticsearch.marvel.test.MarvelIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
@@ -50,8 +51,6 @@ import java.util.Map;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName;
-import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -64,9 +63,6 @@ public class HttpExporterTests extends MarvelIntegTestCase {
     private int webPort;
     private MockWebServer webServer;
 
-    private static final byte[] TIMESTAMPED_TEMPLATE = MarvelTemplateUtils.loadTimestampedIndexTemplate();
-    private static final byte[] DATA_TEMPLATE = MarvelTemplateUtils.loadDataIndexTemplate();
-
     @Before
     public void startWebservice() throws Exception {
         for (webPort = 9250; webPort < 9300; webPort++) {
@@ -91,10 +87,10 @@ public class HttpExporterTests extends MarvelIntegTestCase {
 
     public void testExport() throws Exception {
         enqueueGetClusterVersionResponse(Version.CURRENT);
-        enqueueResponse(404, "template for timestamped indices does not exist");
+        for (String template : monitoringTemplates().keySet()) {
-        enqueueResponse(201, "template for timestamped indices created");
+            enqueueResponse(404, "template [" + template + "] does not exist");
-        enqueueResponse(404, "template for data index does not exist");
+            enqueueResponse(201, "template [" + template + "] created");
-        enqueueResponse(201, "template for data index created");
+        }
         enqueueResponse(200, "{\"errors\": false, \"msg\": \"successful bulk request\"}");
 
         Settings.Builder builder = Settings.builder()
@@ -109,29 +105,22 @@ public class HttpExporterTests extends MarvelIntegTestCase {
         final int nbDocs = randomIntBetween(1, 25);
         export(newRandomMarvelDocs(nbDocs));
 
-        assertThat(webServer.getRequestCount(), equalTo(6));
+        assertThat(webServer.getRequestCount(), equalTo(2 + monitoringTemplates().size() * 2));
 
         RecordedRequest recordedRequest = webServer.takeRequest();
         assertThat(recordedRequest.getMethod(), equalTo("GET"));
         assertThat(recordedRequest.getPath(), equalTo("/"));
 
+        for (Map.Entry<String, String> template : monitoringTemplates().entrySet()) {
             recordedRequest = webServer.takeRequest();
             assertThat(recordedRequest.getMethod(), equalTo("GET"));
-        assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
+            assertThat(recordedRequest.getPath(), equalTo("/_template/" + template.getKey()));
 
             recordedRequest = webServer.takeRequest();
             assertThat(recordedRequest.getMethod(), equalTo("PUT"));
-        assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
+            assertThat(recordedRequest.getPath(), equalTo("/_template/" + template.getKey()));
-        assertThat(recordedRequest.getBody().readByteArray(), equalTo(TIMESTAMPED_TEMPLATE));
+            assertThat(recordedRequest.getBody().readUtf8(), equalTo(template.getValue()));
-
+        }
-        recordedRequest = webServer.takeRequest();
-        assertThat(recordedRequest.getMethod(), equalTo("GET"));
-        assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
-
-        recordedRequest = webServer.takeRequest();
-        assertThat(recordedRequest.getMethod(), equalTo("PUT"));
-        assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
-        assertThat(recordedRequest.getBody().readByteArray(), equalTo(DATA_TEMPLATE));
 
         recordedRequest = webServer.takeRequest();
         assertThat(recordedRequest.getMethod(), equalTo("POST"));
@@ -176,10 +165,10 @@ public class HttpExporterTests extends MarvelIntegTestCase {
         logger.info("--> starting node");
 
         enqueueGetClusterVersionResponse(Version.CURRENT);
-        enqueueResponse(404, "template for timestamped indices does not exist");
+        for (String template : monitoringTemplates().keySet()) {
-        enqueueResponse(201, "template for timestamped indices created");
+            enqueueResponse(404, "template [" + template + "] does not exist");
-        enqueueResponse(404, "template for data index does not exist");
+            enqueueResponse(201, "template [" + template + "] created");
-        enqueueResponse(201, "template for data index created");
+        }
         enqueueResponse(200, "{\"errors\": false, \"msg\": \"successful bulk request\"}");
 
         String agentNode = internalCluster().startNode(builder);
@@ -190,29 +179,22 @@ public class HttpExporterTests extends MarvelIntegTestCase {
         export(Collections.singletonList(newRandomMarvelDoc()));
 
         assertThat(exporter.supportedClusterVersion, is(true));
-        assertThat(webServer.getRequestCount(), equalTo(6));
+        assertThat(webServer.getRequestCount(), equalTo(2 + monitoringTemplates().size() * 2));
 
         RecordedRequest recordedRequest = webServer.takeRequest();
         assertThat(recordedRequest.getMethod(), equalTo("GET"));
         assertThat(recordedRequest.getPath(), equalTo("/"));
 
+        for (Map.Entry<String, String> template : monitoringTemplates().entrySet()) {
             recordedRequest = webServer.takeRequest();
             assertThat(recordedRequest.getMethod(), equalTo("GET"));
-        assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
+            assertThat(recordedRequest.getPath(), equalTo("/_template/" + template.getKey()));
 
             recordedRequest = webServer.takeRequest();
             assertThat(recordedRequest.getMethod(), equalTo("PUT"));
-        assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
+            assertThat(recordedRequest.getPath(), equalTo("/_template/" + template.getKey()));
-        assertThat(recordedRequest.getBody().readByteArray(), equalTo(TIMESTAMPED_TEMPLATE));
+            assertThat(recordedRequest.getBody().readUtf8(), equalTo(template.getValue()));
-
+        }
-        recordedRequest = webServer.takeRequest();
-        assertThat(recordedRequest.getMethod(), equalTo("GET"));
-        assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
-
-        recordedRequest = webServer.takeRequest();
-        assertThat(recordedRequest.getMethod(), equalTo("PUT"));
-        assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
-        assertThat(recordedRequest.getBody().readByteArray(), equalTo(DATA_TEMPLATE));
 
         recordedRequest = webServer.takeRequest();
         assertThat(recordedRequest.getMethod(), equalTo("POST"));
@@ -246,32 +228,37 @@ public class HttpExporterTests extends MarvelIntegTestCase {
             exporter = getExporter(agentNode);
 
             enqueueGetClusterVersionResponse(secondWebServer, Version.CURRENT);
-            enqueueResponse(secondWebServer, 404, "template for timestamped indices does not exist");
+            for (String template : monitoringTemplates().keySet()) {
-            enqueueResponse(secondWebServer, 201, "template for timestamped indices created");
+                if (template.contains(MonitoringBulkResolver.Data.DATA)) {
-            enqueueResponse(secondWebServer, 200, "template for data index exist");
+                    enqueueResponse(secondWebServer, 200, "template [" + template + "] exist");
+                } else {
+                    enqueueResponse(secondWebServer, 404, "template [" + template + "] does not exist");
+                    enqueueResponse(secondWebServer, 201, "template [" + template + "] created");
+                }
+            }
             enqueueResponse(secondWebServer, 200, "{\"errors\": false, \"msg\": \"successful bulk request\"}");
 
             logger.info("--> exporting a second event");
             export(Collections.singletonList(newRandomMarvelDoc()));
 
-            assertThat(secondWebServer.getRequestCount(), equalTo(5));
+            assertThat(secondWebServer.getRequestCount(), equalTo(2 + monitoringTemplates().size() * 2 - 1));
 
             recordedRequest = secondWebServer.takeRequest();
             assertThat(recordedRequest.getMethod(), equalTo("GET"));
             assertThat(recordedRequest.getPath(), equalTo("/"));
 
+            for (Map.Entry<String, String> template : monitoringTemplates().entrySet()) {
                 recordedRequest = secondWebServer.takeRequest();
                 assertThat(recordedRequest.getMethod(), equalTo("GET"));
-            assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
+                assertThat(recordedRequest.getPath(), equalTo("/_template/" + template.getKey()));
 
+                if (template.getKey().contains(MonitoringBulkResolver.Data.DATA) == false) {
                     recordedRequest = secondWebServer.takeRequest();
                     assertThat(recordedRequest.getMethod(), equalTo("PUT"));
-            assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
+                    assertThat(recordedRequest.getPath(), equalTo("/_template/" + template.getKey()));
-            assertThat(recordedRequest.getBody().readByteArray(), equalTo(TIMESTAMPED_TEMPLATE));
+                    assertThat(recordedRequest.getBody().readUtf8(), equalTo(template.getValue()));
-
+                }
-            recordedRequest = secondWebServer.takeRequest();
+            }
-            assertThat(recordedRequest.getMethod(), equalTo("GET"));
-            assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));;
 
             recordedRequest = secondWebServer.takeRequest();
             assertThat(recordedRequest.getMethod(), equalTo("POST"));
@@ -325,10 +312,10 @@ public class HttpExporterTests extends MarvelIntegTestCase {
         logger.info("--> exporting a first event");
 
         enqueueGetClusterVersionResponse(Version.CURRENT);
-        enqueueResponse(404, "template for timestamped indices does not exist");
+        for (String template : monitoringTemplates().keySet()) {
-        enqueueResponse(201, "template for timestamped indices created");
+            enqueueResponse(404, "template [" + template + "] does not exist");
-        enqueueResponse(404, "template for data index does not exist");
+            enqueueResponse(201, "template [" + template + "] created");
-        enqueueResponse(201, "template for data index created");
+        }
         enqueueResponse(200, "{\"errors\": false, \"msg\": \"successful bulk request\"}");
 
         HttpExporter exporter = getExporter(agentNode);
@@ -336,29 +323,23 @@ public class HttpExporterTests extends MarvelIntegTestCase {
         MonitoringDoc doc = newRandomMarvelDoc();
         export(Collections.singletonList(doc));
 
-        assertThat(webServer.getRequestCount(), equalTo(6));
+        final int expectedRequests = 2 + monitoringTemplates().size() * 2;
+        assertThat(webServer.getRequestCount(), equalTo(expectedRequests));
 
         RecordedRequest recordedRequest = webServer.takeRequest();
         assertThat(recordedRequest.getMethod(), equalTo("GET"));
         assertThat(recordedRequest.getPath(), equalTo("/"));
 
+        for (Map.Entry<String, String> template : monitoringTemplates().entrySet()) {
             recordedRequest = webServer.takeRequest();
             assertThat(recordedRequest.getMethod(), equalTo("GET"));
-        assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
+            assertThat(recordedRequest.getPath(), equalTo("/_template/" + template.getKey()));
 
             recordedRequest = webServer.takeRequest();
             assertThat(recordedRequest.getMethod(), equalTo("PUT"));
-        assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
+            assertThat(recordedRequest.getPath(), equalTo("/_template/" + template.getKey()));
-        assertThat(recordedRequest.getBody().readByteArray(), equalTo(TIMESTAMPED_TEMPLATE));
+            assertThat(recordedRequest.getBody().readUtf8(), equalTo(template.getValue()));
-
+        }
-        recordedRequest = webServer.takeRequest();
-        assertThat(recordedRequest.getMethod(), equalTo("GET"));
-        assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
-
-        recordedRequest = webServer.takeRequest();
-        assertThat(recordedRequest.getMethod(), equalTo("PUT"));
-        assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
-        assertThat(recordedRequest.getBody().readByteArray(), equalTo(DATA_TEMPLATE));
 
         recordedRequest = webServer.takeRequest();
         assertThat(recordedRequest.getMethod(), equalTo("POST"));
@@ -381,8 +362,9 @@ public class HttpExporterTests extends MarvelIntegTestCase {
         logger.info("--> exporting a second event");
 
         enqueueGetClusterVersionResponse(Version.CURRENT);
-        enqueueResponse(200, "template for timestamped indices exist");
+        for (String template : monitoringTemplates().keySet()) {
-        enqueueResponse(200, "template for data index exist");
+            enqueueResponse(200, "template [" + template + "] exist");
+        }
         enqueueResponse(200, "{\"errors\": false, \"msg\": \"successful bulk request\"}");
 
         doc = newRandomMarvelDoc();
@@ -391,19 +373,17 @@ public class HttpExporterTests extends MarvelIntegTestCase {
         String expectedMarvelIndex = ".monitoring-es-" + MarvelTemplateUtils.TEMPLATE_VERSION + "-"
                 + DateTimeFormat.forPattern(newTimeFormat).withZoneUTC().print(doc.getTimestamp());
 
-        assertThat(webServer.getRequestCount(), equalTo(6 + 4));
+        assertThat(webServer.getRequestCount(), equalTo(expectedRequests + 2 + monitoringTemplates().size()));
 
         recordedRequest = webServer.takeRequest();
         assertThat(recordedRequest.getMethod(), equalTo("GET"));
         assertThat(recordedRequest.getPath(), equalTo("/"));
 
+        for (String template : monitoringTemplates().keySet()) {
             recordedRequest = webServer.takeRequest();
             assertThat(recordedRequest.getMethod(), equalTo("GET"));
-        assertThat(recordedRequest.getPath(), equalTo("/_template/" + indexTemplateName()));
+            assertThat(recordedRequest.getPath(), equalTo("/_template/" + template));
-
+        }
-        recordedRequest = webServer.takeRequest();
-        assertThat(recordedRequest.getMethod(), equalTo("GET"));
-        assertThat(recordedRequest.getPath(), equalTo("/_template/" + dataTemplateName()));
 
         recordedRequest = webServer.takeRequest();
         assertThat(recordedRequest.getMethod(), equalTo("POST"));

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/local/LocalExporterTests.java

@@ -36,12 +36,9 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
-import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName;
-import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -51,7 +48,6 @@ import static org.hamcrest.Matchers.notNullValue;
 
 @ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
 public class LocalExporterTests extends MarvelIntegTestCase {
-    private final static AtomicLong timeStampGenerator = new AtomicLong();
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
@@ -108,8 +104,7 @@ public class LocalExporterTests extends MarvelIntegTestCase {
         updateMarvelInterval(3L, TimeUnit.SECONDS);
 
         // lets wait until the monitoring template will be installed
-        waitForMarvelTemplate(indexTemplateName());
+        waitForMarvelTemplates();
-        waitForMarvelTemplate(dataTemplateName());
     }
 
     public void testIndexTimestampFormat() throws Exception {

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/bulk/MonitoringBulkResolverTests.java

@@ -56,7 +56,7 @@ public class MonitoringBulkResolverTests extends MonitoringIndexNameResolverTest
         }
 
         MonitoringBulkResolver resolver = newResolver();
-        assertThat(resolver.index(doc), equalTo(".monitoring-kibana-0-2015.07.22"));
+        assertThat(resolver.index(doc), equalTo(".monitoring-kibana-2-2015.07.22"));
         assertThat(resolver.type(doc), equalTo(doc.getType()));
         assertThat(resolver.id(doc), nullValue());
 

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/resolver/cluster/ClusterInfoTests.java

@@ -23,8 +23,6 @@ import org.junit.Before;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
-import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.dataTemplateName;
-import static org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils.indexTemplateName;
 import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.equalTo;
@@ -119,8 +117,7 @@ public class ClusterInfoTests extends MarvelIntegTestCase {
         assertThat(clusterStats, instanceOf(Map.class));
         assertThat(((Map) clusterStats).size(), greaterThan(0));
 
-        waitForMarvelTemplate(indexTemplateName());
+        waitForMarvelTemplates();
-        waitForMarvelTemplate(dataTemplateName());
 
         logger.debug("--> check that the cluster_info is not indexed");
         securedFlush();

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/shield/MarvelInternalClientTests.java

@@ -11,11 +11,12 @@ import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.marvel.MarvelSettings;
-import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
 import org.elasticsearch.marvel.test.MarvelIntegTestCase;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.shield.InternalClient;
 
+import java.util.ArrayList;
+
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.is;
 
@@ -48,15 +49,13 @@ public class MarvelInternalClientTests extends MarvelIntegTestCase {
         assertAccessIsAllowed(internalClient.admin().indices().prepareDelete(MONITORING_INDICES_PREFIX + "*"));
         assertAccessIsAllowed(internalClient.admin().indices().prepareCreate(MONITORING_INDICES_PREFIX + "test"));
 
-        assertAccessIsAllowed(internalClient.admin().indices().preparePutTemplate("foo")
+        assertAccessIsAllowed(internalClient.admin().indices().preparePutTemplate("foo").setSource(randomTemplateSource()));
-                .setSource(MarvelTemplateUtils.loadTimestampedIndexTemplate()));
         assertAccessIsAllowed(internalClient.admin().indices().prepareGetTemplates("foo"));
     }
 
     public void testAllowAllAccess() {
         InternalClient internalClient = internalCluster().getInstance(InternalClient.class);
-        assertAcked(internalClient.admin().indices().preparePutTemplate("foo")
+        assertAcked(internalClient.admin().indices().preparePutTemplate("foo").setSource(randomTemplateSource()).get());
-                .setSource(MarvelTemplateUtils.loadDataIndexTemplate()).get());
 
         assertAccessIsAllowed(internalClient.admin().indices().prepareDeleteTemplate("foo"));
         assertAccessIsAllowed(internalClient.admin().cluster().prepareGetRepositories());
@@ -83,5 +82,12 @@ public class MarvelInternalClientTests extends MarvelIntegTestCase {
             assertThat(e.status(), is(RestStatus.FORBIDDEN));
         }
     }
+
+    /**
+     * @return the source of a random monitoring template
+     */
+    private String randomTemplateSource() {
+        return randomFrom(new ArrayList<>(monitoringTemplates().values()));
+    }
 }
 

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java

@@ -20,9 +20,9 @@ import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.marvel.MarvelSettings;
 import org.elasticsearch.marvel.MonitoredSystem;
 import org.elasticsearch.marvel.agent.AgentService;
-import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
 import org.elasticsearch.marvel.agent.exporter.MonitoringDoc;
 import org.elasticsearch.marvel.agent.resolver.MonitoringIndexNameResolver;
+import org.elasticsearch.marvel.agent.resolver.ResolversRegistry;
 import org.elasticsearch.marvel.client.MonitoringClient;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.shield.authc.file.FileRealm;
@@ -57,6 +57,8 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 
 import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -164,10 +166,7 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
 
     @Override
     protected Set<String> excludeTemplates() {
-        Set<String> templates = new HashSet<>();
+        return monitoringTemplates().keySet();
-        templates.add(MarvelTemplateUtils.indexTemplateName());
-        templates.add(MarvelTemplateUtils.dataTemplateName());
-        return templates;
     }
 
     @Before
@@ -275,6 +274,11 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
         }
     }
 
+    protected Map<String, String> monitoringTemplates() {
+        return StreamSupport.stream(new ResolversRegistry(Settings.EMPTY).spliterator(), false)
+                .collect(Collectors.toMap(MonitoringIndexNameResolver::templateName, MonitoringIndexNameResolver::template, (a, b) -> a));
+    }
+
     protected void assertTemplateInstalled(String name) {
         boolean found = false;
         for (IndexTemplateMetaData template : client().admin().indices().prepareGetTemplates().get().getIndexTemplates()) {
@@ -294,6 +298,10 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
         }, 30, TimeUnit.SECONDS);
     }
 
+    protected void waitForMarvelTemplates() throws Exception {
+        assertBusy(() -> monitoringTemplates().keySet().forEach(this::assertTemplateInstalled), 30, TimeUnit.SECONDS);
+    }
+
     protected void waitForMarvelIndices() throws Exception {
         awaitIndexExists(MONITORING_INDICES_PREFIX + "*");
         assertBusy(this::ensureMarvelIndicesYellow);

elasticsearch/x-pack/marvel/src/test/resources/monitoring-test.json

@@ -0,0 +1,10 @@
+{
+  "template": ".monitoring-data-${monitoring.template.version}",
+  "mappings": {
+    "type_1": {
+      "_meta": {
+        "template.version": "${monitoring.template.version}"
+      }
+    }
+  }
+}