[Monitoring] Remove monitoring resolvers (elastic/x-pack-elasticsearch#2566)

Now all monitoring documents are indexed in timestamped indices dedicated 
to each product, the old monitoring's resolvers and all the associated plumbing 
can be removed from the Elasticsearch monitoring plugin.

This pull request merges the feature branch feature/monitoring-summer-cleanup 
into master. All commits have been independently reviewed as part of elastic/x-pack-elasticsearch#2226:

* [Monitoring] Clean up monitoring doc (elastic/x-pack-elasticsearch#2208)
* [Monitoring] Remove ClusterStatsResolver (elastic/x-pack-elasticsearch#2209)
* [Monitoring] Remove IndexRecoveryResolver (elastic/x-pack-elasticsearch#2229)
* [Monitoring] Remove JobStatsResolver (elastic/x-pack-elasticsearch#2230)
* [Monitoring] Remove ShardsResolver (elastic/x-pack-elasticsearch#2233)
* [Monitoring] Fix missing instanceof in exporter bulks
* [Monitoring] Remove NodeStatsResolver (elastic/x-pack-elasticsearch#2244)
* [Monitoring] Remove IndexStatsResolver & IndicesStatsResolver  (elastic/x-pack-elasticsearch#2247)
* [Monitoring] Remove source node attributes and add timestamp (elastic/x-pack-elasticsearch#2285)
* [Monitoring] Remove last resolver (elastic/x-pack-elasticsearch#2330)
* [Monitoring] Make MonitoringDoc an abstract class (elastic/x-pack-elasticsearch#2451)
* [Monitoring] Fix TransportMonitoringBulkActionTests
* [Monitoring] Remove old resolver tests and add a single integration test (elastic/x-pack-elasticsearch#2453)

It does not introduce any new feature, so it could also go in 6.0.

relates elastic/x-pack-elasticsearch#2226


Original commit: elastic/x-pack-elasticsearch@e3d5cec28c

plugin/src/main/java/org/elasticsearch/xpack/monitoring/MonitoredSystem.java

@@ -5,14 +5,17 @@
  */
 package org.elasticsearch.xpack.monitoring;
 
+import java.util.Arrays;
 import java.util.Locale;
+import java.util.stream.Stream;
 
 public enum MonitoredSystem {
 
     ES("es"),
     KIBANA("kibana"),
     LOGSTASH("logstash"),
-    BEATS("beats");
+    BEATS("beats"),
+    UNKNOWN("unknown");
 
     private final String system;
 
@@ -35,7 +38,20 @@ public enum MonitoredSystem {
             case "beats":
                 return BEATS;
             default:
-                throw new IllegalArgumentException("Unknown monitoring system [" + system + "]");
+                // Return an "unknown" monitored system
+                // that can easily be filtered out if
+                // a node receives documents for a new
+                // system it does not know yet
+                return UNKNOWN;
         }
     }
+
+    /**
+     * Get all {@code MonitoredSystem}s except {@linkplain MonitoredSystem#UNKNOWN UNKNOWN}.
+     *
+     * @return Never {@code null}. A filtered {@code Stream} that removes the {@code UNKNOWN} {@code MonitoredSystem}.
+     */
+    public static Stream<MonitoredSystem> allSystems() {
+        return Arrays.stream(MonitoredSystem.values()).filter(s -> s != MonitoredSystem.UNKNOWN);
+    }
 }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringService.java

@@ -171,7 +171,9 @@ public class MonitoringService extends AbstractLifecycleComponent {
             threadPool.executor(threadPoolName()).submit(new AbstractRunnable() {
                 @Override
                 protected void doRun() throws Exception {
-                    Collection<MonitoringDoc> results = new ArrayList<>();
+                    final long timestamp = System.currentTimeMillis();
+
+                    final Collection<MonitoringDoc> results = new ArrayList<>();
                     for (Collector collector : collectors) {
                         if (isStarted() == false) {
                             // Do not collect more data if the the monitoring service is stopping
@@ -180,7 +182,7 @@ public class MonitoringService extends AbstractLifecycleComponent {
                         }
 
                         try {
-                            Collection<MonitoringDoc> result = collector.collect();
+                            Collection<MonitoringDoc> result = collector.collect(timestamp);
                             if (result != null) {
                                 results.addAll(result);
                             }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringSettings.java

@@ -23,6 +23,7 @@ import static org.elasticsearch.common.settings.Setting.groupSetting;
 import static org.elasticsearch.common.settings.Setting.listSetting;
 import static org.elasticsearch.common.settings.Setting.timeSetting;
 
+// TODO Remove this class and put the settings in Monitoring class
 public class MonitoringSettings extends AbstractComponent {
 
     public static final String HISTORY_DURATION_SETTING_NAME = "history.duration";
@@ -79,6 +80,12 @@ public class MonitoringSettings extends AbstractComponent {
     public static final Setting<TimeValue> JOB_STATS_TIMEOUT =
             timeSetting(collectionKey("ml.job.stats.timeout"), TimeValue.timeValueSeconds(10), Property.Dynamic, Property.NodeScope);
 
+    /**
+     * Timeout value when collecting the nodes statistics (default to 10s)
+     */
+    public static final Setting<TimeValue> NODE_STATS_TIMEOUT =
+            timeSetting(collectionKey("node.stats.timeout"), TimeValue.timeValueSeconds(10), Property.Dynamic, Property.NodeScope);
+
     /**
      * Timeout value when collecting the recovery information (default to 10s)
      */
@@ -123,6 +130,7 @@ public class MonitoringSettings extends AbstractComponent {
                 CLUSTER_STATE_TIMEOUT,
                 CLUSTER_STATS_TIMEOUT,
                 JOB_STATS_TIMEOUT,
+                NODE_STATS_TIMEOUT,
                 HISTORY_DURATION,
                 EXPORTERS_SETTINGS);
     }
@@ -137,6 +145,7 @@ public class MonitoringSettings extends AbstractComponent {
     private volatile TimeValue clusterStatsTimeout;
     private volatile TimeValue recoveryTimeout;
     private volatile TimeValue jobStatsTimeout;
+    private volatile TimeValue nodeStatsTimeout;
     private volatile boolean recoveryActiveOnly;
     private volatile String[] indices;
 
@@ -153,6 +162,8 @@ public class MonitoringSettings extends AbstractComponent {
         clusterSettings.addSettingsUpdateConsumer(CLUSTER_STATS_TIMEOUT, this::setClusterStatsTimeout);
         setJobStatsTimeout(JOB_STATS_TIMEOUT.get(settings));
         clusterSettings.addSettingsUpdateConsumer(JOB_STATS_TIMEOUT, this::setJobStatsTimeout);
+        setNodeStatsTimeout(NODE_STATS_TIMEOUT.get(settings));
+        clusterSettings.addSettingsUpdateConsumer(NODE_STATS_TIMEOUT, this::setNodeStatsTimeout);
         setRecoveryTimeout(INDEX_RECOVERY_TIMEOUT.get(settings));
         clusterSettings.addSettingsUpdateConsumer(INDEX_RECOVERY_TIMEOUT, this::setRecoveryTimeout);
         setRecoveryActiveOnly(INDEX_RECOVERY_ACTIVE_ONLY.get(settings));
@@ -179,6 +190,10 @@ public class MonitoringSettings extends AbstractComponent {
         return jobStatsTimeout;
     }
 
+    public TimeValue nodeStatsTimeout() {
+        return nodeStatsTimeout;
+    }
+
     public TimeValue recoveryTimeout() {
         return recoveryTimeout;
     }
@@ -203,6 +218,10 @@ public class MonitoringSettings extends AbstractComponent {
         this.jobStatsTimeout = jobStatsTimeout;
     }
 
+    public void setNodeStatsTimeout(TimeValue nodeStatsTimeout) {
+        this.nodeStatsTimeout = nodeStatsTimeout;
+    }
+
     private void setRecoveryTimeout(TimeValue recoveryTimeout) {
         this.recoveryTimeout = recoveryTimeout;
     }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkDoc.java

@@ -6,123 +6,150 @@
 package org.elasticsearch.xpack.monitoring.action;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
+import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
 
 import java.io.IOException;
+import java.util.Objects;
 
-// TODO remove inherance from MonitoringDoc once all resolvers are removed
-public class MonitoringBulkDoc extends MonitoringDoc implements Writeable {
+public class MonitoringBulkDoc implements Writeable {
 
-    private MonitoringIndex index;
-    private String type;
-    private String id;
-    private BytesReference source;
-    private XContentType xContentType;
+    private final MonitoredSystem system;
+    private final String type;
+    private final String id;
+    private final long timestamp;
+    private final long interval;
+    private final BytesReference source;
+    private final XContentType xContentType;
 
-    public MonitoringBulkDoc(String monitoringId, String monitoringVersion,
-                              MonitoringIndex index, String type, String id,
-                              String clusterUUID, long timestamp, MonitoringDoc.Node sourceNode,
-                              BytesReference source, XContentType xContentType) {
-        super(monitoringId, monitoringVersion, type, id, clusterUUID, timestamp, sourceNode);
-        this.index = index != null ? index : MonitoringIndex.TIMESTAMPED;
-        this.type = type;
-        this.id = id;
-        this.source = source;
-        this.xContentType = xContentType;
-    }
+    public MonitoringBulkDoc(final MonitoredSystem system,
+                             final String type,
+                             @Nullable final String id,
+                             final long timestamp,
+                             final long interval,
+                             final BytesReference source,
+                             final XContentType xContentType) {
 
-    public MonitoringBulkDoc(String monitoringId, String monitoringVersion,
-                             MonitoringIndex index, String type, String id,
-                             BytesReference source, XContentType xContentType) {
-        this(monitoringId, monitoringVersion, index, type, id, null, 0, null, source, xContentType);
+        this.system = Objects.requireNonNull(system);
+        this.type = Objects.requireNonNull(type);
+        // We allow strings to be "" because Logstash 5.2 - 5.3 would submit empty _id values for time-based documents
+        this.id = Strings.isNullOrEmpty(id) ? null : id;
+        this.timestamp = timestamp;
+        this.interval = interval;
+        this.source = Objects.requireNonNull(source);
+        this.xContentType = Objects.requireNonNull(xContentType);
     }
 
     /**
      * Read from a stream.
-     *
-     * Here we use a static helper method to read a serialized {@link MonitoringBulkDoc} instance
-     * instead of the usual <code>MonitoringBulkDoc(StreamInput in)</code> constructor because
-     * MonitoringBulkDoc still inherits from MonitoringDoc and using ctors would make things
-     * cumbersome. This will be replaced by a ctor <code>MonitoringBulkDoc(StreamInput in)</code>
-     * once MonitoringBulkDoc does not inherit from MonitoringDoc anymore.
      */
     public static MonitoringBulkDoc readFrom(StreamInput in) throws IOException {
-        String monitoringId = in.readOptionalString();
-        String monitoringVersion = in.readOptionalString();
-        String clusterUUID = in.readOptionalString();
-        long timestamp = in.readVLong();
-        MonitoringDoc.Node sourceNode = in.readOptionalWriteable(MonitoringDoc.Node::new);
-        MonitoringIndex index = MonitoringIndex.readFrom(in);
-        String type = in.readOptionalString();
-        String id = in.readOptionalString();
-        BytesReference source = in.readBytesReference();
-        XContentType xContentType;
-        if (source != BytesArray.EMPTY && in.getVersion().onOrAfter(Version.V_5_3_0)) {
-            xContentType = XContentType.readFrom(in);
-        } else {
-            xContentType = XContentFactory.xContentType(source);
+        final MonitoredSystem system = MonitoredSystem.fromSystem(in.readOptionalString());
+
+        if (in.getVersion().before(Version.V_7_0_0_alpha1)) {
+            in.readOptionalString(); // Monitoring version, removed in 7.0
+            in.readOptionalString(); // Cluster UUID, removed in 7.0
         }
-        return new MonitoringBulkDoc(monitoringId, monitoringVersion, index, type, id, clusterUUID,
-                timestamp, sourceNode, source, xContentType);
+
+        final long timestamp = in.readVLong();
+
+        if (in.getVersion().before(Version.V_7_0_0_alpha1)) {
+            in.readOptionalWriteable(MonitoringDoc.Node::new);// Source node, removed in 7.0
+            MonitoringIndex.readFrom(in);// Monitoring index, removed in 7.0
+        }
+
+        final String type = in.readOptionalString();
+        final String id = in.readOptionalString();
+        final BytesReference source = in.readBytesReference();
+        final XContentType xContentType = (source != BytesArray.EMPTY) ? XContentType.readFrom(in) : XContentType.JSON;
+
+        long interval = 0L;
+        if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            interval = in.readVLong();
+        }
+        return new MonitoringBulkDoc(system, type, id, timestamp, interval, source, xContentType);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeOptionalString(getMonitoringId());
-        out.writeOptionalString(getMonitoringVersion());
-        out.writeOptionalString(getClusterUUID());
-        out.writeVLong(getTimestamp());
-        out.writeOptionalWriteable(getSourceNode());
-        index.writeTo(out);
+        out.writeOptionalString(system.getSystem());
+        if (out.getVersion().before(Version.V_7_0_0_alpha1)) {
+            out.writeOptionalString(MonitoringTemplateUtils.TEMPLATE_VERSION);
+            out.writeOptionalString(null);
+        }
+        out.writeVLong(timestamp);
+        if (out.getVersion().before(Version.V_7_0_0_alpha1)) {
+            out.writeOptionalWriteable(null);
+            MonitoringIndex.IGNORED_DATA.writeTo(out);
+        }
         out.writeOptionalString(type);
         out.writeOptionalString(id);
         out.writeBytesReference(source);
-        if (source != null && source != BytesArray.EMPTY && out.getVersion().onOrAfter(Version.V_5_3_0)) {
+        if (source != BytesArray.EMPTY) {
             xContentType.writeTo(out);
         }
+        if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            out.writeVLong(interval);
+        }
     }
 
-    public MonitoringIndex getIndex() {
-        return index;
-    }
-
-    public void setIndex(MonitoringIndex index) {
-        this.index = index;
+    public MonitoredSystem getSystem() {
+        return system;
     }
 
     public String getType() {
         return type;
     }
 
-    public void setType(String type) {
-        this.type = type;
-    }
-
     public String getId() {
         return id;
     }
 
-    public void setId(String id) {
-        this.id = id;
+    public long getTimestamp() {
+        return timestamp;
+    }
+
+    public long getInterval() {
+        return interval;
     }
 
     public BytesReference getSource() {
         return source;
     }
 
-    public void setSource(BytesReference source, XContentType xContentType) {
-        this.source = source;
-        this.xContentType = xContentType;
-    }
-
     public XContentType getXContentType() {
         return xContentType;
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        MonitoringBulkDoc that = (MonitoringBulkDoc) o;
+        return timestamp == that.timestamp
+                && interval == that.interval
+                && system == that.system
+                && Objects.equals(type, that.type)
+                && Objects.equals(id, that.id)
+                && Objects.equals(source, that.source)
+                && xContentType == that.xContentType;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(system, type, id, timestamp, interval, source, xContentType);
+    }
 }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequest.java

@@ -11,11 +11,11 @@ import org.elasticsearch.action.DocWriteRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.Requests;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -28,19 +28,15 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
 /**
  * A monitoring bulk request holds one or more {@link MonitoringBulkDoc}s.
  * <p>
- * Every monitoring document added to the request is associated to a monitoring system id and version. If this {id, version} pair is
- * supported by the monitoring plugin, the monitoring documents will be indexed in a single batch using a normal bulk request.
- * <p>
- * The monitoring {id, version} pair is used by MonitoringIndexNameResolver to resolve the index,
- * type and id of the final document to be indexed. A {@link MonitoringBulkDoc} can also hold its own index/type/id values but there's no
- * guarantee that these information will be effectively used.
+ * Every monitoring document added to the request is associated to a {@link MonitoredSystem}. The monitored system is used
+ * to resolve the index name in which the document will be indexed into.
  */
 public class MonitoringBulkRequest extends ActionRequest {
 
-    final List<MonitoringBulkDoc> docs = new ArrayList<>();
+    private final List<MonitoringBulkDoc> docs = new ArrayList<>();
 
     /**
-     * @return the list of monitoring documents to be indexed
+     * @return the list of {@link MonitoringBulkDoc} to be indexed
      */
     public Collection<MonitoringBulkDoc> getDocs() {
         return Collections.unmodifiableCollection(new ArrayList<>(this.docs));
@@ -52,26 +48,12 @@ public class MonitoringBulkRequest extends ActionRequest {
         if (docs.isEmpty()) {
             validationException = addValidationError("no monitoring documents added", validationException);
         }
-
         for (int i = 0; i < docs.size(); i++) {
             MonitoringBulkDoc doc = docs.get(i);
-            if (Strings.hasLength(doc.getMonitoringId()) == false) {
-                validationException = addValidationError("monitored system id is missing for monitoring document [" + i + "]",
-                        validationException);
-            }
-            if (Strings.hasLength(doc.getMonitoringVersion()) == false) {
-                validationException = addValidationError("monitored system API version is missing for monitoring document [" + i + "]",
-                        validationException);
-            }
-            if (Strings.hasLength(doc.getType()) == false) {
-                validationException = addValidationError("type is missing for monitoring document [" + i + "]",
-                        validationException);
-            }
             if (doc.getSource() == null || doc.getSource().length() == 0) {
                 validationException = addValidationError("source is missing for monitoring document [" + i + "]", validationException);
             }
         }
-
         return validationException;
     }
 
@@ -86,11 +68,16 @@ public class MonitoringBulkRequest extends ActionRequest {
     /**
      * Parses a monitoring bulk request and builds the list of documents to be indexed.
      */
-    public MonitoringBulkRequest add(BytesReference content, String defaultMonitoringId, String defaultMonitoringApiVersion,
-                                     String defaultType, XContentType xContentType) throws IOException {
+    public MonitoringBulkRequest add(final MonitoredSystem system,
+                                     final String defaultType,
+                                     final BytesReference content,
+                                     final XContentType xContentType,
+                                     final long timestamp,
+                                     final long intervalMillis) throws IOException {
+
         // MonitoringBulkRequest accepts a body request that has the same format as the BulkRequest:
         // instead of duplicating the parsing logic here we use a new BulkRequest instance to parse the content.
-        BulkRequest bulkRequest = Requests.bulkRequest().add(content, null, defaultType, xContentType);
+        final BulkRequest bulkRequest = Requests.bulkRequest().add(content, null, defaultType, xContentType);
 
         for (DocWriteRequest request : bulkRequest.requests()) {
             if (request instanceof IndexRequest) {
@@ -102,17 +89,14 @@ public class MonitoringBulkRequest extends ActionRequest {
                     continue;
                 }
 
-                // builds a new monitoring document based on the index request
-                MonitoringBulkDoc doc =
-                        new MonitoringBulkDoc(defaultMonitoringId,
-                                              defaultMonitoringApiVersion,
-                                              MonitoringIndex.TIMESTAMPED,
-                                              indexRequest.type(),
-                                              indexRequest.id(),
-                                              indexRequest.source(),
-                                              xContentType);
+                final BytesReference source = indexRequest.source();
+                if (source.length() == 0) {
+                    throw new IllegalArgumentException("source is missing for monitoring document ["
+                            + indexRequest.index() + "][" + indexRequest.type() + "][" + indexRequest.id() + "]");
+                }
 
-                add(doc);
+                // builds a new monitoring document based on the index request
+                add(new MonitoringBulkDoc(system, indexRequest.type(), indexRequest.id(), timestamp, intervalMillis, source, xContentType));
             } else {
                 throw new IllegalArgumentException("monitoring bulk requests should only contain index requests");
             }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestBuilder.java

@@ -9,6 +9,7 @@ import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
 
 import java.io.IOException;
 
@@ -24,9 +25,13 @@ public class MonitoringBulkRequestBuilder
         return this;
     }
 
-    public MonitoringBulkRequestBuilder add(BytesReference content, String defaultId, String defaultApiVersion, String defaultType,
-                                            XContentType xContentType) throws IOException {
-        request.add(content, defaultId, defaultApiVersion, defaultType, xContentType);
+    public MonitoringBulkRequestBuilder add(final MonitoredSystem system,
+                                            final String type,
+                                            final BytesReference content,
+                                            final XContentType xContentType,
+                                            final long timestamp,
+                                            final long intervalMillis) throws IOException {
+        request.add(system, type, content, xContentType, timestamp, intervalMillis);
         return this;
     }
 

plugin/src/main/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkAction.java

@@ -12,18 +12,20 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+import org.elasticsearch.xpack.monitoring.exporter.BytesReferenceMonitoringDoc;
 import org.elasticsearch.xpack.monitoring.exporter.Exporters;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
 
-import java.util.ArrayList;
 import java.util.Collection;
-import java.util.List;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 public class TransportMonitoringBulkAction extends HandledTransportAction<MonitoringBulkRequest, MonitoringBulkResponse> {
 
@@ -43,77 +45,75 @@ public class TransportMonitoringBulkAction extends HandledTransportAction<Monito
     @Override
     protected void doExecute(MonitoringBulkRequest request, ActionListener<MonitoringBulkResponse> listener) {
         clusterService.state().blocks().globalBlockedRaiseException(ClusterBlockLevel.WRITE);
-        new AsyncAction(request, listener, exportService, clusterService).start();
+
+        final long timestamp = System.currentTimeMillis();
+        final String cluster = clusterService.state().metaData().clusterUUID();
+
+        final DiscoveryNode discoveryNode = clusterService.localNode();
+        final MonitoringDoc.Node node = new MonitoringDoc.Node(discoveryNode.getId(),
+                                                               discoveryNode.getHostName(),
+                                                               discoveryNode.getAddress().toString(),
+                                                               discoveryNode.getHostAddress(),
+                                                               discoveryNode.getName(), timestamp);
+
+        new AsyncAction(request, listener, exportService, cluster, timestamp, node).start();
     }
 
-    class AsyncAction {
+    static class AsyncAction {
 
         private final MonitoringBulkRequest request;
         private final ActionListener<MonitoringBulkResponse> listener;
         private final Exporters exportService;
-        private final ClusterService clusterService;
+        private final String defaultClusterUUID;
+        private final long defaultTimestamp;
+        private final MonitoringDoc.Node defaultNode;
 
-        AsyncAction(MonitoringBulkRequest request, ActionListener<MonitoringBulkResponse> listener,
-                           Exporters exportService, ClusterService clusterService) {
+        AsyncAction(MonitoringBulkRequest request, ActionListener<MonitoringBulkResponse> listener, Exporters exportService,
+                    String defaultClusterUUID, long defaultTimestamp, MonitoringDoc.Node defaultNode) {
             this.request = request;
             this.listener = listener;
             this.exportService = exportService;
-            this.clusterService = clusterService;
+            this.defaultClusterUUID = defaultClusterUUID;
+            this.defaultTimestamp = defaultTimestamp;
+            this.defaultNode = defaultNode;
         }
 
         void start() {
-            executeExport(prepareForExport(request.getDocs()), System.nanoTime(), listener);
+            executeExport(createMonitoringDocs(request.getDocs()), System.nanoTime(), listener);
         }
 
         /**
-         * Iterate over the documents and set the values of common fields if needed:
-         * - cluster UUID
-         * - timestamp
-         * - source node
+         * Iterate over the list of {@link MonitoringBulkDoc} to create the corresponding
+         * list of {@link MonitoringDoc}.
          */
-        Collection<MonitoringDoc> prepareForExport(Collection<MonitoringBulkDoc> bulkDocs) {
-            final long defaultTimestamp = System.currentTimeMillis();
-            final String defaultClusterUUID = clusterService.state().metaData().clusterUUID();
+        Collection<MonitoringDoc> createMonitoringDocs(Collection<MonitoringBulkDoc> bulkDocs) {
+            return bulkDocs.stream()
+                           .filter(bulkDoc -> bulkDoc.getSystem() != MonitoredSystem.UNKNOWN)
+                           .map(this::createMonitoringDoc)
+                           .collect(Collectors.toList());
+        }
 
-            DiscoveryNode discoveryNode = clusterService.localNode();
-            final MonitoringDoc.Node defaultNode = new MonitoringDoc.Node(discoveryNode.getId(),
-                    discoveryNode.getHostName(), discoveryNode.getAddress().toString(),
-                    discoveryNode.getHostAddress(), discoveryNode.getName(),
-                    discoveryNode.getAttributes());
+        /**
+         * Create a {@link MonitoringDoc} from a {@link MonitoringBulkDoc}.
+         *
+         * @param bulkDoc the {@link MonitoringBulkDoc}
+         * @return the {@link MonitoringDoc} to export
+         */
+        MonitoringDoc createMonitoringDoc(final MonitoringBulkDoc bulkDoc) {
+            final MonitoredSystem system = bulkDoc.getSystem();
+            final String type = bulkDoc.getType();
+            final String id = bulkDoc.getId();
+            final XContentType xContentType = bulkDoc.getXContentType();
+            final BytesReference source = bulkDoc.getSource();
 
-            List<MonitoringDoc> docs = new ArrayList<>();
-            for (MonitoringBulkDoc bulkDoc : bulkDocs) {
-                String clusterUUID;
-                if (Strings.hasLength(bulkDoc.getClusterUUID())) {
-                    clusterUUID = bulkDoc.getClusterUUID();
-                } else {
-                    clusterUUID = defaultClusterUUID;
-                }
-
-                long timestamp;
-                if (bulkDoc.getTimestamp() != 0L) {
-                    timestamp = bulkDoc.getTimestamp();
-                } else {
-                    timestamp = defaultTimestamp;
-                }
-
-                MonitoringDoc.Node node;
-                if (bulkDoc.getSourceNode() != null) {
-                    node = bulkDoc.getSourceNode();
-                } else {
-                    node = defaultNode;
-                }
-
-                // TODO Convert MonitoringBulkDoc to a simple MonitoringDoc when all resolvers are
-                // removed and MonitoringBulkDoc does not inherit from MonitoringDoc anymore.
-                // Monitoring indices will be resolved here instead of being resolved at export
-                // time.
-                docs.add(new MonitoringBulkDoc(bulkDoc.getMonitoringId(),
-                        bulkDoc.getMonitoringVersion(), bulkDoc.getIndex(), bulkDoc.getType(),
-                        bulkDoc.getId(), clusterUUID, timestamp, node, bulkDoc.getSource(),
-                        bulkDoc.getXContentType()));
+            final long timestamp;
+            if (bulkDoc.getTimestamp() != 0L) {
+                timestamp = bulkDoc.getTimestamp();
+            } else {
+                timestamp = defaultTimestamp;
             }
-            return docs;
+
+            return new BytesReferenceMonitoringDoc(defaultClusterUUID, timestamp, defaultNode, system, type, id, xContentType, source);
         }
 
         /**
@@ -131,15 +131,15 @@ public class TransportMonitoringBulkAction extends HandledTransportAction<Monito
         }
     }
 
-    private MonitoringBulkResponse response(final long start) {
+    private static MonitoringBulkResponse response(final long start) {
         return new MonitoringBulkResponse(took(start));
     }
 
-    private MonitoringBulkResponse response(final long start, final Exception e) {
+    private static MonitoringBulkResponse response(final long start, final Exception e) {
         return new MonitoringBulkResponse(took(start), new MonitoringBulkResponse.Error(e));
     }
 
-    private long took(final long start) {
+    private static long took(final long start) {
         return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
     }
 

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/Collector.java

@@ -8,13 +8,12 @@ package org.elasticsearch.xpack.monitoring.collector;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.ElasticsearchTimeoutException;
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.inject.internal.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.xpack.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.monitoring.MonitoringSettings;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
 
@@ -64,11 +63,11 @@ public abstract class Collector extends AbstractComponent {
         return clusterService.state().nodes().isLocalNodeElectedMaster();
     }
 
-    public Collection<MonitoringDoc> collect() {
+    public Collection<MonitoringDoc> collect(final long timestamp) {
         try {
             if (shouldCollect()) {
                 logger.trace("collector [{}] - collecting data...", name());
-                return doCollect();
+                return doCollect(convertNode(timestamp, clusterService.localNode()));
             }
         } catch (ElasticsearchTimeoutException e) {
             logger.error((Supplier<?>) () -> new ParameterizedMessage("collector [{}] timed out when collecting data", name()));
@@ -78,23 +77,39 @@ public abstract class Collector extends AbstractComponent {
         return null;
     }
 
-    protected abstract Collection<MonitoringDoc> doCollect() throws Exception;
+    protected abstract Collection<MonitoringDoc> doCollect(MonitoringDoc.Node sourceNode) throws Exception;
 
     protected String clusterUUID() {
         return clusterService.state().metaData().clusterUUID();
     }
 
-    protected DiscoveryNode localNode() {
-        return clusterService.localNode();
+    /**
+     * Returns a timestamp to use in {@link MonitoringDoc}
+     *
+     * @return the timestamp
+     */
+    protected static long timestamp() {
+        return System.currentTimeMillis();
     }
 
-    protected static String monitoringId() {
-        // Collectors always collects data for Elasticsearch
-        return MonitoredSystem.ES.getSystem();
-    }
-
-    protected static String monitoringVersion() {
-        // Collectors always collects data for the current version of Elasticsearch
-        return Version.CURRENT.toString();
+    /**
+     * Creates a {@link MonitoringDoc.Node} from a {@link DiscoveryNode} and a timestamp, copying over the
+     * required information.
+     *
+     * @param timestamp the node's timestamp
+     * @param node the {@link DiscoveryNode}
+     *
+     * @return a {@link MonitoringDoc.Node} instance, or {@code null} if the given discovery node is null.
+     */
+    public static MonitoringDoc.Node convertNode(final long timestamp, final @Nullable DiscoveryNode node) {
+        if (node == null) {
+            return null;
+        }
+        return new MonitoringDoc.Node(node.getId(),
+                                      node.getHostName(),
+                                      node.getAddress().toString(),
+                                      node.getHostAddress(),
+                                      node.getName(),
+                                      timestamp);
     }
 }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollector.java

@@ -12,7 +12,6 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
@@ -46,10 +45,12 @@ public class ClusterStatsCollector extends Collector {
     private final LicenseService licenseService;
     private final Client client;
 
-    public ClusterStatsCollector(Settings settings, ClusterService clusterService,
-                                 MonitoringSettings monitoringSettings,
-                                 XPackLicenseState licenseState, InternalClient client,
-                                 LicenseService licenseService) {
+    public ClusterStatsCollector(final Settings settings,
+                                 final ClusterService clusterService,
+                                 final MonitoringSettings monitoringSettings,
+                                 final XPackLicenseState licenseState,
+                                 final Client client,
+                                 final LicenseService licenseService) {
         super(settings, ClusterStatsMonitoringDoc.TYPE, clusterService, monitoringSettings, licenseState);
         this.client = client;
         this.licenseService = licenseService;
@@ -62,7 +63,7 @@ public class ClusterStatsCollector extends Collector {
     }
 
     @Override
-    protected Collection<MonitoringDoc> doCollect() throws Exception {
+    protected Collection<MonitoringDoc> doCollect(final MonitoringDoc.Node node) throws Exception {
         final Supplier<ClusterStatsResponse> clusterStatsSupplier =
                 () -> client.admin().cluster().prepareClusterStats()
                         .get(monitoringSettings.clusterStatsTimeout());
@@ -71,9 +72,6 @@ public class ClusterStatsCollector extends Collector {
 
         final ClusterStatsResponse clusterStats = clusterStatsSupplier.get();
 
-        final long timestamp = System.currentTimeMillis();
-        final String clusterUUID = clusterUUID();
-        final DiscoveryNode sourceNode = localNode();
         final String clusterName = clusterService.getClusterName().value();
         final String version = Version.CURRENT.toString();
         final ClusterState clusterState = clusterService.state();
@@ -82,9 +80,8 @@ public class ClusterStatsCollector extends Collector {
 
         // Adds a cluster stats document
         return Collections.singleton(
-                new ClusterStatsMonitoringDoc(monitoringId(), monitoringVersion(),
-                                              clusterUUID, timestamp, sourceNode, clusterName, version, license, usage,
-                                              clusterStats, clusterState, clusterStats.getStatus()));
+                new ClusterStatsMonitoringDoc(clusterUUID(), timestamp(), node, clusterName, version,  clusterStats.getStatus(),
+                        license, usage, clusterStats, clusterState));
     }
 
     @Nullable

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDoc.java

@@ -8,12 +8,23 @@ package org.elasticsearch.xpack.monitoring.collector.cluster;
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.license.License;
 import org.elasticsearch.xpack.XPackFeatureSet;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
 
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
 
 /**
  * Monitoring document collected by {@link ClusterStatsCollector}.
@@ -26,58 +37,122 @@ import java.util.List;
  */
 public class ClusterStatsMonitoringDoc extends MonitoringDoc {
 
+    private static final ToXContent.MapParams CLUSTER_STATS_PARAMS =
+            new ToXContent.MapParams(
+                    Collections.singletonMap("metric",
+                                ClusterState.Metric.VERSION + "," +
+                                ClusterState.Metric.MASTER_NODE + "," +
+                                ClusterState.Metric.NODES));
+
     public static final String TYPE = "cluster_stats";
 
     private final String clusterName;
     private final String version;
     private final License license;
-    private final List<XPackFeatureSet.Usage> usage;
+    private final List<XPackFeatureSet.Usage> usages;
     private final ClusterStatsResponse clusterStats;
     private final ClusterState clusterState;
     private final ClusterHealthStatus status;
 
-    public ClusterStatsMonitoringDoc(String monitoringId, String monitoringVersion,
-                                     String clusterUUID, long timestamp, DiscoveryNode node,
-                                     String clusterName, String version, License license,
-                                     List<XPackFeatureSet.Usage> usage,
-                                     ClusterStatsResponse clusterStats,
-                                     ClusterState clusterState, ClusterHealthStatus status) {
-        super(monitoringId, monitoringVersion, TYPE, null, clusterUUID, timestamp, node);
-        this.clusterName = clusterName;
-        this.version = version;
+    ClusterStatsMonitoringDoc(final String cluster,
+                              final long timestamp,
+                              final MonitoringDoc.Node node,
+                              final String clusterName,
+                              final String version,
+                              final ClusterHealthStatus status,
+                              @Nullable final License license,
+                              @Nullable final List<XPackFeatureSet.Usage> usages,
+                              @Nullable final ClusterStatsResponse clusterStats,
+                              @Nullable final ClusterState clusterState) {
+
+        super(cluster, timestamp, node, MonitoredSystem.ES, TYPE, null);
+        this.clusterName = Objects.requireNonNull(clusterName);
+        this.version = Objects.requireNonNull(version);
+        this.status = Objects.requireNonNull(status);
         this.license = license;
-        this.usage = usage;
+        this.usages = usages;
         this.clusterStats = clusterStats;
         this.clusterState = clusterState;
-        this.status = status;
     }
 
-    public String getClusterName() {
+    String getClusterName() {
         return clusterName;
     }
 
-    public String getVersion() {
+    String getVersion() {
         return version;
     }
 
-    public License getLicense() {
+    License getLicense() {
         return license;
     }
 
-    public List<XPackFeatureSet.Usage> getUsage() {
-        return usage;
+    List<XPackFeatureSet.Usage> getUsages() {
+        return usages;
     }
 
-    public ClusterStatsResponse getClusterStats() {
+    ClusterStatsResponse getClusterStats() {
         return clusterStats;
     }
 
-    public ClusterState getClusterState() {
+    ClusterState getClusterState() {
         return clusterState;
     }
 
-    public ClusterHealthStatus getStatus() {
+    ClusterHealthStatus getStatus() {
         return status;
     }
 
+    @Override
+    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field("cluster_name", clusterName);
+        builder.field("version", version);
+
+        if (license != null) {
+            builder.startObject("license");
+            Map<String, String> extraParams = new MapBuilder<String, String>()
+                    .put(License.REST_VIEW_MODE, "true")
+                    .map();
+            params = new ToXContent.DelegatingMapParams(extraParams, params);
+            license.toInnerXContent(builder, params);
+            builder.field("hkey", hash(license, getCluster()));
+            builder.endObject();
+        }
+
+        if (clusterStats != null) {
+            builder.startObject("cluster_stats");
+            clusterStats.toXContent(builder, params);
+            builder.endObject();
+        }
+
+        if (clusterState != null) {
+            builder.startObject("cluster_state");
+            builder.field("status", status.name().toLowerCase(Locale.ROOT));
+            clusterState.toXContent(builder, CLUSTER_STATS_PARAMS);
+            builder.endObject();
+        }
+
+        if (usages != null) {
+            // in the future we may choose to add other usages under the stack_stats section, but it is only xpack for now
+            // it may also be combined on the UI side of phone-home to add things like "kibana" and "logstash" under "stack_stats"
+            builder.startObject("stack_stats");
+            {
+                builder.startObject("xpack");
+                for (final XPackFeatureSet.Usage usage : usages) {
+                    builder.field(usage.name(), usage);
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+        }
+    }
+
+    public static String hash(License license, String clusterName) {
+        return hash(license.status().label(), license.uid(), license.type(), String.valueOf(license.expiryDate()), clusterName);
+    }
+
+    public static String hash(String licenseStatus, String licenseUid, String licenseType, String licenseExpiryDate, String clusterUUID) {
+        String toHash = licenseStatus + licenseUid + licenseType + licenseExpiryDate + clusterUUID;
+        return MessageDigests.toHexString(MessageDigests.sha256().digest(toHash.getBytes(StandardCharsets.UTF_8)));
+    }
 }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryCollector.java

@@ -14,12 +14,12 @@ import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.monitoring.MonitoringSettings;
 import org.elasticsearch.xpack.monitoring.collector.Collector;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
-import org.elasticsearch.xpack.security.InternalClient;
 
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * Collector for the Recovery API.
@@ -31,11 +31,14 @@ public class IndexRecoveryCollector extends Collector {
 
     private final Client client;
 
-    public IndexRecoveryCollector(Settings settings, ClusterService clusterService,
-                                  MonitoringSettings monitoringSettings,
-                                  XPackLicenseState licenseState, InternalClient client) {
-        super(settings, "index-recovery", clusterService, monitoringSettings, licenseState);
-        this.client = client;
+    public IndexRecoveryCollector(final Settings settings,
+                                  final ClusterService clusterService,
+                                  final MonitoringSettings monitoringSettings,
+                                  final XPackLicenseState licenseState,
+                                  final Client client) {
+
+        super(settings, IndexRecoveryMonitoringDoc.TYPE, clusterService, monitoringSettings, licenseState);
+        this.client = Objects.requireNonNull(client);
     }
 
     @Override
@@ -44,7 +47,7 @@ public class IndexRecoveryCollector extends Collector {
     }
 
     @Override
-    protected Collection<MonitoringDoc> doCollect() throws Exception {
+    protected Collection<MonitoringDoc> doCollect(final MonitoringDoc.Node node) throws Exception {
         List<MonitoringDoc> results = new ArrayList<>(1);
         RecoveryResponse recoveryResponse = client.admin().indices().prepareRecoveries()
                 .setIndices(monitoringSettings.indices())
@@ -53,8 +56,7 @@ public class IndexRecoveryCollector extends Collector {
                 .get(monitoringSettings.recoveryTimeout());
 
         if (recoveryResponse.hasRecoveries()) {
-            results.add(new IndexRecoveryMonitoringDoc(monitoringId(), monitoringVersion(),
-                    clusterUUID(), System.currentTimeMillis(), localNode(), recoveryResponse));
+            results.add(new IndexRecoveryMonitoringDoc(clusterUUID(), timestamp(), node, recoveryResponse));
         }
         return Collections.unmodifiableCollection(results);
     }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryMonitoringDoc.java

@@ -6,9 +6,16 @@
 package org.elasticsearch.xpack.monitoring.collector.indices;
 
 import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
-import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.indices.recovery.RecoveryState;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
 
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
 /**
  * Monitoring document collected by {@link IndexRecoveryCollector}
  */
@@ -18,14 +25,46 @@ public class IndexRecoveryMonitoringDoc extends MonitoringDoc {
 
     private final RecoveryResponse recoveryResponse;
 
-    public IndexRecoveryMonitoringDoc(String monitoringId, String monitoringVersion,
-                                      String clusterUUID, long timestamp, DiscoveryNode node,
-                                      RecoveryResponse recoveryResponse) {
-        super(monitoringId, monitoringVersion, TYPE, null, clusterUUID, timestamp, node);
-        this.recoveryResponse = recoveryResponse;
+    public IndexRecoveryMonitoringDoc(final String cluster,
+                                      final long timestamp,
+                                      final MonitoringDoc.Node node,
+                                      final RecoveryResponse recoveryResponse) {
+
+        super(cluster, timestamp, node, MonitoredSystem.ES, TYPE, null);
+        this.recoveryResponse = Objects.requireNonNull(recoveryResponse);
     }
 
-    public RecoveryResponse getRecoveryResponse() {
+    RecoveryResponse getRecoveryResponse() {
         return recoveryResponse;
     }
+
+    @Override
+    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(TYPE);
+        {
+            if (recoveryResponse != null) {
+                builder.startArray("shards");
+                {
+                    Map<String, List<RecoveryState>> shards = recoveryResponse.shardRecoveryStates();
+                    if (shards != null) {
+                        for (Map.Entry<String, List<RecoveryState>> shard : shards.entrySet()) {
+                            List<RecoveryState> indexShards = shard.getValue();
+                            if (indexShards != null) {
+                                for (RecoveryState indexShard : indexShards) {
+                                    builder.startObject();
+                                    {
+                                        builder.field("index_name", shard.getKey());
+                                        indexShard.toXContent(builder, params);
+                                    }
+                                    builder.endObject();
+                                }
+                            }
+                        }
+                    }
+                }
+                builder.endArray();
+            }
+        }
+        builder.endObject();
+    }
 }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsCollector.java

@@ -9,14 +9,12 @@ import org.elasticsearch.action.admin.indices.stats.IndexStats;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.monitoring.MonitoringSettings;
 import org.elasticsearch.xpack.monitoring.collector.Collector;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
-import org.elasticsearch.xpack.security.InternalClient;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -33,9 +31,11 @@ public class IndexStatsCollector extends Collector {
 
     private final Client client;
 
-    public IndexStatsCollector(Settings settings, ClusterService clusterService,
-                               MonitoringSettings monitoringSettings,
-                               XPackLicenseState licenseState, InternalClient client) {
+    public IndexStatsCollector(final Settings settings,
+                               final ClusterService clusterService,
+                               final MonitoringSettings monitoringSettings,
+                               final XPackLicenseState licenseState,
+                               final Client client) {
         super(settings, "index-stats", clusterService, monitoringSettings, licenseState);
         this.client = client;
     }
@@ -46,7 +46,7 @@ public class IndexStatsCollector extends Collector {
     }
 
     @Override
-    protected Collection<MonitoringDoc> doCollect() throws Exception {
+    protected Collection<MonitoringDoc> doCollect(final MonitoringDoc.Node node) throws Exception {
         final List<MonitoringDoc> results = new ArrayList<>();
         final IndicesStatsResponse indicesStats = client.admin().indices().prepareStats()
                 .setIndices(monitoringSettings.indices())
@@ -64,16 +64,15 @@ public class IndexStatsCollector extends Collector {
                 .setRequestCache(true)
                 .get(monitoringSettings.indexStatsTimeout());
 
-        final long timestamp = System.currentTimeMillis();
-        final String clusterUUID = clusterUUID();
-        final DiscoveryNode sourceNode = localNode();
+        final long timestamp = timestamp();
+        final String clusterUuid = clusterUUID();
 
         // add the indices stats that we use to collect the index stats
-        results.add(new IndicesStatsMonitoringDoc(monitoringId(), monitoringVersion(), clusterUUID, timestamp, sourceNode, indicesStats));
+        results.add(new IndicesStatsMonitoringDoc(clusterUuid, timestamp, node, indicesStats));
 
         // collect each index stats document
         for (IndexStats indexStats : indicesStats.getIndices().values()) {
-            results.add(new IndexStatsMonitoringDoc(monitoringId(), monitoringVersion(), clusterUUID, timestamp, sourceNode, indexStats));
+            results.add(new IndexStatsMonitoringDoc(clusterUuid, timestamp, node, indexStats));
         }
 
         return Collections.unmodifiableCollection(results);

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDoc.java

@@ -5,27 +5,126 @@
  */
 package org.elasticsearch.xpack.monitoring.collector.indices;
 
+import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
-import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+import org.elasticsearch.xpack.monitoring.exporter.FilteredMonitoringDoc;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
 
+import java.io.IOException;
+import java.util.Objects;
+import java.util.Set;
+
 /**
  * Monitoring document collected by {@link IndexStatsCollector}
  */
-public class IndexStatsMonitoringDoc extends MonitoringDoc {
+public class IndexStatsMonitoringDoc extends FilteredMonitoringDoc {
 
     public static final String TYPE = "index_stats";
 
     private final IndexStats indexStats;
 
-    public IndexStatsMonitoringDoc(String monitoringId, String monitoringVersion,
-                                   String clusterUUID, long timestamp, DiscoveryNode node,
-                                   IndexStats indexStats) {
-        super(monitoringId, monitoringVersion, TYPE, null, clusterUUID, timestamp, node);
-        this.indexStats = indexStats;
+    IndexStatsMonitoringDoc(final String cluster,
+                            final long timestamp,
+                            final MonitoringDoc.Node node,
+                            final IndexStats indexStats) {
+        super(cluster, timestamp, node, MonitoredSystem.ES, TYPE, null, XCONTENT_FILTERS);
+        this.indexStats = Objects.requireNonNull(indexStats);
     }
 
-    public IndexStats getIndexStats() {
+    IndexStats getIndexStats() {
         return indexStats;
     }
+
+    @Override
+    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(TYPE);
+        {
+            builder.field("index", indexStats.getIndex());
+
+            final CommonStats totalStats = indexStats.getTotal();
+            if (totalStats != null) {
+                builder.startObject("total");
+                {
+                    totalStats.toXContent(builder, params);
+                }
+                builder.endObject();
+            }
+
+            final CommonStats primariesStats = indexStats.getPrimaries();
+            if (primariesStats != null) {
+                builder.startObject("primaries");
+                {
+                    primariesStats.toXContent(builder, params);
+                }
+                builder.endObject();
+            }
+        }
+        builder.endObject();
+    }
+
+    public static final Set<String> XCONTENT_FILTERS =
+        Sets.newHashSet("index_stats.index",
+                        "index_stats.primaries.docs.count",
+                        "index_stats.primaries.fielddata.memory_size_in_bytes",
+                        "index_stats.primaries.fielddata.evictions",
+                        "index_stats.primaries.indexing.index_total",
+                        "index_stats.primaries.indexing.index_time_in_millis",
+                        "index_stats.primaries.indexing.throttle_time_in_millis",
+                        "index_stats.primaries.merges.total_size_in_bytes",
+                        "index_stats.primaries.query_cache.memory_size_in_bytes",
+                        "index_stats.primaries.query_cache.evictions",
+                        "index_stats.primaries.query_cache.hit_count",
+                        "index_stats.primaries.query_cache.miss_count",
+                        "index_stats.primaries.request_cache.memory_size_in_bytes",
+                        "index_stats.primaries.request_cache.evictions",
+                        "index_stats.primaries.request_cache.hit_count",
+                        "index_stats.primaries.request_cache.miss_count",
+                        "index_stats.primaries.search.query_total",
+                        "index_stats.primaries.search.query_time_in_millis",
+                        "index_stats.primaries.segments.count",
+                        "index_stats.primaries.segments.memory_in_bytes",
+                        "index_stats.primaries.segments.terms_memory_in_bytes",
+                        "index_stats.primaries.segments.stored_fields_memory_in_bytes",
+                        "index_stats.primaries.segments.term_vectors_memory_in_bytes",
+                        "index_stats.primaries.segments.norms_memory_in_bytes",
+                        "index_stats.primaries.segments.points_memory_in_bytes",
+                        "index_stats.primaries.segments.doc_values_memory_in_bytes",
+                        "index_stats.primaries.segments.index_writer_memory_in_bytes",
+                        "index_stats.primaries.segments.version_map_memory_in_bytes",
+                        "index_stats.primaries.segments.fixed_bit_set_memory_in_bytes",
+                        "index_stats.primaries.store.size_in_bytes",
+                        "index_stats.primaries.refresh.total_time_in_millis",
+                        "index_stats.total.docs.count",
+                        "index_stats.total.fielddata.memory_size_in_bytes",
+                        "index_stats.total.fielddata.evictions",
+                        "index_stats.total.indexing.index_total",
+                        "index_stats.total.indexing.index_time_in_millis",
+                        "index_stats.total.indexing.throttle_time_in_millis",
+                        "index_stats.total.merges.total_size_in_bytes",
+                        "index_stats.total.query_cache.memory_size_in_bytes",
+                        "index_stats.total.query_cache.evictions",
+                        "index_stats.total.query_cache.hit_count",
+                        "index_stats.total.query_cache.miss_count",
+                        "index_stats.total.request_cache.memory_size_in_bytes",
+                        "index_stats.total.request_cache.evictions",
+                        "index_stats.total.request_cache.hit_count",
+                        "index_stats.total.request_cache.miss_count",
+                        "index_stats.total.search.query_total",
+                        "index_stats.total.search.query_time_in_millis",
+                        "index_stats.total.segments.count",
+                        "index_stats.total.segments.memory_in_bytes",
+                        "index_stats.total.segments.terms_memory_in_bytes",
+                        "index_stats.total.segments.stored_fields_memory_in_bytes",
+                        "index_stats.total.segments.term_vectors_memory_in_bytes",
+                        "index_stats.total.segments.norms_memory_in_bytes",
+                        "index_stats.total.segments.points_memory_in_bytes",
+                        "index_stats.total.segments.doc_values_memory_in_bytes",
+                        "index_stats.total.segments.index_writer_memory_in_bytes",
+                        "index_stats.total.segments.version_map_memory_in_bytes",
+                        "index_stats.total.segments.fixed_bit_set_memory_in_bytes",
+                        "index_stats.total.store.size_in_bytes",
+                        "index_stats.total.refresh.total_time_in_millis");
 }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDoc.java

@@ -6,26 +6,59 @@
 package org.elasticsearch.xpack.monitoring.collector.indices;
 
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
-import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+import org.elasticsearch.xpack.monitoring.exporter.FilteredMonitoringDoc;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
 
+import java.io.IOException;
+import java.util.Objects;
+import java.util.Set;
+
 /**
  * Monitoring document collected by {@link IndexStatsCollector}
  */
-public class IndicesStatsMonitoringDoc extends MonitoringDoc {
+public class IndicesStatsMonitoringDoc extends FilteredMonitoringDoc {
 
     public static final String TYPE = "indices_stats";
 
     private final IndicesStatsResponse indicesStats;
 
-    public IndicesStatsMonitoringDoc(String monitoringId, String monitoringVersion,
-                                     String clusterUUID, long timestamp, DiscoveryNode node,
-                                     IndicesStatsResponse indicesStats) {
-        super(monitoringId, monitoringVersion, TYPE, null, clusterUUID, timestamp, node);
-        this.indicesStats = indicesStats;
+    IndicesStatsMonitoringDoc(final String cluster,
+                              final long timestamp,
+                              final MonitoringDoc.Node node,
+                              final IndicesStatsResponse indicesStats) {
+        super(cluster, timestamp, node, MonitoredSystem.ES, TYPE, null, XCONTENT_FILTERS);
+        this.indicesStats = Objects.requireNonNull(indicesStats);
     }
 
-    public IndicesStatsResponse getIndicesStats() {
+    IndicesStatsResponse getIndicesStats() {
         return indicesStats;
     }
+
+    @Override
+    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(TYPE);
+        indicesStats.toXContent(builder, params);
+        builder.endObject();
+    }
+
+    public static final Set<String> XCONTENT_FILTERS =
+        Sets.newHashSet("indices_stats._all.primaries.docs.count",
+                        "indices_stats._all.primaries.indexing.index_time_in_millis",
+                        "indices_stats._all.primaries.indexing.index_total",
+                        "indices_stats._all.primaries.indexing.is_throttled",
+                        "indices_stats._all.primaries.indexing.throttle_time_in_millis",
+                        "indices_stats._all.primaries.search.query_time_in_millis",
+                        "indices_stats._all.primaries.search.query_total",
+                        "indices_stats._all.primaries.store.size_in_bytes",
+                        "indices_stats._all.total.docs.count",
+                        "indices_stats._all.total.indexing.index_time_in_millis",
+                        "indices_stats._all.total.indexing.index_total",
+                        "indices_stats._all.total.indexing.is_throttled",
+                        "indices_stats._all.total.indexing.throttle_time_in_millis",
+                        "indices_stats._all.total.search.query_time_in_millis",
+                        "indices_stats._all.total.search.query_total",
+                        "indices_stats._all.total.store.size_in_bytes");
 }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsCollector.java

@@ -57,15 +57,14 @@ public class JobStatsCollector extends Collector {
     }
 
     @Override
-    protected List<MonitoringDoc> doCollect() throws Exception {
+    protected List<MonitoringDoc> doCollect(final MonitoringDoc.Node node) throws Exception {
         // fetch details about all jobs
         final GetJobsStatsAction.Response jobs =
                 client.getJobsStats(new GetJobsStatsAction.Request(MetaData.ALL))
                       .actionGet(monitoringSettings.jobStatsTimeout());
 
-        final long timestamp = System.currentTimeMillis();
+        final long timestamp = timestamp();
         final String clusterUuid = clusterUUID();
-        final DiscoveryNode node = localNode();
 
         return jobs.getResponse().results().stream()
                    .map(jobStats -> new JobStatsMonitoringDoc(clusterUuid, timestamp, node, jobStats))

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsMonitoringDoc.java

@@ -5,12 +5,12 @@
  */
 package org.elasticsearch.xpack.monitoring.collector.ml;
 
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.ml.action.GetJobsStatsAction.Response.JobStats;
 import org.elasticsearch.xpack.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
 
+import java.io.IOException;
 import java.util.Objects;
 
 /**
@@ -24,15 +24,21 @@ public class JobStatsMonitoringDoc extends MonitoringDoc {
 
     private final JobStats jobStats;
 
-    public JobStatsMonitoringDoc(final String clusterUuid, final long timestamp, final DiscoveryNode node,
-                                 final JobStats jobStats) {
-        super(MonitoredSystem.ES.getSystem(), Version.CURRENT.toString(), TYPE, null, clusterUuid, timestamp, node);
-
+    public JobStatsMonitoringDoc(final String cluster, final long timestamp, final MonitoringDoc.Node node, final JobStats jobStats) {
+        super(cluster, timestamp, node, MonitoredSystem.ES, TYPE, null);
         this.jobStats = Objects.requireNonNull(jobStats);
     }
 
-    public JobStats getJobStats() {
+    JobStats getJobStats() {
         return jobStats;
     }
 
+    @Override
+    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(TYPE);
+        {
+            jobStats.toUnwrappedXContent(builder);
+        }
+        builder.endObject();
+    }
 }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsCollector.java

@@ -11,17 +11,16 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.bootstrap.BootstrapInfo;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.monitoring.MonitoringSettings;
 import org.elasticsearch.xpack.monitoring.collector.Collector;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
-import org.elasticsearch.xpack.security.InternalClient;
 
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Objects;
 
 /**
  * Collector for nodes statistics.
@@ -43,15 +42,23 @@ public class NodeStatsCollector extends Collector {
 
     private final Client client;
 
-    public NodeStatsCollector(Settings settings, ClusterService clusterService,
-                              MonitoringSettings monitoringSettings,
-                              XPackLicenseState licenseState, InternalClient client) {
-        super(settings, "node-stats", clusterService, monitoringSettings, licenseState);
-        this.client = client;
+    public NodeStatsCollector(final Settings settings,
+                              final ClusterService clusterService,
+                              final MonitoringSettings monitoringSettings,
+                              final XPackLicenseState licenseState,
+                              final Client client) {
+        super(settings, NodeStatsMonitoringDoc.TYPE, clusterService, monitoringSettings, licenseState);
+        this.client = Objects.requireNonNull(client);
+    }
+
+    // For testing purpose
+    @Override
+    protected boolean shouldCollect() {
+        return super.shouldCollect();
     }
 
     @Override
-    protected Collection<MonitoringDoc> doCollect() throws Exception {
+    protected Collection<MonitoringDoc> doCollect(final MonitoringDoc.Node node) throws Exception {
         NodesStatsRequest request = new NodesStatsRequest("_local");
         request.indices(FLAGS);
         request.os(true);
@@ -60,7 +67,7 @@ public class NodeStatsCollector extends Collector {
         request.threadPool(true);
         request.fs(true);
 
-        NodesStatsResponse response = client.admin().cluster().nodesStats(request).actionGet();
+        final NodesStatsResponse response = client.admin().cluster().nodesStats(request).actionGet(monitoringSettings.nodeStatsTimeout());
 
         // if there's a failure, then we failed to work with the
         // _local node (guaranteed a single exception)
@@ -68,14 +75,10 @@ public class NodeStatsCollector extends Collector {
             throw response.failures().get(0);
         }
 
-        NodeStats nodeStats = response.getNodes().get(0);
-        DiscoveryNode sourceNode = localNode();
+        final NodeStats nodeStats = response.getNodes().get(0);
 
-        NodeStatsMonitoringDoc nodeStatsDoc = new NodeStatsMonitoringDoc(monitoringId(),
-                monitoringVersion(), clusterUUID(), sourceNode, isLocalNodeMaster(),
-                nodeStats, BootstrapInfo.isMemoryLocked());
-
-        return Collections.singletonList(nodeStatsDoc);
+        return Collections.singletonList(new NodeStatsMonitoringDoc(clusterUUID(), nodeStats.getTimestamp(), node,
+                node.getUUID(), isLocalNodeMaster(), nodeStats, BootstrapInfo.isMemoryLocked()));
     }
 
 }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDoc.java

@@ -6,13 +6,20 @@
 package org.elasticsearch.xpack.monitoring.collector.node;
 
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
-import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+import org.elasticsearch.xpack.monitoring.exporter.FilteredMonitoringDoc;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
 
+import java.io.IOException;
+import java.util.Objects;
+import java.util.Set;
+
 /**
  * Monitoring document collected by {@link NodeStatsCollector}
  */
-public class NodeStatsMonitoringDoc extends MonitoringDoc {
+public class NodeStatsMonitoringDoc extends FilteredMonitoringDoc {
 
     public static final String TYPE = "node_stats";
 
@@ -21,30 +28,131 @@ public class NodeStatsMonitoringDoc extends MonitoringDoc {
     private final NodeStats nodeStats;
     private final boolean mlockall;
 
-    public NodeStatsMonitoringDoc(String monitoringId, String monitoringVersion,
-                                  String clusterUUID, DiscoveryNode node,
-                                  boolean isMaster, NodeStats nodeStats, boolean mlockall) {
-        super(monitoringId, monitoringVersion, TYPE, null, clusterUUID,
-                nodeStats.getTimestamp(), node);
-        this.nodeId = node.getId();
+    NodeStatsMonitoringDoc(final String cluster,
+                           final long timestamp,
+                           final MonitoringDoc.Node node,
+                           final String nodeId,
+                           final boolean isMaster,
+                           final NodeStats nodeStats,
+                           final boolean mlockall) {
+
+        super(cluster, timestamp, node, MonitoredSystem.ES, TYPE, null, XCONTENT_FILTERS);
+        this.nodeId = Objects.requireNonNull(nodeId);
+        this.nodeStats = Objects.requireNonNull(nodeStats);
         this.nodeMaster = isMaster;
-        this.nodeStats = nodeStats;
         this.mlockall = mlockall;
     }
 
-    public String getNodeId() {
+    String getNodeId() {
         return nodeId;
     }
 
-    public boolean isNodeMaster() {
+    boolean isNodeMaster() {
         return nodeMaster;
     }
 
-    public NodeStats getNodeStats() {
+    NodeStats getNodeStats() {
         return nodeStats;
     }
 
-    public boolean isMlockall() {
+    boolean isMlockall() {
         return mlockall;
     }
+
+    @Override
+    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(TYPE);
+        {
+            builder.field("node_id", nodeId);
+            builder.field("node_master", nodeMaster);
+            builder.field("mlockall", mlockall);
+            nodeStats.toXContent(builder, params);
+        }
+        builder.endObject();
+    }
+
+    public static final Set<String> XCONTENT_FILTERS =
+        Sets.newHashSet("node_stats.node_id",
+                        "node_stats.node_master",
+                        "node_stats.mlockall",
+                        "node_stats.indices.docs.count",
+                        "node_stats.indices.fielddata.memory_size_in_bytes",
+                        "node_stats.indices.fielddata.evictions",
+                        "node_stats.indices.store.size_in_bytes",
+                        "node_stats.indices.indexing.throttle_time_in_millis",
+                        "node_stats.indices.indexing.index_total",
+                        "node_stats.indices.indexing.index_time_in_millis",
+                        "node_stats.indices.query_cache.memory_size_in_bytes",
+                        "node_stats.indices.query_cache.evictions",
+                        "node_stats.indices.query_cache.hit_count",
+                        "node_stats.indices.query_cache.miss_count",
+                        "node_stats.indices.request_cache.memory_size_in_bytes",
+                        "node_stats.indices.request_cache.evictions",
+                        "node_stats.indices.request_cache.hit_count",
+                        "node_stats.indices.request_cache.miss_count",
+                        "node_stats.indices.search.query_total",
+                        "node_stats.indices.search.query_time_in_millis",
+                        "node_stats.indices.segments.count",
+                        "node_stats.indices.segments.memory_in_bytes",
+                        "node_stats.indices.segments.terms_memory_in_bytes",
+                        "node_stats.indices.segments.stored_fields_memory_in_bytes",
+                        "node_stats.indices.segments.term_vectors_memory_in_bytes",
+                        "node_stats.indices.segments.norms_memory_in_bytes",
+                        "node_stats.indices.segments.points_memory_in_bytes",
+                        "node_stats.indices.segments.doc_values_memory_in_bytes",
+                        "node_stats.indices.segments.index_writer_memory_in_bytes",
+                        "node_stats.indices.segments.version_map_memory_in_bytes",
+                        "node_stats.indices.segments.fixed_bit_set_memory_in_bytes",
+                        "node_stats.fs.io_stats.total.operations",
+                        "node_stats.fs.io_stats.total.read_operations",
+                        "node_stats.fs.io_stats.total.write_operations",
+                        "node_stats.fs.io_stats.total.read_kilobytes",
+                        "node_stats.fs.io_stats.total.write_kilobytes",
+                        "node_stats.fs.total.total_in_bytes",
+                        "node_stats.fs.total.free_in_bytes",
+                        "node_stats.fs.total.available_in_bytes",
+                        "node_stats.os.cgroup.cpuacct.control_group",
+                        "node_stats.os.cgroup.cpuacct.usage_nanos",
+                        "node_stats.os.cgroup.cpu.control_group",
+                        "node_stats.os.cgroup.cpu.cfs_period_micros",
+                        "node_stats.os.cgroup.cpu.cfs_quota_micros",
+                        "node_stats.os.cgroup.cpu.stat.number_of_elapsed_periods",
+                        "node_stats.os.cgroup.cpu.stat.number_of_times_throttled",
+                        "node_stats.os.cgroup.cpu.stat.time_throttled_nanos",
+                        "node_stats.os.cpu.load_average.1m",
+                        "node_stats.os.cpu.load_average.5m",
+                        "node_stats.os.cpu.load_average.15m",
+                        "node_stats.process.cpu.percent",
+                        "node_stats.process.max_file_descriptors",
+                        "node_stats.process.open_file_descriptors",
+                        "node_stats.jvm.mem.heap_max_in_bytes",
+                        "node_stats.jvm.mem.heap_used_in_bytes",
+                        "node_stats.jvm.mem.heap_used_percent",
+                        "node_stats.jvm.gc.collectors.young",
+                        "node_stats.jvm.gc.collectors.young.collection_count",
+                        "node_stats.jvm.gc.collectors.young.collection_time_in_millis",
+                        "node_stats.jvm.gc.collectors.old",
+                        "node_stats.jvm.gc.collectors.old.collection_count",
+                        "node_stats.jvm.gc.collectors.old.collection_time_in_millis",
+                        "node_stats.thread_pool.bulk.threads",
+                        "node_stats.thread_pool.bulk.queue",
+                        "node_stats.thread_pool.bulk.rejected",
+                        "node_stats.thread_pool.generic.threads",
+                        "node_stats.thread_pool.generic.queue",
+                        "node_stats.thread_pool.generic.rejected",
+                        "node_stats.thread_pool.get.threads",
+                        "node_stats.thread_pool.get.queue",
+                        "node_stats.thread_pool.get.rejected",
+                        "node_stats.thread_pool.index.threads",
+                        "node_stats.thread_pool.index.queue",
+                        "node_stats.thread_pool.index.rejected",
+                        "node_stats.thread_pool.management.threads",
+                        "node_stats.thread_pool.management.queue",
+                        "node_stats.thread_pool.management.rejected",
+                        "node_stats.thread_pool.search.threads",
+                        "node_stats.thread_pool.search.queue",
+                        "node_stats.thread_pool.search.rejected",
+                        "node_stats.thread_pool.watcher.threads",
+                        "node_stats.thread_pool.watcher.queue",
+                        "node_stats.thread_pool.watcher.rejected");
 }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardMonitoringDoc.java

@@ -5,37 +5,56 @@
  */
 package org.elasticsearch.xpack.monitoring.collector.shards;
 
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+import org.elasticsearch.xpack.monitoring.exporter.FilteredMonitoringDoc;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
 
+import java.io.IOException;
+import java.util.Objects;
+import java.util.Set;
+
 /**
  * Monitoring document collected by {@link ShardsCollector}
  */
-public class ShardMonitoringDoc extends MonitoringDoc {
+public class ShardMonitoringDoc extends FilteredMonitoringDoc {
 
     public static final String TYPE = "shards";
 
-    private ShardRouting shardRouting;
-    private String clusterStateUUID;
+    private final ShardRouting shardRouting;
+    private final String clusterStateUUID;
 
-    public ShardMonitoringDoc(String monitoringId, String monitoringVersion,
-                              String clusterUUID, long timestamp, DiscoveryNode node,
-                              ShardRouting shardRouting, String clusterStateUUID) {
-        super(monitoringId, monitoringVersion, TYPE, id(clusterStateUUID, shardRouting),
-                clusterUUID, timestamp, node);
-        this.shardRouting = shardRouting;
-        this.clusterStateUUID = clusterStateUUID;
+    ShardMonitoringDoc(final String cluster,
+                       final long timestamp,
+                       final MonitoringDoc.Node node,
+                       final ShardRouting shardRouting,
+                       final String clusterStateUUID) {
+
+        super(cluster, timestamp, node, MonitoredSystem.ES, TYPE, id(clusterStateUUID, shardRouting), XCONTENT_FILTERS);
+        this.shardRouting = Objects.requireNonNull(shardRouting);
+        this.clusterStateUUID = Objects.requireNonNull(clusterStateUUID);
     }
 
-    public ShardRouting getShardRouting() {
+    ShardRouting getShardRouting() {
         return shardRouting;
     }
 
-    public String getClusterStateUUID() {
+    String getClusterStateUUID() {
         return clusterStateUUID;
     }
 
+    @Override
+    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field("state_uuid", clusterStateUUID);
+        if (shardRouting != null) {
+            // ShardRouting is rendered inside a startObject() / endObject() but without a name,
+            // so we must use XContentBuilder.field(String, ToXContent, ToXContent.Params) here
+            builder.field("shard", shardRouting, params);
+        }
+    }
+
     /**
      * Compute an id that has the format:
      *
@@ -62,4 +81,13 @@ public class ShardMonitoringDoc extends MonitoringDoc {
         }
         return builder.toString();
     }
+
+    public static final Set<String> XCONTENT_FILTERS =
+        Sets.newHashSet("state_uuid",
+                        "shard.state",
+                        "shard.primary",
+                        "shard.node",
+                        "shard.relocating_node",
+                        "shard.shard",
+                        "shard.index");
 }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsCollector.java

@@ -7,7 +7,6 @@ package org.elasticsearch.xpack.monitoring.collector.shards;
 
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -32,9 +31,12 @@ import java.util.List;
  */
 public class ShardsCollector extends Collector {
 
-    public ShardsCollector(Settings settings, ClusterService clusterService,
-                           MonitoringSettings monitoringSettings, XPackLicenseState licenseState) {
-        super(settings, "shards", clusterService, monitoringSettings, licenseState);
+    public ShardsCollector(final Settings settings,
+                           final ClusterService clusterService,
+                           final MonitoringSettings monitoringSettings,
+                           final XPackLicenseState licenseState) {
+
+        super(settings, ShardMonitoringDoc.TYPE, clusterService, monitoringSettings, licenseState);
     }
 
     @Override
@@ -43,8 +45,8 @@ public class ShardsCollector extends Collector {
     }
 
     @Override
-    protected Collection<MonitoringDoc> doCollect() throws Exception {
-        List<MonitoringDoc> results = new ArrayList<>(1);
+    protected Collection<MonitoringDoc> doCollect(final MonitoringDoc.Node node) throws Exception {
+        final List<MonitoringDoc> results = new ArrayList<>(1);
 
         ClusterState clusterState = clusterService.state();
         if (clusterState != null) {
@@ -52,20 +54,18 @@ public class ShardsCollector extends Collector {
             if (routingTable != null) {
                 List<ShardRouting> shards = routingTable.allShards();
                 if (shards != null) {
-                    String clusterUUID = clusterUUID();
-                    String stateUUID = clusterState.stateUUID();
-                    long timestamp = System.currentTimeMillis();
+                    final String clusterUUID = clusterUUID();
+                    final String stateUUID = clusterState.stateUUID();
+                    final long timestamp = timestamp();
 
                     for (ShardRouting shard : shards) {
                         if (match(shard.getIndexName())) {
-                            DiscoveryNode node = null;
+                            MonitoringDoc.Node shardNode = null;
                             if (shard.assignedToNode()) {
-                                // If the shard is assigned to a node, the shard monitoring document
-                                // refers to this node
-                                node = clusterState.getNodes().get(shard.currentNodeId());
+                                // If the shard is assigned to a node, the shard monitoring document refers to this node
+                                shardNode = convertNode(node.getTimestamp(), clusterState.getNodes().get(shard.currentNodeId()));
                             }
-                            results.add(new ShardMonitoringDoc(monitoringId(), monitoringVersion(),
-                                    clusterUUID, timestamp, node, shard, stateUUID));
+                            results.add(new ShardMonitoringDoc(clusterUUID, timestamp, shardNode, shard, stateUUID));
                         }
                     }
                 }
@@ -74,9 +74,8 @@ public class ShardsCollector extends Collector {
         return Collections.unmodifiableCollection(results);
     }
 
-    private boolean match(String indexName) {
-        String[] indices = monitoringSettings.indices();
-        return IndexNameExpressionResolver.isAllIndices(Arrays.asList(monitoringSettings.indices()))
-                || Regex.simpleMatch(indices, indexName);
+    private boolean match(final String indexName) {
+        final String[] indices = monitoringSettings.indices();
+        return IndexNameExpressionResolver.isAllIndices(Arrays.asList(indices)) || Regex.simpleMatch(indices, indexName);
     }
 }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/BytesReferenceMonitoringDoc.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.monitoring.exporter;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * {@link BytesReferenceMonitoringDoc} is a {@link MonitoringDoc} that prints out a {@link BytesReference}
+ * source when its XContent is rendered.
+ */
+public class BytesReferenceMonitoringDoc extends MonitoringDoc {
+
+    private final XContentType xContentType;
+    private final BytesReference source;
+
+    public BytesReferenceMonitoringDoc(final String cluster,
+                                       final long timestamp,
+                                       @Nullable final Node node,
+                                       final MonitoredSystem system,
+                                       final String type,
+                                       @Nullable final String id,
+                                       final XContentType xContentType,
+                                       final BytesReference source) {
+        super(cluster, timestamp, node, system, type, id);
+        this.xContentType = Objects.requireNonNull(xContentType);
+        this.source = Objects.requireNonNull(source);
+    }
+
+    XContentType getXContentType() {
+        return xContentType;
+    }
+
+    BytesReference getSource() {
+        return source;
+    }
+
+    @Override
+    protected void innerToXContent(final XContentBuilder builder, final Params params) throws IOException {
+        if (source.length() > 0) {
+            builder.rawField(getType(), source, xContentType);
+        } else {
+            builder.nullField(getType());
+        }
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        if (!super.equals(o)) {
+            return false;
+        }
+        BytesReferenceMonitoringDoc that = (BytesReferenceMonitoringDoc) o;
+        return xContentType == that.xContentType && Objects.equals(source, that.source);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), xContentType, source);
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporter.java

@@ -7,8 +7,11 @@ package org.elasticsearch.xpack.monitoring.exporter;
 
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.monitoring.MonitoringSettings;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
 
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -24,6 +27,10 @@ public abstract class Exporter implements AutoCloseable {
      * Every {@code Exporter} allows users to explicitly disable cluster alerts.
      */
     public static final String CLUSTER_ALERTS_MANAGEMENT_SETTING = "cluster_alerts.management.enabled";
+    /**
+     * Every {@code Exporter} allows users to use a different index time format.
+     */
+    public static final String INDEX_NAME_TIME_FORMAT_SETTING = "index.name.time_format";
 
     protected final Config config;
 
@@ -73,6 +80,16 @@ public abstract class Exporter implements AutoCloseable {
         return MonitoringSettings.EXPORTERS_SETTINGS.getKey() + config.name + "." + setting;
     }
 
+    protected static DateTimeFormatter dateTimeFormatter(final Config config) {
+        String format = config.settings().get(INDEX_NAME_TIME_FORMAT_SETTING, "YYYY.MM.dd");
+        try {
+            return DateTimeFormat.forPattern(format).withZoneUTC();
+        } catch (IllegalArgumentException e) {
+            throw new SettingsException("[" + settingFQN(config, INDEX_NAME_TIME_FORMAT_SETTING)
+                    + "] invalid index name time format: [" + format + "]", e);
+        }
+    }
+
     public static class Config {
 
         private final String name;

plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/FilteredMonitoringDoc.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.monitoring.exporter;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+
+import java.io.IOException;
+import java.util.Set;
+
+import static org.elasticsearch.common.xcontent.NamedXContentRegistry.EMPTY;
+
+/**
+ * {@link FilteredMonitoringDoc} are a kind of {@link MonitoringDoc} whose XContent
+ * is filtered when the document is printed out.
+ */
+public abstract class FilteredMonitoringDoc extends MonitoringDoc {
+
+    /**
+     * List of common XContent fields that exist in all monitoring documents
+     */
+    static final Set<String> COMMON_XCONTENT_FILTERS = Sets.newHashSet("cluster_uuid", "timestamp", "type", "source_node");
+
+    private final Set<String> filters;
+
+    public FilteredMonitoringDoc(final String cluster,
+                                 final long timestamp,
+                                 @Nullable final Node node,
+                                 final MonitoredSystem system,
+                                 final String type,
+                                 @Nullable final String id,
+                                 final Set<String> xContentFilters) {
+        super(cluster, timestamp, node, system, type, id);
+        if (xContentFilters.isEmpty()) {
+            throw new IllegalArgumentException("xContentFilters must not be empty");
+        }
+
+        filters = Sets.union(COMMON_XCONTENT_FILTERS, xContentFilters);
+    }
+
+    Set<String> getFilters() {
+        return filters;
+    }
+
+    @Override
+    public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        final XContent xContent = builder.contentType().xContent();
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            try (XContentBuilder filteredBuilder = new XContentBuilder(xContent, out, filters)) {
+                super.toXContent(filteredBuilder, params);
+            }
+            try (XContentParser parser = xContent.createParser(EMPTY, out.bytes())) {
+                return builder.copyCurrentStructure(parser);
+            }
+        }
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/MonitoringDoc.java

@@ -5,68 +5,61 @@
  */
 package org.elasticsearch.xpack.monitoring.exporter;
 
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.internal.Nullable;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ToXContent.Params;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
 import java.util.Objects;
 
-import static org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc.Node.fromDiscoveryNode;
-
 /**
  * Base class for all monitoring documents.
  */
-public class MonitoringDoc {
-
-    @Deprecated private final String monitoringId;
-    @Deprecated private final String monitoringVersion;
+public abstract class MonitoringDoc implements ToXContentObject {
 
+    private final String cluster;
+    private final long timestamp;
+    private final Node node;
+    private final MonitoredSystem system;
     private final String type;
     private final String id;
 
-    private final String clusterUUID;
-    private final long timestamp;
-    private final Node sourceNode;
+    public MonitoringDoc(final String cluster,
+                         final long timestamp,
+                         @Nullable final Node node,
+                         final MonitoredSystem system,
+                         final String type,
+                         @Nullable final String id) {
 
-    private final BytesReference source;
-    private final XContentType sourceXContentType;
-
-    protected MonitoringDoc(String monitoringId, String monitoringVersion,
-                            String type, @Nullable String id,
-                            String clusterUUID, long timestamp, Node node,
-                            BytesReference source, XContentType sourceXContentType) {
-        this.monitoringId = monitoringId;
-        this.monitoringVersion = monitoringVersion;
-        this.type = type;
-        // We allow strings to be "" because Logstash 5.2 - 5.3 would submit empty _id values for time-based documents
-        this.id = Strings.isNullOrEmpty(id) ? null : id;
-        this.clusterUUID = clusterUUID;
+        this.cluster = Objects.requireNonNull(cluster);
         this.timestamp = timestamp;
-        this.sourceNode = node;
-        this.source = source;
-        this.sourceXContentType = sourceXContentType;
+        this.node = node;
+        this.system = Objects.requireNonNull(system);
+        this.type = Objects.requireNonNull(type);
+        this.id = id;
     }
 
-    public MonitoringDoc(String monitoringId, String monitoringVersion, String type, String id,
-                            String clusterUUID, long timestamp, Node node) {
-        this(monitoringId, monitoringVersion, type, id, clusterUUID, timestamp, node, null, null);
+    public String getCluster() {
+        return cluster;
     }
 
-    public MonitoringDoc(String monitoringId, String monitoringVersion, String type, String id,
-                         String clusterUUID, long timestamp, DiscoveryNode discoveryNode) {
-        this(monitoringId, monitoringVersion, type, id, clusterUUID, timestamp, fromDiscoveryNode(discoveryNode));
+    public long getTimestamp() {
+        return timestamp;
+    }
+
+    public Node getNode() {
+        return node;
+    }
+
+    public MonitoredSystem getSystem() {
+        return system;
     }
 
     public String getType() {
@@ -77,36 +70,6 @@ public class MonitoringDoc {
         return id;
     }
 
-    public String getClusterUUID() {
-        return clusterUUID;
-    }
-
-    @Deprecated
-    public String getMonitoringId() {
-        return monitoringId;
-    }
-
-    @Deprecated
-    public String getMonitoringVersion() {
-        return monitoringVersion;
-    }
-
-    public Node getSourceNode() {
-        return sourceNode;
-    }
-
-    public long getTimestamp() {
-        return timestamp;
-    }
-
-    public BytesReference getSource() {
-        return source;
-    }
-
-    public XContentType getXContentType() {
-        return sourceXContentType;
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) {
@@ -116,21 +79,42 @@ public class MonitoringDoc {
             return false;
         }
         MonitoringDoc that = (MonitoringDoc) o;
-        return timestamp == that.timestamp &&
-                Objects.equals(monitoringId, that.monitoringId)
-                && Objects.equals(monitoringVersion, that.monitoringVersion)
+        return timestamp == that.timestamp
+                && Objects.equals(cluster, that.cluster)
+                && Objects.equals(node, that.node)
+                && system == that.system
                 && Objects.equals(type, that.type)
-                && Objects.equals(id, that.id)
-                && Objects.equals(clusterUUID, that.clusterUUID)
-                && Objects.equals(sourceNode, that.sourceNode)
-                && Objects.equals(sourceNode, that.sourceNode)
-                && Objects.equals(sourceXContentType, that.sourceXContentType);
+                && Objects.equals(id, that.id);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(monitoringId, monitoringVersion, type, id, clusterUUID, timestamp,
-                sourceNode, source, sourceXContentType);
+        return Objects.hash(cluster, timestamp, node, system, type, id);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        {
+            builder.field("cluster_uuid", cluster);
+            builder.field("timestamp", toUTC(timestamp));
+            builder.field("type", type);
+            builder.field("source_node", node);
+            innerToXContent(builder, params);
+        }
+        return builder.endObject();
+    }
+
+    protected abstract void innerToXContent(XContentBuilder builder, Params params) throws IOException;
+
+    /**
+     * Converts a timestamp in milliseconds to its {@link String} representation in UTC time.
+     *
+     * @param timestamp the timestamp to convert
+     * @return a string representing the timestamp
+     */
+    static String toUTC(final long timestamp) {
+        return new DateTime(timestamp, DateTimeZone.UTC).toString();
     }
 
     /**
@@ -139,25 +123,25 @@ public class MonitoringDoc {
      */
     public static class Node implements Writeable, ToXContentObject {
 
-        private String uuid;
-        private String host;
-        private String transportAddress;
-        private String ip;
-        private String name;
-        private Map<String, String> attributes;
+        private final String uuid;
+        private final String host;
+        private final String transportAddress;
+        private final String ip;
+        private final String name;
+        private final long timestamp;
 
-        public Node(String uuid, String host, String transportAddress, String ip, String name,
-                    Map<String, String> attributes) {
+        public Node(final String uuid,
+                    final String host,
+                    final String transportAddress,
+                    final String ip,
+                    final String name,
+                    final long timestamp) {
             this.uuid = uuid;
             this.host = host;
             this.transportAddress = transportAddress;
             this.ip = ip;
             this.name = name;
-            if (attributes == null) {
-                this.attributes = new HashMap<>();
-            } else {
-                this.attributes = Collections.unmodifiableMap(attributes);
-            }
+            this.timestamp = timestamp;
         }
 
         /**
@@ -169,10 +153,16 @@ public class MonitoringDoc {
             transportAddress = in.readOptionalString();
             ip = in.readOptionalString();
             name = in.readOptionalString();
-            int size = in.readVInt();
-            this.attributes = new HashMap<>();
-            for (int i = 0; i < size; i++) {
-                attributes.put(in.readString(), in.readString());
+            if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+                timestamp = in.readVLong();
+            } else {
+                // Read the node attributes (removed in 7.0 alpha1)
+                int size = in.readVInt();
+                for (int i = 0; i < size; i++) {
+                    in.readString();
+                    in.readString();
+                }
+                timestamp = 0L;
             }
         }
 
@@ -183,18 +173,14 @@ public class MonitoringDoc {
             out.writeOptionalString(transportAddress);
             out.writeOptionalString(ip);
             out.writeOptionalString(name);
-            if (attributes != null) {
-                out.writeVInt(attributes.size());
-                for (Map.Entry<String, String> entry : attributes.entrySet()) {
-                    out.writeString(entry.getKey());
-                    out.writeString(entry.getValue());
-                }
+            if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+                out.writeVLong(timestamp);
             } else {
+                // Write an empty map of node attributes (removed in 7.0 alpha1)
                 out.writeVInt(0);
             }
         }
 
-
         public String getUUID() {
             return uuid;
         }
@@ -215,25 +201,21 @@ public class MonitoringDoc {
             return name;
         }
 
-        public Map<String, String> getAttributes() {
-            return attributes;
+        public long getTimestamp() {
+            return timestamp;
         }
 
         @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params)
-                throws IOException {
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
-            builder.field(Fields.UUID, getUUID());
-            builder.field(Fields.HOST, getHost());
-            builder.field(Fields.TRANSPORT_ADDRESS, getTransportAddress());
-            builder.field(Fields.IP, getIp());
-            builder.field(Fields.NAME, getName());
-
-            builder.startObject(Fields.ATTRIBUTES);
-            for (Map.Entry<String, String> entry : getAttributes().entrySet()) {
-                builder.field(entry.getKey(), entry.getValue());
+            {
+                builder.field("uuid", uuid);
+                builder.field("host", host);
+                builder.field("transport_address", transportAddress);
+                builder.field("ip", ip);
+                builder.field("name", name);
+                builder.field("timestamp", toUTC(timestamp));
             }
-            builder.endObject();
             return builder.endObject();
         }
 
@@ -251,39 +233,12 @@ public class MonitoringDoc {
                     && Objects.equals(transportAddress, node.transportAddress)
                     && Objects.equals(ip, node.ip)
                     && Objects.equals(name, node.name)
-                    && Objects.equals(attributes, node.attributes);
+                    && Objects.equals(timestamp, node.timestamp);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(uuid, host, transportAddress, ip, name, attributes);
-        }
-
-        /**
-         * Creates a {@link MonitoringDoc.Node} from a {@link DiscoveryNode}, copying over the
-         * required information.
-         *
-         * @param discoveryNode the {@link DiscoveryNode}
-         * @return a {@link MonitoringDoc.Node} instance, or {@code null} if the given discovery
-         *         node is null.
-         */
-        static Node fromDiscoveryNode(@Nullable DiscoveryNode discoveryNode) {
-            MonitoringDoc.Node node = null;
-            if (discoveryNode != null) {
-                node = new MonitoringDoc.Node(discoveryNode.getId(), discoveryNode.getHostName(),
-                        discoveryNode.getAddress().toString(), discoveryNode.getHostAddress(),
-                        discoveryNode.getName(), discoveryNode.getAttributes());
-            }
-            return node;
-        }
-
-        static final class Fields {
-            static final String UUID = "uuid";
-            static final String HOST = "host";
-            static final String TRANSPORT_ADDRESS = "transport_address";
-            static final String IP = "ip";
-            static final String NAME = "name";
-            static final String ATTRIBUTES = "attributes";
+            return Objects.hash(uuid, host, transportAddress, ip, name, timestamp);
         }
     }
 }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/MonitoringTemplateUtils.java

@@ -8,7 +8,9 @@ package org.elasticsearch.xpack.monitoring.exporter;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.template.TemplateUtils;
+import org.joda.time.format.DateTimeFormatter;
 
 import java.io.IOException;
 import java.util.Locale;
@@ -37,11 +39,9 @@ public final class MonitoringTemplateUtils {
     public static final String OLD_TEMPLATE_VERSION = "2";
 
     /**
-     * IDs of templates that can be used with {@linkplain #loadTemplate(String) loadTemplate} that are not managed by a Resolver.
-     * <p>
-     * This will be the complete list of template IDs when resolvers are removed.
+     * IDs of templates that can be used with {@linkplain #loadTemplate(String) loadTemplate}.
      */
-    public static final String[] TEMPLATE_IDS = { "alerts" };
+    public static final String[] TEMPLATE_IDS = { "alerts", "es", "kibana", "logstash", "beats" };
 
     /**
      * IDs of templates that can be used with {@linkplain #createEmptyTemplate(String) createEmptyTemplate} that are not managed by a
@@ -243,4 +243,16 @@ public final class MonitoringTemplateUtils {
             throw new RuntimeException("Failed to create empty pipeline", e);
         }
     }
+
+    /**
+     * Get the index name given a specific date format, a monitored system and a timestamp.
+     *
+     * @param formatter the {@link DateTimeFormatter} to use to compute the timestamped index name
+     * @param system the {@link MonitoredSystem} for which the index name is computed
+     * @param timestamp the timestamp value to use to compute the timestamped index name
+     * @return the index name as a @{link String}
+     */
+    public static String indexName(final DateTimeFormatter formatter, final MonitoredSystem system, final long timestamp) {
+        return ".monitoring-" + system.getSystem() + "-" + TEMPLATE_VERSION + "-" + formatter.print(timestamp);
+    }
 }

plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulk.java

@@ -22,12 +22,13 @@ import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.xpack.monitoring.exporter.ExportBulk;
 import org.elasticsearch.xpack.monitoring.exporter.ExportException;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
-import org.elasticsearch.xpack.monitoring.resolver.ResolversRegistry;
+import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
+import org.joda.time.format.DateTimeFormatter;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -51,9 +52,9 @@ class HttpExportBulk extends ExportBulk {
     private final Map<String, String> params;
 
     /**
-     * Resolvers are used to render monitoring documents into JSON.
+     * {@link DateTimeFormatter} used to resolve timestamped index name.
      */
-    private final ResolversRegistry registry;
+    private final DateTimeFormatter formatter;
 
     /**
      * The bytes payload that represents the bulk body is created via {@link #doAdd(Collection)}.
@@ -61,12 +62,12 @@ class HttpExportBulk extends ExportBulk {
     private byte[] payload = null;
 
     HttpExportBulk(final String name, final RestClient client, final Map<String, String> parameters,
-                   final ResolversRegistry registry, ThreadContext threadContext) {
+                   final DateTimeFormatter dateTimeFormatter, final ThreadContext threadContext) {
         super(name, threadContext);
 
         this.client = client;
         this.params = parameters;
-        this.registry = registry;
+        this.formatter = dateTimeFormatter;
     }
 
     @Override
@@ -131,41 +132,39 @@ class HttpExportBulk extends ExportBulk {
         final XContentType xContentType = XContentType.JSON;
         final XContent xContent = xContentType.xContent();
 
+        final String index = MonitoringTemplateUtils.indexName(formatter, doc.getSystem(), doc.getTimestamp());
+        final String id = doc.getId();
+
         try (BytesStreamOutput out = new BytesStreamOutput()) {
-            MonitoringIndexNameResolver<MonitoringDoc> resolver = registry.getResolver(doc);
-
-            if (resolver != null) {
-                String index = resolver.index(doc);
-                String id = doc.getId();
-
-                try (XContentBuilder builder = new XContentBuilder(xContent, out)) {
-                    // Builds the bulk action metadata line
-                    builder.startObject();
+            try (XContentBuilder builder = new XContentBuilder(xContent, out)) {
+                // Builds the bulk action metadata line
+                builder.startObject();
+                {
                     builder.startObject("index");
-                    builder.field("_index", index);
-                    builder.field("_type", "doc");
-                    if (id != null) {
-                        builder.field("_id", id);
+                    {
+                        builder.field("_index", index);
+                        builder.field("_type", "doc");
+                        if (id != null) {
+                            builder.field("_id", id);
+                        }
                     }
                     builder.endObject();
-                    builder.endObject();
                 }
-
-                // Adds action metadata line bulk separator
-                out.write(xContent.streamSeparator());
-
-                // Render the monitoring document
-                BytesRef bytesRef = resolver.source(doc, xContentType).toBytesRef();
-                out.write(bytesRef.bytes, bytesRef.offset, bytesRef.length);
-
-                // Adds final bulk separator
-                out.write(xContent.streamSeparator());
-
-                logger.trace("added index request [index={}, type={}, id={}]", index, doc.getType(), id);
-            } else {
-                logger.error("no resolver found for monitoring document [class={}]", doc.getClass().getName());
+                builder.endObject();
             }
 
+            // Adds action metadata line bulk separator
+            out.write(xContent.streamSeparator());
+
+            // Adds the source of the monitoring document
+            final BytesRef source = XContentHelper.toXContent(doc, xContentType, false).toBytesRef();
+            out.write(source.bytes, source.offset, source.length);
+
+            // Adds final bulk separator
+            out.write(xContent.streamSeparator());
+
+            logger.trace("added index request [index={}, type={}, id={}]", index, doc.getType(), id);
+
             return BytesReference.toBytes(out.bytes());
         } catch (Exception e) {
             logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to render document [{}], skipping it [{}]", doc, name), e);

plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java

@@ -35,15 +35,13 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
 import org.elasticsearch.xpack.monitoring.exporter.Exporter;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
-import org.elasticsearch.xpack.monitoring.resolver.ResolversRegistry;
 import org.elasticsearch.xpack.ssl.SSLService;
+import org.joda.time.format.DateTimeFormatter;
 
 import javax.net.ssl.SSLContext;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -162,8 +160,8 @@ public class HttpExporter extends Exporter {
      */
     private final AtomicBoolean clusterAlertsAllowed = new AtomicBoolean(false);
 
-    private final ResolversRegistry resolvers;
     private final ThreadContext threadContext;
+    private final DateTimeFormatter dateTimeFormatter;
 
     /**
      * Create an {@link HttpExporter}.
@@ -197,7 +195,7 @@ public class HttpExporter extends Exporter {
      * @throws SettingsException if any setting is malformed
      */
     HttpExporter(final Config config, final RestClient client, final ThreadContext threadContext, final NodeFailureListener listener) {
-        this(config, client, createSniffer(config, client, listener), threadContext, listener, new ResolversRegistry(config.settings()));
+        this(config, client, createSniffer(config, client, listener), threadContext, listener);
     }
 
     /**
@@ -206,12 +204,11 @@ public class HttpExporter extends Exporter {
      * @param config The HTTP Exporter's configuration
      * @param client The REST Client used to make all requests to the remote Elasticsearch cluster
      * @param listener The node failure listener used to notify an optional sniffer and resources
-     * @param resolvers The resolver registry used to load templates and resolvers
      * @throws SettingsException if any setting is malformed
      */
     HttpExporter(final Config config, final RestClient client, @Nullable final Sniffer sniffer, final ThreadContext threadContext,
-                 final NodeFailureListener listener, final ResolversRegistry resolvers) {
-        this(config, client, sniffer, threadContext, listener, resolvers, createResources(config, resolvers));
+                 final NodeFailureListener listener) {
+        this(config, client, sniffer, threadContext, listener, createResources(config));
     }
 
     /**
@@ -221,20 +218,19 @@ public class HttpExporter extends Exporter {
      * @param client The REST Client used to make all requests to the remote Elasticsearch cluster
      * @param sniffer The optional sniffer, which has already been associated with the {@code listener}
      * @param listener The node failure listener used to notify resources
-     * @param resolvers The resolver registry used to load templates and resolvers
      * @param resource Blocking HTTP resource to prevent bulks until all requirements are met
      * @throws SettingsException if any setting is malformed
      */
     HttpExporter(final Config config, final RestClient client, @Nullable final Sniffer sniffer, final ThreadContext threadContext,
-                 final NodeFailureListener listener, final ResolversRegistry resolvers, final HttpResource resource) {
+                 final NodeFailureListener listener, final HttpResource resource) {
         super(config);
 
         this.client = Objects.requireNonNull(client);
         this.sniffer = sniffer;
-        this.resolvers = resolvers;
         this.resource = resource;
         this.defaultParams = createDefaultParams(config);
         this.threadContext = threadContext;
+        this.dateTimeFormatter = dateTimeFormatter(config);
 
         // mark resources as dirty after any node failure or license change
         listener.setResource(resource);
@@ -308,10 +304,9 @@ public class HttpExporter extends Exporter {
      * Create a {@link MultiHttpResource} that can be used to block bulk exporting until all expected resources are available.
      *
      * @param config The HTTP Exporter's configuration
-     * @param resolvers The resolvers that contain all known templates.
      * @return Never {@code null}.
      */
-    static MultiHttpResource createResources(final Config config, final ResolversRegistry resolvers) {
+    static MultiHttpResource createResources(final Config config) {
         final String resourceOwnerName = settingFQN(config);
         // order controls the order that each is checked; more direct checks should always happen first (e.g., version checks)
         final List<HttpResource> resources = new ArrayList<>();
@@ -319,7 +314,7 @@ public class HttpExporter extends Exporter {
         // block the exporter from working against a monitoring cluster with the wrong version
         resources.add(new VersionHttpResource(resourceOwnerName, MIN_SUPPORTED_CLUSTER_VERSION));
         // load all templates (template bodies are lazily loaded on demand)
-        configureTemplateResources(config, resolvers, resourceOwnerName, resources);
+        configureTemplateResources(config, resourceOwnerName, resources);
         // load the pipeline (this will get added to as the monitoring API version increases)
         configurePipelineResources(config, resourceOwnerName, resources);
 
@@ -521,15 +516,14 @@ public class HttpExporter extends Exporter {
      * Adds the {@code resources} necessary for checking and publishing monitoring templates.
      *
      * @param config The HTTP Exporter's configuration
-     * @param resolvers The resolvers that contain all known templates.
      * @param resourceOwnerName The resource owner name to display for any logging messages.
      * @param resources The resources to add too.
      */
-    private static void configureTemplateResources(final Config config, final ResolversRegistry resolvers, final String resourceOwnerName,
+    private static void configureTemplateResources(final Config config,
+                                                   final String resourceOwnerName,
                                                    final List<HttpResource> resources) {
         final Settings settings = config.settings();
         final TimeValue templateTimeout = settings.getAsTime(TEMPLATE_CHECK_TIMEOUT_SETTING, null);
-        final Set<String> templateNames = new HashSet<>();
 
         // add templates not managed by resolvers
         for (final String templateId : MonitoringTemplateUtils.TEMPLATE_IDS) {
@@ -539,16 +533,6 @@ public class HttpExporter extends Exporter {
             resources.add(new TemplateHttpResource(resourceOwnerName, templateTimeout, templateName, templateLoader));
         }
 
-        // TODO: when resolvers are removed, all templates managed by this loop should be included in the TEMPLATE_IDS loop above
-        for (final MonitoringIndexNameResolver resolver : resolvers) {
-            final String templateName = resolver.templateName();
-
-            // ignore duplicates
-            if (templateNames.add(templateName)) {
-                resources.add(new TemplateHttpResource(resourceOwnerName, templateTimeout, templateName, resolver::template));
-            }
-        }
-
         // add old templates, like ".monitoring-data-2" and ".monitoring-es-2" so that other versions can continue to work
         if (settings.getAsBoolean(TEMPLATE_CREATE_LEGACY_VERSIONS_SETTING, true)) {
             for (final String templateId : MonitoringTemplateUtils.OLD_TEMPLATE_IDS) {
@@ -639,7 +623,7 @@ public class HttpExporter extends Exporter {
     public HttpExportBulk openBulk() {
         // block until all resources are verified to exist
         if (isExporterReady()) {
-            return new HttpExportBulk(settingFQN(config), client, defaultParams, resolvers, threadContext);
+            return new HttpExportBulk(settingFQN(config), client, defaultParams, dateTimeFormatter, threadContext);
         }
 
         return null;

plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalBulk.java

@@ -11,14 +11,15 @@ import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.xpack.monitoring.exporter.ExportBulk;
 import org.elasticsearch.xpack.monitoring.exporter.ExportException;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
-import org.elasticsearch.xpack.monitoring.resolver.ResolversRegistry;
 import org.elasticsearch.xpack.security.InternalClient;
+import org.joda.time.format.DateTimeFormatter;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -32,17 +33,17 @@ public class LocalBulk extends ExportBulk {
 
     private final Logger logger;
     private final InternalClient client;
-    private final ResolversRegistry resolvers;
+    private final DateTimeFormatter formatter;
     private final boolean usePipeline;
 
     private BulkRequestBuilder requestBuilder;
 
 
-    public LocalBulk(String name, Logger logger, InternalClient client, ResolversRegistry resolvers, boolean usePipeline) {
+    LocalBulk(String name, Logger logger, InternalClient client, DateTimeFormatter dateTimeFormatter, boolean usePipeline) {
         super(name, client.threadPool().getThreadContext());
         this.logger = logger;
         this.client = client;
-        this.resolvers = resolvers;
+        this.formatter = dateTimeFormatter;
         this.usePipeline = usePipeline;
     }
 
@@ -59,12 +60,15 @@ public class LocalBulk extends ExportBulk {
             }
 
             try {
-                MonitoringIndexNameResolver<MonitoringDoc> resolver = resolvers.getResolver(doc);
-                IndexRequest request = new IndexRequest(resolver.index(doc), "doc");
+                final String index = MonitoringTemplateUtils.indexName(formatter, doc.getSystem(), doc.getTimestamp());
+
+                final IndexRequest request = new IndexRequest(index, "doc");
                 if (Strings.hasText(doc.getId())) {
                     request.id(doc.getId());
                 }
-                request.source(resolver.source(doc, XContentType.SMILE), XContentType.SMILE);
+
+                final BytesReference source = XContentHelper.toXContent(doc, XContentType.SMILE, false);
+                request.source(source, XContentType.SMILE);
 
                 // allow the use of ingest pipelines to be completely optional
                 if (usePipeline) {

plugin/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java

@@ -30,7 +30,6 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.internal.Nullable;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.regex.Regex;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.gateway.GatewayService;
@@ -40,13 +39,11 @@ import org.elasticsearch.ingest.PipelineConfiguration;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.XPackClient;
 import org.elasticsearch.xpack.XPackSettings;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.monitoring.cleaner.CleanerService;
 import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
 import org.elasticsearch.xpack.monitoring.exporter.Exporter;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
-import org.elasticsearch.xpack.monitoring.resolver.ResolversRegistry;
 import org.elasticsearch.xpack.security.InternalClient;
 import org.elasticsearch.xpack.watcher.client.WatcherClient;
 import org.elasticsearch.xpack.watcher.transport.actions.delete.DeleteWatchRequest;
@@ -56,6 +53,7 @@ import org.elasticsearch.xpack.watcher.transport.actions.put.PutWatchRequest;
 import org.elasticsearch.xpack.watcher.watch.Watch;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormatter;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -69,7 +67,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
 
 import static org.elasticsearch.common.Strings.collectionToCommaDelimitedString;
 import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.LAST_UPDATED_VERSION;
@@ -87,9 +84,9 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
     private final InternalClient client;
     private final ClusterService clusterService;
     private final XPackLicenseState licenseState;
-    private final ResolversRegistry resolvers;
     private final CleanerService cleanerService;
     private final boolean useIngest;
+    private final DateTimeFormatter dateTimeFormatter;
 
     private final AtomicReference<State> state = new AtomicReference<>(State.INITIALIZED);
     private final AtomicBoolean installingSomething = new AtomicBoolean(false);
@@ -103,24 +100,12 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
         this.licenseState = config.licenseState();
         this.useIngest = config.settings().getAsBoolean(USE_INGEST_PIPELINE_SETTING, true);
         this.cleanerService = cleanerService;
-        this.resolvers = new ResolversRegistry(config.settings());
-
-        // Checks that required templates are loaded
-        for (MonitoringIndexNameResolver resolver : resolvers) {
-            if (resolver.template() == null) {
-                throw new IllegalStateException("unable to find built-in template " + resolver.templateName());
-            }
-        }
-
+        this.dateTimeFormatter = dateTimeFormatter(config);
         clusterService.addListener(this);
         cleanerService.add(this);
         licenseState.addListener(this::licenseChanged);
     }
 
-    ResolversRegistry getResolvers() {
-        return resolvers;
-    }
-
     @Override
     public void clusterChanged(ClusterChangedEvent event) {
         if (state.get() == State.INITIALIZED) {
@@ -178,18 +163,10 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
             return null;
         }
 
-        // List of distinct templates
-        final Map<String, String> templates = StreamSupport.stream(new ResolversRegistry(Settings.EMPTY).spliterator(), false)
-                .collect(Collectors.toMap(MonitoringIndexNameResolver::templateName, MonitoringIndexNameResolver::template, (a, b) -> a));
-
-        // templates not managed by resolvers
-        // TODO: this should just become "templates" when we remove resolvers (and the above templates will disappear as a result)
-        final Map<String, String> nonResolverTemplates = Arrays.stream(MonitoringTemplateUtils.TEMPLATE_IDS)
+        // List of templates
+        final Map<String, String> templates = Arrays.stream(MonitoringTemplateUtils.TEMPLATE_IDS)
                 .collect(Collectors.toMap(MonitoringTemplateUtils::templateName, MonitoringTemplateUtils::loadTemplate));
 
-        // add templates that don't come from resolvers
-        templates.putAll(nonResolverTemplates);
-
         boolean setup = true;
 
         // elected master node needs to setup templates; non-master nodes need to wait for it to be setup
@@ -217,7 +194,7 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
             clusterService.removeListener(this);
         }
 
-        return new LocalBulk(name(), logger, client, resolvers, useIngest);
+        return new LocalBulk(name(), logger, client, dateTimeFormatter, useIngest);
     }
 
     /**
@@ -504,17 +481,16 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
 
             ClusterState clusterState = clusterService.state();
             if (clusterState != null) {
-                long expirationTime = expiration.getMillis();
+                final long expirationTimeMillis = expiration.getMillis();
+                final long currentTimeMillis = System.currentTimeMillis();
 
                 // list of index patterns that we clean up; we may add watcher history in the future
                 final String[] indexPatterns = new String[] { ".monitoring-*" };
 
-                MonitoringDoc monitoringDoc = new MonitoringDoc(null, null, null, null, null,
-                        System.currentTimeMillis(), (MonitoringDoc.Node) null);
 
                 // Get the names of the current monitoring indices
-                Set<String> currents = StreamSupport.stream(getResolvers().spliterator(), false)
-                                                    .map(r -> r.index(monitoringDoc))
+                final Set<String> currents = MonitoredSystem.allSystems()
+                                                    .map(s -> MonitoringTemplateUtils.indexName(dateTimeFormatter, s, currentTimeMillis))
                                                     .collect(Collectors.toSet());
 
                 // avoid deleting the current alerts index, but feel free to delete older ones
@@ -531,7 +507,7 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle
                         }
 
                         long creationDate = index.value.getCreationDate();
-                        if (creationDate <= expirationTime) {
+                        if (creationDate <= expirationTimeMillis) {
                             if (logger.isDebugEnabled()) {
                                 logger.debug("detected expired index [name={}, created={}, expired={}]",
                                         indexName, new DateTime(creationDate, DateTimeZone.UTC), expiration);

plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/MonitoringIndexNameResolver.java

@@ -1,165 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver;
-
-
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Set;
-import java.util.function.Function;
-
-/**
- * MonitoringIndexNameResolver are used to resolve the index name, type name
- * and id of a {@link MonitoringDoc}.
- */
-public abstract class MonitoringIndexNameResolver<T extends MonitoringDoc> {
-
-    public static final String PREFIX = ".monitoring";
-    public static final String DELIMITER = "-";
-
-    /**
-     * Returns the name of the index in which the monitoring document must be indexed.
-     *
-     * @param document the monitoring document
-     * @return the name of the index
-     */
-    public abstract String index(T document);
-
-    /**
-     * Returns the generic part of the index name (ie without any dynamic part) that can be
-     * used to match indices names.
-     *
-     * @return the index pattern
-     */
-    public abstract String indexPattern();
-
-    /**
-     * Builds the source of the document in a given XContentType
-     *
-     * @param document     the monitoring document
-     * @param xContentType the content type
-     * @return the name of the index
-     */
-    public BytesReference source(T document, XContentType xContentType) throws IOException {
-        try (BytesStreamOutput out = new BytesStreamOutput()) {
-            try (XContentBuilder builder = new XContentBuilder(xContentType.xContent(), out, filters())) {
-                builder.startObject();
-
-                builder.field(Fields.CLUSTER_UUID, document.getClusterUUID());
-                DateTime timestampDateTime = new DateTime(document.getTimestamp(), DateTimeZone.UTC);
-                builder.field(Fields.TIMESTAMP, timestampDateTime.toString());
-                builder.field(Fields.TYPE, document.getType());
-
-                MonitoringDoc.Node sourceNode = document.getSourceNode();
-                if (sourceNode != null) {
-                    builder.field(Fields.SOURCE_NODE, sourceNode);
-                }
-
-                buildXContent(document, builder, ToXContent.EMPTY_PARAMS);
-                builder.endObject();
-            }
-            return out.bytes();
-        }
-    }
-
-    /**
-     * @return the template name that is required by the resolver.
-     */
-    public abstract String templateName();
-
-    /**
-     * @return the template source required by the resolver
-     */
-    public abstract String template();
-
-    /**
-     * @return the filters used when rendering the document.
-     * If empty, no filtering is applied.
-     */
-    public Set<String> filters() {
-        // No filtering by default
-        return Collections.emptySet();
-    }
-
-    protected abstract void buildXContent(T document, XContentBuilder builder, ToXContent.Params params) throws IOException;
-
-    public static final class Fields {
-        public static final String CLUSTER_UUID = "cluster_uuid";
-        public static final String TIMESTAMP = "timestamp";
-        public static final String TYPE = "type";
-        public static final String SOURCE_NODE = "source_node";
-    }
-
-    /**
-     * Timestamped index name resolvers are used used to index documents in
-     * a timestamped index (.monitoring-{ID}-{VERSION}-YYYY.MM.dd)
-     */
-    public abstract static class Timestamped<T extends MonitoringDoc> extends MonitoringIndexNameResolver<T> {
-
-        public static final Setting<String> INDEX_NAME_TIME_FORMAT_SETTING = new Setting<>("index.name.time_format", "YYYY.MM.dd",
-                Function.identity(), Setting.Property.NodeScope);
-
-        private final MonitoredSystem system;
-        private final DateTimeFormatter formatter;
-        private final String index;
-
-        public Timestamped(MonitoredSystem system, Settings settings) {
-            this(system, settings, MonitoringTemplateUtils.TEMPLATE_VERSION);
-        }
-
-        // Used in tests
-        protected Timestamped(MonitoredSystem system, Settings settings, String version) {
-            this.system = system;
-            this.index = String.join(DELIMITER, PREFIX, system.getSystem(), version);
-            String format = INDEX_NAME_TIME_FORMAT_SETTING.get(settings);
-            try {
-                this.formatter = DateTimeFormat.forPattern(format).withZoneUTC();
-            } catch (IllegalArgumentException e) {
-                throw new SettingsException("invalid index name time format [" + format + "]", e);
-            }
-        }
-
-        @Override
-        public String index(T document) {
-            return String.join(DELIMITER, index, formatter.print(document.getTimestamp()));
-        }
-
-        @Override
-        public String indexPattern() {
-            return String.join(DELIMITER, index, "*");
-        }
-
-        @Override
-        public String templateName() {
-            return MonitoringTemplateUtils.templateName(getId());
-        }
-
-        @Override
-        public String template() {
-            return MonitoringTemplateUtils.loadTemplate(getId());
-        }
-
-        String getId() {
-            return system.getSystem();
-        }
-    }
-}

plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/ResolversRegistry.java

@@ -1,138 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.action.MonitoringBulkDoc;
-import org.elasticsearch.xpack.monitoring.action.MonitoringIndex;
-import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStatsMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.collector.indices.IndexRecoveryMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.collector.indices.IndexStatsMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.collector.indices.IndicesStatsMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.collector.ml.JobStatsMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.collector.node.NodeStatsMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.collector.shards.ShardMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.monitoring.resolver.bulk.MonitoringBulkTimestampedResolver;
-import org.elasticsearch.xpack.monitoring.resolver.cluster.ClusterStatsResolver;
-import org.elasticsearch.xpack.monitoring.resolver.indices.IndexRecoveryResolver;
-import org.elasticsearch.xpack.monitoring.resolver.indices.IndexStatsResolver;
-import org.elasticsearch.xpack.monitoring.resolver.indices.IndicesStatsResolver;
-import org.elasticsearch.xpack.monitoring.resolver.ml.JobStatsResolver;
-import org.elasticsearch.xpack.monitoring.resolver.node.NodeStatsResolver;
-import org.elasticsearch.xpack.monitoring.resolver.shards.ShardsResolver;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Objects;
-import java.util.function.Predicate;
-
-public class ResolversRegistry implements Iterable<MonitoringIndexNameResolver> {
-
-    private final List<Registration> registrations = new ArrayList<>();
-
-    public ResolversRegistry(Settings settings) {
-        // register built-in defaults resolvers
-        registerBuiltIn(MonitoredSystem.ES, settings);
-
-        final List<String> supportedApiVersions = Arrays.asList(
-                MonitoringTemplateUtils.TEMPLATE_VERSION,
-                MonitoringTemplateUtils.OLD_TEMPLATE_VERSION
-        );
-
-        // register resolvers for monitored systems
-        registerMonitoredSystem(MonitoredSystem.KIBANA, settings, supportedApiVersions);
-        registerMonitoredSystem(MonitoredSystem.LOGSTASH, settings, supportedApiVersions);
-        // Beats did not report data in the 5.x timeline, so it should never send the original version
-        registerMonitoredSystem(MonitoredSystem.BEATS, settings, Collections.singletonList(MonitoringTemplateUtils.TEMPLATE_VERSION));
-    }
-
-    /**
-     * Registers resolvers for elasticsearch documents collected by the monitoring plugin
-     */
-    private void registerBuiltIn(MonitoredSystem id, Settings settings) {
-        registrations.add(resolveByClass(ClusterStatsMonitoringDoc.class, new ClusterStatsResolver(id, settings)));
-        registrations.add(resolveByClass(IndexRecoveryMonitoringDoc.class, new IndexRecoveryResolver(id, settings)));
-        registrations.add(resolveByClass(IndexStatsMonitoringDoc.class, new IndexStatsResolver(id, settings)));
-        registrations.add(resolveByClass(IndicesStatsMonitoringDoc.class, new IndicesStatsResolver(id, settings)));
-        registrations.add(resolveByClass(NodeStatsMonitoringDoc.class, new NodeStatsResolver(id, settings)));
-        registrations.add(resolveByClass(ShardMonitoringDoc.class, new ShardsResolver(id, settings)));
-        registrations.add(resolveByClass(JobStatsMonitoringDoc.class, new JobStatsResolver(id, settings)));
-    }
-
-    /**
-     * Registers resolvers for monitored systems
-     */
-    private void registerMonitoredSystem(final MonitoredSystem id, final Settings settings, final List<String> supportedApiVersions) {
-        final MonitoringBulkTimestampedResolver timestampedResolver =  new MonitoringBulkTimestampedResolver(id, settings);
-
-        // Note: We resolve requests by the API version that is supplied; this allows us to translate and up-convert any older
-        // requests that come through the _xpack/monitoring/_bulk endpoint
-        registrations.add(resolveByClassSystemVersion(id, timestampedResolver, MonitoringIndex.TIMESTAMPED, supportedApiVersions));
-    }
-
-    /**
-     * @return a Resolver that is able to resolver the given monitoring document
-     */
-    public MonitoringIndexNameResolver getResolver(MonitoringDoc document) {
-        for (Registration registration : registrations) {
-            if (registration.support(document)) {
-                return registration.resolver();
-            }
-        }
-        throw new IllegalArgumentException("No resolver found for monitoring document");
-    }
-
-    @Override
-    public Iterator<MonitoringIndexNameResolver> iterator() {
-        return registrations.stream().map(Registration::resolver).iterator();
-    }
-
-    static Registration resolveByClass(Class<? extends MonitoringDoc> type, MonitoringIndexNameResolver resolver) {
-        return new Registration(resolver, type::isInstance);
-    }
-
-    static Registration resolveByClassSystemVersion(MonitoredSystem system, MonitoringIndexNameResolver  resolver, MonitoringIndex index,
-                                                    List<String> supportedApiVersion) {
-        return new Registration(resolver, doc -> {
-            try {
-                if (doc instanceof MonitoringBulkDoc == false || index != ((MonitoringBulkDoc)doc).getIndex()) {
-                    return false;
-                }
-                if (system != MonitoredSystem.fromSystem(doc.getMonitoringId())) {
-                    return false;
-                }
-                return supportedApiVersion.contains(doc.getMonitoringVersion());
-            } catch (Exception e) {
-                return false;
-            }
-        });
-    }
-
-    static class Registration {
-
-        private final MonitoringIndexNameResolver resolver;
-        private final Predicate<MonitoringDoc> predicate;
-
-        Registration(MonitoringIndexNameResolver resolver, Predicate<MonitoringDoc> predicate) {
-            this.resolver = Objects.requireNonNull(resolver);
-            this.predicate = Objects.requireNonNull(predicate);
-        }
-
-        boolean support(MonitoringDoc document) {
-            return predicate.test(document);
-        }
-
-        MonitoringIndexNameResolver resolver() {
-            return resolver;
-        }
-    }
-}

plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/bulk/MonitoringBulkTimestampedResolver.java

@@ -1,32 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.bulk;
-
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.action.MonitoringBulkDoc;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
-
-import java.io.IOException;
-
-public class MonitoringBulkTimestampedResolver extends MonitoringIndexNameResolver.Timestamped<MonitoringBulkDoc> {
-
-    public MonitoringBulkTimestampedResolver(MonitoredSystem id, Settings settings) {
-        super(id, settings);
-    }
-
-    @Override
-    protected void buildXContent(MonitoringBulkDoc document, XContentBuilder builder, ToXContent.Params params) throws IOException {
-        BytesReference source = document.getSource();
-        if (source != null && source.length() > 0) {
-            builder.rawField(document.getType(), source, document.getXContentType());
-        }
-    }
-}

plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterStatsResolver.java

@@ -1,94 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.cluster;
-
-import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.common.collect.MapBuilder;
-import org.elasticsearch.common.hash.MessageDigests;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.license.License;
-import org.elasticsearch.xpack.XPackFeatureSet;
-import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStatsMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-
-public class ClusterStatsResolver extends MonitoringIndexNameResolver.Timestamped<ClusterStatsMonitoringDoc> {
-
-    private static final ToXContent.MapParams CLUSTER_STATS_PARAMS =
-            new ToXContent.MapParams(
-                    Collections.singletonMap("metric",
-                                             ClusterState.Metric.VERSION + "," +
-                                             ClusterState.Metric.MASTER_NODE + "," +
-                                             ClusterState.Metric.NODES));
-
-    public ClusterStatsResolver(MonitoredSystem system, Settings settings) {
-        super(system, settings);
-    }
-
-    @Override
-    protected void buildXContent(ClusterStatsMonitoringDoc document, XContentBuilder builder, ToXContent.Params params) throws IOException {
-        builder.field("cluster_name", document.getClusterName());
-        builder.field("version", document.getVersion());
-
-        final License license = document.getLicense();
-        if (license != null) {
-            builder.startObject("license");
-            Map<String, String> extraParams = new MapBuilder<String, String>()
-                    .put(License.REST_VIEW_MODE, "true")
-                    .map();
-            params = new ToXContent.DelegatingMapParams(extraParams, params);
-            license.toInnerXContent(builder, params);
-            builder.field("hkey", hash(license, document.getClusterUUID()));
-            builder.endObject();
-        }
-
-        final ClusterStatsResponse clusterStats = document.getClusterStats();
-        if (clusterStats != null) {
-            builder.startObject("cluster_stats");
-            clusterStats.toXContent(builder, params);
-            builder.endObject();
-        }
-
-        final ClusterState clusterState = document.getClusterState();
-        if (clusterState != null) {
-            builder.startObject("cluster_state");
-            builder.field("status", document.getStatus().name().toLowerCase(Locale.ROOT));
-            clusterState.toXContent(builder, CLUSTER_STATS_PARAMS);
-            builder.endObject();
-        }
-
-        final List<XPackFeatureSet.Usage> usages = document.getUsage();
-        if (usages != null) {
-            // in the future we may choose to add other usages under the stack_stats section, but it is only xpack for now
-            // it may also be combined on the UI side of phone-home to add things like "kibana" and "logstash" under "stack_stats"
-            builder.startObject("stack_stats").startObject("xpack");
-            for (final XPackFeatureSet.Usage usage : usages) {
-                builder.field(usage.name(), usage);
-            }
-            builder.endObject().endObject();
-        }
-    }
-
-    public static String hash(License license, String clusterName) {
-        return hash(license.status().label(), license.uid(), license.type(), String.valueOf(license.expiryDate()), clusterName);
-    }
-
-    public static String hash(String licenseStatus, String licenseUid, String licenseType, String licenseExpiryDate, String clusterUUID) {
-        String toHash = licenseStatus + licenseUid + licenseType + licenseExpiryDate + clusterUUID;
-        return MessageDigests.toHexString(MessageDigests.sha256().digest(toHash.getBytes(StandardCharsets.UTF_8)));
-    }
-
-}

plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/indices/IndexRecoveryResolver.java

@@ -1,63 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.indices;
-
-import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.indices.recovery.RecoveryState;
-import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.collector.indices.IndexRecoveryMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-public class IndexRecoveryResolver extends MonitoringIndexNameResolver.Timestamped<IndexRecoveryMonitoringDoc> {
-
-    public static final String TYPE = "index_recovery";
-
-    public IndexRecoveryResolver(MonitoredSystem id, Settings settings) {
-        super(id, settings);
-    }
-
-    @Override
-    protected void buildXContent(IndexRecoveryMonitoringDoc document,
-                                 XContentBuilder builder, ToXContent.Params params) throws IOException {
-        builder.startObject(Fields.INDEX_RECOVERY);
-
-        RecoveryResponse recovery = document.getRecoveryResponse();
-        if (recovery != null) {
-            builder.startArray(Fields.SHARDS);
-            Map<String, List<RecoveryState>> shards = recovery.shardRecoveryStates();
-            if (shards != null) {
-                for (Map.Entry<String, List<RecoveryState>> shard : shards.entrySet()) {
-
-                    List<RecoveryState> indexShards = shard.getValue();
-                    if (indexShards != null) {
-                        for (RecoveryState indexShard : indexShards) {
-                            builder.startObject();
-                            builder.field(Fields.INDEX_NAME, shard.getKey());
-                            indexShard.toXContent(builder, params);
-                            builder.endObject();
-                        }
-                    }
-                }
-            }
-            builder.endArray();
-        }
-
-        builder.endObject();
-    }
-
-    static final class Fields {
-        static final String INDEX_RECOVERY = TYPE;
-        static final String SHARDS = "shards";
-        static final String INDEX_NAME = "index_name";
-    }
-}

plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/indices/IndexStatsResolver.java

@@ -1,135 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.indices;
-
-import org.elasticsearch.action.admin.indices.stats.IndexStats;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.collector.indices.IndexStatsMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Set;
-
-public class IndexStatsResolver extends MonitoringIndexNameResolver.Timestamped<IndexStatsMonitoringDoc> {
-
-    public static final String TYPE = "index_stats";
-
-    static final Set<String> FILTERS;
-    static {
-        Set<String> filters = Sets.newHashSet(
-            "cluster_uuid",
-            "timestamp",
-            "type",
-            "source_node",
-            "index_stats.index",
-            "index_stats.primaries.docs.count",
-            "index_stats.primaries.fielddata.memory_size_in_bytes",
-            "index_stats.primaries.fielddata.evictions",
-            "index_stats.primaries.indexing.index_total",
-            "index_stats.primaries.indexing.index_time_in_millis",
-            "index_stats.primaries.indexing.throttle_time_in_millis",
-            "index_stats.primaries.merges.total_size_in_bytes",
-            "index_stats.primaries.query_cache.memory_size_in_bytes",
-            "index_stats.primaries.query_cache.evictions",
-            "index_stats.primaries.query_cache.hit_count",
-            "index_stats.primaries.query_cache.miss_count",
-            "index_stats.primaries.request_cache.memory_size_in_bytes",
-            "index_stats.primaries.request_cache.evictions",
-            "index_stats.primaries.request_cache.hit_count",
-            "index_stats.primaries.request_cache.miss_count",
-            "index_stats.primaries.search.query_total",
-            "index_stats.primaries.search.query_time_in_millis",
-            "index_stats.primaries.segments.count",
-            "index_stats.primaries.segments.memory_in_bytes",
-            "index_stats.primaries.segments.terms_memory_in_bytes",
-            "index_stats.primaries.segments.stored_fields_memory_in_bytes",
-            "index_stats.primaries.segments.term_vectors_memory_in_bytes",
-            "index_stats.primaries.segments.norms_memory_in_bytes",
-            "index_stats.primaries.segments.points_memory_in_bytes",
-            "index_stats.primaries.segments.doc_values_memory_in_bytes",
-            "index_stats.primaries.segments.index_writer_memory_in_bytes",
-            "index_stats.primaries.segments.version_map_memory_in_bytes",
-            "index_stats.primaries.segments.fixed_bit_set_memory_in_bytes",
-            "index_stats.primaries.store.size_in_bytes",
-            "index_stats.primaries.refresh.total_time_in_millis",
-            "index_stats.total.docs.count",
-            "index_stats.total.fielddata.memory_size_in_bytes",
-            "index_stats.total.fielddata.evictions",
-            "index_stats.total.indexing.index_total",
-            "index_stats.total.indexing.index_time_in_millis",
-            "index_stats.total.indexing.throttle_time_in_millis",
-            "index_stats.total.merges.total_size_in_bytes",
-            "index_stats.total.query_cache.memory_size_in_bytes",
-            "index_stats.total.query_cache.evictions",
-            "index_stats.total.query_cache.hit_count",
-            "index_stats.total.query_cache.miss_count",
-            "index_stats.total.request_cache.memory_size_in_bytes",
-            "index_stats.total.request_cache.evictions",
-            "index_stats.total.request_cache.hit_count",
-            "index_stats.total.request_cache.miss_count",
-            "index_stats.total.search.query_total",
-            "index_stats.total.search.query_time_in_millis",
-            "index_stats.total.segments.count",
-            "index_stats.total.segments.memory_in_bytes",
-            "index_stats.total.segments.terms_memory_in_bytes",
-            "index_stats.total.segments.stored_fields_memory_in_bytes",
-            "index_stats.total.segments.term_vectors_memory_in_bytes",
-            "index_stats.total.segments.norms_memory_in_bytes",
-            "index_stats.total.segments.points_memory_in_bytes",
-            "index_stats.total.segments.doc_values_memory_in_bytes",
-            "index_stats.total.segments.index_writer_memory_in_bytes",
-            "index_stats.total.segments.version_map_memory_in_bytes",
-            "index_stats.total.segments.fixed_bit_set_memory_in_bytes",
-            "index_stats.total.store.size_in_bytes",
-            "index_stats.total.refresh.total_time_in_millis");
-        FILTERS = Collections.unmodifiableSet(filters);
-    }
-
-    public IndexStatsResolver(MonitoredSystem id, Settings settings) {
-        super(id, settings);
-    }
-
-    @Override
-    public Set<String> filters() {
-        return FILTERS;
-    }
-
-    @Override
-    protected void buildXContent(IndexStatsMonitoringDoc document, XContentBuilder builder, ToXContent.Params params) throws IOException {
-        builder.startObject(Fields.INDEX_STATS);
-
-        IndexStats indexStats = document.getIndexStats();
-        if (indexStats != null) {
-            builder.field(Fields.INDEX, indexStats.getIndex());
-
-            builder.startObject(Fields.TOTAL);
-            if (indexStats.getTotal() != null) {
-                indexStats.getTotal().toXContent(builder, params);
-            }
-            builder.endObject();
-
-            builder.startObject(Fields.PRIMARIES);
-            if (indexStats.getPrimaries() != null) {
-                indexStats.getPrimaries().toXContent(builder, params);
-            }
-            builder.endObject();
-        }
-
-        builder.endObject();
-    }
-
-    static final class Fields {
-        static final String INDEX_STATS = TYPE;
-        static final String INDEX = "index";
-        static final String TOTAL = "total";
-        static final String PRIMARIES = "primaries";
-    }
-}

plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/indices/IndicesStatsResolver.java

@@ -1,74 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.indices;
-
-import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.collector.indices.IndicesStatsMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Set;
-
-public class IndicesStatsResolver extends MonitoringIndexNameResolver.Timestamped<IndicesStatsMonitoringDoc> {
-
-    public static final String TYPE = "indices_stats";
-
-    static final Set<String> FILTERS;
-    static {
-        Set<String> filters = Sets.newHashSet(
-            "cluster_uuid",
-            "timestamp",
-            "type",
-            "source_node",
-            "indices_stats._all.primaries.docs.count",
-            "indices_stats._all.primaries.indexing.index_time_in_millis",
-            "indices_stats._all.primaries.indexing.index_total",
-            "indices_stats._all.primaries.indexing.is_throttled",
-            "indices_stats._all.primaries.indexing.throttle_time_in_millis",
-            "indices_stats._all.primaries.search.query_time_in_millis",
-            "indices_stats._all.primaries.search.query_total",
-            "indices_stats._all.primaries.store.size_in_bytes",
-            "indices_stats._all.total.docs.count",
-            "indices_stats._all.total.indexing.index_time_in_millis",
-            "indices_stats._all.total.indexing.index_total",
-            "indices_stats._all.total.indexing.is_throttled",
-            "indices_stats._all.total.indexing.throttle_time_in_millis",
-            "indices_stats._all.total.search.query_time_in_millis",
-            "indices_stats._all.total.search.query_total",
-            "indices_stats._all.total.store.size_in_bytes"
-        );
-        FILTERS = Collections.unmodifiableSet(filters);
-    }
-
-    public IndicesStatsResolver(MonitoredSystem id, Settings settings) {
-        super(id, settings);
-    }
-
-    @Override
-    public Set<String> filters() {
-        return FILTERS;
-    }
-
-    @Override
-    protected void buildXContent(IndicesStatsMonitoringDoc document, XContentBuilder builder, ToXContent.Params params) throws IOException {
-        builder.startObject(Fields.INDICES_STATS);
-        IndicesStatsResponse indicesStats = document.getIndicesStats();
-        if (indicesStats != null) {
-            indicesStats.toXContent(builder, params);
-        }
-        builder.endObject();
-    }
-
-    static final class Fields {
-        static final String INDICES_STATS = TYPE;
-    }
-}

plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/ml/JobStatsResolver.java

@@ -1,35 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.ml;
-
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.ml.action.GetJobsStatsAction.Response.JobStats;
-import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.collector.ml.JobStatsMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
-
-import java.io.IOException;
-
-public class JobStatsResolver extends MonitoringIndexNameResolver.Timestamped<JobStatsMonitoringDoc> {
-
-    public JobStatsResolver(MonitoredSystem system, Settings settings) {
-        super(system, settings);
-    }
-
-    @Override
-    protected void buildXContent(JobStatsMonitoringDoc document, XContentBuilder builder, ToXContent.Params params) throws IOException {
-        final JobStats jobStats = document.getJobStats();
-
-        builder.startObject(document.getType());
-        {
-            jobStats.toUnwrappedXContent(builder);
-        }
-        builder.endObject();
-    }
-
-}

plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/node/NodeStatsResolver.java

@@ -1,157 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.node;
-
-import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.collector.node.NodeStatsMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Set;
-
-public class NodeStatsResolver extends MonitoringIndexNameResolver.Timestamped<NodeStatsMonitoringDoc> {
-
-    public static final String TYPE = "node_stats";
-
-    static final Set<String> FILTERS;
-    static {
-        Set<String> filters = Sets.newHashSet(
-            // Common information
-            "cluster_uuid",
-            "timestamp",
-            "type",
-            "source_node",
-            // Extra information
-            "node_stats.node_id",
-            "node_stats.node_master",
-            "node_stats.mlockall",
-            // Node Stats
-            "node_stats.indices.docs.count",
-            "node_stats.indices.fielddata.memory_size_in_bytes",
-            "node_stats.indices.fielddata.evictions",
-            "node_stats.indices.store.size_in_bytes",
-            "node_stats.indices.indexing.throttle_time_in_millis",
-            "node_stats.indices.indexing.index_total",
-            "node_stats.indices.indexing.index_time_in_millis",
-            "node_stats.indices.query_cache.memory_size_in_bytes",
-            "node_stats.indices.query_cache.evictions",
-            "node_stats.indices.query_cache.hit_count",
-            "node_stats.indices.query_cache.miss_count",
-            "node_stats.indices.request_cache.memory_size_in_bytes",
-            "node_stats.indices.request_cache.evictions",
-            "node_stats.indices.request_cache.hit_count",
-            "node_stats.indices.request_cache.miss_count",
-            "node_stats.indices.search.query_total",
-            "node_stats.indices.search.query_time_in_millis",
-            "node_stats.indices.segments.count",
-            "node_stats.indices.segments.memory_in_bytes",
-            "node_stats.indices.segments.terms_memory_in_bytes",
-            "node_stats.indices.segments.stored_fields_memory_in_bytes",
-            "node_stats.indices.segments.term_vectors_memory_in_bytes",
-            "node_stats.indices.segments.norms_memory_in_bytes",
-            "node_stats.indices.segments.points_memory_in_bytes",
-            "node_stats.indices.segments.doc_values_memory_in_bytes",
-            "node_stats.indices.segments.index_writer_memory_in_bytes",
-            "node_stats.indices.segments.version_map_memory_in_bytes",
-            "node_stats.indices.segments.fixed_bit_set_memory_in_bytes",
-            "node_stats.fs.total.total_in_bytes",
-            "node_stats.fs.total.free_in_bytes",
-            "node_stats.fs.total.available_in_bytes",
-            "node_stats.os.cpu.load_average.1m",
-            "node_stats.os.cpu.load_average.5m",
-            "node_stats.os.cpu.load_average.15m",
-            "node_stats.process.cpu.percent",
-            "node_stats.process.max_file_descriptors",
-            "node_stats.process.open_file_descriptors",
-            "node_stats.jvm.mem.heap_max_in_bytes",
-            "node_stats.jvm.mem.heap_used_in_bytes",
-            "node_stats.jvm.mem.heap_used_percent",
-            "node_stats.jvm.gc.collectors.young",
-            "node_stats.jvm.gc.collectors.young.collection_count",
-            "node_stats.jvm.gc.collectors.young.collection_time_in_millis",
-            "node_stats.jvm.gc.collectors.old",
-            "node_stats.jvm.gc.collectors.old.collection_count",
-            "node_stats.jvm.gc.collectors.old.collection_time_in_millis",
-            "node_stats.thread_pool.bulk.threads",
-            "node_stats.thread_pool.bulk.queue",
-            "node_stats.thread_pool.bulk.rejected",
-            "node_stats.thread_pool.generic.threads",
-            "node_stats.thread_pool.generic.queue",
-            "node_stats.thread_pool.generic.rejected",
-            "node_stats.thread_pool.get.threads",
-            "node_stats.thread_pool.get.queue",
-            "node_stats.thread_pool.get.rejected",
-            "node_stats.thread_pool.index.threads",
-            "node_stats.thread_pool.index.queue",
-            "node_stats.thread_pool.index.rejected",
-            "node_stats.thread_pool.management.threads",
-            "node_stats.thread_pool.management.queue",
-            "node_stats.thread_pool.management.rejected",
-            "node_stats.thread_pool.search.threads",
-            "node_stats.thread_pool.search.queue",
-            "node_stats.thread_pool.search.rejected",
-            "node_stats.thread_pool.watcher.threads",
-            "node_stats.thread_pool.watcher.queue",
-            "node_stats.thread_pool.watcher.rejected",
-            // Cgroup data (generally Linux only and only sometimes on those systems)
-            "node_stats.os.cgroup.cpuacct.control_group",
-            "node_stats.os.cgroup.cpuacct.usage_nanos",
-            "node_stats.os.cgroup.cpu.control_group",
-            "node_stats.os.cgroup.cpu.cfs_period_micros",
-            "node_stats.os.cgroup.cpu.cfs_quota_micros",
-            "node_stats.os.cgroup.cpu.stat.number_of_elapsed_periods",
-            "node_stats.os.cgroup.cpu.stat.number_of_times_throttled",
-            "node_stats.os.cgroup.cpu.stat.time_throttled_nanos",
-            // Linux Only (at least for now)
-            // Disk Info
-            "node_stats.fs.data.spins",
-            // Node IO Stats
-            "node_stats.fs.io_stats.total.operations",
-            "node_stats.fs.io_stats.total.read_operations",
-            "node_stats.fs.io_stats.total.write_operations",
-            "node_stats.fs.io_stats.total.read_kilobytes",
-            "node_stats.fs.io_stats.total.write_kilobytes");
-        FILTERS = Collections.unmodifiableSet(filters);
-    }
-
-    public NodeStatsResolver(MonitoredSystem id, Settings settings) {
-        super(id, settings);
-    }
-
-    @Override
-    public Set<String> filters() {
-        return FILTERS;
-    }
-
-    @Override
-    protected void buildXContent(NodeStatsMonitoringDoc document, XContentBuilder builder, ToXContent.Params params) throws IOException {
-        builder.startObject(Fields.NODE_STATS);
-
-        builder.field(Fields.NODE_ID, document.getNodeId());
-        builder.field(Fields.NODE_MASTER, document.isNodeMaster());
-        builder.field(Fields.MLOCKALL, document.isMlockall());
-
-        NodeStats nodeStats = document.getNodeStats();
-        if (nodeStats != null) {
-            nodeStats.toXContent(builder, params);
-        }
-
-        builder.endObject();
-    }
-
-    static final class Fields {
-        static final String NODE_STATS = TYPE;
-        static final String NODE_ID = "node_id";
-        static final String NODE_MASTER = "node_master";
-        static final String MLOCKALL = "mlockall";
-    }
-}

plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/shards/ShardsResolver.java

@@ -1,67 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.shards;
-
-import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.collector.shards.ShardMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Set;
-
-public class ShardsResolver extends MonitoringIndexNameResolver.Timestamped<ShardMonitoringDoc> {
-
-    public static final String TYPE = "shards";
-
-    static final Set<String> FILTERS;
-    static {
-        Set<String> filters = Sets.newHashSet(
-            "cluster_uuid",
-            "timestamp",
-            "type",
-            "source_node",
-            "state_uuid",
-            "shard.state",
-            "shard.primary",
-            "shard.node",
-            "shard.relocating_node",
-            "shard.shard",
-            "shard.index");
-        FILTERS = Collections.unmodifiableSet(filters);
-    }
-
-    public ShardsResolver(MonitoredSystem id, Settings settings) {
-        super(id, settings);
-    }
-
-    @Override
-    public Set<String> filters() {
-        return FILTERS;
-    }
-
-    @Override
-    protected void buildXContent(ShardMonitoringDoc document, XContentBuilder builder, ToXContent.Params params) throws IOException {
-        builder.field(Fields.STATE_UUID, document.getClusterStateUUID());
-
-        ShardRouting shardRouting = document.getShardRouting();
-        if (shardRouting != null) {
-            // ShardRouting is rendered inside a startObject() / endObject() but without a name,
-            // so we must use XContentBuilder.field(String, ToXContent, ToXContent.Params) here
-            builder.field(Fields.SHARD, shardRouting, params);
-        }
-    }
-
-    static final class Fields {
-        static final String SHARD = "shard";
-        static final String STATE_UUID = "state_uuid";
-    }
-}

plugin/src/main/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkAction.java

@@ -15,12 +15,21 @@ import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.RestBuilderListener;
 import org.elasticsearch.xpack.XPackClient;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.monitoring.action.MonitoringBulkRequestBuilder;
 import org.elasticsearch.xpack.monitoring.action.MonitoringBulkResponse;
+import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
 import org.elasticsearch.xpack.monitoring.rest.MonitoringRestHandler;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
+import static java.util.Collections.emptyList;
+import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 
@@ -30,12 +39,26 @@ public class RestMonitoringBulkAction extends MonitoringRestHandler {
     public static final String MONITORING_VERSION = "system_api_version";
     public static final String INTERVAL = "interval";
 
+    private final Map<MonitoredSystem, List<String>> supportedApiVersions;
+
     public RestMonitoringBulkAction(Settings settings, RestController controller) {
         super(settings);
         controller.registerHandler(POST, URI_BASE + "/_bulk", this);
         controller.registerHandler(PUT, URI_BASE + "/_bulk", this);
         controller.registerHandler(POST, URI_BASE + "/{type}/_bulk", this);
         controller.registerHandler(PUT, URI_BASE + "/{type}/_bulk", this);
+
+        final List<String> allVersions = Arrays.asList(
+                MonitoringTemplateUtils.TEMPLATE_VERSION,
+                MonitoringTemplateUtils.OLD_TEMPLATE_VERSION
+        );
+
+        final Map<MonitoredSystem, List<String>> versionsMap = new HashMap<>();
+        versionsMap.put(MonitoredSystem.KIBANA, allVersions);
+        versionsMap.put(MonitoredSystem.LOGSTASH, allVersions);
+        // Beats did not report data in the 5.x timeline, so it should never send the original version
+        versionsMap.put(MonitoredSystem.BEATS, Collections.singletonList(MonitoringTemplateUtils.TEMPLATE_VERSION));
+        supportedApiVersions = Collections.unmodifiableMap(versionsMap);
     }
 
     @Override
@@ -45,19 +68,20 @@ public class RestMonitoringBulkAction extends MonitoringRestHandler {
 
     @Override
     public RestChannelConsumer doPrepareRequest(RestRequest request, XPackClient client) throws IOException {
-        String defaultType = request.param("type");
+        final String defaultType = request.param("type");
 
-        String id = request.param(MONITORING_ID);
+        final String id = request.param(MONITORING_ID);
         if (Strings.isEmpty(id)) {
             throw new IllegalArgumentException("no [" + MONITORING_ID + "] for monitoring bulk request");
         }
-        String version = request.param(MONITORING_VERSION);
+
+        final String version = request.param(MONITORING_VERSION);
         if (Strings.isEmpty(version)) {
             throw new IllegalArgumentException("no [" + MONITORING_VERSION + "] for monitoring bulk request");
         }
-        // we don't currently use the interval, but in future releases we can incorporate it without breaking BWC since it was here from
-        //  the beginning
-        if (Strings.isEmpty(request.param(INTERVAL))) {
+
+        final String intervalAsString = request.param(INTERVAL);
+        if (Strings.isEmpty(intervalAsString)) {
             throw new IllegalArgumentException("no [" + INTERVAL + "] for monitoring bulk request");
         }
 
@@ -65,19 +89,30 @@ public class RestMonitoringBulkAction extends MonitoringRestHandler {
             throw new ElasticsearchParseException("no body content for monitoring bulk request");
         }
 
-        MonitoringBulkRequestBuilder requestBuilder = client.monitoring().prepareMonitoringBulk();
-        requestBuilder.add(request.content(), id, version, defaultType, request.getXContentType());
+        final MonitoredSystem system = MonitoredSystem.fromSystem(id);
+        if (isSupportedSystemVersion(system, version) == false) {
+            throw new IllegalArgumentException(MONITORING_VERSION + " [" + version + "] is not supported by "
+                    + MONITORING_ID + " [" + id + "]");
+        }
+
+        final long timestamp = System.currentTimeMillis();
+        final long intervalMillis = parseTimeValue(intervalAsString, INTERVAL).getMillis();
+
+        final MonitoringBulkRequestBuilder requestBuilder = client.monitoring().prepareMonitoringBulk();
+        requestBuilder.add(system, defaultType, request.content(), request.getXContentType(), timestamp, intervalMillis);
         return channel -> requestBuilder.execute(new RestBuilderListener<MonitoringBulkResponse>(channel) {
             @Override
             public RestResponse buildResponse(MonitoringBulkResponse response, XContentBuilder builder) throws Exception {
                 builder.startObject();
-                builder.field(Fields.TOOK, response.getTookInMillis());
+                {
+                    builder.field("took", response.getTookInMillis());
 
-                MonitoringBulkResponse.Error error = response.getError();
-                builder.field(Fields.ERRORS, error != null);
+                    final MonitoringBulkResponse.Error error = response.getError();
+                    builder.field("errors", error != null);
 
-                if (error != null) {
-                    builder.field(Fields.ERROR, response.getError());
+                    if (error != null) {
+                        builder.field("error", response.getError());
+                    }
                 }
                 builder.endObject();
                 return new BytesRestResponse(response.status(), builder);
@@ -90,9 +125,16 @@ public class RestMonitoringBulkAction extends MonitoringRestHandler {
         return true;
     }
 
-    static final class Fields {
-        static final String TOOK = "took";
-        static final String ERRORS = "errors";
-        static final String ERROR = "error";
+    /**
+     * Indicate if the given {@link MonitoredSystem} and system api version pair is supported by
+     * the Monitoring Bulk API.
+     *
+     * @param system the {@link MonitoredSystem}
+     * @param version the system API version
+     * @return true if supported, false otherwise
+     */
+    private boolean isSupportedSystemVersion(final MonitoredSystem system, final String version) {
+        final List<String> monitoredSystem = supportedApiVersions.getOrDefault(system, emptyList());
+        return monitoredSystem.contains(version);
     }
 }

plugin/src/main/resources/monitoring-beats.json

@@ -1,6 +1,6 @@
 {
   "index_patterns": ".monitoring-beats-${monitoring.template.version}-*",
-  "version": 6000026,
+  "version": 7000001,
   "settings": {
     "index.number_of_shards": 1,
     "index.number_of_replicas": 1,
@@ -37,6 +37,10 @@
             },
             "name": {
               "type": "keyword"
+            },
+            "timestamp": {
+              "type": "date",
+              "format": "date_time"
             }
           }
         },

plugin/src/main/resources/monitoring-es.json

@@ -1,6 +1,6 @@
 {
   "index_patterns": ".monitoring-es-${monitoring.template.version}-*",
-  "version": 6000026,
+  "version": 7000001,
   "settings": {
     "index.number_of_shards": 1,
     "index.number_of_replicas": 1,
@@ -41,6 +41,10 @@
             },
             "name": {
               "type": "keyword"
+            },
+            "timestamp": {
+              "type": "date",
+              "format": "date_time"
             }
           }
         },

plugin/src/main/resources/monitoring-kibana.json

@@ -1,6 +1,6 @@
 {
   "index_patterns": ".monitoring-kibana-${monitoring.template.version}-*",
-  "version": 6000026,
+  "version": 7000001,
   "settings": {
     "index.number_of_shards": 1,
     "index.number_of_replicas": 1,
@@ -37,6 +37,10 @@
             },
             "name": {
               "type": "keyword"
+            },
+            "timestamp": {
+              "type": "date",
+              "format": "date_time"
             }
           }
         },

plugin/src/main/resources/monitoring-logstash.json

@@ -1,6 +1,6 @@
 {
   "index_patterns": ".monitoring-logstash-${monitoring.template.version}-*",
-  "version": 6000026,
+  "version": 7000001,
   "settings": {
     "index.number_of_shards": 1,
     "index.number_of_replicas": 1,
@@ -37,6 +37,10 @@
             },
             "name": {
               "type": "keyword"
+            },
+            "timestamp": {
+              "type": "date",
+              "format": "date_time"
             }
           }
         },

plugin/src/test/java/org/elasticsearch/xpack/monitoring/MonitoredSystemTests.java

@@ -9,7 +9,7 @@ import org.elasticsearch.test.ESTestCase;
 
 import java.util.Locale;
 
-import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 
 /**
  * Tests {@link MonitoredSystem}.
@@ -33,13 +33,7 @@ public class MonitoredSystemTests extends ESTestCase {
     }
 
     public void testFromUnknownSystem() {
-        final String unknownSystem = randomAlphaOfLengthBetween(3, 4);
-
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
-            MonitoredSystem.fromSystem(unknownSystem);
-        });
-
-        assertThat(e.getMessage(), containsString(unknownSystem));
+        assertThat(MonitoredSystem.fromSystem(randomAlphaOfLengthBetween(3, 4)), equalTo(MonitoredSystem.UNKNOWN));
     }
 
 }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringTestUtils.java

@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.monitoring;
+
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.RandomObjects;
+import org.elasticsearch.xpack.monitoring.action.MonitoringBulkDoc;
+import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
+
+import java.io.IOException;
+import java.util.Random;
+
+import static org.elasticsearch.test.ESTestCase.buildNewFakeTransportAddress;
+
+public final class MonitoringTestUtils {
+
+    private MonitoringTestUtils() {
+    }
+
+    /**
+     * Generates a random {@link MonitoringDoc.Node}
+     */
+    public static MonitoringDoc.Node randomMonitoringNode(final Random random) {
+        final String id = RandomStrings.randomAsciiLettersOfLength(random, 5);
+        final String name = RandomStrings.randomAsciiLettersOfLengthBetween(random, 3, 10);
+
+        final TransportAddress fakeTransportAddress = buildNewFakeTransportAddress();
+        final String host = fakeTransportAddress.address().getHostString();
+        final String transportAddress = fakeTransportAddress.toString();
+        final String ip = fakeTransportAddress.getAddress();
+        final long timestamp = RandomNumbers.randomLongBetween(random, 0, Long.MAX_VALUE);
+
+        return new MonitoringDoc.Node(id, host, transportAddress, ip, name, timestamp);
+    }
+
+    /**
+     * Generates a random {@link MonitoringDoc}
+     */
+    public static MonitoringBulkDoc randomMonitoringBulkDoc(Random random) throws IOException {
+        return randomMonitoringBulkDoc(random, RandomPicks.randomFrom(random, XContentType.values()));
+    }
+
+    /**
+     * Generates a random {@link MonitoringDoc} with a given {@link XContentType}
+     */
+    public static MonitoringBulkDoc randomMonitoringBulkDoc(final Random random,
+                                                            final XContentType xContentType) throws IOException {
+        return randomMonitoringBulkDoc(random, xContentType, RandomObjects.randomSource(random, xContentType));
+    }
+
+    /**
+     * Generates a random {@link MonitoringDoc} with a given {@link XContentType} and {@link BytesReference} source
+     */
+    public static MonitoringBulkDoc randomMonitoringBulkDoc(final Random random,
+                                                            final XContentType xContentType,
+                                                            final BytesReference source) throws IOException {
+        return randomMonitoringBulkDoc(random, xContentType, source, RandomPicks.randomFrom(random, MonitoredSystem.values()));
+    }
+
+    /**
+     * Generates a random {@link MonitoringDoc} with a given {@link XContentType}, {@link BytesReference} source and {@link MonitoredSystem}
+     */
+    public static MonitoringBulkDoc randomMonitoringBulkDoc(final Random random,
+                                                            final XContentType xContentType,
+                                                            final BytesReference source,
+                                                            final MonitoredSystem system) throws IOException {
+        final String type = RandomPicks.randomFrom(random, new String[]{"type1", "type2", "type3"});
+        return randomMonitoringBulkDoc(random, xContentType, source, system, type);
+    }
+
+    /**
+     * Generates a random {@link MonitoringDoc} with a given {@link XContentType}, {@link BytesReference} source,
+     * {@link MonitoredSystem} and type.
+     */
+    public static MonitoringBulkDoc randomMonitoringBulkDoc(final Random random,
+                                                            final XContentType xContentType,
+                                                            final BytesReference source,
+                                                            final MonitoredSystem system,
+                                                            final String type) throws IOException {
+        final String id = random.nextBoolean() ? RandomStrings.randomAsciiLettersOfLength(random, 5) : null;
+        final long timestamp = RandomNumbers.randomLongBetween(random, 0L, Long.MAX_VALUE);
+        final long interval = RandomNumbers.randomLongBetween(random, 0L, Long.MAX_VALUE);
+        return new MonitoringBulkDoc(system, type, id, timestamp, interval, source, xContentType);
+    }
+}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/MultiNodesStatsTests.java

@@ -3,11 +3,10 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.monitoring.resolver.node;
+package org.elasticsearch.xpack.monitoring;
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.search.aggregations.Aggregation;
@@ -16,6 +15,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.xpack.monitoring.MonitoringSettings;
+import org.elasticsearch.xpack.monitoring.collector.node.NodeStatsMonitoringDoc;
 import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
 import org.junit.After;
 
@@ -78,12 +78,11 @@ public class MultiNodesStatsTests extends MonitoringIntegTestCase {
         waitForMonitoringIndices();
 
         assertBusy(() -> {
-            String indices = MONITORING_INDICES_PREFIX + "*";
-            flush(indices);
+            flush(ALL_MONITORING_INDICES);
             refresh();
 
-            SearchResponse response = client().prepareSearch(indices)
-                    .setQuery(QueryBuilders.termQuery("type", NodeStatsResolver.TYPE))
+            SearchResponse response = client().prepareSearch(ALL_MONITORING_INDICES)
+                    .setQuery(QueryBuilders.termQuery("type", NodeStatsMonitoringDoc.TYPE))
                     .setSize(0)
                     .addAggregation(AggregationBuilders.terms("nodes_ids").field("node_stats.node_id"))
                     .get();

plugin/src/test/java/org/elasticsearch/xpack/monitoring/OldMonitoringIndicesBackwardsCompatibilityTests.java

@@ -6,7 +6,6 @@
 package org.elasticsearch.xpack.monitoring;
 
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
-
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.AbstractOldXPackIndicesBackwardsCompatibilityTestCase;
 import org.elasticsearch.Version;
@@ -28,10 +27,8 @@ import org.elasticsearch.xpack.monitoring.collector.indices.IndexStatsMonitoring
 import org.elasticsearch.xpack.monitoring.collector.indices.IndicesStatsMonitoringDoc;
 import org.elasticsearch.xpack.monitoring.collector.node.NodeStatsMonitoringDoc;
 import org.elasticsearch.xpack.monitoring.collector.shards.ShardMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
-import org.elasticsearch.xpack.monitoring.resolver.indices.IndexStatsResolver;
 import org.hamcrest.Matcher;
+import org.joda.time.format.DateTimeFormat;
 
 import java.net.InetSocketAddress;
 import java.util.Arrays;
@@ -43,6 +40,7 @@ import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.TEMPLATE_VERSION;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -110,11 +108,12 @@ public class OldMonitoringIndicesBackwardsCompatibilityTests extends AbstractOld
             Settings.Builder settings = Settings.builder().put(MonitoringSettings.INTERVAL.getKey(), timeValueSeconds(3).getStringRep());
             assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings).get());
 
+            final String prefix = ".monitoring-" + MonitoredSystem.ES.getSystem() + "-" + TEMPLATE_VERSION + "-";
+
             // And we wait until data have been indexed locally using either by the local or http exporter
-            MonitoringIndexNameResolver.Timestamped resolver = new IndexStatsResolver(MonitoredSystem.ES, Settings.EMPTY);
-            MonitoringDoc monitoringDoc = new IndexStatsMonitoringDoc(MonitoredSystem.ES.getSystem(), null, null, System.currentTimeMillis(), null, null);
-            final String expectedIndex = resolver.index(monitoringDoc);
-            final String indexPattern = resolver.indexPattern();
+            final String dateTime = DateTimeFormat.forPattern("YYYY.MM.dd").print(System.currentTimeMillis());
+            final String expectedIndex = prefix + dateTime;
+            final String indexPattern = prefix + "*";
 
             logger.info("--> {} Waiting for [{}] to be ready", Thread.currentThread().getName(), expectedIndex);
             assertBusy(() -> {

plugin/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkDocTests.java

@@ -6,95 +6,185 @@
 package org.elasticsearch.xpack.monitoring.action;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.elasticsearch.test.RandomObjects;
 import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
+import org.junit.Before;
 
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.List;
 
-import static org.elasticsearch.test.VersionUtils.randomVersion;
+import static java.util.Collections.emptyList;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.elasticsearch.xpack.monitoring.MonitoringTestUtils.randomMonitoringBulkDoc;
 import static org.hamcrest.Matchers.equalTo;
 
 public class MonitoringBulkDocTests extends ESTestCase {
 
+    private MonitoredSystem system;
+    private String type;
+    private String id;
+    private long timestamp;
+    private long interval;
+    private BytesReference source;
+    private XContentType xContentType;
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        system = randomFrom(MonitoredSystem.values());
+        type = randomAlphaOfLength(5);
+        id = randomBoolean() ? randomAlphaOfLength(10) : null;
+        timestamp = randomNonNegativeLong();
+        interval = randomNonNegativeLong();
+        xContentType = randomFrom(XContentType.values());
+        source = RandomObjects.randomSource(random(), xContentType);
+    }
+
+    public void testConstructorMonitoredSystemMustNotBeNull() {
+        expectThrows(NullPointerException.class, () -> new MonitoringBulkDoc(null, type, id, timestamp, interval, source, xContentType));
+    }
+
+    public void testConstructorTypeMustNotBeNull() {
+        expectThrows(NullPointerException.class, () -> new MonitoringBulkDoc(system, null, id, timestamp, interval, source, xContentType));
+    }
+
+    public void testConstructorSourceMustNotBeNull() {
+        expectThrows(NullPointerException.class, () -> new MonitoringBulkDoc(system, type, id, timestamp, interval, null, xContentType));
+    }
+
+    public void testConstructorXContentTypeMustNotBeNull() {
+        expectThrows(NullPointerException.class, () -> new MonitoringBulkDoc(system, type, id, timestamp, interval, source, null));
+    }
+
+    public void testConstructor() {
+        final MonitoringBulkDoc document = new MonitoringBulkDoc(system, type, id, timestamp, interval, source, xContentType);
+
+        assertThat(document.getSystem(), equalTo(system));
+        assertThat(document.getType(), equalTo(type));
+        assertThat(document.getId(), equalTo(id));
+        assertThat(document.getTimestamp(), equalTo(timestamp));
+        assertThat(document.getInterval(), equalTo(interval));
+        assertThat(document.getSource(), equalTo(source));
+        assertThat(document.getXContentType(), equalTo(xContentType));
+    }
+
+    public void testEqualsAndHashcode() {
+        final EqualsHashCodeTestUtils.CopyFunction<MonitoringBulkDoc> copy =
+                doc -> new MonitoringBulkDoc(doc.getSystem(), doc.getType(), doc.getId(), doc.getTimestamp(), doc.getInterval(),
+                                             doc.getSource(), doc.getXContentType());
+
+        final List<EqualsHashCodeTestUtils.MutateFunction<MonitoringBulkDoc>> mutations = new ArrayList<>();
+        mutations.add(doc -> {
+            MonitoredSystem system;
+            do {
+                system = randomFrom(MonitoredSystem.values());
+            } while (system == doc.getSystem());
+            return new MonitoringBulkDoc(system, doc.getType(), doc.getId(), doc.getTimestamp(), doc.getInterval(),
+                                         doc.getSource(),  doc.getXContentType());
+        });
+        mutations.add(doc -> {
+            String type;
+            do {
+                type = randomAlphaOfLength(5);
+            } while (type.equals(doc.getType()));
+            return new MonitoringBulkDoc(doc.getSystem(), type, doc.getId(), doc.getTimestamp(), doc.getInterval(),
+                                         doc.getSource(), doc.getXContentType());
+        });
+        mutations.add(doc -> {
+            String id;
+            do {
+                id = randomAlphaOfLength(10);
+            } while (id.equals(doc.getId()));
+            return new MonitoringBulkDoc(doc.getSystem(), doc.getType(), id, doc.getTimestamp(), doc.getInterval(),
+                                         doc.getSource(), doc.getXContentType());
+        });
+        mutations.add(doc -> {
+            long timestamp;
+            do {
+                timestamp = randomNonNegativeLong();
+            } while (timestamp == doc.getTimestamp());
+            return new MonitoringBulkDoc(doc.getSystem(), doc.getType(), doc.getId(), timestamp, doc.getInterval(),
+                                         doc.getSource(), doc.getXContentType());
+        });
+        mutations.add(doc -> {
+            long interval;
+            do {
+                interval = randomNonNegativeLong();
+            } while (interval == doc.getInterval());
+            return new MonitoringBulkDoc(doc.getSystem(), doc.getType(), doc.getId(), doc.getTimestamp(), interval,
+                                         doc.getSource(), doc.getXContentType());
+        });
+        mutations.add(doc -> {
+            final BytesReference source = RandomObjects.randomSource(random(), doc.getXContentType());
+            return new MonitoringBulkDoc(doc.getSystem(), doc.getType(), doc.getId(), doc.getTimestamp(), doc.getInterval(),
+                                         source, doc.getXContentType());
+        });
+        mutations.add(doc -> {
+            XContentType xContentType;
+            do {
+                xContentType = randomFrom(XContentType.values());
+            } while (xContentType == doc.getXContentType());
+            return new MonitoringBulkDoc(doc.getSystem(), doc.getType(), doc.getId(), doc.getTimestamp(), doc.getInterval(),
+                                         doc.getSource(), xContentType);
+        });
+
+        final MonitoringBulkDoc document = new MonitoringBulkDoc(system, type, id, timestamp, interval, source, xContentType);
+        checkEqualsAndHashCode(document, copy, randomFrom(mutations));
+    }
+
     public void testSerialization() throws IOException {
-        int iterations = randomIntBetween(5, 50);
+        final NamedWriteableRegistry registry = new NamedWriteableRegistry(emptyList());
+
+        final int iterations = randomIntBetween(5, 50);
         for (int i = 0; i < iterations; i++) {
-            MonitoringBulkDoc doc = newRandomMonitoringBulkDoc();
+            final MonitoringBulkDoc original = randomMonitoringBulkDoc(random());
+            final MonitoringBulkDoc deserialized = copyWriteable(original, registry, MonitoringBulkDoc::readFrom);
 
-            BytesStreamOutput output = new BytesStreamOutput();
-            Version outputVersion = randomVersion(random());
-            output.setVersion(outputVersion);
-            doc.writeTo(output);
-
-            StreamInput streamInput = output.bytes().streamInput();
-            streamInput.setVersion(outputVersion);
-            MonitoringBulkDoc doc2 = MonitoringBulkDoc.readFrom(streamInput);
-
-            assertThat(doc2.getMonitoringId(), equalTo(doc.getMonitoringId()));
-            assertThat(doc2.getMonitoringVersion(), equalTo(doc.getMonitoringVersion()));
-            assertThat(doc2.getClusterUUID(), equalTo(doc.getClusterUUID()));
-            assertThat(doc2.getTimestamp(), equalTo(doc.getTimestamp()));
-            assertThat(doc2.getSourceNode(), equalTo(doc.getSourceNode()));
-            assertThat(doc2.getIndex(), equalTo(doc.getIndex()));
-            assertThat(doc2.getType(), equalTo(doc.getType()));
-            assertThat(doc2.getId(), equalTo(doc.getId()));
-            assertThat(doc2.getXContentType(), equalTo(doc.getXContentType()));
-            if (doc.getSource() == null) {
-                assertThat(doc2.getSource(), equalTo(BytesArray.EMPTY));
-            } else {
-                assertThat(doc2.getSource(), equalTo(doc.getSource()));
-            }
+            assertEquals(original, deserialized);
+            assertEquals(original.hashCode(), deserialized.hashCode());
+            assertNotSame(original, deserialized);
         }
     }
 
-    public static MonitoringBulkDoc newRandomMonitoringBulkDoc() {
-        String monitoringId = randomFrom(MonitoredSystem.values()).getSystem();
-        String monitoringVersion = randomVersion(random()).toString();
-        MonitoringIndex index = randomBoolean() ? randomFrom(MonitoringIndex.values()) : null;
-        String type = randomFrom("type1", "type2", "type3");
-        String id = randomBoolean() ? randomAlphaOfLength(3) : null;
-        String clusterUUID = randomBoolean() ? randomAlphaOfLength(5) : null;
-        long timestamp = randomBoolean() ? randomNonNegativeLong() : 0L;
-        MonitoringDoc.Node sourceNode = randomBoolean() ? newRandomSourceNode() : null;
-        BytesReference source =  new BytesArray("{\"key\" : \"value\"}");
-        XContentType xContentType = XContentType.JSON;
-
-        return new MonitoringBulkDoc(monitoringId, monitoringVersion, index, type, id,
-                clusterUUID, timestamp, sourceNode, source, xContentType);
+    public void testSerializationBwc() throws IOException {
+        final byte[] data = Base64.getDecoder().decode("AQNtSWQBBTUuMS4yAAAAAQEEdHlwZQECaWQNeyJmb28iOiJiYXIifQAAAAAAAAAA");
+        final Version version = randomFrom(Version.V_5_0_0, Version.V_5_0_1, Version.V_5_0_2,
+                Version.V_5_1_1, Version.V_5_1_2, Version.V_5_2_0);
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+            MonitoringBulkDoc bulkDoc = MonitoringBulkDoc.readFrom(in);
+            assertEquals(MonitoredSystem.UNKNOWN, bulkDoc.getSystem());
+            assertEquals("type", bulkDoc.getType());
+            assertEquals("id", bulkDoc.getId());
+            assertEquals(0L, bulkDoc.getTimestamp());
+            assertEquals(0L, bulkDoc.getInterval());
+            assertEquals("{\"foo\":\"bar\"}", bulkDoc.getSource().utf8ToString());
+            assertEquals(XContentType.JSON, bulkDoc.getXContentType());
+        }
     }
 
-    public static MonitoringDoc.Node newRandomSourceNode() {
-        String uuid = null;
-        String name = null;
-        String ip = null;
-        String transportAddress = null;
-        String host = null;
-        Map<String, String> attributes = null;
+    /**
+     *  Test that we allow strings to be "" because Logstash 5.2 - 5.3 would submit empty _id values for time-based documents
+     */
+    public void testEmptyIdBecomesNull() {
+        final String id = randomFrom("", null, randomAlphaOfLength(5));
+        final MonitoringBulkDoc doc = new MonitoringBulkDoc(MonitoredSystem.ES, "_type", id, 1L, 2L, BytesArray.EMPTY, XContentType.JSON);
 
-        if (frequently()) {
-            uuid = randomAlphaOfLength(5);
-            name = randomAlphaOfLength(5);
+        if (Strings.isNullOrEmpty(id)) {
+            assertNull(doc.getId());
+        } else {
+            assertSame(id, doc.getId());
         }
-        if (randomBoolean()) {
-            ip = randomAlphaOfLength(5);
-            transportAddress = randomAlphaOfLength(5);
-            host = randomAlphaOfLength(3);
-        }
-        if (rarely()) {
-            int nbAttributes = randomIntBetween(0, 5);
-            attributes = new HashMap<>();
-            for (int i = 0; i < nbAttributes; i++) {
-                attributes.put("key#" + i, String.valueOf(i));
-            }
-        }
-        return new MonitoringDoc.Node(uuid, host, transportAddress, ip, name, attributes);
     }
 }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java

@@ -5,201 +5,307 @@
  */
 package org.elasticsearch.xpack.monitoring.action;
 
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
-import org.hamcrest.Matcher;
+import org.elasticsearch.test.RandomObjects;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+import org.elasticsearch.xpack.monitoring.MonitoringTestUtils;
 
 import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
 
-import static org.elasticsearch.test.VersionUtils.randomVersion;
+import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasItems;
+import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.nullValue;
-import static org.hamcrest.Matchers.sameInstance;
 
+/**
+ * Tests {@link MonitoringBulkRequest}
+ */
 public class MonitoringBulkRequestTests extends ESTestCase {
 
-    private static final BytesArray SOURCE = new BytesArray("{\"key\" : \"value\"}");
-
-    public void testValidateRequestNoDocs() {
-        assertValidationErrors(new MonitoringBulkRequest(), hasItems("no monitoring documents added"));
+    public void testValidateWithNoDocs() {
+        ActionRequestValidationException validation = new MonitoringBulkRequest().validate();
+        assertNotNull(validation);
+        assertThat(validation.validationErrors(), hasItem("no monitoring documents added"));
     }
 
-    public void testValidateRequestSingleDoc() {
-        MonitoringBulkDoc doc = new MonitoringBulkDoc(null, null, null, null, null, null, null);
+    public void testValidateWithEmptySource() throws IOException {
+        final MonitoringBulkRequest request = new MonitoringBulkRequest();
 
-        assertValidationErrors(new MonitoringBulkRequest().add(doc), hasItems("monitored system id is missing for monitoring document [0]",
-                "monitored system API version is missing for monitoring document [0]",
-                "type is missing for monitoring document [0]",
-                "source is missing for monitoring document [0]"));
+        final int nbDocs = randomIntBetween(0, 5);
+        for (int i = 0; i < nbDocs; i++) {
+            request.add(randomMonitoringBulkDoc());
+        }
 
-        doc = new MonitoringBulkDoc("id", null, null, null, null, null, null);
-        assertValidationErrors(new MonitoringBulkRequest().add(doc),
-                hasItems("monitored system API version is missing for monitoring document [0]",
-                "type is missing for monitoring document [0]",
-                "source is missing for monitoring document [0]"));
+        final int nbEmptyDocs = randomIntBetween(1, 20);
+        for (int i = 0; i < nbEmptyDocs; i++) {
+            request.add(MonitoringTestUtils.randomMonitoringBulkDoc(random(), randomXContentType(), BytesArray.EMPTY));
+        }
 
-        doc = new MonitoringBulkDoc("id", "version", null, null, null, null, null);
-        assertValidationErrors(new MonitoringBulkRequest().add(doc), hasItems("type is missing for monitoring document [0]",
-                "source is missing for monitoring document [0]"));
+        final ActionRequestValidationException validation = request.validate();
+        assertNotNull(validation);
 
-        doc.setType("type");
-        assertValidationErrors(new MonitoringBulkRequest().add(doc), hasItems("source is missing for monitoring document [0]"));
-
-        doc.setSource(SOURCE, XContentType.JSON);
-        assertValidationErrors(new MonitoringBulkRequest().add(doc), nullValue());
+        final List<String> validationErrors = validation.validationErrors();
+        for (int i = 0; i < nbEmptyDocs; i++) {
+            assertThat(validationErrors, hasItem("source is missing for monitoring document [" + String.valueOf(nbDocs + i) + "]"));
+        }
     }
 
+    public void testAdd() throws IOException {
+        final MonitoringBulkRequest request = new MonitoringBulkRequest();
 
-    public void testValidateRequestMultiDocs() {
-        MonitoringBulkRequest request = new MonitoringBulkRequest();
-
-        // Doc0 is complete
-        MonitoringBulkDoc doc0 = new MonitoringBulkDoc(randomAlphaOfLength(2),
-                randomAlphaOfLength(2), MonitoringIndex.TIMESTAMPED, randomAlphaOfLength(5),
-                null, SOURCE, XContentType.JSON);
-        request.add(doc0);
-
-        // Doc1 has no type
-        MonitoringBulkDoc doc1 = new MonitoringBulkDoc(randomAlphaOfLength(2),
-                randomAlphaOfLength(2), MonitoringIndex.TIMESTAMPED, null,
-                null, SOURCE, XContentType.JSON);
-        request.add(doc1);
-
-        // Doc2 has no source
-        MonitoringBulkDoc doc2 = new MonitoringBulkDoc(randomAlphaOfLength(2),
-                randomAlphaOfLength(2), MonitoringIndex.TIMESTAMPED, randomAlphaOfLength(5),
-                null, BytesArray.EMPTY, XContentType.JSON);
-        request.add(doc2);
-
-        // Doc3 has no version
-        MonitoringBulkDoc doc3 = new MonitoringBulkDoc(randomAlphaOfLength(2),
-                null, MonitoringIndex.TIMESTAMPED, randomAlphaOfLength(5),
-                null, SOURCE, XContentType.JSON);
-        request.add(doc3);
-
-        // Doc4 has no id
-        MonitoringBulkDoc doc4 = new MonitoringBulkDoc(null,
-                randomAlphaOfLength(2), MonitoringIndex.TIMESTAMPED, randomAlphaOfLength(5),
-                null, SOURCE, XContentType.JSON);
-        request.add(doc4);
-
-        assertValidationErrors(request, hasItems("type is missing for monitoring document [1]",
-                "source is missing for monitoring document [2]",
-                "monitored system API version is missing for monitoring document [3]",
-                "monitored system id is missing for monitoring document [4]"));
-
-    }
-
-    public void testAddSingleDoc() {
-        MonitoringBulkRequest request = new MonitoringBulkRequest();
         final int nbDocs = randomIntBetween(1, 20);
         for (int i = 0; i < nbDocs; i++) {
-            request.add(new MonitoringBulkDoc(String.valueOf(i), String.valueOf(i),
-                    randomFrom(MonitoringIndex.values()), randomAlphaOfLength(5),
-                    randomAlphaOfLength(5), SOURCE, XContentType.JSON));
+            request.add(randomMonitoringBulkDoc());
         }
+
         assertThat(request.getDocs(), hasSize(nbDocs));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/1353")
-    public void testAddMultipleDocs() throws Exception {
-        final int nbDocs = randomIntBetween(3, 20);
-        final String[] types = new String[nbDocs];
+    public void testAddRequestContent() throws IOException {
         final XContentType xContentType = XContentType.JSON;
-        int dataCount = 0;
-        int i;
+        final String defaultType = rarely() ? randomAlphaOfLength(4) : null;
 
-        try (BytesStreamOutput content = new BytesStreamOutput()) {
-            try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, content)) {
-                for (i = 0; i < nbDocs; i++) {
-                    builder.startObject().startObject("index");
-                    if (rarely()) {
-                        builder.field("_index", "");
+        final int nbDocs = randomIntBetween(1, 20);
+        final String[] types = new String[nbDocs];
+        final String[] ids = new String[nbDocs];
+        final BytesReference[] sources = new BytesReference[nbDocs];
+
+        final BytesStreamOutput content = new BytesStreamOutput();
+        try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, content)) {
+            for (int i = 0; i < nbDocs; i++) {
+                builder.startObject();
+                {
+                    builder.startObject("index");
+                    {
+                        if (rarely()) {
+                            builder.field("_index", "");
+                        }
+                        if (defaultType == null || randomBoolean()) {
+                            types[i] = randomAlphaOfLength(5);
+                            builder.field("_type", types[i]);
+                        }
+                        if (randomBoolean()) {
+                            ids[i] = randomAlphaOfLength(10);
+                            builder.field("_id", ids[i]);
+                        }
                     }
-                    if (randomBoolean()) {
-                        types[i] = randomAlphaOfLength(5);
-                        builder.field("_type", types[i]);
-                    }
-                    builder.endObject().endObject().flush();
-                    content.write(xContentType.xContent().streamSeparator());
-                    builder.startObject().field("foo").value(i).endObject().flush();
-                    content.write(xContentType.xContent().streamSeparator());
+                    builder.endObject();
                 }
+                builder.endObject();
+
+                builder.flush();
+                content.write(xContentType.xContent().streamSeparator());
+
+                sources[i] = RandomObjects.randomSource(random(), xContentType);
+                BytesRef bytes = sources[i].toBytesRef();
+                content.write(bytes.bytes, bytes.offset, bytes.length);
+
+                content.write(xContentType.xContent().streamSeparator());
             }
 
-            String defaultMonitoringId = randomBoolean() ? randomAlphaOfLength(2) : null;
-            String defaultMonitoringVersion = randomBoolean() ? randomAlphaOfLength(3) : null;
-            String defaultType = rarely() ? randomAlphaOfLength(4) : null;
-
-            MonitoringBulkRequest request = new MonitoringBulkRequest();
-            request.add(content.bytes(), defaultMonitoringId, defaultMonitoringVersion, defaultType, xContentType);
-            assertThat(request.getDocs(), hasSize(nbDocs - dataCount));
-
-            i = 0;
-
-            for (final MonitoringBulkDoc doc : request.getDocs()) {
-                final String expectedType = types[i] != null ? types[i] : defaultType;
-
-                assertThat(doc.getMonitoringId(), equalTo(defaultMonitoringId));
-                assertThat(doc.getMonitoringVersion(), equalTo(defaultMonitoringVersion));
-                assertThat(doc.getType(), equalTo(expectedType));
-
-                ++i;
-            }
+            content.write(xContentType.xContent().streamSeparator());
         }
+
+        final MonitoredSystem system = randomFrom(MonitoredSystem.values());
+        final long timestamp = randomNonNegativeLong();
+        final long interval = randomNonNegativeLong();
+
+        final MonitoringBulkRequest bulkRequest = new MonitoringBulkRequest();
+        bulkRequest.add(system, defaultType, content.bytes(), xContentType, timestamp, interval);
+
+        final Collection<MonitoringBulkDoc> bulkDocs = bulkRequest.getDocs();
+        assertNotNull(bulkDocs);
+        assertEquals(nbDocs, bulkDocs.size());
+
+        int count = 0;
+        for (final MonitoringBulkDoc bulkDoc : bulkDocs) {
+            assertThat(bulkDoc.getSystem(), equalTo(system));
+            assertThat(bulkDoc.getType(), equalTo(types[count] != null ? types[count] : defaultType));
+            assertThat(bulkDoc.getId(), equalTo(ids[count]));
+            assertThat(bulkDoc.getTimestamp(), equalTo(timestamp));
+            assertThat(bulkDoc.getInterval(), equalTo(interval));
+            assertThat(bulkDoc.getSource(), equalTo(sources[count]));
+            assertThat(bulkDoc.getXContentType(), equalTo(xContentType));
+            ++count;
+        }
+    }
+
+    public void testAddRequestContentWithEmptySource() throws IOException {
+        final int nbDocs = randomIntBetween(0, 5);
+        final int nbEmptyDocs = randomIntBetween(1, 10);
+        final int totalDocs = nbDocs + nbEmptyDocs;
+
+        final XContentType xContentType = XContentType.JSON;
+        final byte separator = xContentType.xContent().streamSeparator();
+
+        final BytesStreamOutput content = new BytesStreamOutput();
+        try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, content)) {
+            for (int i = 0; i < totalDocs; i++) {
+                builder.startObject();
+                {
+                    builder.startObject("index");
+                    {
+                        builder.field("_index", "");
+                        builder.field("_type", "doc");
+                        builder.field("_id", String.valueOf(i));
+                    }
+                    builder.endObject();
+                }
+                builder.endObject();
+
+                builder.flush();
+                content.write(separator);
+
+                final BytesRef bytes;
+                if (i < nbDocs) {
+                    bytes = RandomObjects.randomSource(random(), xContentType).toBytesRef();
+                } else {
+                    bytes = BytesArray.EMPTY.toBytesRef();
+                }
+                content.write(bytes.bytes, bytes.offset, bytes.length);
+                content.write(separator);
+            }
+
+            content.write(separator);
+        }
+
+        final MonitoringBulkRequest bulkRequest = new MonitoringBulkRequest();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            bulkRequest.add(randomFrom(MonitoredSystem.values()), null, content.bytes(), xContentType, 0L, 0L)
+        );
+
+        assertThat(e.getMessage(), containsString("source is missing for monitoring document [][doc][" + nbDocs + "]"));
+    }
+
+    public void testAddRequestContentWithUnrecognizedIndexName() throws IOException {
+        final String indexName = randomAlphaOfLength(10);
+
+        final XContentType xContentType = XContentType.JSON;
+        final byte separator = xContentType.xContent().streamSeparator();
+
+        final BytesStreamOutput content = new BytesStreamOutput();
+        try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType, content)) {
+            builder.startObject();
+            {
+                builder.startObject("index");
+                {
+                    builder.field("_index", indexName);
+                    builder.field("_type", "doc");
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+
+            builder.flush();
+            content.write(separator);
+
+            final BytesRef bytes = RandomObjects.randomSource(random(), xContentType).toBytesRef();
+            content.write(bytes.bytes, bytes.offset, bytes.length);
+            content.write(separator);
+
+            content.write(separator);
+        }
+
+        final MonitoringBulkRequest bulkRequest = new MonitoringBulkRequest();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+                bulkRequest.add(randomFrom(MonitoredSystem.values()), null, content.bytes(), xContentType, 0L, 0L)
+        );
+
+        assertThat(e.getMessage(), containsString("unrecognized index name [" + indexName + "]"));
     }
 
     public void testSerialization() throws IOException {
-        MonitoringBulkRequest request = new MonitoringBulkRequest();
+        final MonitoringBulkRequest originalRequest = new MonitoringBulkRequest();
 
-        int numDocs = iterations(10, 30);
+        final int numDocs = iterations(10, 30);
         for (int i = 0; i < numDocs; i++) {
-            request.add(MonitoringBulkDocTests.newRandomMonitoringBulkDoc());
+            originalRequest.add(randomMonitoringBulkDoc());
         }
 
-        BytesStreamOutput out = new BytesStreamOutput();
-        out.setVersion(randomVersion(random()));
-        request.writeTo(out);
+        final BytesStreamOutput out = new BytesStreamOutput();
+        originalRequest.writeTo(out);
 
-        StreamInput in = out.bytes().streamInput();
+        final StreamInput in = out.bytes().streamInput();
         in.setVersion(out.getVersion());
-        MonitoringBulkRequest request2 = new MonitoringBulkRequest();
-        request2.readFrom(in);
+
+        final MonitoringBulkRequest deserializedRequest = new MonitoringBulkRequest();
+        deserializedRequest.readFrom(in);
 
         assertThat(in.available(), equalTo(0));
-        assertThat(request2.docs.size(), equalTo(request.docs.size()));
 
-        for (int i = 0; i < request2.docs.size(); i++) {
-            MonitoringBulkDoc doc = request.docs.get(i);
-            MonitoringBulkDoc doc2 = request2.docs.get(i);
-            assertThat(doc2.getMonitoringId(), equalTo(doc.getMonitoringId()));
-            assertThat(doc2.getMonitoringVersion(), equalTo(doc.getMonitoringVersion()));
-            assertThat(doc2.getClusterUUID(), equalTo(doc.getClusterUUID()));
-            assertThat(doc2.getTimestamp(), equalTo(doc.getTimestamp()));
-            assertThat(doc2.getSourceNode(), equalTo(doc.getSourceNode()));
-            assertThat(doc2.getIndex(), equalTo(doc.getIndex()));
-            assertThat(doc2.getType(), equalTo(doc.getType()));
-            assertThat(doc2.getId(), equalTo(doc.getId()));
-            assertThat(doc2.getSource(), equalTo(doc.getSource()));
-            assertThat(doc2.getXContentType(), equalTo(doc.getXContentType()));
+        final MonitoringBulkDoc[] originalBulkDocs = originalRequest.getDocs().toArray(new MonitoringBulkDoc[]{});
+        final MonitoringBulkDoc[] deserializedBulkDocs = deserializedRequest.getDocs().toArray(new MonitoringBulkDoc[]{});
+
+        assertArrayEquals(originalBulkDocs, deserializedBulkDocs);
+    }
+
+    public void testSerializationBwc() throws IOException {
+        final MonitoringBulkRequest originalRequest = new MonitoringBulkRequest();
+
+        final int numDocs = iterations(10, 30);
+        for (int i = 0; i < numDocs; i++) {
+            originalRequest.add(randomMonitoringBulkDoc());
+        }
+
+        final Version version = randomVersionBetween(random(), Version.V_5_0_0, Version.V_7_0_0_alpha1);
+
+        final BytesStreamOutput out = new BytesStreamOutput();
+        out.setVersion(version);
+        originalRequest.writeTo(out);
+
+        final StreamInput in = out.bytes().streamInput();
+        in.setVersion(out.getVersion());
+
+        final MonitoringBulkRequest deserializedRequest = new MonitoringBulkRequest();
+        deserializedRequest.readFrom(in);
+
+        assertThat(in.available(), equalTo(0));
+
+        final MonitoringBulkDoc[] originalBulkDocs = originalRequest.getDocs().toArray(new MonitoringBulkDoc[]{});
+        final MonitoringBulkDoc[] deserializedBulkDocs = deserializedRequest.getDocs().toArray(new MonitoringBulkDoc[]{});
+
+        assertThat(originalBulkDocs.length, equalTo(deserializedBulkDocs.length));
+
+        for (int i = 0; i < originalBulkDocs.length; i++) {
+            final MonitoringBulkDoc original = originalBulkDocs[i];
+            final MonitoringBulkDoc deserialized = deserializedBulkDocs[i];
+
+            assertThat(deserialized.getSystem(), equalTo(original.getSystem()));
+            assertThat(deserialized.getType(), equalTo(original.getType()));
+            assertThat(deserialized.getId(), equalTo(original.getId()));
+            assertThat(deserialized.getTimestamp(), equalTo(original.getTimestamp()));
+            assertThat(deserialized.getSource(), equalTo(original.getSource()));
+            assertThat(deserialized.getXContentType(), equalTo(original.getXContentType()));
+
+            if (version.onOrAfter(Version.V_7_0_0_alpha1)) {
+                assertThat(deserialized.getInterval(), equalTo(original.getInterval()));
+            } else {
+                assertThat(deserialized.getInterval(), equalTo(0L));
+            }
         }
     }
 
-    @SuppressWarnings("unchecked")
-    private static <T> void assertValidationErrors(MonitoringBulkRequest request, Matcher<? super T> matcher) {
-        ActionRequestValidationException validation = request.validate();
-        if (validation != null) {
-            assertThat((T) validation.validationErrors(), matcher);
-        } else {
-            assertThat(null, matcher);
-        }
+    /**
+     * Return a {@link XContentType} supported by the Monitoring Bulk API (JSON or Smile)
+     */
+    private XContentType randomXContentType() {
+        return randomFrom(XContentType.JSON, XContentType.SMILE);
+    }
+
+    private MonitoringBulkDoc randomMonitoringBulkDoc() throws IOException {
+        return MonitoringTestUtils.randomMonitoringBulkDoc(random(), randomXContentType());
     }
 }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkTests.java

@@ -1,198 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.action;
-
-import org.apache.http.Header;
-import org.apache.http.HttpHeaders;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.message.BasicHeader;
-import org.apache.lucene.util.LuceneTestCase;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.client.RestClient;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.collect.MapBuilder;
-import org.elasticsearch.common.network.NetworkModule;
-import org.elasticsearch.common.settings.SecureString;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.monitoring.resolver.bulk.MonitoringBulkTimestampedResolver;
-import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
-import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
-
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.hamcrest.Matchers.empty;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-
-@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/353")
-public class MonitoringBulkTests extends MonitoringIntegTestCase {
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(NetworkModule.HTTP_ENABLED.getKey(), securityEnabled).build();
-    }
-
-    @Override
-    protected Settings transportClientSettings() {
-        return super.transportClientSettings();
-    }
-
-    public void testMonitoringBulkIndexing() throws Exception {
-        MonitoringBulkRequestBuilder requestBuilder = monitoringClient().prepareMonitoringBulk();
-        String[] types = {"type1", "type2", "type3"};
-
-        int numDocs = scaledRandomIntBetween(100, 5000);
-        for (int i = 0; i < numDocs; i++) {
-            MonitoringBulkDoc doc = new MonitoringBulkDoc(MonitoredSystem.KIBANA.getSystem(),
-                    MonitoringTemplateUtils.TEMPLATE_VERSION, null, randomFrom(types), null,
-                    jsonBuilder().startObject().field("num", numDocs).endObject().bytes(),
-                    XContentType.JSON);
-            requestBuilder.add(doc);
-        }
-
-        MonitoringBulkResponse response = requestBuilder.get();
-        assertThat(response.getError(), is(nullValue()));
-        refresh();
-
-        SearchResponse searchResponse = client().prepareSearch().setTypes(types).setSize(numDocs).get();
-        assertHitCount(searchResponse, numDocs);
-
-        for (SearchHit searchHit : searchResponse.getHits()) {
-            Map<String, Object> source = searchHit.getSourceAsMap();
-            assertNotNull(source.get(MonitoringBulkTimestampedResolver.Fields.CLUSTER_UUID));
-            assertNotNull(source.get(MonitoringBulkTimestampedResolver.Fields.TIMESTAMP));
-            assertNotNull(source.get(MonitoringBulkTimestampedResolver.Fields.SOURCE_NODE));
-        }
-    }
-
-    /**
-     * This test creates N threads that execute a random number of monitoring bulk requests.
-     */
-    public void testConcurrentRequests() throws Exception {
-        final int numberThreads = randomIntBetween(3, 5);
-        final Thread[] threads = new Thread[numberThreads];
-        final CountDownLatch latch = new CountDownLatch(numberThreads + 1);
-        final List<Throwable> exceptions = new CopyOnWriteArrayList<>();
-
-        AtomicLong total = new AtomicLong(0);
-
-        logger.info("--> using {} concurrent clients to execute requests", threads.length);
-        for (int i = 0; i < threads.length; i++) {
-            final int nbRequests = randomIntBetween(1, 5);
-
-            threads[i] = new Thread(new AbstractRunnable() {
-                @Override
-                public void onFailure(Exception e) {
-                    logger.error("unexpected error in exporting thread", e);
-                    exceptions.add(e);
-                }
-
-                @Override
-                protected void doRun() throws Exception {
-                    latch.countDown();
-                    latch.await();
-                    for (int j = 0; j < nbRequests; j++) {
-                        MonitoringBulkRequestBuilder requestBuilder = monitoringClient().prepareMonitoringBulk();
-
-                        int numDocs = scaledRandomIntBetween(10, 50);
-                        for (int k = 0; k < numDocs; k++) {
-                            BytesReference source =  jsonBuilder().startObject()
-                                                                    .field("num", k)
-                                                                  .endObject().bytes();
-                            MonitoringBulkDoc doc =
-                                    new MonitoringBulkDoc(MonitoredSystem.KIBANA.getSystem(),
-                                            MonitoringTemplateUtils.TEMPLATE_VERSION, null,
-                                            "concurrent", null, source, XContentType.JSON);
-                            requestBuilder.add(doc);
-                        }
-
-                        total.addAndGet(numDocs);
-                        MonitoringBulkResponse response = requestBuilder.get();
-                        assertNull (response.getError());
-                    }
-                }
-            }, "export_thread_" + i);
-            threads[i].start();
-        }
-
-        // wait for all threads to be ready
-        latch.countDown();
-        latch.await();
-
-        // wait for all threads to finish
-        for (Thread thread : threads) {
-            thread.join();
-        }
-
-        assertThat(exceptions, empty());
-        awaitMonitoringDocsCountOnPrimary(greaterThanOrEqualTo(total.get()), "concurrent");
-    }
-
-    public void testUnsupportedSystem() throws Exception {
-        MonitoringBulkRequestBuilder requestBuilder = monitoringClient().prepareMonitoringBulk();
-        String[] types = {"type1", "type2", "type3"};
-
-        int totalDocs = randomIntBetween(10, 1000);
-        int unsupportedDocs = 0;
-
-        for (int i = 0; i < totalDocs; i++) {
-            BytesReference source = jsonBuilder().startObject().field("num", i).endObject().bytes();
-            MonitoringBulkDoc doc;
-            if (randomBoolean()) {
-                doc = new MonitoringBulkDoc("unknown", MonitoringTemplateUtils.TEMPLATE_VERSION,
-                        null, randomFrom(types), null, source, XContentType.JSON);
-                unsupportedDocs++;
-            } else {
-                doc = new MonitoringBulkDoc(MonitoredSystem.KIBANA.getSystem(), MonitoringTemplateUtils.TEMPLATE_VERSION,
-                        null, randomFrom(types), null, source, XContentType.JSON);
-            }
-            requestBuilder.add(doc);
-        }
-
-        MonitoringBulkResponse response = requestBuilder.get();
-        if (unsupportedDocs == 0) {
-            assertThat(response.getError(), is(nullValue()));
-        } else {
-            assertThat(response.getError(), is(notNullValue()));
-        }
-        refresh();
-
-        SearchResponse countResponse = client().prepareSearch().setTypes(types).setSize(0).get();
-        assertHitCount(countResponse, totalDocs - unsupportedDocs);
-    }
-
-    public void testOverHttp() throws Exception {
-        final String contentType = randomFrom("application/json", "application/x-ndjson");
-        assumeTrue("security needs to be enabled for the http transport type to be set", securityEnabled);
-        RestClient restClient = getRestClient();
-        final Header authorization = new BasicHeader(HttpHeaders.AUTHORIZATION,
-                    UsernamePasswordToken.basicAuthHeaderValue(SecuritySettings.TEST_USERNAME,
-                            new SecureString(SecuritySettings.TEST_PASSWORD.toCharArray())));
-        Response response = restClient.performRequest("POST", "/_xpack/monitoring/_bulk",
-                MapBuilder.<String, String>newMapBuilder().put("system_id", MonitoredSystem.KIBANA.getSystem())
-                        .put("system_api_version", MonitoringTemplateUtils.TEMPLATE_VERSION)
-                        .put("interval", "10s").immutableMap(),
-                new StringEntity("{\"index\":{\"_index\":\"\",\"_type\":\"kibana\",\"_id\":\"1\"} }\n" +
-                "{ \"field1\" : \"value1\" }\n", ContentType.create(contentType)), authorization);
-        assertEquals(RestStatus.OK.getStatus(), response.getStatusLine().getStatusCode());
-    }
-}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java

@@ -5,261 +5,351 @@
  */
 package org.elasticsearch.xpack.monitoring.action;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilter;
 import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.settings.ClusterSettings;
-import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.test.ClusterServiceUtils;
+import org.elasticsearch.tasks.TaskAwareRequest;
+import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.transport.CapturingTransport;
-import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.test.RandomObjects;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.MonitoringSettings;
-import org.elasticsearch.xpack.monitoring.exporter.ExportException;
+import org.elasticsearch.xpack.monitoring.MonitoringTestUtils;
+import org.elasticsearch.xpack.monitoring.exporter.BytesReferenceMonitoringDoc;
 import org.elasticsearch.xpack.monitoring.exporter.Exporters;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
-import org.junit.After;
-import org.junit.AfterClass;
 import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
-import org.junit.rules.ExpectedException;
+import org.mockito.ArgumentCaptor;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
+import java.util.List;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Consumer;
 
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.emptySet;
-import static org.elasticsearch.test.VersionUtils.randomVersion;
+import static org.elasticsearch.Version.CURRENT;
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-import static org.hamcrest.core.IsEqual.equalTo;
+import static org.hamcrest.Matchers.startsWith;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
+/**
+ * Tests {@link TransportMonitoringBulkAction}
+ */
 public class TransportMonitoringBulkActionTests extends ESTestCase {
 
-    private static ThreadPool threadPool;
-
-    @Rule
-    public ExpectedException expectedException = ExpectedException.none();
-
+    private ActionListener<MonitoringBulkResponse> listener;
+    private Exporters exporters;
+    private ThreadPool threadPool;
     private ClusterService clusterService;
-    private final XPackLicenseState licenseState = mock(XPackLicenseState.class);
     private TransportService transportService;
-    private CapturingExporters exportService;
-    private TransportMonitoringBulkAction action;
-
-    @BeforeClass
-    public static void beforeClass() {
-        threadPool = new TestThreadPool(TransportMonitoringBulkActionTests.class.getSimpleName());
-    }
-
-    @AfterClass
-    public static void afterClass() {
-        ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS);
-        threadPool = null;
-    }
+    private ActionFilters filters;
+    private IndexNameExpressionResolver resolver;
+    private XPackLicenseState licenseState;
+    private TaskManager taskManager;
 
     @Before
-    public void setUp() throws Exception {
-        super.setUp();
-        CapturingTransport transport = new CapturingTransport();
-        Set<Setting<?>>  clusterSettings = new HashSet<>();
-        clusterSettings.addAll(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        clusterSettings.add(MonitoringSettings.EXPORTERS_SETTINGS);
-        final DiscoveryNode node = new DiscoveryNode("node", buildNewFakeTransportAddress(), emptyMap(), emptySet(),
-                Version.CURRENT);
-        clusterService = ClusterServiceUtils.createClusterService(threadPool, node, new ClusterSettings(Settings.EMPTY, clusterSettings));
-        transportService = new TransportService(clusterService.getSettings(), transport, threadPool,
-                TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> node, null);
-        transportService.start();
-        transportService.acceptIncomingRequests();
-        exportService = new CapturingExporters();
-        action = new TransportMonitoringBulkAction(
-                Settings.EMPTY,
-                threadPool,
-                clusterService,
-                transportService,
-                new ActionFilters(Collections.emptySet()),
-                new IndexNameExpressionResolver(Settings.EMPTY),
-                exportService
-        );
+    @SuppressWarnings("unchecked")
+    public void setUpMocks() throws Exception {
+        listener = mock(ActionListener.class);
+        exporters = mock(Exporters.class);
+        threadPool = mock(ThreadPool.class);
+        clusterService = mock(ClusterService.class);
+        transportService = mock(TransportService.class);
+        filters = mock(ActionFilters.class);
+        resolver = mock(IndexNameExpressionResolver.class);
+        licenseState = mock(XPackLicenseState.class);
+        taskManager = mock(TaskManager.class);
+
+        when(transportService.getTaskManager()).thenReturn(taskManager);
+        when(taskManager.register(anyString(), eq(MonitoringBulkAction.NAME), any(TaskAwareRequest.class))).thenReturn(null);
+        when(filters.filters()).thenReturn(new ActionFilter[0]);
     }
 
-    @After
-    public void tearDown() throws Exception {
-        super.tearDown();
-        clusterService.close();
-        transportService.close();
-    }
+    public void testExecuteWithGlobalBlock() throws Exception {
+        final ClusterBlocks.Builder clusterBlock = ClusterBlocks.builder().addGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ALL);
+        when(clusterService.state()).thenReturn(ClusterState.builder(ClusterName.DEFAULT).blocks(clusterBlock).build());
 
-    public void testGlobalBlock() throws Exception {
-        expectedException.expect(ExecutionException.class);
-        expectedException.expect(hasToString(containsString("ClusterBlockException[blocked by: [SERVICE_UNAVAILABLE/2/no master]")));
+        final TransportMonitoringBulkAction action = new TransportMonitoringBulkAction(Settings.EMPTY, threadPool, clusterService,
+                                                                                       transportService, filters, resolver, exporters);
 
-        final ClusterBlocks.Builder block = ClusterBlocks.builder().addGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ALL);
-        ClusterState currentState = clusterService.state();
-        ClusterServiceUtils.setState(clusterService,
-                ClusterState.builder(currentState).blocks(block).version(currentState.version() + 1).build());
-
-        MonitoringBulkRequest request = randomRequest();
-        action.execute(request).get();
-    }
-
-    public void testEmptyRequest() throws Exception {
-        expectedException.expect(ExecutionException.class);
-        expectedException.expect(hasToString(containsString("no monitoring documents added")));
-
-        MonitoringBulkRequest request = randomRequest(0);
-        action.execute(request).get();
-
-        assertThat(exportService.getExported(), hasSize(0));
-    }
-
-    public void testBasicRequest() throws Exception {
-        MonitoringBulkRequest request = randomRequest();
-        action.execute(request).get();
-
-        assertThat(exportService.getExported(), hasSize(request.getDocs().size()));
-    }
-
-    public void testAsyncActionPrepareDocs() throws Exception {
-        final PlainActionFuture<MonitoringBulkResponse> listener = new PlainActionFuture<>();
         final MonitoringBulkRequest request = randomRequest();
+        final ExecutionException e = expectThrows(ExecutionException.class, () -> action.execute(request).get());
 
-        Collection<MonitoringDoc> results = action.new AsyncAction(request, listener, exportService, clusterService)
-                                                            .prepareForExport(request.getDocs());
+        assertThat(e, hasToString(containsString("ClusterBlockException[blocked by: [SERVICE_UNAVAILABLE/2/no master]")));
+    }
 
-        assertThat(results, hasSize(request.getDocs().size()));
-        for (MonitoringDoc exported : results) {
-            assertThat(exported.getClusterUUID(), equalTo(clusterService.state().metaData().clusterUUID()));
-            assertThat(exported.getTimestamp(), greaterThan(0L));
-            assertThat(exported.getSourceNode(), notNullValue());
-            assertThat(exported.getSourceNode().getUUID(), equalTo(clusterService.localNode().getId()));
-            assertThat(exported.getSourceNode().getName(), equalTo(clusterService.localNode().getName()));
+    public void testExecuteEmptyRequest() throws Exception {
+        final TransportMonitoringBulkAction action = new TransportMonitoringBulkAction(Settings.EMPTY, threadPool, clusterService,
+                                                                                       transportService, filters, resolver, exporters);
+
+        final MonitoringBulkRequest request = new MonitoringBulkRequest();
+        final ExecutionException e = expectThrows(ExecutionException.class, () -> action.execute(request).get());
+
+        assertThat(e, hasToString(containsString("no monitoring documents added")));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testExecuteRequest() throws Exception {
+        final DiscoveryNode discoveryNode =  new DiscoveryNode("_id", new TransportAddress(TransportAddress.META_ADDRESS, 9300), CURRENT);
+        when(clusterService.localNode()).thenReturn(discoveryNode);
+
+        final String clusterUUID = UUIDs.randomBase64UUID();
+        when(clusterService.state()).thenReturn(ClusterState.builder(ClusterName.DEFAULT)
+                                                            .metaData(MetaData.builder().clusterUUID(clusterUUID).build())
+                                                            .build());
+
+        final MonitoringBulkRequest request = new MonitoringBulkRequest();
+
+        final MonitoredSystem system = randomFrom(MonitoredSystem.KIBANA, MonitoredSystem.BEATS, MonitoredSystem.LOGSTASH);
+        final String type = randomAlphaOfLength(5);
+        final String id = randomBoolean() ? randomAlphaOfLength(5) : null;
+        final long timestamp = randomNonNegativeLong();
+        final XContentType xContentType = randomFrom(XContentType.values());
+
+        final int nbDocs = randomIntBetween(1, 50);
+        for (int i = 0; i < nbDocs; i++) {
+            MonitoringBulkDoc mockBulkDoc = mock(MonitoringBulkDoc.class);
+            request.add(mockBulkDoc);
+
+            when(mockBulkDoc.getSystem()).thenReturn(system);
+            when(mockBulkDoc.getType()).thenReturn(type);
+            when(mockBulkDoc.getId()).thenReturn(id + String.valueOf(i));
+            when(mockBulkDoc.getTimestamp()).thenReturn(timestamp);
+            when(mockBulkDoc.getSource()).thenReturn(RandomObjects.randomSource(random(), xContentType));
+            when(mockBulkDoc.getXContentType()).thenReturn(xContentType);
         }
+
+        doAnswer((i) -> {
+            final Collection<MonitoringDoc> exportedDocs = (Collection) i.getArguments()[0];
+            assertEquals(nbDocs, exportedDocs.size());
+            exportedDocs.forEach(exportedDoc -> {
+                assertThat(exportedDoc, instanceOf(BytesReferenceMonitoringDoc.class));
+                assertThat(exportedDoc.getSystem(), equalTo(system));
+                assertThat(exportedDoc.getType(), equalTo(type));
+                assertThat(exportedDoc.getId(), startsWith(id));
+                assertThat(exportedDoc.getTimestamp(), equalTo(timestamp));
+                assertThat(exportedDoc.getNode().getUUID(), equalTo(discoveryNode.getId()));
+                assertThat(exportedDoc.getCluster(), equalTo(clusterUUID));
+            });
+
+            final ActionListener listener = (ActionListener) i.getArguments()[1];
+            listener.onResponse(null);
+            return Void.TYPE;
+        }).when(exporters).export(any(Collection.class), any(ActionListener.class));
+
+        final TransportMonitoringBulkAction action = new TransportMonitoringBulkAction(Settings.EMPTY, threadPool, clusterService,
+                                                                                       transportService, filters, resolver, exporters);
+        action.execute(request).get();
+
+        verify(exporters).export(any(Collection.class), any(ActionListener.class));
+        verify(clusterService, times(2)).state();
+        verify(clusterService).localNode();
     }
 
-    public void testAsyncActionExecuteExport() throws Exception {
-        final PlainActionFuture<MonitoringBulkResponse> listener = new PlainActionFuture<>();
-        final MonitoringBulkRequest request = randomRequest();
-        final Collection<MonitoringDoc> docs = Collections.unmodifiableCollection(request.getDocs());
+    public void testAsyncActionCreateMonitoringDocsWithNoDocs() throws Exception {
+        final Collection<MonitoringBulkDoc> bulkDocs = new ArrayList<>();
+        if (randomBoolean()) {
+            final int nbDocs = randomIntBetween(1, 50);
+            for (int i = 0; i < nbDocs; i++) {
+                MonitoringBulkDoc mockBulkDoc = mock(MonitoringBulkDoc.class);
+                bulkDocs.add(mockBulkDoc);
+                when(mockBulkDoc.getSystem()).thenReturn(MonitoredSystem.UNKNOWN);
+            }
+        }
 
-        action.new AsyncAction(request, listener, exportService, clusterService).executeExport(docs, 0L, listener);
-        assertThat(listener.get().getError(), nullValue());
+        final Collection<MonitoringDoc> results =
+                new TransportMonitoringBulkAction.AsyncAction(null, null, null, null, 0L, null).createMonitoringDocs(bulkDocs);
 
-        Collection<MonitoringDoc> exported = exportService.getExported();
-        assertThat(exported, hasSize(request.getDocs().size()));
+        assertThat(results, notNullValue());
+        assertThat(results.size(), equalTo(0));
     }
 
-    public void testAsyncActionExportThrowsException() throws Exception {
-        final PlainActionFuture<MonitoringBulkResponse> listener = new PlainActionFuture<>();
-        final MonitoringBulkRequest request = randomRequest();
+    public void testAsyncActionCreateMonitoringDocs() throws Exception {
+        final List<MonitoringBulkDoc> docs = new ArrayList<>();
 
-        final Exporters exporters = new ConsumingExporters(docs -> {
-            throw new IllegalStateException();
+        final MonitoredSystem system = randomFrom(MonitoredSystem.KIBANA, MonitoredSystem.BEATS, MonitoredSystem.LOGSTASH);
+        final String type = randomAlphaOfLength(5);
+        final String id = randomBoolean() ? randomAlphaOfLength(5) : null;
+        final long timestamp = randomBoolean() ? randomNonNegativeLong() : 0L;
+        final XContentType xContentType = randomFrom(XContentType.values());
+        final MonitoringDoc.Node node = MonitoringTestUtils.randomMonitoringNode(random());
+
+        final int nbDocs = randomIntBetween(1, 50);
+        for (int i = 0; i < nbDocs; i++) {
+            MonitoringBulkDoc mockBulkDoc = mock(MonitoringBulkDoc.class);
+            docs.add(mockBulkDoc);
+
+            when(mockBulkDoc.getSystem()).thenReturn(system);
+            when(mockBulkDoc.getType()).thenReturn(type);
+            when(mockBulkDoc.getId()).thenReturn(id);
+            when(mockBulkDoc.getTimestamp()).thenReturn(timestamp);
+            when(mockBulkDoc.getSource()).thenReturn(BytesArray.EMPTY);
+            when(mockBulkDoc.getXContentType()).thenReturn(xContentType);
+        }
+
+        final Collection<MonitoringDoc> exportedDocs =
+                new TransportMonitoringBulkAction.AsyncAction(null, null, null, "_cluster", 123L, node).createMonitoringDocs(docs);
+
+        assertThat(exportedDocs, notNullValue());
+        assertThat(exportedDocs.size(), equalTo(nbDocs));
+        exportedDocs.forEach(exportedDoc -> {
+            assertThat(exportedDoc.getSystem(), equalTo(system));
+            assertThat(exportedDoc.getType(), equalTo(type));
+            assertThat(exportedDoc.getId(), equalTo(id));
+            assertThat(exportedDoc.getTimestamp(), equalTo(timestamp != 0L ? timestamp : 123L));
+            assertThat(exportedDoc.getNode(), equalTo(node));
+            assertThat(exportedDoc.getCluster(), equalTo("_cluster"));
         });
+    }
 
-        action.new AsyncAction(request, listener, exporters, clusterService).start();
-        assertThat(listener.get().getError(), notNullValue());
-        assertThat(listener.get().getError().getCause(), instanceOf(IllegalStateException.class));
+    public void testAsyncActionCreateMonitoringDocWithNoTimestamp() throws Exception {
+        final MonitoringBulkDoc monitoringBulkDoc =
+            new MonitoringBulkDoc(MonitoredSystem.BEATS, "_type", "_id", 0L, 0L, BytesArray.EMPTY, XContentType.JSON);
+
+        final MonitoringDoc monitoringDoc =
+            new TransportMonitoringBulkAction.AsyncAction(null, null, null, "", 456L, null).createMonitoringDoc(monitoringBulkDoc);
+        assertThat(monitoringDoc.getTimestamp(), equalTo(456L));
+    }
+
+    public void testAsyncActionCreateMonitoringDoc() throws Exception {
+        final MonitoringDoc.Node node = new MonitoringDoc.Node("_uuid", "_host", "_addr", "_ip", "_name", 1504169190855L);
+
+        final XContentType xContentType = randomFrom(XContentType.values());
+        final BytesReference source = XContentBuilder.builder(xContentType.xContent())
+                                                        .startObject()
+                                                            .startObject("_foo")
+                                                                .field("_bar", "_baz")
+                                                            .endObject()
+                                                        .endObject().bytes();
+
+        final MonitoringBulkDoc monitoringBulkDoc =
+                new MonitoringBulkDoc(MonitoredSystem.LOGSTASH, "_type", "_id", 1502107402133L, 15_000L, source, xContentType);
+
+        final MonitoringDoc monitoringDoc =
+                new TransportMonitoringBulkAction.AsyncAction(null, null, null, "_cluster_uuid", 3L, node)
+                        .createMonitoringDoc(monitoringBulkDoc);
+
+        final BytesReference xContent = XContentHelper.toXContent(monitoringDoc, XContentType.JSON, randomBoolean());
+        assertEquals("{"
+                    + "\"cluster_uuid\":\"_cluster_uuid\","
+                    + "\"timestamp\":\"2017-08-07T12:03:22.133Z\","
+                    + "\"type\":\"_type\","
+                    + "\"source_node\":{"
+                        + "\"uuid\":\"_uuid\","
+                        + "\"host\":\"_host\","
+                        + "\"transport_address\":\"_addr\","
+                        + "\"ip\":\"_ip\","
+                        + "\"name\":\"_name\","
+                        + "\"timestamp\":\"2017-08-31T08:46:30.855Z\""
+                    + "},"
+                    + "\"_type\":{"
+                        + "\"_foo\":{"
+                            + "\"_bar\":\"_baz\""
+                        + "}"
+                    + "}"
+                + "}" , xContent.utf8ToString());
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testAsyncActionExecuteExport() throws Exception {
+        final int nbDocs = randomIntBetween(1, 25);
+        final Collection<MonitoringDoc> docs = new ArrayList<>(nbDocs);
+        for (int i = 0; i < nbDocs; i++) {
+            docs.add(mock(MonitoringDoc.class));
+        }
+
+        doAnswer((i) -> {
+            final Collection<MonitoringDoc> exportedDocs = (Collection) i.getArguments()[0];
+            assertThat(exportedDocs, is(docs));
+
+            final ActionListener listener = (ActionListener) i.getArguments()[1];
+            listener.onResponse(null);
+            return Void.TYPE;
+        }).when(exporters).export(any(Collection.class), any(ActionListener.class));
+
+        final TransportMonitoringBulkAction.AsyncAction asyncAction =
+                new TransportMonitoringBulkAction.AsyncAction(null, null, exporters, null, 0L, null);
+
+        asyncAction.executeExport(docs, randomNonNegativeLong(), listener);
+
+        verify(exporters).export(eq(docs), any(ActionListener.class));
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testAsyncActionExportThrowsException() throws Exception {
+        final int nbDocs = randomIntBetween(1, 25);
+        final Collection<MonitoringDoc> docs = new ArrayList<>(nbDocs);
+        for (int i = 0; i < nbDocs; i++) {
+            docs.add(mock(MonitoringDoc.class));
+        }
+
+        doThrow(new IllegalStateException("something went wrong"))
+                .when(exporters)
+                    .export(any(Collection.class), any(ActionListener.class));
+
+        final TransportMonitoringBulkAction.AsyncAction asyncAction =
+                new TransportMonitoringBulkAction.AsyncAction(null, null, exporters, null, 0L, null);
+
+        asyncAction.executeExport(docs, randomNonNegativeLong(), listener);
+
+        final ArgumentCaptor<Collection<MonitoringDoc>> argDocs = ArgumentCaptor.forClass((Class) Collection.class);
+        verify(exporters).export(argDocs.capture(), any(ActionListener.class));
+        assertThat(argDocs.getValue(), is(docs));
+
+        final ArgumentCaptor<MonitoringBulkResponse> argResponse = ArgumentCaptor.forClass(MonitoringBulkResponse.class);
+        verify(listener).onResponse(argResponse.capture());
+        assertThat(argResponse.getValue().getError(), notNullValue());
+        assertThat(argResponse.getValue().getError().getCause(), instanceOf(IllegalStateException.class));
     }
 
     /**
      * @return a new MonitoringBulkRequest instance with random number of documents
      */
-    private static MonitoringBulkRequest randomRequest() throws IOException {
+    private MonitoringBulkRequest randomRequest() throws IOException {
         return randomRequest(scaledRandomIntBetween(1, 100));
     }
 
     /**
-     * @return a new MonitoringBulkRequest instance with given number of documents
+     * Creates a {@link MonitoringBulkRequest} with the given number of {@link MonitoringBulkDoc} in it.
+     *
+     * @return the {@link MonitoringBulkRequest}
      */
-    private static MonitoringBulkRequest randomRequest(final int numDocs) throws IOException {
-        MonitoringBulkRequest request = new MonitoringBulkRequest();
+    private MonitoringBulkRequest randomRequest(final int numDocs) throws IOException {
+        final MonitoringBulkRequest request = new MonitoringBulkRequest();
         for (int i = 0; i < numDocs; i++) {
-            String monitoringId = randomFrom(MonitoredSystem.values()).getSystem();
-            String monitoringVersion = randomVersion(random()).toString();
-            MonitoringIndex index = randomBoolean() ? randomFrom(MonitoringIndex.values()) : null;
-            String type = randomFrom("type1", "type2", "type3");
-            String id = randomAlphaOfLength(10);
-
-            BytesReference source;
-            XContentType xContentType = randomFrom(XContentType.values());
-            try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
-                source = builder.startObject().field("key", i).endObject().bytes();
-            }
-            request.add(new MonitoringBulkDoc(monitoringId, monitoringVersion, index, type, id,
-                    null, 0L, null, source, xContentType));
+            request.add(MonitoringTestUtils.randomMonitoringBulkDoc(random()));
         }
         return request;
     }
-
-    /**
-     * A Exporters implementation that captures the documents to export
-     */
-    class CapturingExporters extends Exporters {
-
-        private final Collection<MonitoringDoc> exported = ConcurrentCollections.newConcurrentSet();
-
-        CapturingExporters() {
-            super(Settings.EMPTY, Collections.emptyMap(), clusterService, licenseState, threadPool.getThreadContext());
-        }
-
-        @Override
-        public synchronized void export(Collection<MonitoringDoc> docs, ActionListener<Void> listener) throws ExportException {
-            exported.addAll(docs);
-            listener.onResponse(null);
-        }
-
-        public Collection<MonitoringDoc> getExported() {
-            return exported;
-        }
-    }
-
-    /**
-     * A Exporters implementation that applies a Consumer when exporting documents
-     */
-    class ConsumingExporters extends Exporters {
-
-        private final Consumer<Collection<? extends MonitoringDoc>> consumer;
-
-        ConsumingExporters(Consumer<Collection<? extends MonitoringDoc>> consumer) {
-            super(Settings.EMPTY, Collections.emptyMap(), clusterService, licenseState, threadPool.getThreadContext());
-            this.consumer = consumer;
-        }
-
-        @Override
-        public synchronized void export(Collection<MonitoringDoc> docs, ActionListener<Void> listener) throws ExportException {
-            consumer.accept(docs);
-            listener.onResponse(null);
-        }
-    }
 }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/cleaner/AbstractIndicesCleanerTestCase.java

@@ -8,16 +8,15 @@ package org.elasticsearch.xpack.monitoring.cleaner;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.xpack.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.monitoring.MonitoringSettings;
 import org.elasticsearch.xpack.monitoring.exporter.Exporter;
 import org.elasticsearch.xpack.monitoring.exporter.Exporters;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
 import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
 
 import java.util.Locale;
 
@@ -202,12 +201,9 @@ public abstract class AbstractIndicesCleanerTestCase extends MonitoringIntegTest
      * Creates a monitoring timestamped index using a given template version.
      */
     protected void createTimestampedIndex(DateTime creationDate, String version) {
-        MonitoringDoc monitoringDoc = new MonitoringDoc(null, null, null, null, null,
-                creationDate.getMillis(), (MonitoringDoc.Node) null);
-
-        MonitoringIndexNameResolver.Timestamped resolver =
-                new MockTimestampedIndexNameResolver(MonitoredSystem.ES, Settings.EMPTY, version);
-        createIndex(resolver.index(monitoringDoc), creationDate);
+        final DateTimeFormatter formatter = DateTimeFormat.forPattern("YYYY.MM.dd").withZoneUTC();
+        final String index = ".monitoring-es-" + version + "-" + formatter.print(creationDate.getMillis());
+        createIndex(index, creationDate);
     }
 
     protected abstract void createIndex(String name, DateTime creationDate);

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/AbstractCollectorTestCase.java

@@ -1,65 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.collector;
-
-import java.util.Collection;
-import java.util.concurrent.TimeUnit;
-
-import org.elasticsearch.cluster.block.ClusterBlocks;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.xpack.monitoring.MonitoringSettings;
-import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
-import org.elasticsearch.xpack.security.InternalClient;
-
-@ClusterScope(scope = ESIntegTestCase.Scope.SUITE, transportClientRatio = 0.0)
-public abstract class AbstractCollectorTestCase extends MonitoringIntegTestCase {
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder()
-                .put(super.nodeSettings(nodeOrdinal))
-                .put(MonitoringSettings.INTERVAL.getKey(), "-1")
-                .build();
-    }
-
-    public InternalClient securedClient() {
-        return internalCluster().getInstance(InternalClient.class);
-    }
-
-    public InternalClient securedClient(String nodeId) {
-        return internalCluster().getInstance(InternalClient.class, nodeId);
-    }
-
-    protected void assertCanCollect(Collector collector) {
-        assertNotNull(collector);
-        assertTrue("collector [" + collector.name() + "] should be able to collect data", collector.shouldCollect());
-        Collection results = collector.collect();
-        assertNotNull(results);
-    }
-
-    public void waitForNoBlocksOnNodes() throws Exception {
-        assertBusy(() -> {
-            for (String nodeId : internalCluster().getNodeNames()) {
-                try {
-                    waitForNoBlocksOnNode(nodeId);
-                } catch (Exception e) {
-                    fail("failed to wait for no blocks on node [" + nodeId + "]: " + e.getMessage());
-                }
-            }
-        });
-    }
-
-    public void waitForNoBlocksOnNode(final String nodeId) throws Exception {
-        assertBusy(() -> {
-            ClusterBlocks clusterBlocks =
-                    client(nodeId).admin().cluster().prepareState().setLocal(true).execute().actionGet().getState().blocks();
-            assertTrue(clusterBlocks.global().isEmpty());
-            assertTrue(clusterBlocks.indices().values().isEmpty());
-        }, 30L, TimeUnit.SECONDS);
-    }
-}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/BaseCollectorTestCase.java

@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.monitoring.collector;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.monitoring.MonitoringSettings;
+import org.elasticsearch.xpack.security.InternalClient;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public abstract class BaseCollectorTestCase extends ESTestCase {
+
+    protected ClusterName clusterName;
+    protected ClusterService clusterService;
+    protected ClusterState clusterState;
+    protected DiscoveryNodes nodes;
+    protected MetaData metaData;
+    protected MonitoringSettings monitoringSettings;
+    protected XPackLicenseState licenseState;
+    protected InternalClient client;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        clusterName = mock(ClusterName.class);
+        clusterService = mock(ClusterService.class);
+        clusterState = mock(ClusterState.class);
+        nodes = mock(DiscoveryNodes.class);
+        metaData = mock(MetaData.class);
+        monitoringSettings = mock(MonitoringSettings.class);
+        licenseState = mock(XPackLicenseState.class);
+        client = mock(InternalClient.class);
+    }
+
+    protected void whenLocalNodeElectedMaster(final boolean electedMaster) {
+        when(clusterService.state()).thenReturn(clusterState);
+        when(clusterState.nodes()).thenReturn(nodes);
+        when(nodes.isLocalNodeElectedMaster()).thenReturn(electedMaster);
+    }
+
+    protected void whenClusterStateWithName(final String name) {
+        when(clusterName.value()).thenReturn(name);
+        when(clusterService.getClusterName()).thenReturn(clusterName);
+        when(clusterState.getClusterName()).thenReturn(clusterName);
+    }
+
+    protected void whenClusterStateWithUUID(final String clusterUUID) {
+        when(clusterService.state()).thenReturn(clusterState);
+        when(clusterState.metaData()).thenReturn(metaData);
+        when(metaData.clusterUUID()).thenReturn(clusterUUID);
+    }
+
+    protected static DiscoveryNode localNode(final String uuid) {
+        return new DiscoveryNode(uuid, new TransportAddress(TransportAddress.META_ADDRESS, 9300), Version.CURRENT);
+    }
+}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/CollectorTests.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.monitoring.collector;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+public class CollectorTests extends ESTestCase {
+
+    public void testConvertNullNode() {
+        assertEquals(null, Collector.convertNode(randomNonNegativeLong(), null));
+    }
+
+    public void testConvertNode() {
+        final String name = randomBoolean() ? randomAlphaOfLength(5) : "";
+        final String nodeId = randomAlphaOfLength(5);
+        final TransportAddress address = buildNewFakeTransportAddress();
+        final Version version = randomFrom(Version.V_5_0_1, Version.V_5_3_0, Version.CURRENT);
+        final long timestamp = randomNonNegativeLong();
+
+        final Set<DiscoveryNode.Role> roles = new HashSet<>();
+        if (randomBoolean()) {
+            roles.addAll(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values())));
+        }
+
+        final MonitoringDoc.Node expectedNode = new MonitoringDoc.Node(nodeId, address.address().getHostString(), address.toString(),
+                                                                       address.getAddress(), name, timestamp);
+
+        DiscoveryNode discoveryNode = new DiscoveryNode(name, nodeId, address, Collections.emptyMap(), roles, version);
+        assertEquals(expectedNode, Collector.convertNode(timestamp, discoveryNode));
+    }
+}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollectorTests.java

@@ -5,64 +5,173 @@
  */
 package org.elasticsearch.xpack.monitoring.collector.cluster;
 
-import java.util.Collection;
-
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.admin.cluster.stats.ClusterStatsIndices;
+import org.elasticsearch.action.admin.cluster.stats.ClusterStatsNodes;
+import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequestBuilder;
+import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.ClusterAdminClient;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.license.License;
 import org.elasticsearch.license.LicenseService;
-import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.xpack.action.XPackUsageAction;
+import org.elasticsearch.xpack.action.XPackUsageRequest;
+import org.elasticsearch.xpack.action.XPackUsageResponse;
+import org.elasticsearch.xpack.logstash.Logstash;
+import org.elasticsearch.xpack.logstash.LogstashFeatureSet;
 import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.MonitoringSettings;
-import org.elasticsearch.xpack.monitoring.collector.AbstractCollectorTestCase;
+import org.elasticsearch.xpack.monitoring.MonitoringTestUtils;
+import org.elasticsearch.xpack.monitoring.collector.BaseCollectorTestCase;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
 
+import java.util.Collection;
+import java.util.UUID;
+
+import static java.util.Collections.singletonList;
+import static org.elasticsearch.xpack.monitoring.MonitoringTestUtils.randomMonitoringNode;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Matchers.same;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
-public class ClusterStatsCollectorTests extends AbstractCollectorTestCase {
+public class ClusterStatsCollectorTests extends BaseCollectorTestCase {
 
-    public void testClusterStatsCollector() throws Exception {
-        Collection<MonitoringDoc> results = newClusterStatsCollector().doCollect();
-        assertThat(results, hasSize(1));
+    private LicenseService licenseService;
 
-        // validate the document
-        ClusterStatsMonitoringDoc clusterInfoMonitoringDoc =
-                (ClusterStatsMonitoringDoc)results.stream().filter(o -> o instanceof ClusterStatsMonitoringDoc).findFirst().get();
-
-        assertThat(clusterInfoMonitoringDoc.getMonitoringId(), equalTo(MonitoredSystem.ES.getSystem()));
-        assertThat(clusterInfoMonitoringDoc.getMonitoringVersion(), equalTo(Version.CURRENT.toString()));
-        assertThat(clusterInfoMonitoringDoc.getClusterUUID(),
-                equalTo(client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID()));
-        assertThat(clusterInfoMonitoringDoc.getTimestamp(), greaterThan(0L));
-        assertThat(clusterInfoMonitoringDoc.getSourceNode(), notNullValue());
-
-        assertThat(clusterInfoMonitoringDoc.getClusterName(),
-                equalTo(client().admin().cluster().prepareState().setMetaData(true).get().getClusterName().value()));
-        assertThat(clusterInfoMonitoringDoc.getVersion(),
-                equalTo(client().admin().cluster().prepareNodesInfo().get().getNodes().get(0).getVersion().toString()));
-
-        assertThat(clusterInfoMonitoringDoc.getLicense(), notNullValue());
-
-        assertNotNull(clusterInfoMonitoringDoc.getClusterStats());
-        assertThat(clusterInfoMonitoringDoc.getClusterStats().getNodesStats().getCounts().getTotal(),
-                equalTo(internalCluster().getNodeNames().length));
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        licenseService = mock(LicenseService.class);
     }
 
-    private ClusterStatsCollector newClusterStatsCollector() {
-        // This collector runs on master node only
-        return newClusterStatsCollector(internalCluster().getMasterName());
+    public void testShouldCollectReturnsFalseIfNotMaster() {
+        // this controls the blockage
+        whenLocalNodeElectedMaster(false);
+
+        final ClusterStatsCollector collector =
+                new ClusterStatsCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState, client, licenseService);
+
+        assertThat(collector.shouldCollect(), is(false));
+        verify(nodes).isLocalNodeElectedMaster();
     }
 
-    private ClusterStatsCollector newClusterStatsCollector(String nodeId) {
-        assertNotNull(nodeId);
-        return new ClusterStatsCollector(internalCluster().getInstance(Settings.class, nodeId),
-                internalCluster().getInstance(ClusterService.class, nodeId),
-                internalCluster().getInstance(MonitoringSettings.class, nodeId),
-                internalCluster().getInstance(XPackLicenseState.class, nodeId),
-                securedClient(nodeId),
-                internalCluster().getInstance(LicenseService.class, nodeId));
+    public void testShouldCollectReturnsTrue() {
+        whenLocalNodeElectedMaster(true);
+
+        final ClusterStatsCollector collector =
+                new ClusterStatsCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState, client, licenseService);
+
+        assertThat(collector.shouldCollect(), is(true));
+        verify(nodes).isLocalNodeElectedMaster();
+    }
+
+    public void testDoCollect() throws Exception {
+        whenLocalNodeElectedMaster(true);
+
+        final String clusterName = randomAlphaOfLength(10);
+        whenClusterStateWithName(clusterName);
+
+        final String clusterUUID = UUID.randomUUID().toString();
+        whenClusterStateWithUUID(clusterUUID);
+
+        final MonitoringDoc.Node node = randomMonitoringNode(random());
+
+        final License license = License.builder()
+                                        .uid(UUID.randomUUID().toString())
+                                        .type("trial")
+                                        .issuer("elasticsearch")
+                                        .issuedTo("elastic")
+                                        .issueDate(System.currentTimeMillis())
+                                        .expiryDate(System.currentTimeMillis() + TimeValue.timeValueHours(24L).getMillis())
+                                        .maxNodes(2)
+                                        .build();
+        when(licenseService.getLicense()).thenReturn(license);
+
+        final TimeValue timeout = mock(TimeValue.class);
+        when(monitoringSettings.clusterStatsTimeout()).thenReturn(timeout);
+
+        final ClusterStatsResponse mockClusterStatsResponse = mock(ClusterStatsResponse.class);
+
+        final ClusterHealthStatus clusterStatus = randomFrom(ClusterHealthStatus.values());
+        when(mockClusterStatsResponse.getStatus()).thenReturn(clusterStatus);
+        when(mockClusterStatsResponse.getNodesStats()).thenReturn(mock(ClusterStatsNodes.class));
+
+        final ClusterStatsIndices mockClusterStatsIndices = mock(ClusterStatsIndices.class);
+
+        final int nbIndices = randomIntBetween(0, 100);
+        when(mockClusterStatsIndices.getIndexCount()).thenReturn(nbIndices);
+        when(mockClusterStatsResponse.getIndicesStats()).thenReturn(mockClusterStatsIndices);
+
+        final ClusterStatsRequestBuilder clusterStatsRequestBuilder = mock(ClusterStatsRequestBuilder.class);
+        when(clusterStatsRequestBuilder.get(eq(timeout))).thenReturn(mockClusterStatsResponse);
+
+        final ClusterAdminClient clusterAdminClient = mock(ClusterAdminClient.class);
+        when(clusterAdminClient.prepareClusterStats()).thenReturn(clusterStatsRequestBuilder);
+
+        final AdminClient adminClient = mock(AdminClient.class);
+        when(adminClient.cluster()).thenReturn(clusterAdminClient);
+
+        final Client client = mock(Client.class);
+        when(client.admin()).thenReturn(adminClient);
+
+        final XPackUsageResponse xPackUsageResponse = new XPackUsageResponse(singletonList(new LogstashFeatureSet.Usage(true, true)));
+
+        @SuppressWarnings("unchecked")
+        final ActionFuture<XPackUsageResponse> xPackUsageFuture = (ActionFuture<XPackUsageResponse>) mock(ActionFuture.class);
+        when(client.execute(same(XPackUsageAction.INSTANCE), any(XPackUsageRequest.class))).thenReturn(xPackUsageFuture);
+        when(xPackUsageFuture.actionGet()).thenReturn(xPackUsageResponse);
+
+        final ClusterStatsCollector collector =
+                new ClusterStatsCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState, client, licenseService);
+
+        final Collection<MonitoringDoc> results = collector.doCollect(node);
+        assertEquals(1, results.size());
+
+        final MonitoringDoc monitoringDoc = results.iterator().next();
+        assertThat(monitoringDoc, instanceOf(ClusterStatsMonitoringDoc.class));
+
+        final ClusterStatsMonitoringDoc document = (ClusterStatsMonitoringDoc) monitoringDoc;
+        assertThat(document.getCluster(), equalTo(clusterUUID));
+        assertThat(document.getTimestamp(), greaterThan(0L));
+        assertThat(document.getNode(), equalTo(node));
+        assertThat(document.getSystem(), is(MonitoredSystem.ES));
+        assertThat(document.getType(), equalTo(ClusterStatsMonitoringDoc.TYPE));
+        assertThat(document.getId(), nullValue());
+
+        assertThat(document.getClusterName(), equalTo(clusterName));
+        assertThat(document.getVersion(), equalTo(Version.CURRENT.toString()));
+        assertThat(document.getLicense(), equalTo(license));
+        assertThat(document.getStatus(), equalTo(clusterStatus));
+
+        assertThat(document.getClusterStats(), notNullValue());
+        assertThat(document.getClusterStats().getStatus(), equalTo(clusterStatus));
+        assertThat(document.getClusterStats().getIndicesStats().getIndexCount(), equalTo(nbIndices));
+
+        assertThat(document.getUsages(), hasSize(1));
+        assertThat(document.getUsages().iterator().next().name(), equalTo(Logstash.NAME));
+
+        assertThat(document.getClusterState().getClusterName().value(), equalTo(clusterName));
+        assertThat(document.getClusterState().stateUUID(), equalTo(clusterState.stateUUID()));
+
+        verify(clusterService, times(1)).getClusterName();
+        verify(clusterService, times(2)).state();
+        verify(licenseService, times(1)).getLicense();
+        verify(clusterAdminClient).prepareClusterStats();
+        verify(client).execute(same(XPackUsageAction.INSTANCE), any(XPackUsageRequest.class));
     }
 }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java

@@ -0,0 +1,501 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.monitoring.collector.cluster;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
+import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
+import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
+import org.elasticsearch.action.admin.cluster.stats.ClusterStatsNodeResponse;
+import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
+import org.elasticsearch.action.admin.indices.stats.CommonStats;
+import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
+import org.elasticsearch.action.admin.indices.stats.ShardStats;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.BoundTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.license.License;
+import org.elasticsearch.monitor.fs.FsInfo;
+import org.elasticsearch.monitor.jvm.JvmInfo;
+import org.elasticsearch.monitor.jvm.JvmStats;
+import org.elasticsearch.monitor.os.OsInfo;
+import org.elasticsearch.monitor.os.OsStats;
+import org.elasticsearch.monitor.process.ProcessStats;
+import org.elasticsearch.plugins.PluginInfo;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.transport.TransportInfo;
+import org.elasticsearch.xpack.XPackFeatureSet;
+import org.elasticsearch.xpack.logstash.LogstashFeatureSet;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+import org.elasticsearch.xpack.monitoring.exporter.BaseMonitoringDocTestCase;
+import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singleton;
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<ClusterStatsMonitoringDoc> {
+
+    private String clusterName;
+    private String version;
+    private ClusterHealthStatus clusterStatus;
+    private List<XPackFeatureSet.Usage> usages;
+    private ClusterStatsResponse clusterStats;
+    private ClusterState clusterState;
+    private License license;
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        clusterName = randomAlphaOfLength(5);
+        version = VersionUtils.randomVersion(random()).toString();
+        clusterStatus = randomFrom(ClusterHealthStatus.values());
+        usages = emptyList();
+        clusterStats = mock(ClusterStatsResponse.class);
+        clusterState = mock(ClusterState.class);
+        license = License.builder()
+                         .uid(randomAlphaOfLength(5))
+                         .type(randomFrom(License.OperationMode.values()).name())
+                         .issuer(randomAlphaOfLength(5))
+                         .issuedTo(randomAlphaOfLength(5))
+                         .issueDate(timestamp)
+                         .expiryDate(timestamp + randomIntBetween(1, 10) * 1_000L)
+                         .maxNodes(randomIntBetween(1, 5))
+                         .build();
+    }
+
+    @Override
+    protected ClusterStatsMonitoringDoc createMonitoringDoc(String cluster, long timestamp, MonitoringDoc.Node node,
+                                                            MonitoredSystem system, String type, String id) {
+        return new ClusterStatsMonitoringDoc(cluster, timestamp, node,
+                                             clusterName, version, clusterStatus, license, usages, clusterStats, clusterState);
+    }
+
+    @Override
+    protected void assertMonitoringDoc(final ClusterStatsMonitoringDoc document) {
+        assertThat(document.getSystem(), is(MonitoredSystem.ES));
+        assertThat(document.getType(), is(ClusterStatsMonitoringDoc.TYPE));
+        assertThat(document.getId(), nullValue());
+
+        assertThat(document.getClusterName(), equalTo(clusterName));
+        assertThat(document.getVersion(), equalTo(version));
+        assertThat(document.getStatus(), equalTo(clusterStatus));
+        assertThat(document.getLicense(), equalTo(license));
+        assertThat(document.getUsages(), is(usages));
+        assertThat(document.getClusterStats(), is(clusterStats));
+        assertThat(document.getClusterState(), is(clusterState));
+    }
+
+    public void testConstructorClusterNameMustNotBeNull() {
+        expectThrows(NullPointerException.class,
+                () -> new ClusterStatsMonitoringDoc(cluster, timestamp, node,
+                        null, version, clusterStatus, license, usages, clusterStats, clusterState));
+    }
+
+    public void testConstructorVersionMustNotBeNull() {
+        expectThrows(NullPointerException.class,
+                () -> new ClusterStatsMonitoringDoc(cluster, timestamp, node,
+                        clusterName, null, clusterStatus, license, usages, clusterStats, clusterState));
+    }
+
+    public void testConstructorClusterHealthStatusMustNotBeNull() {
+        expectThrows(NullPointerException.class,
+                () -> new ClusterStatsMonitoringDoc(cluster, timestamp, node,
+                        clusterName, version, null, license, usages, clusterStats, clusterState));
+    }
+
+    public void testHash() {
+        assertEquals("cc6628dbcfd052fba57870dc4eed4bc9f5cd5d43b4df46e97867aeb2dd7bd2e8",
+                ClusterStatsMonitoringDoc.hash("licenseStatus", "licenseUid", "licenseType", "licenseExpiryDate", "clusterUUID"));
+    }
+
+    public void testHashWithLicense() {
+        final StringBuilder builder = new StringBuilder();
+
+        final License license = mock(License.class);
+        final License.Status status = randomFrom(License.Status.values());
+        when(license.status()).thenReturn(status);
+        builder.append(status.label());
+
+        final String uuid = randomAlphaOfLength(10);
+        when(license.uid()).thenReturn(uuid);
+        builder.append(uuid);
+
+        final String type = randomAlphaOfLength(10);
+        when(license.type()).thenReturn(type);
+        builder.append(type);
+
+        final long expiryDate = randomNonNegativeLong();
+        when(license.expiryDate()).thenReturn(expiryDate);
+        builder.append(String.valueOf(expiryDate));
+
+        final String cluster = randomAlphaOfLength(5);
+        builder.append(cluster);
+
+        assertEquals(MessageDigests.toHexString(MessageDigests.sha256().digest(builder.toString().getBytes(StandardCharsets.UTF_8))),
+                ClusterStatsMonitoringDoc.hash(license, cluster));
+    }
+
+    @Override
+    public void testToXContent() throws IOException {
+        final ClusterName clusterName = new ClusterName("_cluster_name");
+        final TransportAddress transportAddress = new TransportAddress(TransportAddress.META_ADDRESS, 9300);
+        final DiscoveryNode discoveryNode = new DiscoveryNode("_node_name",
+                                                                "_node_id",
+                                                                "_ephemeral_id",
+                                                                "_host_name",
+                                                                "_host_address",
+                                                                transportAddress,
+                                                                singletonMap("attr", "value"),
+                                                                singleton(DiscoveryNode.Role.MASTER),
+                                                                Version.V_6_0_0_beta1);
+
+        final ClusterState clusterState = ClusterState.builder(clusterName)
+                                                        .stateUUID("_state_uuid")
+                                                        .version(12L)
+                                                        .nodes(DiscoveryNodes.builder()
+                                                                             .masterNodeId("_node")
+                                                                             .localNodeId("_node")
+                                                                             .add(discoveryNode)
+                                                                             .build())
+                                                        .build();
+
+        final License license = License.builder()
+                                        .uid("442ca961-9c00-4bb2-b5c9-dfaacd547403")
+                                        .type("trial")
+                                        .issuer("elasticsearch")
+                                        .issuedTo("customer")
+                                        .issueDate(1451606400000L)
+                                        .expiryDate(1502107402133L)
+                                        .maxNodes(2)
+                                        .build();
+
+        List<XPackFeatureSet.Usage> usages = singletonList(new LogstashFeatureSet.Usage(false, true));
+
+        final NodeInfo mockNodeInfo = mock(NodeInfo.class);
+        when(mockNodeInfo.getVersion()).thenReturn(Version.V_6_0_0_alpha2);
+        when(mockNodeInfo.getNode()).thenReturn(discoveryNode);
+
+        final TransportInfo mockTransportInfo = mock(TransportInfo.class);
+        when(mockNodeInfo.getTransport()).thenReturn(mockTransportInfo);
+
+        final BoundTransportAddress bound = new BoundTransportAddress(new TransportAddress[]{transportAddress}, transportAddress);
+        when(mockTransportInfo.address()).thenReturn(bound);
+        when(mockNodeInfo.getSettings()).thenReturn(Settings.builder()
+                                                            .put(NetworkModule.TRANSPORT_TYPE_KEY, "_transport")
+                                                            .put(NetworkModule.HTTP_TYPE_KEY, "_http")
+                                                            .build());
+
+        final PluginsAndModules mockPluginsAndModules = mock(PluginsAndModules.class);
+        when(mockNodeInfo.getPlugins()).thenReturn(mockPluginsAndModules);
+        final PluginInfo pluginInfo = new PluginInfo("_plugin", "_plugin_desc", "_plugin_version", "_plugin_class", false, false);
+        when(mockPluginsAndModules.getPluginInfos()).thenReturn(singletonList(pluginInfo));
+
+        final OsInfo mockOsInfo = mock(OsInfo.class);
+        when(mockNodeInfo.getOs()).thenReturn(mockOsInfo);
+        when(mockOsInfo.getAvailableProcessors()).thenReturn(32);
+        when(mockOsInfo.getAllocatedProcessors()).thenReturn(16);
+        when(mockOsInfo.getName()).thenReturn("_os_name");
+
+        final JvmInfo mockJvmInfo = mock(JvmInfo.class);
+        when(mockNodeInfo.getJvm()).thenReturn(mockJvmInfo);
+        when(mockJvmInfo.version()).thenReturn("_jvm_version");
+        when(mockJvmInfo.getVmName()).thenReturn("_jvm_vm_name");
+        when(mockJvmInfo.getVmVersion()).thenReturn("_jvm_vm_version");
+        when(mockJvmInfo.getVmVendor()).thenReturn("_jvm_vm_vendor");
+
+        final NodeStats mockNodeStats = mock(NodeStats.class);
+        when(mockNodeStats.getTimestamp()).thenReturn(0L);
+
+        final FsInfo mockFsInfo = mock(FsInfo.class);
+        when(mockNodeStats.getFs()).thenReturn(mockFsInfo);
+        when(mockFsInfo.getTotal()).thenReturn(new FsInfo.Path("_fs_path","_fs_mount", 100L, 49L, 51L));
+
+        final OsStats mockOsStats = mock(OsStats.class);
+        when(mockNodeStats.getOs()).thenReturn(mockOsStats);
+        when(mockOsStats.getMem()).thenReturn(new OsStats.Mem(100, 79));
+
+        final ProcessStats mockProcessStats = mock(ProcessStats.class);
+        when(mockNodeStats.getProcess()).thenReturn(mockProcessStats);
+        when(mockProcessStats.getOpenFileDescriptors()).thenReturn(42L);
+        when(mockProcessStats.getCpu()).thenReturn(new ProcessStats.Cpu((short) 3, 32L));
+
+        final JvmStats.Threads mockThreads = mock(JvmStats.Threads.class);
+        when(mockThreads.getCount()).thenReturn(9);
+
+        final JvmStats.Mem mockMem = mock(JvmStats.Mem.class);
+        when(mockMem.getHeapUsed()).thenReturn(new ByteSizeValue(512, ByteSizeUnit.MB));
+        when(mockMem.getHeapMax()).thenReturn(new ByteSizeValue(24, ByteSizeUnit.GB));
+
+        final JvmStats mockJvmStats = mock(JvmStats.class);
+        when(mockNodeStats.getJvm()).thenReturn(mockJvmStats);
+        when(mockJvmStats.getThreads()).thenReturn(mockThreads);
+        when(mockJvmStats.getMem()).thenReturn(mockMem);
+        when(mockJvmStats.getUptime()).thenReturn(TimeValue.timeValueHours(3));
+
+        final ShardId shardId = new ShardId("_index", "_index_id", 7);
+        final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "_message");
+        final ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, EXISTING_STORE_INSTANCE, unassignedInfo);
+
+        final ShardStats mockShardStats = mock(ShardStats.class);
+        when(mockShardStats.getShardRouting()).thenReturn(shardRouting);
+        when(mockShardStats.getStats()).thenReturn(new CommonStats(CommonStatsFlags.ALL));
+
+        final ClusterStatsNodeResponse mockNodeResponse = mock(ClusterStatsNodeResponse.class);
+        when(mockNodeResponse.clusterStatus()).thenReturn(ClusterHealthStatus.RED);
+        when(mockNodeResponse.nodeInfo()).thenReturn(mockNodeInfo);
+        when(mockNodeResponse.nodeStats()).thenReturn(mockNodeStats);
+        when(mockNodeResponse.shardsStats()).thenReturn(new ShardStats[]{mockShardStats});
+
+        final ClusterStatsResponse clusterStats = new ClusterStatsResponse(1451606400000L,
+                                                                            clusterName,
+                                                                            singletonList(mockNodeResponse),
+                                                                            emptyList());
+
+        final MonitoringDoc.Node node = new MonitoringDoc.Node("_uuid", "_host", "_addr", "_ip", "_name", 1504169190855L);
+
+        final ClusterStatsMonitoringDoc doc = new ClusterStatsMonitoringDoc("_cluster",
+                                                                            1502107402133L,
+                                                                            node,
+                                                                            clusterName.value(),
+                                                                            "_version",
+                                                                            ClusterHealthStatus.GREEN,
+                                                                            license,
+                                                                            usages,
+                                                                            clusterStats,
+                                                                            clusterState);
+
+        final BytesReference xContent = XContentHelper.toXContent(doc, XContentType.JSON, false);
+        assertEquals("{"
+                  + "\"cluster_uuid\":\"_cluster\","
+                  + "\"timestamp\":\"2017-08-07T12:03:22.133Z\","
+                  + "\"type\":\"cluster_stats\","
+                  + "\"source_node\":{"
+                    + "\"uuid\":\"_uuid\","
+                    + "\"host\":\"_host\","
+                    + "\"transport_address\":\"_addr\","
+                    + "\"ip\":\"_ip\","
+                    + "\"name\":\"_name\","
+                    + "\"timestamp\":\"2017-08-31T08:46:30.855Z\""
+                  + "},"
+                  + "\"cluster_name\":\"_cluster_name\","
+                  + "\"version\":\"_version\","
+                  + "\"license\":{"
+                    + "\"status\":\"expired\","
+                    + "\"uid\":\"442ca961-9c00-4bb2-b5c9-dfaacd547403\","
+                    + "\"type\":\"trial\","
+                    + "\"issue_date\":\"2016-01-01T00:00:00.000Z\","
+                    + "\"issue_date_in_millis\":1451606400000,"
+                    + "\"expiry_date\":\"2017-08-07T12:03:22.133Z\","
+                    + "\"expiry_date_in_millis\":1502107402133,"
+                    + "\"max_nodes\":2,"
+                    + "\"issued_to\":\"customer\","
+                    + "\"issuer\":\"elasticsearch\","
+                    + "\"start_date_in_millis\":-1,"
+                    + "\"hkey\":\"e05627254d639cf36346bf99934dc4a4ac9f37bdc9100cee450c10fa6322a6dd\""
+                  + "},"
+                  + "\"cluster_stats\":{"
+                    + "\"timestamp\":1451606400000,"
+                    + "\"status\":\"red\","
+                    + "\"indices\":{"
+                      + "\"count\":1,"
+                      + "\"shards\":{"
+                        + "\"total\":1,"
+                        + "\"primaries\":1,"
+                        + "\"replication\":0.0,"
+                        + "\"index\":{"
+                          + "\"shards\":{"
+                            + "\"min\":1,"
+                            + "\"max\":1,"
+                            + "\"avg\":1.0"
+                          + "},"
+                          + "\"primaries\":{"
+                            + "\"min\":1,"
+                            + "\"max\":1,"
+                            + "\"avg\":1.0"
+                          + "},"
+                          + "\"replication\":{"
+                            + "\"min\":0.0,"
+                            + "\"max\":0.0,"
+                            + "\"avg\":0.0"
+                          + "}"
+                        + "}"
+                      + "},"
+                      + "\"docs\":{"
+                        + "\"count\":0,"
+                        + "\"deleted\":0"
+                      + "},"
+                      + "\"store\":{"
+                        + "\"size_in_bytes\":0"
+                      + "},"
+                      + "\"fielddata\":{"
+                        + "\"memory_size_in_bytes\":0,"
+                        + "\"evictions\":0"
+                      + "},"
+                      + "\"query_cache\":{"
+                        + "\"memory_size_in_bytes\":0,"
+                        + "\"total_count\":0,"
+                        + "\"hit_count\":0,"
+                        + "\"miss_count\":0,"
+                        + "\"cache_size\":0,"
+                        + "\"cache_count\":0,"
+                        + "\"evictions\":0"
+                      + "},"
+                      + "\"completion\":{"
+                        + "\"size_in_bytes\":0"
+                      + "},"
+                      + "\"segments\":{"
+                        + "\"count\":0,"
+                        + "\"memory_in_bytes\":0,"
+                        + "\"terms_memory_in_bytes\":0,"
+                        + "\"stored_fields_memory_in_bytes\":0,"
+                        + "\"term_vectors_memory_in_bytes\":0,"
+                        + "\"norms_memory_in_bytes\":0,"
+                        + "\"points_memory_in_bytes\":0,"
+                        + "\"doc_values_memory_in_bytes\":0,"
+                        + "\"index_writer_memory_in_bytes\":0,"
+                        + "\"version_map_memory_in_bytes\":0,"
+                        + "\"fixed_bit_set_memory_in_bytes\":0,"
+                        + "\"max_unsafe_auto_id_timestamp\":-9223372036854775808,"
+                        + "\"file_sizes\":{}"
+                      + "}"
+                    + "},"
+                    + "\"nodes\":{"
+                      + "\"count\":{"
+                        + "\"total\":1,"
+                        + "\"data\":0,"
+                        + "\"coordinating_only\":0,"
+                        + "\"master\":1,"
+                        + "\"ingest\":0"
+                      + "},"
+                      + "\"versions\":["
+                        + "\"6.0.0-alpha2\""
+                      + "],"
+                      + "\"os\":{"
+                        + "\"available_processors\":32,"
+                        + "\"allocated_processors\":16,"
+                        + "\"names\":["
+                          + "{"
+                            + "\"name\":\"_os_name\","
+                            + "\"count\":1"
+                          + "}"
+                        + "],"
+                        + "\"mem\":{"
+                          + "\"total_in_bytes\":100,"
+                          + "\"free_in_bytes\":79,"
+                          + "\"used_in_bytes\":21,"
+                          + "\"free_percent\":79,"
+                          + "\"used_percent\":21"
+                        + "}"
+                      + "},"
+                      + "\"process\":{"
+                        + "\"cpu\":{"
+                          + "\"percent\":3"
+                        + "},"
+                        + "\"open_file_descriptors\":{"
+                          + "\"min\":42,"
+                          + "\"max\":42,"
+                          + "\"avg\":42"
+                        + "}"
+                      + "},"
+                      + "\"jvm\":{"
+                        + "\"max_uptime_in_millis\":10800000,"
+                        + "\"versions\":["
+                          + "{"
+                            + "\"version\":\"_jvm_version\","
+                            + "\"vm_name\":\"_jvm_vm_name\","
+                            + "\"vm_version\":\"_jvm_vm_version\","
+                            + "\"vm_vendor\":\"_jvm_vm_vendor\","
+                            + "\"count\":1"
+                          + "}"
+                        + "],"
+                        + "\"mem\":{"
+                          + "\"heap_used_in_bytes\":536870912,"
+                          + "\"heap_max_in_bytes\":25769803776"
+                        + "},"
+                        + "\"threads\":9"
+                      + "},"
+                      + "\"fs\":{"
+                        + "\"total_in_bytes\":100,"
+                        + "\"free_in_bytes\":49,"
+                        + "\"available_in_bytes\":51"
+                      + "},"
+                      + "\"plugins\":["
+                        + "{"
+                          + "\"name\":\"_plugin\","
+                          + "\"version\":\"_plugin_version\","
+                          + "\"description\":\"_plugin_desc\","
+                          + "\"classname\":\"_plugin_class\","
+                          + "\"has_native_controller\":false,"
+                          + "\"requires_keystore\":false"
+                        + "}"
+                      + "],"
+                      + "\"network_types\":{"
+                        + "\"transport_types\":{"
+                          + "\"_transport\":1"
+                        + "},"
+                        + "\"http_types\":{"
+                          + "\"_http\":1"
+                        + "}"
+                      + "}"
+                    + "}"
+                  + "},"
+                  + "\"cluster_state\":{"
+                    + "\"status\":\"green\","
+                    + "\"version\":12,"
+                    + "\"state_uuid\":\"_state_uuid\","
+                    + "\"master_node\":\"_node\","
+                    + "\"nodes\":{"
+                      + "\"_node_id\":{"
+                        + "\"name\":\"_node_name\","
+                        + "\"ephemeral_id\":\"_ephemeral_id\","
+                        + "\"transport_address\":\"0.0.0.0:9300\","
+                        + "\"attributes\":{"
+                          + "\"attr\":\"value\""
+                        + "}"
+                      + "}"
+                    + "}"
+                  + "},"
+                  + "\"stack_stats\":{"
+                    + "\"xpack\":{"
+                      + "\"logstash\":{"
+                        + "\"available\":false,"
+                        + "\"enabled\":true"
+                      + "}"
+                    + "}"
+                  + "}"
+                + "}" , xContent.utf8ToString());
+    }
+}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryCollectorTests.java

@@ -5,148 +5,173 @@
  */
 package org.elasticsearch.xpack.monitoring.collector.indices;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.recovery.RecoveryAction;
+import org.elasticsearch.action.admin.indices.recovery.RecoveryRequestBuilder;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
-import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Strings;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.IndicesAdminClient;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.recovery.RecoveryState;
-import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.MonitoringSettings;
-import org.elasticsearch.xpack.monitoring.collector.AbstractCollectorTestCase;
+import org.elasticsearch.xpack.monitoring.collector.BaseCollectorTestCase;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
 
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.hamcrest.Matchers.empty;
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+import static org.elasticsearch.xpack.monitoring.MonitoringTestUtils.randomMonitoringNode;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
-@ClusterScope(numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
-public class IndexRecoveryCollectorTests extends AbstractCollectorTestCase {
+public class IndexRecoveryCollectorTests extends BaseCollectorTestCase {
 
-    private final boolean activeOnly = false;
-    private final String indexName = "test";
+    public void testShouldCollectReturnsFalseIfMonitoringNotAllowed() {
+        // this controls the blockage
+        when(licenseState.isMonitoringAllowed()).thenReturn(false);
+        whenLocalNodeElectedMaster(randomBoolean());
 
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder()
-                .put(super.nodeSettings(nodeOrdinal))
-                .put(MonitoringSettings.INDEX_RECOVERY_ACTIVE_ONLY.getKey(), activeOnly)
-                .put(MonitoringSettings.INDICES.getKey(), indexName)
-                .build();
+        final IndexRecoveryCollector collector =
+                new IndexRecoveryCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState, client);
+
+        assertThat(collector.shouldCollect(), is(false));
+        verify(licenseState).isMonitoringAllowed();
     }
 
-    public void testIndexRecoveryCollector() throws Exception {
-        logger.info("--> start first node");
-        final String node1 = internalCluster().startNode();
-        waitForNoBlocksOnNode(node1);
+    public void testShouldCollectReturnsFalseIfNotMaster() {
+        when(licenseState.isMonitoringAllowed()).thenReturn(true);
+        // this controls the blockage
+        whenLocalNodeElectedMaster(false);
 
-        logger.info("--> collect index recovery data");
-        Collection<MonitoringDoc> results = newIndexRecoveryCollector(node1).doCollect();
+        final IndexRecoveryCollector collector =
+                new IndexRecoveryCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState, client);
 
-        logger.info("--> no indices created, expecting 0 monitoring documents");
-        assertNotNull(results);
-        assertThat(results, is(empty()));
+        assertThat(collector.shouldCollect(), is(false));
+        verify(licenseState).isMonitoringAllowed();
+        verify(nodes).isLocalNodeElectedMaster();
+    }
 
-        logger.info("--> create index [{}] on node [{}]", indexName, node1);
-        ElasticsearchAssertions.assertAcked(prepareCreate(indexName, 1,
-                Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 3).put(SETTING_NUMBER_OF_REPLICAS, 1)));
+    public void testShouldCollectReturnsTrue() {
+        when(licenseState.isMonitoringAllowed()).thenReturn(true);
+        whenLocalNodeElectedMaster(true);
 
-        logger.info("--> indexing sample data");
-        final int numDocs = between(50, 150);
-        for (int i = 0; i < numDocs; i++) {
-            client().prepareIndex(indexName, "foo").setSource("value", randomInt()).get();
+        final IndexRecoveryCollector collector =
+                new IndexRecoveryCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState, client);
+
+        assertThat(collector.shouldCollect(), is(true));
+        verify(licenseState).isMonitoringAllowed();
+        verify(nodes).isLocalNodeElectedMaster();
+    }
+
+    public void testDoCollect() throws Exception {
+        whenLocalNodeElectedMaster(true);
+
+        final String clusterName = randomAlphaOfLength(10);
+        whenClusterStateWithName(clusterName);
+
+        final String clusterUUID = UUID.randomUUID().toString();
+        whenClusterStateWithUUID(clusterUUID);
+
+        final DiscoveryNode localNode = localNode(randomAlphaOfLength(5));
+        when(clusterService.localNode()).thenReturn(localNode);
+
+        final MonitoringDoc.Node node = randomMonitoringNode(random());
+
+        final boolean recoveryOnly = randomBoolean();
+        when(monitoringSettings.recoveryActiveOnly()).thenReturn(recoveryOnly);
+
+        final String[] indices;
+        if (randomBoolean()) {
+            indices = null;
+        } else {
+            indices = new String[randomIntBetween(1, 5)];
+            for (int i = 0; i < indices.length; i++) {
+                indices[i] = randomAlphaOfLengthBetween(5, 10);
+            }
         }
+        when(monitoringSettings.indices()).thenReturn(indices);
+        when(monitoringSettings.recoveryTimeout()).thenReturn(TimeValue.timeValueSeconds(12));
 
-        logger.info("--> create a second index [{}] on node [{}] that won't be part of stats collection", indexName, node1);
-        client().prepareIndex("other", "bar").setSource("value", randomInt()).get();
+        final int nbRecoveries = randomBoolean() ? 0 : randomIntBetween(1, 3);
+        final Map<String, List<RecoveryState>> recoveryStates = new HashMap<>();
+        for (int i = 0; i < nbRecoveries; i++) {
+            ShardId shardId = new ShardId("_index_" + i, "_uuid_" + i, i);
+            RecoverySource source = RecoverySource.PeerRecoverySource.INSTANCE;
+            final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "_index_info_" + i);
+            final ShardRouting shardRouting = ShardRouting
+                                                .newUnassigned(shardId, true, source, unassignedInfo)
+                                                .initialize(localNode.getId(), "_allocation_id", 10 * i);
 
-        flushAndRefresh();
-        assertHitCount(client().prepareSearch(indexName).setSize(0).get(), numDocs);
-        assertHitCount(client().prepareSearch("other").setSize(0).get(), 1L);
-
-        logger.info("--> start second node");
-        final String node2 = internalCluster().startNode();
-        waitForNoBlocksOnNode(node2);
-        waitForRelocation();
-
-        for (MonitoringSettings monitoringSettings : internalCluster().getInstances(MonitoringSettings.class)) {
-            assertThat(monitoringSettings.recoveryActiveOnly(), equalTo(activeOnly));
+            final RecoveryState recoveryState = new RecoveryState(shardRouting, localNode, localNode);
+            recoveryStates.put("_index_" + i, singletonList(recoveryState));
         }
+        final RecoveryResponse recoveryResponse =
+                new RecoveryResponse(randomInt(), randomInt(), randomInt(), randomBoolean(), recoveryStates, emptyList());
 
-        logger.info("--> collect index recovery data");
-        results = newIndexRecoveryCollector(null).doCollect();
+        final TimeValue timeout = mock(TimeValue.class);
+        when(monitoringSettings.recoveryTimeout()).thenReturn(timeout);
 
-        logger.info("--> we should have at least 1 shard in relocation state");
-        assertNotNull(results);
-        assertThat(results, hasSize(1));
+        final RecoveryRequestBuilder recoveryRequestBuilder =
+                spy(new RecoveryRequestBuilder(mock(ElasticsearchClient.class), RecoveryAction.INSTANCE));
+        doReturn(recoveryResponse).when(recoveryRequestBuilder).get(eq(timeout));
 
-        MonitoringDoc monitoringDoc = results.iterator().next();
-        assertNotNull(monitoringDoc);
-        assertThat(monitoringDoc, instanceOf(IndexRecoveryMonitoringDoc.class));
+        final IndicesAdminClient indicesAdminClient = mock(IndicesAdminClient.class);
+        when(indicesAdminClient.prepareRecoveries()).thenReturn(recoveryRequestBuilder);
 
-        IndexRecoveryMonitoringDoc indexRecoveryMonitoringDoc = (IndexRecoveryMonitoringDoc) monitoringDoc;
-        assertThat(indexRecoveryMonitoringDoc.getMonitoringId(), equalTo(MonitoredSystem.ES.getSystem()));
-        assertThat(indexRecoveryMonitoringDoc.getMonitoringVersion(), equalTo(Version.CURRENT.toString()));
-        assertThat(indexRecoveryMonitoringDoc.getClusterUUID(),
-                equalTo(client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID()));
-        assertThat(indexRecoveryMonitoringDoc.getTimestamp(), greaterThan(0L));
-        assertThat(indexRecoveryMonitoringDoc.getSourceNode(), notNullValue());
+        final AdminClient adminClient = mock(AdminClient.class);
+        when(adminClient.indices()).thenReturn(indicesAdminClient);
 
-        RecoveryResponse recovery = indexRecoveryMonitoringDoc.getRecoveryResponse();
-        assertNotNull(recovery);
+        final Client client = mock(Client.class);
+        when(client.admin()).thenReturn(adminClient);
 
-        Map<String, List<RecoveryState>> shards = recovery.shardRecoveryStates();
-        assertThat(shards.size(), greaterThan(0));
+        final IndexRecoveryCollector collector =
+                new IndexRecoveryCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState, client);
 
-        for (Map.Entry<String, List<RecoveryState>> shard : shards.entrySet()) {
-            List<RecoveryState> shardRecoveries = shard.getValue();
-            assertNotNull(shardRecoveries);
-            assertThat(shardRecoveries.size(), greaterThan(0));
-            assertThat(shard.getKey(), equalTo(indexName));
+        final Collection<MonitoringDoc> results = collector.doCollect(node);
+        verify(indicesAdminClient).prepareRecoveries();
+
+        if (nbRecoveries == 0) {
+            assertEquals(0, results.size());
+        } else {
+            assertEquals(1, results.size());
+
+            final MonitoringDoc monitoringDoc = results.iterator().next();
+            assertThat(monitoringDoc, instanceOf(IndexRecoveryMonitoringDoc.class));
+
+            final IndexRecoveryMonitoringDoc document = (IndexRecoveryMonitoringDoc) monitoringDoc;
+            assertThat(document.getCluster(), equalTo(clusterUUID));
+            assertThat(document.getTimestamp(), greaterThan(0L));
+            assertThat(document.getNode(), equalTo(node));
+            assertThat(document.getSystem(), is(MonitoredSystem.ES));
+            assertThat(document.getType(), equalTo(IndexRecoveryMonitoringDoc.TYPE));
+            assertThat(document.getId(), nullValue());
+
+            final RecoveryResponse recoveries = document.getRecoveryResponse();
+            assertThat(recoveries, notNullValue());
+            assertThat(recoveries.hasRecoveries(), equalTo(true));
+            assertThat(recoveries.shardRecoveryStates().size(), equalTo(nbRecoveries));
         }
     }
-
-    public void testEmptyCluster() throws Exception {
-        final String node = internalCluster().startNode(Settings.builder().put(MonitoringSettings.INDICES.getKey(), Strings.EMPTY_ARRAY));
-        waitForNoBlocksOnNode(node);
-        assertThat(newIndexRecoveryCollector(node).doCollect(), hasSize(0));
-    }
-
-    public void testEmptyClusterAllIndices() throws Exception {
-        final String node = internalCluster().startNode(Settings.builder().put(MonitoringSettings.INDICES.getKey(), MetaData.ALL));
-        waitForNoBlocksOnNode(node);
-        assertThat(newIndexRecoveryCollector(node).doCollect(), hasSize(0));
-    }
-
-    public void testEmptyClusterMissingIndex() throws Exception {
-        final String node = internalCluster().startNode(Settings.builder().put(MonitoringSettings.INDICES.getKey(), "unknown"));
-        waitForNoBlocksOnNode(node);
-        assertThat(newIndexRecoveryCollector(node).doCollect(), hasSize(0));
-    }
-
-    private IndexRecoveryCollector newIndexRecoveryCollector(String nodeId) {
-        if (!Strings.hasText(nodeId)) {
-            nodeId = randomFrom(internalCluster().getNodeNames());
-        }
-        return new IndexRecoveryCollector(internalCluster().getInstance(Settings.class, nodeId),
-                internalCluster().getInstance(ClusterService.class, nodeId),
-                internalCluster().getInstance(MonitoringSettings.class, nodeId),
-                internalCluster().getInstance(XPackLicenseState.class, nodeId),
-                securedClient(nodeId));
-    }
-}
+}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexRecoveryMonitoringDocTests.java

@@ -0,0 +1,186 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.monitoring.collector.indices;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ShardOperationFailedException;
+import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
+import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RecoverySource;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.recovery.RecoveryState;
+import org.elasticsearch.transport.NodeDisconnectedException;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+import org.elasticsearch.xpack.monitoring.exporter.BaseMonitoringDocTestCase;
+import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.singleton;
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+
+public class IndexRecoveryMonitoringDocTests extends BaseMonitoringDocTestCase<IndexRecoveryMonitoringDoc> {
+
+    private RecoveryResponse recoveryResponse;
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        recoveryResponse = mock(RecoveryResponse.class);
+    }
+
+    @Override
+    protected IndexRecoveryMonitoringDoc createMonitoringDoc(String cluster, long timestamp, MonitoringDoc.Node node,
+                                                             MonitoredSystem system, String type, String id) {
+        return new IndexRecoveryMonitoringDoc(cluster, timestamp, node, recoveryResponse);
+    }
+
+    @Override
+    protected void assertMonitoringDoc(final IndexRecoveryMonitoringDoc document) {
+        assertThat(document.getSystem(), is(MonitoredSystem.ES));
+        assertThat(document.getType(), is(IndexRecoveryMonitoringDoc.TYPE));
+        assertThat(document.getId(), nullValue());
+
+        assertThat(document.getRecoveryResponse(), is(recoveryResponse));
+    }
+
+    public void testConstructorRecoveryResponseMustNotBeNull() {
+        expectThrows(NullPointerException.class,
+                () -> new IndexRecoveryMonitoringDoc(cluster, timestamp, node, null));
+    }
+
+    @Override
+    public void testToXContent() throws IOException {
+        final DiscoveryNode discoveryNodeZero = new DiscoveryNode("_node_0",
+                                                                    "_node_id_0",
+                                                                    "_ephemeral_id_0",
+                                                                    "_host_name_0",
+                                                                    "_host_address_0",
+                                                                    new TransportAddress(TransportAddress.META_ADDRESS, 9300),
+                                                                    singletonMap("attr", "value_0"),
+                                                                    singleton(DiscoveryNode.Role.MASTER),
+                                                                    Version.V_6_0_0_beta1);
+
+        final DiscoveryNode discoveryNodeOne = new DiscoveryNode("_node_1",
+                                                                    "_node_id_1",
+                                                                    "_ephemeral_id_1",
+                                                                    "_host_name_1",
+                                                                    "_host_address_1",
+                                                                    new TransportAddress(TransportAddress.META_ADDRESS, 9301),
+                                                                    singletonMap("attr", "value_1"),
+                                                                    singleton(DiscoveryNode.Role.DATA),
+                                                                    Version.V_6_0_0_alpha1);
+
+        final ShardId shardId = new ShardId("_index_a", "_uuid_a", 0);
+        final RecoverySource source = RecoverySource.PeerRecoverySource.INSTANCE;
+        final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "_index_info_a");
+        final ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, source, unassignedInfo)
+                                                      .initialize("_node_id", "_allocation_id", 123L);
+
+        final Map<String, List<RecoveryState>> shardRecoveryStates = new HashMap<>();
+        final RecoveryState recoveryState = new RecoveryState(shardRouting, discoveryNodeOne, discoveryNodeOne);
+        shardRecoveryStates.put("_shard_0", singletonList(recoveryState));
+
+        final RecoveryState.Timer timer = recoveryState.getTimer();
+        timer.stop();
+
+        final List<ShardOperationFailedException> shardFailures = new ArrayList<>();
+        final Throwable reason = new NodeDisconnectedException(discoveryNodeZero, "");
+        shardFailures.add(new IndicesShardStoresResponse.Failure("_failed_node_id", "_failed_index", 1, reason));
+
+        final RecoveryResponse recoveryResponse = new RecoveryResponse(10, 7, 3, true, shardRecoveryStates, shardFailures);
+        final MonitoringDoc.Node node = new MonitoringDoc.Node("_uuid", "_host", "_addr", "_ip", "_name", 1504169190855L);
+        final IndexRecoveryMonitoringDoc document = new IndexRecoveryMonitoringDoc("_cluster", 1502266739402L, node, recoveryResponse);
+
+        final BytesReference xContent = XContentHelper.toXContent(document, XContentType.JSON, false);
+        assertEquals("{"
+                    + "\"cluster_uuid\":\"_cluster\","
+                    + "\"timestamp\":\"2017-08-09T08:18:59.402Z\","
+                    + "\"type\":\"index_recovery\","
+                    + "\"source_node\":{"
+                        + "\"uuid\":\"_uuid\","
+                        + "\"host\":\"_host\","
+                        + "\"transport_address\":\"_addr\","
+                        + "\"ip\":\"_ip\","
+                        + "\"name\":\"_name\","
+                        + "\"timestamp\":\"2017-08-31T08:46:30.855Z\""
+                    + "},"
+                    + "\"index_recovery\":{"
+                        + "\"shards\":["
+                        + "{"
+                          + "\"index_name\":\"_shard_0\","
+                          + "\"id\":0,"
+                          + "\"type\":\"PEER\","
+                          + "\"stage\":\"INIT\","
+                          + "\"primary\":true,"
+                          + "\"start_time_in_millis\":" + timer.startTime() + ","
+                          + "\"stop_time_in_millis\":" + timer.stopTime() + ","
+                          + "\"total_time_in_millis\":" + timer.time() + ","
+                          + "\"source\":{"
+                            + "\"id\":\"_node_id_1\","
+                            + "\"host\":\"_host_name_1\","
+                              + "\"transport_address\":\"0.0.0.0:9301\","
+                            + "\"ip\":\"_host_address_1\","
+                            + "\"name\":\"_node_1\""
+                          + "},"
+                          + "\"target\":{"
+                            + "\"id\":\"_node_id_1\","
+                            + "\"host\":\"_host_name_1\","
+                            + "\"transport_address\":\"0.0.0.0:9301\","
+                            + "\"ip\":\"_host_address_1\","
+                            + "\"name\":\"_node_1\""
+                          + "},"
+                          + "\"index\":{"
+                            + "\"size\":{"
+                              + "\"total_in_bytes\":0,"
+                              + "\"reused_in_bytes\":0,"
+                              + "\"recovered_in_bytes\":0,"
+                              + "\"percent\":\"0.0%\""
+                            + "},"
+                            + "\"files\":{"
+                              + "\"total\":0,"
+                              + "\"reused\":0,"
+                              + "\"recovered\":0,"
+                              + "\"percent\":\"0.0%\""
+                            + "},"
+                            + "\"total_time_in_millis\":0,"
+                            + "\"source_throttle_time_in_millis\":0,"
+                            + "\"target_throttle_time_in_millis\":0"
+                          + "},"
+                          + "\"translog\":{"
+                            + "\"recovered\":0,"
+                            + "\"total\":-1,"
+                            + "\"percent\":\"-1.0%\","
+                            + "\"total_on_start\":-1,"
+                            + "\"total_time_in_millis\":0"
+                          + "},"
+                          + "\"verify_index\":{"
+                            + "\"check_index_time_in_millis\":0,"
+                            + "\"total_time_in_millis\":0"
+                          + "}"
+                        + "}"
+                      + "]"
+                    + "}"
+                + "}", xContent.utf8ToString());
+    }
+}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsCollectorTests.java

@@ -5,226 +5,139 @@
  */
 package org.elasticsearch.xpack.monitoring.collector.indices;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
-import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.IndicesAdminClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.MonitoringSettings;
-import org.elasticsearch.xpack.monitoring.collector.AbstractCollectorTestCase;
+import org.elasticsearch.xpack.monitoring.collector.BaseCollectorTestCase;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
 
 import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
 
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.hamcrest.Matchers.arrayWithSize;
+import static org.elasticsearch.xpack.monitoring.MonitoringTestUtils.randomMonitoringNode;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
-@ClusterScope(numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
-public class IndexStatsCollectorTests extends AbstractCollectorTestCase {
+public class IndexStatsCollectorTests extends BaseCollectorTestCase {
 
-    @Override
-    protected int numberOfReplicas() {
-        return 0;
+    public void testShouldCollectReturnsFalseIfMonitoringNotAllowed() {
+        // this controls the blockage
+        when(licenseState.isMonitoringAllowed()).thenReturn(false);
+        whenLocalNodeElectedMaster(randomBoolean());
+
+        final IndexStatsCollector collector =
+                new IndexStatsCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState, client);
+
+        assertThat(collector.shouldCollect(), is(false));
+        verify(licenseState).isMonitoringAllowed();
     }
 
-    public void testEmptyCluster() throws Exception {
-        final String node = internalCluster().startNode();
-        waitForNoBlocksOnNode(node);
-        assertThat(newIndexStatsCollector(node).doCollect(), hasSize(1));
+    public void testShouldCollectReturnsFalseIfNotMaster() {
+        when(licenseState.isMonitoringAllowed()).thenReturn(true);
+        // this controls the blockage
+        whenLocalNodeElectedMaster(false);
+
+        final IndexStatsCollector collector =
+                new IndexStatsCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState, client);
+
+        assertThat(collector.shouldCollect(), is(false));
+        verify(licenseState).isMonitoringAllowed();
+        verify(nodes).isLocalNodeElectedMaster();
     }
 
-    public void testEmptyClusterAllIndices() throws Exception {
-        final String node = internalCluster().startNode(Settings.builder().put(MonitoringSettings.INDICES.getKey(), MetaData.ALL));
-        waitForNoBlocksOnNode(node);
-        assertThat(newIndexStatsCollector(node).doCollect(), hasSize(1));
+    public void testShouldCollectReturnsTrue() {
+        when(licenseState.isMonitoringAllowed()).thenReturn(true);
+        whenLocalNodeElectedMaster(true);
+
+        final IndexStatsCollector collector =
+                new IndexStatsCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState, client);
+
+        assertThat(collector.shouldCollect(), is(true));
+        verify(licenseState).isMonitoringAllowed();
+        verify(nodes).isLocalNodeElectedMaster();
     }
 
-    public void testEmptyClusterMissingIndex() throws Exception {
-        final String node = internalCluster().startNode(Settings.builder().put(MonitoringSettings.INDICES.getKey(), "unknown"));
-        waitForNoBlocksOnNode(node);
-        assertThat(newIndexStatsCollector(node).doCollect(), hasSize(1));
-    }
+    public void testDoCollect() throws Exception {
+        whenLocalNodeElectedMaster(true);
 
-    public void testIndexStatsCollectorOneIndex() throws Exception {
-        final String node = internalCluster().startNode();
-        waitForNoBlocksOnNode(node);
+        final String clusterName = randomAlphaOfLength(10);
+        whenClusterStateWithName(clusterName);
 
-        final String indexName = "one-index";
-        createIndex(indexName);
-        ensureGreen(indexName);
+        final String clusterUUID = UUID.randomUUID().toString();
+        whenClusterStateWithUUID(clusterUUID);
 
-        final int nbDocs = randomIntBetween(1, 20);
-        for (int i = 0; i < nbDocs; i++) {
-            client().prepareIndex(indexName, "test").setSource("num", i).get();
+        final MonitoringDoc.Node node = randomMonitoringNode(random());
+
+        final TimeValue timeout = mock(TimeValue.class);
+        when(monitoringSettings.indexStatsTimeout()).thenReturn(timeout);
+
+        final Map<String, IndexStats> indicesStats = new HashMap<>();
+        final int indices = randomIntBetween(0, 10);
+        for (int i = 0; i < indices; i++) {
+            String index = "_index_" + i;
+            IndexStats indexStats = mock(IndexStats.class);
+            when(indexStats.getIndex()).thenReturn(index);
+            indicesStats.put(index, indexStats);
         }
 
-        refresh();
+        final IndicesStatsResponse indicesStatsResponse = mock(IndicesStatsResponse.class);
+        when(indicesStatsResponse.getIndices()).thenReturn(indicesStats);
 
-        assertHitCount(client().prepareSearch().setSize(0).get(), nbDocs);
+        final IndicesStatsRequestBuilder indicesStatsRequestBuilder =
+                spy(new IndicesStatsRequestBuilder(mock(ElasticsearchClient.class), IndicesStatsAction.INSTANCE));
+        doReturn(indicesStatsResponse).when(indicesStatsRequestBuilder).get(eq(timeout));
 
-        Collection<MonitoringDoc> results = newIndexStatsCollector().doCollect();
-        assertThat(results, hasSize(2));
+        final IndicesAdminClient indicesAdminClient = mock(IndicesAdminClient.class);
+        when(indicesAdminClient.prepareStats()).thenReturn(indicesStatsRequestBuilder);
 
-        // indices stats
-        final Optional<IndicesStatsMonitoringDoc> indicesStatsDoc =
-            results.stream().filter(doc -> doc instanceof IndicesStatsMonitoringDoc).map(doc -> (IndicesStatsMonitoringDoc)doc).findFirst();
+        final AdminClient adminClient = mock(AdminClient.class);
+        when(adminClient.indices()).thenReturn(indicesAdminClient);
 
-        assertThat(indicesStatsDoc.isPresent(), is(true));
+        final Client client = mock(Client.class);
+        when(client.admin()).thenReturn(adminClient);
 
-        IndicesStatsMonitoringDoc indicesStatsMonitoringDoc = indicesStatsDoc.get();
-        assertThat(indicesStatsMonitoringDoc.getClusterUUID(), equalTo(client().admin().cluster().
-                prepareState().setMetaData(true).get().getState().metaData().clusterUUID()));
-        assertThat(indicesStatsMonitoringDoc.getTimestamp(), greaterThan(0L));
-        assertThat(indicesStatsMonitoringDoc.getSourceNode(), notNullValue());
+        final IndexStatsCollector collector =
+                new IndexStatsCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState, client);
 
-        IndicesStatsResponse indicesStats = indicesStatsMonitoringDoc.getIndicesStats();
-        assertNotNull(indicesStats);
-        assertThat(indicesStats.getIndices().keySet(), hasSize(1));
-        assertThat(indicesStats.getIndex(indexName).getShards(), arrayWithSize(getNumShards(indexName).totalNumShards));
+        final Collection<MonitoringDoc> results = collector.doCollect(node);
+        verify(indicesAdminClient).prepareStats();
 
-        // index stats
-        final Optional<IndexStatsMonitoringDoc> indexStatsDoc =
-                results.stream()
-                       .filter(doc -> doc instanceof IndexStatsMonitoringDoc)
-                       .map(doc -> (IndexStatsMonitoringDoc)doc)
-                       .findFirst();
+        assertEquals(1 + indices, results.size());
 
-        assertThat(indexStatsDoc.isPresent(), is(true));
+        for (MonitoringDoc document : results) {
+            assertThat(document.getCluster(), equalTo(clusterUUID));
+            assertThat(document.getTimestamp(), greaterThan(0L));
+            assertThat(document.getNode(), equalTo(node));
+            assertThat(document.getSystem(), is(MonitoredSystem.ES));
+            assertThat(document.getId(), nullValue());
 
-        IndexStatsMonitoringDoc indexStatsMonitoringDoc = indexStatsDoc.get();
-        assertThat(indexStatsMonitoringDoc.getMonitoringId(), equalTo(MonitoredSystem.ES.getSystem()));
-        assertThat(indexStatsMonitoringDoc.getMonitoringVersion(), equalTo(Version.CURRENT.toString()));
-        assertThat(indexStatsMonitoringDoc.getClusterUUID(),
-                equalTo(client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID()));
-        assertThat(indexStatsMonitoringDoc.getTimestamp(), greaterThan(0L));
-        assertThat(indexStatsMonitoringDoc.getSourceNode(), notNullValue());
+            if (document instanceof IndicesStatsMonitoringDoc) {
+                assertThat(document.getType(), equalTo(IndicesStatsMonitoringDoc.TYPE));
+                assertThat(((IndicesStatsMonitoringDoc) document).getIndicesStats(), is(indicesStatsResponse));
+            } else {
+                assertThat(document.getType(), equalTo(IndexStatsMonitoringDoc.TYPE));
 
-        IndexStats indexStats = indexStatsMonitoringDoc.getIndexStats();
-        assertNotNull(indexStats);
-
-        assertThat(indexStats.getIndex(), equalTo(indexName));
-        assertThat(indexStats.getPrimaries().getDocs().getCount(), equalTo((long) nbDocs));
-        assertNotNull(indexStats.getTotal().getStore());
-        assertThat(indexStats.getTotal().getStore().getSizeInBytes(), greaterThan(0L));
-        assertNotNull(indexStats.getTotal().getIndexing());
-        assertThat(indexStats.getTotal().getIndexing().getTotal().getThrottleTime().millis(), equalTo(0L));
-    }
-
-    public void testIndexStatsCollectorMultipleIndices() throws Exception {
-        final String node = internalCluster().startNode();
-        waitForNoBlocksOnNode(node);
-
-        final String indexPrefix = "multi-indices-";
-        final int nbIndices = randomIntBetween(1, 5);
-        int[] docsPerIndex = new int[nbIndices];
-
-        for (int i = 0; i < nbIndices; i++) {
-            String index = indexPrefix + i;
-            createIndex(index);
-            ensureGreen(index);
-
-            docsPerIndex[i] = randomIntBetween(1, 20);
-            for (int j = 0; j < docsPerIndex[i]; j++) {
-                client().prepareIndex(index, "test").setSource("num", i).get();
+                IndexStats indexStats = ((IndexStatsMonitoringDoc) document).getIndexStats();
+                assertThat(indexStats, is(indicesStats.get(indexStats.getIndex())));
             }
         }
-
-        refresh();
-
-        for (int i = 0; i < nbIndices; i++) {
-            assertHitCount(client().prepareSearch(indexPrefix + i).setSize(0).get(), docsPerIndex[i]);
-        }
-
-        String clusterUUID = client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID();
-
-        Collection<MonitoringDoc> results = newIndexStatsCollector().doCollect();
-        // extra document is for the IndicesStatsMonitoringDoc
-        assertThat(results, hasSize(nbIndices + 1));
-
-        // indices stats
-        final Optional<IndicesStatsMonitoringDoc> indicesStatsDoc =
-                results.stream()
-                       .filter(doc -> doc instanceof IndicesStatsMonitoringDoc)
-                       .map(doc -> (IndicesStatsMonitoringDoc)doc)
-                       .findFirst();
-
-        assertThat(indicesStatsDoc.isPresent(), is(true));
-
-        IndicesStatsMonitoringDoc indicesStatsMonitoringDoc = indicesStatsDoc.get();
-        assertThat(indicesStatsMonitoringDoc.getMonitoringId(), equalTo(MonitoredSystem.ES.getSystem()));
-        assertThat(indicesStatsMonitoringDoc.getMonitoringVersion(), equalTo(Version.CURRENT.toString()));
-        assertThat(indicesStatsMonitoringDoc.getClusterUUID(),
-                equalTo(client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID()));
-        assertThat(indicesStatsMonitoringDoc.getTimestamp(), greaterThan(0L));
-
-        IndicesStatsResponse indicesStats = indicesStatsMonitoringDoc.getIndicesStats();
-        assertNotNull(indicesStats);
-        assertThat(indicesStats.getIndices().keySet(), hasSize(nbIndices));
-
-        // index stats
-        final List<IndexStatsMonitoringDoc> indexStatsDocs =
-                results.stream()
-                       .filter(doc -> doc instanceof IndexStatsMonitoringDoc)
-                       .map(doc -> (IndexStatsMonitoringDoc)doc)
-                       .collect(Collectors.toList());
-
-        assertThat(indexStatsDocs, hasSize(nbIndices));
-
-        for (int i = 0; i < nbIndices; i++) {
-            String indexName = indexPrefix + i;
-            boolean found = false;
-
-            Iterator<IndexStatsMonitoringDoc> it = indexStatsDocs.iterator();
-            while (!found && it.hasNext()) {
-                IndexStatsMonitoringDoc indexStatsMonitoringDoc = it.next();
-                IndexStats indexStats = indexStatsMonitoringDoc.getIndexStats();
-                assertNotNull(indexStats);
-
-                if (indexStats.getIndex().equals(indexPrefix + i)) {
-                    assertThat(indexStatsMonitoringDoc.getClusterUUID(), equalTo(clusterUUID));
-                    assertThat(indexStatsMonitoringDoc.getTimestamp(), greaterThan(0L));
-                    assertThat(indexStatsMonitoringDoc.getSourceNode(), notNullValue());
-
-                    assertThat(indexStats.getIndex(), equalTo(indexName));
-                    assertNotNull(indexStats.getTotal().getDocs());
-                    assertThat(indexStats.getPrimaries().getDocs().getCount(), equalTo((long) docsPerIndex[i]));
-                    assertNotNull(indexStats.getTotal().getStore());
-                    assertThat(indexStats.getTotal().getStore().getSizeInBytes(), greaterThanOrEqualTo(0L));
-                    assertNotNull(indexStats.getTotal().getIndexing());
-                    assertThat(indexStats.getTotal().getIndexing().getTotal().getThrottleTime().millis(), equalTo(0L));
-                    found = true;
-                }
-            }
-            assertThat("could not find collected stats for index [" + indexPrefix + i + "]", found, is(true));
-        }
-    }
-
-    private IndexStatsCollector newIndexStatsCollector() {
-        // This collector runs on master node only
-        return newIndexStatsCollector(internalCluster().getMasterName());
-    }
-
-    private IndexStatsCollector newIndexStatsCollector(String nodeId) {
-        assertNotNull(nodeId);
-        return new IndexStatsCollector(internalCluster().getInstance(Settings.class, nodeId),
-                internalCluster().getInstance(ClusterService.class, nodeId),
-                internalCluster().getInstance(MonitoringSettings.class, nodeId),
-                internalCluster().getInstance(XPackLicenseState.class, nodeId),
-                securedClient(nodeId));
     }
 }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndexStatsMonitoringDocTests.java

@@ -0,0 +1,271 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.monitoring.collector.indices;
+
+import org.elasticsearch.action.admin.indices.stats.CommonStats;
+import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
+import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.cache.query.QueryCacheStats;
+import org.elasticsearch.index.cache.request.RequestCacheStats;
+import org.elasticsearch.index.engine.SegmentsStats;
+import org.elasticsearch.index.fielddata.FieldDataStats;
+import org.elasticsearch.index.refresh.RefreshStats;
+import org.elasticsearch.index.search.stats.SearchStats;
+import org.elasticsearch.index.shard.DocsStats;
+import org.elasticsearch.index.shard.IndexingStats;
+import org.elasticsearch.index.store.StoreStats;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+import org.elasticsearch.xpack.monitoring.exporter.BaseFilteredMonitoringDocTestCase;
+import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class IndexStatsMonitoringDocTests extends BaseFilteredMonitoringDocTestCase<IndexStatsMonitoringDoc> {
+
+
+    private IndexStats indexStats;
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        indexStats = mock(IndexStats.class);
+    }
+
+    @Override
+    protected IndexStatsMonitoringDoc createMonitoringDoc(String cluster, long timestamp, MonitoringDoc.Node node,
+                                                          MonitoredSystem system, String type, String id) {
+        return new IndexStatsMonitoringDoc(cluster, timestamp, node, indexStats);
+    }
+
+    @Override
+    protected void assertFilteredMonitoringDoc(final IndexStatsMonitoringDoc document) {
+        assertThat(document.getSystem(), is(MonitoredSystem.ES));
+        assertThat(document.getType(), is(IndexStatsMonitoringDoc.TYPE));
+        assertThat(document.getId(), nullValue());
+
+        assertThat(document.getIndexStats(), is(indexStats));
+    }
+
+    @Override
+    protected Set<String> getExpectedXContentFilters() {
+        return IndexStatsMonitoringDoc.XCONTENT_FILTERS;
+    }
+
+    public void testConstructorIndexStatsMustNotBeNull() {
+        expectThrows(NullPointerException.class, () -> new IndexStatsMonitoringDoc(cluster, timestamp, node, null));
+    }
+
+    @Override
+    public void testToXContent() throws IOException {
+        final MonitoringDoc.Node node = new MonitoringDoc.Node("_uuid", "_host", "_addr", "_ip", "_name", 1504169190855L);
+        when(indexStats.getIndex()).thenReturn("_index");
+        when(indexStats.getTotal()).thenReturn(mockCommonStats());
+        when(indexStats.getPrimaries()).thenReturn(mockCommonStats());
+
+        final IndexStatsMonitoringDoc document = new IndexStatsMonitoringDoc("_cluster", 1502266739402L, node, indexStats);
+
+        final BytesReference xContent = XContentHelper.toXContent(document, XContentType.JSON, false);
+        assertEquals("{"
+                     + "\"cluster_uuid\":\"_cluster\","
+                     + "\"timestamp\":\"2017-08-09T08:18:59.402Z\","
+                     + "\"type\":\"index_stats\","
+                     + "\"source_node\":{"
+                       + "\"uuid\":\"_uuid\","
+                       + "\"host\":\"_host\","
+                       + "\"transport_address\":\"_addr\","
+                       + "\"ip\":\"_ip\","
+                       + "\"name\":\"_name\","
+                       + "\"timestamp\":\"2017-08-31T08:46:30.855Z\""
+                     + "},"
+                     + "\"index_stats\":{"
+                       + "\"index\":\"_index\","
+                       + "\"total\":{"
+                         + "\"docs\":{"
+                           + "\"count\":1"
+                         + "},"
+                         + "\"store\":{"
+                           + "\"size_in_bytes\":13"
+                         + "},"
+                         + "\"indexing\":{"
+                           + "\"index_total\":15,"
+                           + "\"index_time_in_millis\":16,"
+                           + "\"throttle_time_in_millis\":17"
+                         + "},"
+                         + "\"search\":{"
+                           + "\"query_total\":18,"
+                           + "\"query_time_in_millis\":19"
+                         + "},"
+                         + "\"merges\":{"
+                           + "\"total_size_in_bytes\":4"
+                         + "},"
+                         + "\"refresh\":{"
+                           + "\"total_time_in_millis\":14"
+                         + "},"
+                         + "\"query_cache\":{"
+                           + "\"memory_size_in_bytes\":5,"
+                           + "\"hit_count\":6,"
+                           + "\"miss_count\":7,"
+                           + "\"evictions\":9"
+                         + "},"
+                         + "\"fielddata\":{"
+                           + "\"memory_size_in_bytes\":2,"
+                           + "\"evictions\":3"
+                         + "},"
+                         + "\"segments\":{"
+                           + "\"count\":20,"
+                           + "\"memory_in_bytes\":21,"
+                           + "\"terms_memory_in_bytes\":22,"
+                           + "\"stored_fields_memory_in_bytes\":23,"
+                           + "\"term_vectors_memory_in_bytes\":24,"
+                           + "\"norms_memory_in_bytes\":25,"
+                           + "\"points_memory_in_bytes\":26,"
+                           + "\"doc_values_memory_in_bytes\":27,"
+                           + "\"index_writer_memory_in_bytes\":28,"
+                           + "\"version_map_memory_in_bytes\":29,"
+                           + "\"fixed_bit_set_memory_in_bytes\":30"
+                         + "},"
+                         + "\"request_cache\":{"
+                           + "\"memory_size_in_bytes\":9,"
+                           + "\"evictions\":10,"
+                           + "\"hit_count\":11,"
+                           + "\"miss_count\":12"
+                         + "}"
+                       + "},"
+                       + "\"primaries\":{"
+                         + "\"docs\":{"
+                           + "\"count\":1"
+                         + "},"
+                         + "\"store\":{"
+                           + "\"size_in_bytes\":13"
+                         + "},"
+                         + "\"indexing\":{"
+                           + "\"index_total\":15,"
+                           + "\"index_time_in_millis\":16,"
+                           + "\"throttle_time_in_millis\":17"
+                         + "},"
+                         + "\"search\":{"
+                           + "\"query_total\":18,"
+                           + "\"query_time_in_millis\":19"
+                         + "},"
+                         + "\"merges\":{"
+                           + "\"total_size_in_bytes\":4"
+                         + "},"
+                         + "\"refresh\":{"
+                           + "\"total_time_in_millis\":14"
+                         + "},"
+                         + "\"query_cache\":{"
+                           + "\"memory_size_in_bytes\":5,"
+                           + "\"hit_count\":6,"
+                           + "\"miss_count\":7,"
+                           + "\"evictions\":9"
+                         + "},"
+                         + "\"fielddata\":{"
+                           + "\"memory_size_in_bytes\":2,"
+                           + "\"evictions\":3"
+                         + "},"
+                         + "\"segments\":{"
+                           + "\"count\":20,"
+                           + "\"memory_in_bytes\":21,"
+                           + "\"terms_memory_in_bytes\":22,"
+                           + "\"stored_fields_memory_in_bytes\":23,"
+                           + "\"term_vectors_memory_in_bytes\":24,"
+                           + "\"norms_memory_in_bytes\":25,"
+                           + "\"points_memory_in_bytes\":26,"
+                           + "\"doc_values_memory_in_bytes\":27,"
+                           + "\"index_writer_memory_in_bytes\":28,"
+                           + "\"version_map_memory_in_bytes\":29,"
+                           + "\"fixed_bit_set_memory_in_bytes\":30"
+                         + "},"
+                         + "\"request_cache\":{"
+                           + "\"memory_size_in_bytes\":9,"
+                           + "\"evictions\":10,"
+                           + "\"hit_count\":11,"
+                           + "\"miss_count\":12"
+                        + "}"
+                        + "}"
+                      + "}"
+                + "}", xContent.utf8ToString());
+    }
+
+    public void testToXContentWithNullStats() throws IOException {
+        final MonitoringDoc.Node node = new MonitoringDoc.Node("_uuid", "_host", "_addr", "_ip", "_name", 1504169190855L);
+        when(indexStats.getIndex()).thenReturn("_index");
+        when(indexStats.getTotal()).thenReturn(null);
+        when(indexStats.getPrimaries()).thenReturn(null);
+
+        final IndexStatsMonitoringDoc document = new IndexStatsMonitoringDoc("_cluster", 1502266739402L, node, indexStats);
+
+        final BytesReference xContent = XContentHelper.toXContent(document, XContentType.JSON, false);
+        assertEquals("{"
+                     + "\"cluster_uuid\":\"_cluster\","
+                     + "\"timestamp\":\"2017-08-09T08:18:59.402Z\","
+                     + "\"type\":\"index_stats\","
+                     + "\"source_node\":{"
+                       + "\"uuid\":\"_uuid\","
+                       + "\"host\":\"_host\","
+                       + "\"transport_address\":\"_addr\","
+                       + "\"ip\":\"_ip\","
+                       + "\"name\":\"_name\","
+                       + "\"timestamp\":\"2017-08-31T08:46:30.855Z\""
+                     + "},"
+                     + "\"index_stats\":{"
+                       + "\"index\":\"_index\""
+                      + "}"
+                + "}", xContent.utf8ToString());
+    }
+
+    private static CommonStats mockCommonStats() {
+        // This value is used in constructors of various stats objects,
+        // when the value is not printed out in the final XContent.
+        final long no = -1;
+
+        // This value is used in constructors of various stats objects,
+        // when the value is printed out in the XContent. Must be
+        // incremented for each usage.
+        long iota = 0L;
+
+        final CommonStats commonStats = new CommonStats(CommonStatsFlags.ALL);
+        commonStats.getDocs().add(new DocsStats(++iota, no));
+        commonStats.getFieldData().add(new FieldDataStats(++iota, ++iota, null));
+        commonStats.getMerge().add(no, no, no, ++iota, no, no, no, no, no, no);
+        commonStats.getQueryCache().add(new QueryCacheStats(++iota, ++iota, ++iota, ++iota, no));
+        commonStats.getRequestCache().add(new RequestCacheStats(++iota, ++iota, ++iota, ++iota));
+        commonStats.getStore().add(new StoreStats(++iota));
+        commonStats.getRefresh().add(new RefreshStats(no, ++iota, (int) no));
+
+        final IndexingStats.Stats indexingStats = new IndexingStats.Stats(++iota, ++iota, no, no, no, no, no, no, false, ++iota);
+        commonStats.getIndexing().add(new IndexingStats(indexingStats, null));
+
+        final SearchStats.Stats searchStats = new SearchStats.Stats(++iota, ++iota, no, no, no, no, no, no, no, no, no, no);
+        commonStats.getSearch().add(new SearchStats(searchStats, no, null));
+
+        final SegmentsStats segmentsStats = new SegmentsStats();
+        segmentsStats.add(++iota, ++iota);
+        segmentsStats.addTermsMemoryInBytes(++iota);
+        segmentsStats.addStoredFieldsMemoryInBytes(++iota);
+        segmentsStats.addTermVectorsMemoryInBytes(++iota);
+        segmentsStats.addNormsMemoryInBytes(++iota);
+        segmentsStats.addPointsMemoryInBytes(++iota);
+        segmentsStats.addDocValuesMemoryInBytes(++iota);
+        segmentsStats.addIndexWriterMemoryInBytes(++iota);
+        segmentsStats.addVersionMapMemoryInBytes(++iota);
+        segmentsStats.addBitsetMemoryInBytes(++iota);
+        commonStats.getSegments().add(segmentsStats);
+
+        return commonStats;
+    }
+}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java

@@ -0,0 +1,194 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.monitoring.collector.indices;
+
+import org.elasticsearch.action.admin.indices.stats.CommonStats;
+import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.elasticsearch.action.admin.indices.stats.ShardStats;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.search.stats.SearchStats;
+import org.elasticsearch.index.shard.DocsStats;
+import org.elasticsearch.index.shard.IndexingStats;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.shard.ShardPath;
+import org.elasticsearch.index.store.StoreStats;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+import org.elasticsearch.xpack.monitoring.exporter.BaseFilteredMonitoringDocTestCase;
+import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Set;
+
+import static java.util.Collections.emptyList;
+import static org.elasticsearch.action.admin.indices.stats.IndicesStatsResponseTestUtils.newIndicesStatsResponse;
+import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class IndicesStatsMonitoringDocTests extends BaseFilteredMonitoringDocTestCase<IndicesStatsMonitoringDoc> {
+
+    private IndicesStatsResponse indicesStatsResponse;
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        indicesStatsResponse = mock(IndicesStatsResponse.class);
+        when(indicesStatsResponse.toXContent(any(XContentBuilder.class), any(ToXContent.Params.class))).thenCallRealMethod();
+        when(indicesStatsResponse.getPrimaries()).thenReturn(mockCommonStats());
+        when(indicesStatsResponse.getTotal()).thenReturn(mockCommonStats());
+    }
+
+    @Override
+    protected IndicesStatsMonitoringDoc createMonitoringDoc(String cluster, long timestamp, MonitoringDoc.Node node,
+                                                            MonitoredSystem system, String type, String id) {
+        return new IndicesStatsMonitoringDoc(cluster, timestamp, node, indicesStatsResponse);
+    }
+
+    @Override
+    protected void assertFilteredMonitoringDoc(final IndicesStatsMonitoringDoc document) {
+        assertThat(document.getSystem(), is(MonitoredSystem.ES));
+        assertThat(document.getType(), is(IndicesStatsMonitoringDoc.TYPE));
+        assertThat(document.getId(), nullValue());
+
+        assertThat(document.getIndicesStats(), is(indicesStatsResponse));
+    }
+
+    @Override
+    protected Set<String> getExpectedXContentFilters() {
+        return IndicesStatsMonitoringDoc.XCONTENT_FILTERS;
+    }
+
+    public void testConstructorIndexStatsMustNotBeNull() {
+        expectThrows(NullPointerException.class, () -> new IndicesStatsMonitoringDoc(cluster, timestamp, node, null));
+    }
+
+    @Override
+    public void testToXContent() throws IOException {
+        final MonitoringDoc.Node node = new MonitoringDoc.Node("_uuid", "_host", "_addr", "_ip", "_name", 1504169190855L);
+
+        final ShardStats[] shards = new ShardStats[] {
+                // Primaries
+                new ShardStats(mockShardRouting(true), mockShardPath(), mockCommonStats(), null, null),
+                new ShardStats(mockShardRouting(true), mockShardPath(), mockCommonStats(), null, null),
+                // Replica
+                new ShardStats(mockShardRouting(false), mockShardPath(), mockCommonStats(), null, null)
+        };
+        final IndicesStatsResponse indicesStatsResponse = newIndicesStatsResponse(shards, -1, -1, -1, emptyList());
+
+        final IndicesStatsMonitoringDoc document = new IndicesStatsMonitoringDoc("_cluster", 1502266739402L, node, indicesStatsResponse);
+
+        final BytesReference xContent = XContentHelper.toXContent(document, XContentType.JSON, false);
+        assertEquals("{"
+                     + "\"cluster_uuid\":\"_cluster\","
+                     + "\"timestamp\":\"2017-08-09T08:18:59.402Z\","
+                     + "\"type\":\"indices_stats\","
+                     + "\"source_node\":{"
+                       + "\"uuid\":\"_uuid\","
+                       + "\"host\":\"_host\","
+                       + "\"transport_address\":\"_addr\","
+                       + "\"ip\":\"_ip\","
+                       + "\"name\":\"_name\","
+                       + "\"timestamp\":\"2017-08-31T08:46:30.855Z\""
+                      + "},"
+                     + "\"indices_stats\":{"
+                       + "\"_all\":{"
+                         + "\"primaries\":{"
+                           + "\"docs\":{"
+                             + "\"count\":2"
+                            + "},"
+                           + "\"store\":{"
+                             + "\"size_in_bytes\":4"
+                            + "},"
+                           + "\"indexing\":{"
+                             + "\"index_total\":6,"
+                             + "\"index_time_in_millis\":8,"
+                             + "\"is_throttled\":true,"
+                             + "\"throttle_time_in_millis\":10"
+                            + "},"
+                           + "\"search\":{"
+                             + "\"query_total\":12,"
+                             + "\"query_time_in_millis\":14"
+                          + "}"
+                          + "},"
+                         + "\"total\":{"
+                           + "\"docs\":{"
+                             + "\"count\":3"
+                            + "},"
+                           + "\"store\":{"
+                             + "\"size_in_bytes\":6"
+                            + "},"
+                           + "\"indexing\":{"
+                             + "\"index_total\":9,"
+                             + "\"index_time_in_millis\":12,"
+                             + "\"is_throttled\":true,"
+                             + "\"throttle_time_in_millis\":15"
+                            + "},"
+                           + "\"search\":{"
+                             + "\"query_total\":18,"
+                             + "\"query_time_in_millis\":21"
+                            + "}"
+                        + "}"
+                    + "}"
+                + "}"
+            + "}", xContent.utf8ToString());
+    }
+
+    private CommonStats mockCommonStats() {
+        final CommonStats commonStats = new CommonStats(CommonStatsFlags.ALL);
+        commonStats.getDocs().add(new DocsStats(1L, -1L));
+        commonStats.getStore().add(new StoreStats(2L));
+
+        final IndexingStats.Stats indexingStats = new IndexingStats.Stats(3L, 4L, -1L, -1L, -1L, -1L, -1L, -1L, true, 5L);
+        commonStats.getIndexing().add(new IndexingStats(indexingStats, null));
+
+        final SearchStats.Stats searchStats = new SearchStats.Stats(6L, 7L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L, -1L);
+        commonStats.getSearch().add(new SearchStats(searchStats, -1L, null));
+
+        return commonStats;
+    }
+
+    private ShardPath mockShardPath() {
+        // Mock paths in a way that pass ShardPath constructor assertions
+        final int shardId = randomIntBetween(0, 10);
+        final Path getFileNameShardId = mock(Path.class);
+        when(getFileNameShardId.toString()).thenReturn(Integer.toString(shardId));
+
+        final String shardUuid =  randomAlphaOfLength(5);
+        final Path getFileNameShardUuid = mock(Path.class);
+        when(getFileNameShardUuid.toString()).thenReturn(shardUuid);
+
+        final Path getParent = mock(Path.class);
+        when(getParent.getFileName()).thenReturn(getFileNameShardUuid);
+
+        final Path path = mock(Path.class);
+        when(path.getParent()).thenReturn(getParent);
+        when(path.getFileName()).thenReturn(getFileNameShardId);
+
+        // Mock paths for ShardPath#getRootDataPath()
+        final Path getParentOfParent = mock(Path.class);
+        when(getParent.getParent()).thenReturn(getParentOfParent);
+        when(getParentOfParent.getParent()).thenReturn(mock(Path.class));
+
+        return new ShardPath(false, path, path, new ShardId(randomAlphaOfLength(5), shardUuid, shardId));
+    }
+
+    private ShardRouting mockShardRouting(final boolean primary) {
+        return TestShardRouting.newShardRouting(randomAlphaOfLength(5), randomInt(), randomAlphaOfLength(5), primary, INITIALIZING);
+    }
+}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsCollectorTests.java

@@ -5,18 +5,10 @@
  */
 package org.elasticsearch.xpack.monitoring.collector.ml;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.XPackSettings;
 import org.elasticsearch.xpack.ml.action.GetJobsStatsAction.Request;
 import org.elasticsearch.xpack.ml.action.GetJobsStatsAction.Response;
@@ -24,16 +16,19 @@ import org.elasticsearch.xpack.ml.action.GetJobsStatsAction.Response.JobStats;
 import org.elasticsearch.xpack.ml.action.util.QueryPage;
 import org.elasticsearch.xpack.ml.client.MachineLearningClient;
 import org.elasticsearch.xpack.ml.job.config.Job;
-import org.elasticsearch.xpack.monitoring.MonitoringSettings;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+import org.elasticsearch.xpack.monitoring.collector.BaseCollectorTestCase;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
-import org.elasticsearch.xpack.security.InternalClient;
 
 import java.util.ArrayList;
 import java.util.List;
 
+import static org.elasticsearch.xpack.monitoring.MonitoringTestUtils.randomMonitoringNode;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
@@ -42,14 +37,7 @@ import static org.mockito.Mockito.when;
 /**
  * Tests {@link JobStatsCollector}.
  */
-public class JobStatsCollectorTests extends ESTestCase {
-
-    private final ClusterService clusterService = mock(ClusterService.class);
-    private final ClusterState clusterState = mock(ClusterState.class);
-    private final DiscoveryNodes nodes = mock(DiscoveryNodes.class);
-    private final MonitoringSettings monitoringSettings = mock(MonitoringSettings.class);
-    private final XPackLicenseState licenseState = mock(XPackLicenseState.class);
-    private final InternalClient client = mock(InternalClient.class);
+public class JobStatsCollectorTests extends BaseCollectorTestCase {
 
     public void testShouldCollectReturnsFalseIfMonitoringNotAllowed() {
         final Settings settings = randomFrom(mlEnabledSettings(), mlDisabledSettings());
@@ -130,8 +118,7 @@ public class JobStatsCollectorTests extends ESTestCase {
         final TimeValue timeout = mock(TimeValue.class);
         final MetaData metaData = mock(MetaData.class);
         final String clusterUuid = randomAlphaOfLength(5);
-        final String nodeUuid = randomAlphaOfLength(5);
-        final DiscoveryNode localNode = localNode(nodeUuid);
+        final MonitoringDoc.Node node = randomMonitoringNode(random());
         final MachineLearningClient client = mock(MachineLearningClient.class);
 
         when(monitoringSettings.jobStatsTimeout()).thenReturn(timeout);
@@ -140,8 +127,6 @@ public class JobStatsCollectorTests extends ESTestCase {
         when(clusterState.metaData()).thenReturn(metaData);
         when(metaData.clusterUUID()).thenReturn(clusterUuid);
 
-        when(clusterService.localNode()).thenReturn(localNode);
-
         final JobStatsCollector collector =
                 new JobStatsCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState, client);
 
@@ -154,7 +139,7 @@ public class JobStatsCollectorTests extends ESTestCase {
         when(client.getJobsStats(eq(new Request(MetaData.ALL)))).thenReturn(future);
         when(future.actionGet(timeout)).thenReturn(response);
 
-        final List<MonitoringDoc> monitoringDocs = collector.doCollect();
+        final List<MonitoringDoc> monitoringDocs = collector.doCollect(node);
 
         assertThat(monitoringDocs, hasSize(jobStats.size()));
 
@@ -162,10 +147,12 @@ public class JobStatsCollectorTests extends ESTestCase {
             final JobStatsMonitoringDoc jobStatsMonitoringDoc = (JobStatsMonitoringDoc)monitoringDocs.get(i);
             final JobStats jobStat = jobStats.get(i);
 
+            assertThat(jobStatsMonitoringDoc.getCluster(), is(clusterUuid));
+            assertThat(jobStatsMonitoringDoc.getTimestamp(), greaterThan(0L));
+            assertThat(jobStatsMonitoringDoc.getNode(), equalTo(node));
+            assertThat(jobStatsMonitoringDoc.getSystem(), is(MonitoredSystem.ES));
             assertThat(jobStatsMonitoringDoc.getType(), is(JobStatsMonitoringDoc.TYPE));
-            assertThat(jobStatsMonitoringDoc.getSourceNode(), notNullValue());
-            assertThat(jobStatsMonitoringDoc.getSourceNode().getUUID(), is(nodeUuid));
-            assertThat(jobStatsMonitoringDoc.getClusterUUID(), is(clusterUuid));
+            assertThat(jobStatsMonitoringDoc.getId(), nullValue());
 
             assertThat(jobStatsMonitoringDoc.getJobStats(), is(jobStat));
         }
@@ -182,16 +169,6 @@ public class JobStatsCollectorTests extends ESTestCase {
         return jobStats;
     }
 
-    private DiscoveryNode localNode(final String uuid) {
-        return new DiscoveryNode(uuid, new TransportAddress(TransportAddress.META_ADDRESS, 9300), Version.CURRENT);
-    }
-
-    private void whenLocalNodeElectedMaster(final boolean electedMaster) {
-        when(clusterService.state()).thenReturn(clusterState);
-        when(clusterState.nodes()).thenReturn(nodes);
-        when(nodes.isLocalNodeElectedMaster()).thenReturn(electedMaster);
-    }
-
     private Settings mlEnabledSettings() {
         // since it's the default, we want to ensure we test both with/without it
         return randomBoolean() ? Settings.EMPTY : Settings.builder().put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), true).build();

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/ml/JobStatsMonitoringDocTests.java

@@ -7,45 +7,162 @@ package org.elasticsearch.xpack.monitoring.collector.ml;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.xpack.ml.action.GetJobsStatsAction.Response.JobStats;
+import org.elasticsearch.xpack.ml.job.config.JobState;
+import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
+import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats;
 import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+import org.elasticsearch.xpack.monitoring.exporter.BaseMonitoringDocTestCase;
+import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
+import org.joda.time.DateTime;
+import org.junit.Before;
 
+import java.io.IOException;
+import java.util.Date;
+
+import static java.util.Collections.singleton;
+import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Mockito.mock;
 
 /**
  * Tests {@link JobStatsMonitoringDocTests}.
  */
-public class JobStatsMonitoringDocTests extends ESTestCase {
+public class JobStatsMonitoringDocTests extends BaseMonitoringDocTestCase<JobStatsMonitoringDoc> {
 
-    private final String clusterUuid = randomAlphaOfLength(5);
-    private final long timestamp = System.currentTimeMillis();
-    private final String nodeUuid = randomAlphaOfLength(5);
-    private final DiscoveryNode node =
-            new DiscoveryNode(nodeUuid, new TransportAddress(TransportAddress.META_ADDRESS, 9300), Version.CURRENT);
-    private final JobStats jobStats = mock(JobStats.class);
+    private JobStats jobStats;
 
-    private final JobStatsMonitoringDoc doc = new JobStatsMonitoringDoc(clusterUuid, timestamp, node, jobStats);
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        jobStats = mock(JobStats.class);
+    }
+
+    @Override
+    protected JobStatsMonitoringDoc createMonitoringDoc(final String cluster, final long timestamp, final MonitoringDoc.Node node,
+                                                        final MonitoredSystem system, final String type, final String id) {
+        return new JobStatsMonitoringDoc(cluster, timestamp, node, jobStats);
+    }
+
+    @Override
+    protected void assertMonitoringDoc(final JobStatsMonitoringDoc document) {
+        assertThat(document.getSystem(), is(MonitoredSystem.ES));
+        assertThat(document.getType(), is(JobStatsMonitoringDoc.TYPE));
+        assertThat(document.getId(), nullValue());
+
+        assertThat(document.getJobStats(), is(jobStats));
+    }
 
     public void testConstructorJobStatsMustNotBeNull() {
         expectThrows(NullPointerException.class,
-                     () -> new JobStatsMonitoringDoc(clusterUuid, timestamp, node, null));
+                     () -> new JobStatsMonitoringDoc(cluster, timestamp, node, null));
     }
 
-    public void testConstructor() {
-        assertThat(doc.getMonitoringId(), is(MonitoredSystem.ES.getSystem()));
-        assertThat(doc.getMonitoringVersion(), is(Version.CURRENT.toString()));
-        assertThat(doc.getType(), is(JobStatsMonitoringDoc.TYPE));
-        assertThat(doc.getId(), nullValue());
-        assertThat(doc.getClusterUUID(), is(clusterUuid));
-        assertThat(doc.getTimestamp(), is(timestamp));
-        assertThat(doc.getSourceNode(), notNullValue());
-        assertThat(doc.getSourceNode().getUUID(), is(nodeUuid));
-        assertThat(doc.getJobStats(), is(jobStats));
-    }
+    @Override
+    public void testToXContent() throws IOException {
+        final TimeValue time = TimeValue.timeValueHours(13L);
+        final Date date1 = DateTime.parse("2017-01-01T01:01:01.001+01").toDate();
+        final Date date2 = DateTime.parse("2017-01-02T02:02:02.002+02").toDate();
+        final Date date3 = DateTime.parse("2017-01-03T03:03:03.003+03").toDate();
+        final Date date4 = DateTime.parse("2017-01-04T04:04:04.004+04").toDate();
+        final Date date5 = DateTime.parse("2017-01-05T05:05:05.005+05").toDate();
+        final Date date6 = DateTime.parse("2017-01-06T06:06:06.006+06").toDate();
+        final Date date7 = DateTime.parse("2017-01-07T07:07:07.007+07").toDate();
 
+
+        final DiscoveryNode discoveryNode = new DiscoveryNode("_node_name",
+                                                             "_node_id",
+                                                             "_ephemeral_id",
+                                                             "_host_name",
+                                                             "_host_address",
+                                                             new TransportAddress(TransportAddress.META_ADDRESS, 9300),
+                                                             singletonMap("attr", "value"),
+                                                             singleton(DiscoveryNode.Role.MASTER),
+                                                             Version.V_6_0_0_beta1);
+
+        final ModelSizeStats modelStats = new ModelSizeStats.Builder("_model")
+                                                            .setModelBytes(100L)
+                                                            .setTotalByFieldCount(101L)
+                                                            .setTotalOverFieldCount(102L)
+                                                            .setTotalPartitionFieldCount(103L)
+                                                            .setBucketAllocationFailuresCount(104L)
+                                                            .setMemoryStatus(ModelSizeStats.MemoryStatus.OK)
+                                                            .setTimestamp(date1)
+                                                            .setLogTime(date2)
+                                                            .build();
+
+        final DataCounts dataCounts = new DataCounts("_job_id", 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, date3, date4, date5, date6, date7);
+        final JobStats jobStats = new JobStats("_job", dataCounts, modelStats, JobState.OPENED, discoveryNode, "_explanation", time);
+        final MonitoringDoc.Node node = new MonitoringDoc.Node("_uuid", "_host", "_addr", "_ip", "_name", 1504169190855L);
+
+        final JobStatsMonitoringDoc document = new JobStatsMonitoringDoc("_cluster", 1502266739402L, node, jobStats);
+
+        final BytesReference xContent = XContentHelper.toXContent(document, XContentType.JSON, false);
+        assertEquals("{"
+                     + "\"cluster_uuid\":\"_cluster\","
+                     + "\"timestamp\":\"2017-08-09T08:18:59.402Z\","
+                     + "\"type\":\"job_stats\","
+                     + "\"source_node\":{"
+                       + "\"uuid\":\"_uuid\","
+                       + "\"host\":\"_host\","
+                       + "\"transport_address\":\"_addr\","
+                       + "\"ip\":\"_ip\","
+                       + "\"name\":\"_name\","
+                       + "\"timestamp\":\"2017-08-31T08:46:30.855Z\""
+                      + "},"
+                     + "\"job_stats\":{"
+                       + "\"job_id\":\"_job\","
+                       + "\"data_counts\":{"
+                         + "\"job_id\":\"_job_id\","
+                         + "\"processed_record_count\":0,"
+                         + "\"processed_field_count\":1,"
+                         + "\"input_bytes\":2,"
+                         + "\"input_field_count\":3,"
+                         + "\"invalid_date_count\":4,"
+                         + "\"missing_field_count\":5,"
+                         + "\"out_of_order_timestamp_count\":6,"
+                         + "\"empty_bucket_count\":7,"
+                         + "\"sparse_bucket_count\":8,"
+                         + "\"bucket_count\":9,"
+                         + "\"earliest_record_timestamp\":1483401783003,"
+                         + "\"latest_record_timestamp\":1483488244004,"
+                         + "\"last_data_time\":1483574705005,"
+                         + "\"latest_empty_bucket_timestamp\":1483661166006,"
+                         + "\"latest_sparse_bucket_timestamp\":1483747627007,"
+                         + "\"input_record_count\":10"
+                        + "},"
+                       + "\"model_size_stats\":{"
+                         + "\"job_id\":\"_model\","
+                         + "\"result_type\":\"model_size_stats\","
+                         + "\"model_bytes\":100,"
+                         + "\"total_by_field_count\":101,"
+                         + "\"total_over_field_count\":102,"
+                         + "\"total_partition_field_count\":103,"
+                         + "\"bucket_allocation_failures_count\":104,"
+                         + "\"memory_status\":\"ok\","
+                         + "\"log_time\":1483315322002,"
+                         + "\"timestamp\":1483228861001"
+                        + "},"
+                       + "\"state\":\"opened\","
+                       + "\"node\":{"
+                         + "\"id\":\"_node_id\","
+                         + "\"name\":\"_node_name\","
+                         + "\"ephemeral_id\":\"_ephemeral_id\","
+                         + "\"transport_address\":\"0.0.0.0:9300\","
+                         + "\"attributes\":{"
+                           + "\"attr\":\"value\""
+                          + "}"
+                        + "},"
+                       + "\"assignment_explanation\":\"_explanation\","
+                       + "\"open_time\":\"13h\""
+                      + "}"
+                    + "}", xContent.utf8ToString());
+    }
 }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsCollectorTests.java

@@ -5,64 +5,135 @@
  */
 package org.elasticsearch.xpack.monitoring.collector.node;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.FailedNodeException;
+import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.bootstrap.BootstrapInfo;
-import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.ClusterAdminClient;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.MonitoringSettings;
-import org.elasticsearch.xpack.monitoring.collector.AbstractCollectorTestCase;
+import org.elasticsearch.xpack.monitoring.collector.BaseCollectorTestCase;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
-import org.elasticsearch.xpack.security.InternalClient;
 
 import java.util.Collection;
+import java.util.Collections;
+import java.util.UUID;
 
+import static org.elasticsearch.xpack.monitoring.MonitoringTestUtils.randomMonitoringNode;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
-// numClientNodes is set to 0 in this test because the NodeStatsCollector never collects data on client nodes:
-// the NodeStatsCollector.shouldCollect() method checks if the node has node files and client nodes don't have
-// such files.
-@ClusterScope(numClientNodes = 0)
-public class NodeStatsCollectorTests extends AbstractCollectorTestCase {
+public class NodeStatsCollectorTests extends BaseCollectorTestCase {
 
-    public void testNodeStatsCollector() throws Exception {
-        String[] nodes = internalCluster().getNodeNames();
-        for (String node : nodes) {
-            Collection<MonitoringDoc> results = newNodeStatsCollector(node).doCollect();
-            assertThat(results, hasSize(1));
+    public void testShouldCollectReturnsFalseIfMonitoringNotAllowed() {
+        // this controls the blockage
+        when(licenseState.isMonitoringAllowed()).thenReturn(false);
+        whenLocalNodeElectedMaster(randomBoolean());
 
-            MonitoringDoc monitoringDoc = results.iterator().next();
-            assertNotNull(monitoringDoc);
-            assertThat(monitoringDoc, instanceOf(NodeStatsMonitoringDoc.class));
+        final NodeStatsCollector collector =
+                new NodeStatsCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState, client);
 
-            NodeStatsMonitoringDoc nodeStatsMonitoringDoc = (NodeStatsMonitoringDoc) monitoringDoc;
-            assertThat(nodeStatsMonitoringDoc.getMonitoringId(), equalTo(MonitoredSystem.ES.getSystem()));
-            assertThat(nodeStatsMonitoringDoc.getMonitoringVersion(), equalTo(Version.CURRENT.toString()));
-            assertThat(nodeStatsMonitoringDoc.getClusterUUID(),
-                    equalTo(client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID()));
-            assertThat(nodeStatsMonitoringDoc.getTimestamp(), greaterThan(0L));
-            assertThat(nodeStatsMonitoringDoc.getSourceNode(), notNullValue());
-
-            assertThat(nodeStatsMonitoringDoc.getNodeId(),
-                    equalTo(internalCluster().getInstance(ClusterService.class, node).localNode().getId()));
-            assertThat(nodeStatsMonitoringDoc.isNodeMaster(), equalTo(node.equals(internalCluster().getMasterName())));
-            assertThat(nodeStatsMonitoringDoc.isMlockall(), equalTo(BootstrapInfo.isMemoryLocked()));
-
-            assertNotNull(nodeStatsMonitoringDoc.getNodeStats());
-        }
+        assertThat(collector.shouldCollect(), is(false));
+        verify(licenseState).isMonitoringAllowed();
     }
 
-    private NodeStatsCollector newNodeStatsCollector(final String nodeId) {
-        return new NodeStatsCollector(internalCluster().getInstance(Settings.class, nodeId),
-                internalCluster().getInstance(ClusterService.class, nodeId),
-                internalCluster().getInstance(MonitoringSettings.class, nodeId),
-                internalCluster().getInstance(XPackLicenseState.class, nodeId),
-                internalCluster().getInstance(InternalClient.class, nodeId));
+    public void testShouldCollectReturnsTrue() {
+        when(licenseState.isMonitoringAllowed()).thenReturn(true);
+        whenLocalNodeElectedMaster(true);
+
+        final NodeStatsCollector collector =
+                new NodeStatsCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState, client);
+
+        assertThat(collector.shouldCollect(), is(true));
+        verify(licenseState).isMonitoringAllowed();
+    }
+
+    public void testDoCollectWithFailures() throws Exception {
+        when(licenseState.isMonitoringAllowed()).thenReturn(true);
+
+        final NodesStatsResponse nodesStatsResponse = mock(NodesStatsResponse.class);
+        when(nodesStatsResponse.hasFailures()).thenReturn(true);
+
+        final FailedNodeException exception = new FailedNodeException("_node_id", "_msg", new Exception());
+        when(nodesStatsResponse.failures()).thenReturn(Collections.singletonList(exception));
+
+        final Client client = mock(Client.class);
+        thenReturnNodeStats(client, nodesStatsResponse);
+
+        final NodeStatsCollector collector =
+                new NodeStatsCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState, client);
+
+        final FailedNodeException e = expectThrows(FailedNodeException.class, () -> collector.doCollect(randomMonitoringNode(random())));
+        assertEquals(exception, e);
+    }
+
+    public void testDoCollect() throws Exception {
+        when(licenseState.isMonitoringAllowed()).thenReturn(true);
+
+        final boolean isMaster = randomBoolean();
+        whenLocalNodeElectedMaster(isMaster);
+
+        final String clusterUUID = UUID.randomUUID().toString();
+        whenClusterStateWithUUID(clusterUUID);
+
+        final MonitoringDoc.Node node = randomMonitoringNode(random());
+
+        final NodesStatsResponse nodesStatsResponse = mock(NodesStatsResponse.class);
+        when(nodesStatsResponse.hasFailures()).thenReturn(false);
+
+        final NodeStats nodeStats = mock(NodeStats.class);
+        when(nodesStatsResponse.getNodes()).thenReturn(Collections.singletonList(nodeStats));
+
+        final long timestamp = randomNonNegativeLong();
+        when(nodeStats.getTimestamp()).thenReturn(timestamp);
+
+        final Client client = mock(Client.class);
+        thenReturnNodeStats(client, nodesStatsResponse);
+
+        final NodeStatsCollector collector =
+                new NodeStatsCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState, client);
+
+        final Collection<MonitoringDoc> results = collector.doCollect(node);
+        assertEquals(1, results.size());
+
+        final MonitoringDoc monitoringDoc = results.iterator().next();
+        assertThat(monitoringDoc, instanceOf(NodeStatsMonitoringDoc.class));
+
+        final NodeStatsMonitoringDoc document = (NodeStatsMonitoringDoc) monitoringDoc;
+        assertThat(document.getCluster(), equalTo(clusterUUID));
+        assertThat(document.getTimestamp(), equalTo(timestamp));
+        assertThat(document.getNode(), equalTo(node));
+        assertThat(document.getSystem(), is(MonitoredSystem.ES));
+        assertThat(document.getType(), equalTo(NodeStatsMonitoringDoc.TYPE));
+        assertThat(document.getId(), nullValue());
+
+        assertThat(document.isNodeMaster(), equalTo(isMaster));
+        assertThat(document.getNodeId(), equalTo(node.getUUID()));
+        assertThat(document.getNodeStats(), is(nodeStats));
+        assertThat(document.isMlockall(), equalTo(BootstrapInfo.isMemoryLocked()));
+    }
+
+    private void thenReturnNodeStats(final Client client, final NodesStatsResponse nodesStatsResponse) {
+        @SuppressWarnings("unchecked")
+        final ActionFuture<NodesStatsResponse> future = (ActionFuture<NodesStatsResponse>) mock(ActionFuture.class);
+        when(future.actionGet(eq(monitoringSettings.nodeStatsTimeout()))).thenReturn(nodesStatsResponse);
+
+        final ClusterAdminClient clusterAdminClient = mock(ClusterAdminClient.class);
+        when(clusterAdminClient.nodesStats(any(NodesStatsRequest.class))).thenReturn(future);
+
+        final AdminClient adminClient = mock(AdminClient.class);
+        when(adminClient.cluster()).thenReturn(clusterAdminClient);
+        when(client.admin()).thenReturn(adminClient);
     }
 }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/node/NodeStatsMonitoringDocTests.java

@@ -0,0 +1,372 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.monitoring.collector.node;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
+import org.elasticsearch.action.admin.indices.stats.CommonStats;
+import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.cache.query.QueryCacheStats;
+import org.elasticsearch.index.cache.request.RequestCacheStats;
+import org.elasticsearch.index.engine.SegmentsStats;
+import org.elasticsearch.index.fielddata.FieldDataStats;
+import org.elasticsearch.index.search.stats.SearchStats;
+import org.elasticsearch.index.shard.DocsStats;
+import org.elasticsearch.index.shard.IndexingStats;
+import org.elasticsearch.index.store.StoreStats;
+import org.elasticsearch.indices.NodeIndicesStats;
+import org.elasticsearch.monitor.fs.FsInfo;
+import org.elasticsearch.monitor.jvm.JvmStats;
+import org.elasticsearch.monitor.os.OsStats;
+import org.elasticsearch.monitor.process.ProcessStats;
+import org.elasticsearch.threadpool.ThreadPoolStats;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+import org.elasticsearch.xpack.monitoring.exporter.BaseFilteredMonitoringDocTestCase;
+import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.emptySet;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
+
+public class NodeStatsMonitoringDocTests extends BaseFilteredMonitoringDocTestCase<NodeStatsMonitoringDoc> {
+
+    private String nodeId;
+    private boolean isMaster;
+    private NodeStats nodeStats;
+    private boolean mlockall;
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        nodeId = randomAlphaOfLength(5);
+        isMaster = randomBoolean();
+        nodeStats = mock(NodeStats.class);
+        mlockall = randomBoolean();
+    }
+
+    @Override
+    protected NodeStatsMonitoringDoc createMonitoringDoc(String cluster, long timestamp, MonitoringDoc.Node node,
+                                                         MonitoredSystem system, String type, String id) {
+        return new NodeStatsMonitoringDoc(cluster, timestamp, node, nodeId, isMaster, nodeStats, mlockall);
+    }
+
+    @Override
+    protected void assertFilteredMonitoringDoc(final NodeStatsMonitoringDoc document) {
+        assertThat(document.getSystem(), is(MonitoredSystem.ES));
+        assertThat(document.getType(), is(NodeStatsMonitoringDoc.TYPE));
+        assertThat(document.getId(), nullValue());
+
+        assertThat(document.isNodeMaster(), equalTo(isMaster));
+        assertThat(document.getNodeId(), equalTo(nodeId));
+        assertThat(document.getNodeStats(), is(nodeStats));
+        assertThat(document.isMlockall(), equalTo(mlockall));
+    }
+
+    @Override
+    protected Set<String> getExpectedXContentFilters() {
+        return NodeStatsMonitoringDoc.XCONTENT_FILTERS;
+    }
+
+    public void testConstructorNodeIdMustNotBeNull() {
+        expectThrows(NullPointerException.class, () ->
+                new NodeStatsMonitoringDoc(cluster, timestamp, node, null, isMaster, nodeStats, mlockall));
+    }
+
+    public void testConstructorNodeStatsMustNotBeNull() {
+        expectThrows(NullPointerException.class, () ->
+                new NodeStatsMonitoringDoc(cluster, timestamp, node, nodeId, isMaster, null, mlockall));
+    }
+
+    @Override
+    public void testToXContent() throws IOException {
+        final MonitoringDoc.Node node = new MonitoringDoc.Node("_uuid", "_host", "_addr", "_ip", "_name", 1504169190855L);
+        final NodeStats nodeStats = mockNodeStats();
+
+        final NodeStatsMonitoringDoc doc = new NodeStatsMonitoringDoc("_cluster", 1502107402133L, node, "_node_id", true, nodeStats, false);
+
+        final BytesReference xContent = XContentHelper.toXContent(doc, XContentType.JSON, false);
+        assertEquals("{"
+                     + "\"cluster_uuid\":\"_cluster\","
+                     + "\"timestamp\":\"2017-08-07T12:03:22.133Z\","
+                     + "\"type\":\"node_stats\","
+                     + "\"source_node\":{"
+                       + "\"uuid\":\"_uuid\","
+                       + "\"host\":\"_host\","
+                       + "\"transport_address\":\"_addr\","
+                       + "\"ip\":\"_ip\","
+                       + "\"name\":\"_name\","
+                       + "\"timestamp\":\"2017-08-31T08:46:30.855Z\""
+                     + "},"
+                     + "\"node_stats\":{"
+                       + "\"node_id\":\"_node_id\","
+                       + "\"node_master\":true,"
+                       + "\"mlockall\":false,"
+                       + "\"indices\":{"
+                         + "\"docs\":{"
+                           + "\"count\":1"
+                         + "},"
+                         + "\"store\":{"
+                           + "\"size_in_bytes\":4"
+                         + "},"
+                         + "\"indexing\":{"
+                           + "\"index_total\":5,"
+                           + "\"index_time_in_millis\":6,"
+                           + "\"throttle_time_in_millis\":8"
+                         + "},"
+                         + "\"search\":{"
+                           + "\"query_total\":17,"
+                           + "\"query_time_in_millis\":18"
+                         + "},"
+                         + "\"query_cache\":{"
+                           + "\"memory_size_in_bytes\":9,"
+                           + "\"hit_count\":10,"
+                           + "\"miss_count\":11,"
+                           + "\"evictions\":13"
+                         + "},"
+                         + "\"fielddata\":{"
+                           + "\"memory_size_in_bytes\":2,"
+                           + "\"evictions\":3"
+                         + "},"
+                         + "\"segments\":{"
+                           + "\"count\":19,"
+                           + "\"memory_in_bytes\":20,"
+                           + "\"terms_memory_in_bytes\":21,"
+                           + "\"stored_fields_memory_in_bytes\":22,"
+                           + "\"term_vectors_memory_in_bytes\":23,"
+                           + "\"norms_memory_in_bytes\":24,"
+                           + "\"points_memory_in_bytes\":25,"
+                           + "\"doc_values_memory_in_bytes\":26,"
+                           + "\"index_writer_memory_in_bytes\":27,"
+                           + "\"version_map_memory_in_bytes\":28,"
+                           + "\"fixed_bit_set_memory_in_bytes\":29"
+                         + "},"
+                         + "\"request_cache\":{"
+                           + "\"memory_size_in_bytes\":13,"
+                           + "\"evictions\":14,"
+                           + "\"hit_count\":15,"
+                           + "\"miss_count\":16"
+                         + "}"
+                       + "},"
+                       + "\"os\":{"
+                         + "\"cpu\":{"
+                           + "\"load_average\":{"
+                             + "\"1m\":41.0,"
+                             + "\"5m\":42.0,"
+                             + "\"15m\":43.0"
+                           + "}"
+                         + "},"
+                         + "\"cgroup\":{"
+                           + "\"cpuacct\":{"
+                             + "\"control_group\":\"_cpu_acct_ctrl_group\","
+                             + "\"usage_nanos\":47"
+                           + "},"
+                           + "\"cpu\":{"
+                             + "\"control_group\":\"_cpu_ctrl_group\","
+                             + "\"cfs_period_micros\":48,"
+                             + "\"cfs_quota_micros\":49,"
+                             + "\"stat\":{"
+                               + "\"number_of_elapsed_periods\":44,"
+                               + "\"number_of_times_throttled\":45,"
+                               + "\"time_throttled_nanos\":46"
+                             + "}"
+                           + "}"
+                        + "}"
+                       + "},"
+                       + "\"process\":{"
+                         + "\"open_file_descriptors\":51,"
+                         + "\"max_file_descriptors\":52,"
+                         + "\"cpu\":{"
+                           + "\"percent\":50"
+                         + "}"
+                       + "},"
+                       + "\"jvm\":{"
+                         + "\"mem\":{"
+                           + "\"heap_used_in_bytes\":53,"
+                           + "\"heap_used_percent\":98,"
+                           + "\"heap_max_in_bytes\":54"
+                         + "},"
+                         + "\"gc\":{"
+                           + "\"collectors\":{"
+                             + "\"young\":{"
+                               + "\"collection_count\":55,"
+                               + "\"collection_time_in_millis\":56"
+                             + "},"
+                             + "\"old\":{"
+                               + "\"collection_count\":57,"
+                               + "\"collection_time_in_millis\":58"
+                             + "}"
+                            + "}"
+                        + "}"
+                       + "},"
+                       + "\"thread_pool\":{"
+                         + "\"bulk\":{"
+                           + "\"threads\":59,"
+                           + "\"queue\":60,"
+                           + "\"rejected\":61"
+                         + "},"
+                         + "\"generic\":{"
+                           + "\"threads\":62,"
+                           + "\"queue\":63,"
+                           + "\"rejected\":64"
+                         + "},"
+                         + "\"get\":{"
+                           + "\"threads\":65,"
+                           + "\"queue\":66,"
+                           + "\"rejected\":67"
+                         + "},"
+                         + "\"index\":{"
+                           + "\"threads\":68,"
+                           + "\"queue\":69,"
+                           + "\"rejected\":70"
+                         + "},"
+                         + "\"management\":{"
+                           + "\"threads\":71,"
+                           + "\"queue\":72,"
+                           + "\"rejected\":73"
+                         + "},"
+                         + "\"search\":{"
+                           + "\"threads\":74,"
+                           + "\"queue\":75,"
+                           + "\"rejected\":76"
+                         + "},"
+                         + "\"watcher\":{"
+                           + "\"threads\":77,"
+                           + "\"queue\":78,"
+                           + "\"rejected\":79"
+                          + "}"
+                       + "},"
+                       + "\"fs\":{"
+                         + "\"total\":{"
+                           + "\"total_in_bytes\":38,"
+                           + "\"free_in_bytes\":39,"
+                           + "\"available_in_bytes\":40"
+                         + "},"
+                         + "\"io_stats\":{"
+                           + "\"total\":{"
+                             + "\"operations\":8,"
+                             + "\"read_operations\":4,"
+                             + "\"write_operations\":4,"
+                             + "\"read_kilobytes\":2,"
+                             + "\"write_kilobytes\":2"
+                           + "}"
+                          + "}"
+                        + "}"
+                      + "}"
+                    + "}", xContent.utf8ToString());
+    }
+
+    private static NodeStats mockNodeStats() {
+
+        // This value is used in constructors of various stats objects,
+        // when the value is not printed out in the final XContent.
+        final long no = -1;
+
+        // This value is used in constructors of various stats objects,
+        // when the value is printed out in the XContent. Must be
+        // incremented for each usage.
+        long iota = 0L;
+
+        // Indices
+        final CommonStats indicesCommonStats = new CommonStats(CommonStatsFlags.ALL);
+        indicesCommonStats.getDocs().add(new DocsStats(++iota, no));
+        indicesCommonStats.getFieldData().add(new FieldDataStats(++iota, ++iota, null));
+        indicesCommonStats.getStore().add(new StoreStats(++iota));
+
+        final IndexingStats.Stats indexingStats = new IndexingStats.Stats(++iota, ++iota, ++iota, no, no, no, no, no, false, ++iota);
+        indicesCommonStats.getIndexing().add(new IndexingStats(indexingStats, null));
+        indicesCommonStats.getQueryCache().add(new QueryCacheStats(++iota, ++iota, ++iota, ++iota, no));
+        indicesCommonStats.getRequestCache().add(new RequestCacheStats(++iota, ++iota, ++iota, ++iota));
+
+        final SearchStats.Stats searchStats = new SearchStats.Stats(++iota, ++iota, no, no, no, no, no, no, no, no, no, no);
+        indicesCommonStats.getSearch().add(new SearchStats(searchStats, no, null));
+
+        final SegmentsStats segmentsStats = new SegmentsStats();
+        segmentsStats.add(++iota, ++iota);
+        segmentsStats.addTermsMemoryInBytes(++iota);
+        segmentsStats.addStoredFieldsMemoryInBytes(++iota);
+        segmentsStats.addTermVectorsMemoryInBytes(++iota);
+        segmentsStats.addNormsMemoryInBytes(++iota);
+        segmentsStats.addPointsMemoryInBytes(++iota);
+        segmentsStats.addDocValuesMemoryInBytes(++iota);
+        segmentsStats.addIndexWriterMemoryInBytes(++iota);
+        segmentsStats.addVersionMapMemoryInBytes(++iota);
+        segmentsStats.addBitsetMemoryInBytes(++iota);
+        indicesCommonStats.getSegments().add(segmentsStats);
+
+        final NodeIndicesStats indices = new NodeIndicesStats(indicesCommonStats, emptyMap());
+
+        // Filesystem
+        final FsInfo.DeviceStats ioStatsOne = new FsInfo.DeviceStats((int) no, (int) no, null, ++iota, ++iota, ++iota, ++iota, null);
+        final FsInfo.DeviceStats ioStatsTwo = new FsInfo.DeviceStats((int) no, (int) no, null, ++iota, ++iota, ++iota, ++iota, ioStatsOne);
+
+        final FsInfo.IoStats ioStats = new FsInfo.IoStats(new FsInfo.DeviceStats[]{ioStatsTwo});
+        final FsInfo fs = new FsInfo(no, ioStats, new FsInfo.Path[]{new FsInfo.Path(null, null, ++iota, ++iota, ++iota)});
+
+        // TODO: Looks like this filter is not used anymore...
+        // "node_stats.fs.data.spins",
+
+        // Os
+        final OsStats.Cpu osCpu = new OsStats.Cpu((short) no, new double[]{++iota, ++iota, ++iota});
+        final OsStats.Cgroup.CpuStat osCpuStat = new OsStats.Cgroup.CpuStat(++iota, ++iota, ++iota);
+        final OsStats.Cgroup osCgroup = new OsStats.Cgroup("_cpu_acct_ctrl_group", ++iota, "_cpu_ctrl_group", ++iota, ++iota, osCpuStat);
+
+        final OsStats.Mem osMem = new OsStats.Mem(no, no);
+        final OsStats.Swap osSwap = new OsStats.Swap(no, no);
+        final OsStats os = new OsStats(no, osCpu, osMem, osSwap, osCgroup);
+
+        // Process
+        final ProcessStats.Cpu processCpu = new ProcessStats.Cpu((short) ++iota, no);
+        final ProcessStats process = new ProcessStats(no, ++iota, ++iota, processCpu, null);
+
+        // Jvm
+        final JvmStats.Threads jvmThreads = new JvmStats.Threads((int) no, (int) no);
+        final JvmStats.Classes jvmClasses = new JvmStats.Classes(no, no, no);
+        final JvmStats.Mem jvmMem = new JvmStats.Mem(no, ++iota, ++iota, no, no, emptyList());
+        final JvmStats.GarbageCollectors gcs = new JvmStats.GarbageCollectors(new JvmStats.GarbageCollector[]{
+                new JvmStats.GarbageCollector("young", ++iota, ++iota),
+                new JvmStats.GarbageCollector("old", ++iota, ++iota)});
+        final JvmStats jvm = new JvmStats(no, no, jvmMem, jvmThreads, gcs, emptyList(), jvmClasses);
+
+        // Threadpools
+        final List<ThreadPoolStats.Stats> threadpools = new ArrayList<>();
+        threadpools.add(new ThreadPoolStats.Stats("bulk", (int) ++iota, (int) ++iota, (int) no, ++iota, (int) no, no));
+        threadpools.add(new ThreadPoolStats.Stats("generic", (int) ++iota, (int) ++iota, (int) no, ++iota, (int) no, no));
+        threadpools.add(new ThreadPoolStats.Stats("get", (int) ++iota, (int) ++iota, (int) no, ++iota, (int) no, no));
+        threadpools.add(new ThreadPoolStats.Stats("index", (int) ++iota, (int) ++iota, (int) no, ++iota, (int) no, no));
+        threadpools.add(new ThreadPoolStats.Stats("management", (int) ++iota, (int) ++iota, (int) no, ++iota, (int) no, no));
+        threadpools.add(new ThreadPoolStats.Stats("search", (int) ++iota, (int) ++iota, (int) no, ++iota, (int) no, no));
+        threadpools.add(new ThreadPoolStats.Stats("watcher", (int) ++iota, (int) ++iota, (int) no, ++iota, (int) no, no));
+        final ThreadPoolStats threadPool = new ThreadPoolStats(threadpools);
+
+        final DiscoveryNode discoveryNode = new DiscoveryNode("_node_name",
+                                                                "_node_id",
+                                                                "_ephemeral_id",
+                                                                "_host_name",
+                                                                "_host_address",
+                                                                new TransportAddress(TransportAddress.META_ADDRESS, 1234),
+                                                                emptyMap(),
+                                                                emptySet(),
+                                                                Version.V_6_0_0_beta1);
+
+        return new NodeStats(discoveryNode, no, indices, os, process, jvm, threadPool, fs, null, null, null, null, null, null);
+    }
+}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsCollectorTests.java

@@ -5,172 +5,148 @@
  */
 package org.elasticsearch.xpack.monitoring.collector.shards;
 
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.MonitoringSettings;
-import org.elasticsearch.xpack.monitoring.collector.AbstractCollectorTestCase;
+import org.elasticsearch.xpack.monitoring.collector.BaseCollectorTestCase;
+import org.elasticsearch.xpack.monitoring.collector.Collector;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
 
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
 
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
+import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
+import static org.elasticsearch.xpack.monitoring.MonitoringTestUtils.randomMonitoringNode;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
-public class ShardsCollectorTests extends AbstractCollectorTestCase {
+public class ShardsCollectorTests extends BaseCollectorTestCase {
 
-    public void testShardsCollectorNoIndices() throws Exception {
-        Collection<MonitoringDoc> results = newShardsCollector().doCollect();
-        assertThat(results, hasSize(0));
+    /** Used to match no indices when collecting shards information **/
+    private static final String[] NONE = new String[]{"_none"};
+
+    public void testShouldCollectReturnsFalseIfMonitoringNotAllowed() {
+        // this controls the blockage
+        when(licenseState.isMonitoringAllowed()).thenReturn(false);
+        whenLocalNodeElectedMaster(randomBoolean());
+
+        final ShardsCollector collector = new ShardsCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState);
+
+        assertThat(collector.shouldCollect(), is(false));
+        verify(licenseState).isMonitoringAllowed();
     }
 
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder()
-                .put(super.nodeSettings(nodeOrdinal))
-                .put(MonitoringSettings.INDICES.getKey(), "test-shards*")
-                .build();
+    public void testShouldCollectReturnsFalseIfNotMaster() {
+        when(licenseState.isMonitoringAllowed()).thenReturn(true);
+        // this controls the blockage
+        whenLocalNodeElectedMaster(false);
+
+        final ShardsCollector collector = new ShardsCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState);
+
+        assertThat(collector.shouldCollect(), is(false));
+        verify(licenseState).isMonitoringAllowed();
+        verify(nodes).isLocalNodeElectedMaster();
     }
 
-    public void testShardsCollectorOneIndex() throws Exception {
-        int nbDocs = randomIntBetween(1, 20);
-        for (int i = 0; i < nbDocs; i++) {
-            client().prepareIndex("test-shards", "test").setSource("num", i).get();
-        }
+    public void testShouldCollectReturnsTrue() {
+        when(licenseState.isMonitoringAllowed()).thenReturn(true);
+        whenLocalNodeElectedMaster(true);
 
-        waitForRelocation();
-        ensureGreen();
-        refresh();
+        final ShardsCollector collector = new ShardsCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState);
 
-        assertHitCount(client().prepareSearch().setSize(0).get(), nbDocs);
+        assertThat(collector.shouldCollect(), is(true));
+        verify(licenseState).isMonitoringAllowed();
+        verify(nodes).isLocalNodeElectedMaster();
+    }
 
-        Collection<MonitoringDoc> results = newShardsCollector().doCollect();
-        assertThat(results, hasSize(getNumShards("test-shards").totalNumShards));
+    public void testDoCollectWhenNoClusterState() throws Exception {
+        when(clusterService.state()).thenReturn(null);
 
-        final ClusterState clusterState = client().admin().cluster().prepareState().setMetaData(true).get().getState();
+        final ShardsCollector collector = new ShardsCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState);
 
-        int primaries = 0;
-        int replicas = 0;
+        final Collection<MonitoringDoc> results = collector.doCollect(randomMonitoringNode(random()));
+        assertThat(results, notNullValue());
+        assertThat(results.size(), equalTo(0));
+        verify(clusterService).state();
+    }
+
+    public void testDoCollect() throws Exception {
+        final String clusterName = randomAlphaOfLength(10);
+        whenClusterStateWithName(clusterName);
+
+        final String clusterUUID = UUID.randomUUID().toString();
+        whenClusterStateWithUUID(clusterUUID);
+
+        final String stateUUID = UUID.randomUUID().toString();
+        when(clusterState.stateUUID()).thenReturn(stateUUID);
+
+        final String[] indices = randomFrom(NONE, Strings.EMPTY_ARRAY, new String[]{"_all"}, new String[]{"_index*"});
+        when(monitoringSettings.indices()).thenReturn(indices);
+
+        final RoutingTable routingTable = mockRoutingTable();
+        when(clusterState.routingTable()).thenReturn(routingTable);
+
+        final DiscoveryNode localNode = localNode("_current");
+        final MonitoringDoc.Node node = Collector.convertNode(randomNonNegativeLong(), localNode);
+        when(nodes.get(eq("_current"))).thenReturn(localNode);
+        when(clusterState.getNodes()).thenReturn(nodes);
+
+        final ShardsCollector collector = new ShardsCollector(Settings.EMPTY, clusterService, monitoringSettings, licenseState);
+
+        final Collection<MonitoringDoc> results = collector.doCollect(node);
+        assertThat(results, notNullValue());
+        assertThat(results.size(), equalTo((indices != NONE) ? routingTable.allShards().size() : 0));
 
         for (MonitoringDoc monitoringDoc : results) {
             assertNotNull(monitoringDoc);
             assertThat(monitoringDoc, instanceOf(ShardMonitoringDoc.class));
 
-            ShardMonitoringDoc shardMonitoringDoc = (ShardMonitoringDoc) monitoringDoc;
-            assertThat(shardMonitoringDoc.getMonitoringId(), equalTo(MonitoredSystem.ES.getSystem()));
-            assertThat(shardMonitoringDoc.getMonitoringVersion(), equalTo(Version.CURRENT.toString()));
-            assertThat(shardMonitoringDoc.getClusterUUID(), equalTo(clusterState.metaData().clusterUUID()));
-            assertThat(shardMonitoringDoc.getTimestamp(), greaterThan(0L));
-            assertThat(shardMonitoringDoc.getSourceNode(), notNullValue());
-            assertThat(shardMonitoringDoc.getClusterStateUUID(), equalTo(clusterState.stateUUID()));
+            final ShardMonitoringDoc document = (ShardMonitoringDoc) monitoringDoc;
+            assertThat(document.getCluster(), equalTo(clusterUUID));
+            assertThat(document.getTimestamp(), greaterThan(0L));
+            assertThat(document.getSystem(), is(MonitoredSystem.ES));
+            assertThat(document.getType(), equalTo(ShardMonitoringDoc.TYPE));
+            assertThat(document.getId(), equalTo(ShardMonitoringDoc.id(stateUUID, document.getShardRouting())));
+            assertThat(document.getClusterStateUUID(), equalTo(stateUUID));
 
-            ShardRouting shardRouting = shardMonitoringDoc.getShardRouting();
-            assertNotNull(shardRouting);
-            assertThat(shardMonitoringDoc.getShardRouting().assignedToNode(), is(true));
-
-            if (shardRouting.primary()) {
-                primaries++;
+            if (document.getShardRouting().assignedToNode()) {
+                assertThat(document.getNode(), equalTo(node));
             } else {
-                replicas++;
+                assertThat(document.getNode(), nullValue());
             }
-        }
 
-        int expectedPrimaries = getNumShards("test-shards").numPrimaries;
-        int expectedReplicas = expectedPrimaries * getNumShards("test-shards").numReplicas;
-        assertThat(primaries, equalTo(expectedPrimaries));
-        assertThat(replicas, equalTo(expectedReplicas));
-    }
-
-    @AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/96")
-    public void testShardsCollectorMultipleIndices() throws Exception {
-        final String indexPrefix = "test-shards-";
-        final int nbIndices = randomIntBetween(1, 3);
-        final int[] nbDocsPerIndex = new int[nbIndices];
-
-        for (int i = 0; i < nbIndices; i++) {
-            String index = indexPrefix + String.valueOf(i);
-            assertAcked(prepareCreate(index));
-
-            nbDocsPerIndex[i] = randomIntBetween(1, 20);
-            for (int j = 0; j < nbDocsPerIndex[i]; j++) {
-                client().prepareIndex(index, "test").setSource("num", i).get();
-            }
-        }
-
-        waitForRelocation();
-        refresh();
-
-        int totalShards = 0;
-        for (int i = 0; i < nbIndices; i++) {
-            String index = indexPrefix + String.valueOf(i);
-
-            assertHitCount(client().prepareSearch(index).setSize(0).get(), nbDocsPerIndex[i]);
-            disableAllocation(index);
-            totalShards += getNumShards(index).totalNumShards;
-        }
-
-        Collection<MonitoringDoc> results = newShardsCollector().doCollect();
-        assertThat(results, hasSize(totalShards));
-
-        final ClusterState clusterState = client().admin().cluster().prepareState().setMetaData(true).get().getState();
-
-        for (MonitoringDoc doc : results) {
-            assertNotNull(doc);
-            assertThat(doc, instanceOf(ShardMonitoringDoc.class));
-
-            ShardMonitoringDoc shardDoc = (ShardMonitoringDoc) doc;
-            assertThat(shardDoc.getMonitoringId(), equalTo(MonitoredSystem.ES.getSystem()));
-            assertThat(shardDoc.getMonitoringVersion(), equalTo(Version.CURRENT.toString()));
-            assertThat(shardDoc.getClusterUUID(), equalTo(clusterState.metaData().clusterUUID()));
-            assertThat(shardDoc.getTimestamp(), greaterThan(0L));
-            assertThat(shardDoc.getClusterStateUUID(), equalTo(clusterState.stateUUID()));
-
-            ShardRouting shardRouting = shardDoc.getShardRouting();
-            assertNotNull(shardRouting);
-
-            if (shardRouting.assignedToNode()) {
-                assertThat(shardDoc.getSourceNode(), notNullValue());
-            } else {
-                assertThat(shardDoc.getSourceNode(), nullValue());
-            }
-        }
-
-        // Checks that a correct number of ShardMonitoringDoc documents has been created for each index
-        int[] shards = new int[nbIndices];
-        for (MonitoringDoc monitoringDoc : results) {
-            ShardRouting routing = ((ShardMonitoringDoc) monitoringDoc).getShardRouting();
-            int index = Integer.parseInt(routing.getIndexName().substring(indexPrefix.length()));
-            shards[index]++;
-        }
-
-        for (int i = 0; i < nbIndices; i++) {
-            String index = indexPrefix + String.valueOf(i);
-            int total = getNumShards(index).totalNumShards;
-            assertThat("expecting " + total + " shards monitoring documents for index [" + index + "]", shards[i], equalTo(total));
         }
     }
 
-    private ShardsCollector newShardsCollector() {
-        // This collector runs on master node only
-        return newShardsCollector(internalCluster().getMasterName());
-    }
+    private static RoutingTable mockRoutingTable() {
+        final List<ShardRouting> allShards = new ArrayList<>();
 
-    private ShardsCollector newShardsCollector(String nodeId) {
-        assertNotNull(nodeId);
-        return new ShardsCollector(internalCluster().getInstance(Settings.class, nodeId),
-                internalCluster().getInstance(ClusterService.class, nodeId),
-                internalCluster().getInstance(MonitoringSettings.class, nodeId),
-                internalCluster().getInstance(XPackLicenseState.class, nodeId));
+        final int nbShards = randomIntBetween(0, 10);
+        for (int i = 0; i < nbShards; i++) {
+            ShardRoutingState state = randomFrom(STARTED, UNASSIGNED);
+            ShardId shardId = new ShardId("_index", randomAlphaOfLength(12), i);
+            allShards.add(TestShardRouting.newShardRouting(shardId, state == STARTED ? "_current" : null, true, state));
+        }
+
+        final RoutingTable routingTable = mock(RoutingTable.class);
+        when(routingTable.allShards()).thenReturn(allShards);
+        return routingTable;
     }
 }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/shards/ShardsMonitoringDocTests.java

@@ -0,0 +1,132 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.monitoring.collector.shards;
+
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+import org.elasticsearch.xpack.monitoring.MonitoringTestUtils;
+import org.elasticsearch.xpack.monitoring.exporter.BaseFilteredMonitoringDocTestCase;
+import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Set;
+
+import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
+import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
+import static org.elasticsearch.cluster.routing.TestShardRouting.newShardRouting;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+
+public class ShardsMonitoringDocTests extends BaseFilteredMonitoringDocTestCase<ShardMonitoringDoc> {
+
+    private String stateUuid;
+    private boolean assignedToNode;
+    private ShardRouting shardRouting;
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        stateUuid = randomAlphaOfLength(5);
+        assignedToNode = randomBoolean();
+        node = assignedToNode ? MonitoringTestUtils.randomMonitoringNode(random()) : null;
+        shardRouting = newShardRouting(randomAlphaOfLength(5),
+                randomIntBetween(0, 5),
+                assignedToNode ? node.getUUID() : null,
+                randomBoolean(),
+                assignedToNode ? INITIALIZING : UNASSIGNED);
+    }
+
+    @Override
+    protected ShardMonitoringDoc createMonitoringDoc(String cluster, long timestamp, MonitoringDoc.Node node,
+                                                     MonitoredSystem system, String type, String id) {
+        return new ShardMonitoringDoc(cluster, timestamp, node, shardRouting, stateUuid);
+    }
+
+    @Override
+    protected void assertFilteredMonitoringDoc(final ShardMonitoringDoc document) {
+        assertThat(document.getSystem(), is(MonitoredSystem.ES));
+        assertThat(document.getType(), is(ShardMonitoringDoc.TYPE));
+        assertThat(document.getId(), equalTo(ShardMonitoringDoc.id(stateUuid, shardRouting)));
+
+        assertThat(document.getShardRouting(), is(shardRouting));
+        if (assignedToNode) {
+            assertThat(document.getShardRouting().assignedToNode(), is(true));
+            assertThat(document.getNode(), is(node));
+        } else {
+            assertThat(document.getNode(), nullValue());
+        }
+    }
+
+    @Override
+    protected Set<String> getExpectedXContentFilters() {
+        return ShardMonitoringDoc.XCONTENT_FILTERS;
+    }
+
+    public void testConstructorShardRoutingMustNotBeNull() {
+        expectThrows(NullPointerException.class, () -> new ShardMonitoringDoc(cluster, timestamp, node, null, stateUuid));
+    }
+
+    public void testConstructorStateUuidMustNotBeNull() {
+        expectThrows(NullPointerException.class, () -> new ShardMonitoringDoc(cluster, timestamp, node, shardRouting, null));
+    }
+
+    public void testIdWithPrimaryShardAssigned() {
+        final ShardRouting shardRouting = newShardRouting("_index_0", 123, "_node_0", randomAlphaOfLength(5), true, INITIALIZING);
+        assertEquals("_state_uuid_0:_node_0:_index_0:123:p", ShardMonitoringDoc.id("_state_uuid_0", shardRouting));
+    }
+
+    public void testIdWithReplicaShardAssigned() {
+        final ShardRouting shardRouting = newShardRouting("_index_1", 456, "_node_1", randomAlphaOfLength(5), false, INITIALIZING);
+        assertEquals("_state_uuid_1:_node_1:_index_1:456:r", ShardMonitoringDoc.id("_state_uuid_1", shardRouting));
+    }
+
+    public void testIdWithPrimaryShardUnassigned() {
+        final ShardRouting shardRouting = newShardRouting("_index_2", 789, null, randomAlphaOfLength(5), true, UNASSIGNED);
+        assertEquals("_state_uuid_2:_na:_index_2:789:p", ShardMonitoringDoc.id("_state_uuid_2", shardRouting));
+    }
+
+    public void testIdWithReplicaShardUnassigned() {
+        final ShardRouting shardRouting = newShardRouting("_index_3", 159, null, randomAlphaOfLength(5), false, UNASSIGNED);
+        assertEquals("_state_uuid_3:_na:_index_3:159:r", ShardMonitoringDoc.id("_state_uuid_3", shardRouting));
+    }
+
+    @Override
+    public void testToXContent() throws IOException {
+        final ShardRouting shardRouting = newShardRouting("_index", 1, "_index_uuid", "_node_uuid", true, INITIALIZING);
+        final MonitoringDoc.Node node = new MonitoringDoc.Node("_uuid", "_host", "_addr", "_ip", "_name", 1504169190855L);
+        final ShardMonitoringDoc doc =  new ShardMonitoringDoc("_cluster", 1502107402133L, node, shardRouting, "_state_uuid");
+
+        final BytesReference xContent = XContentHelper.toXContent(doc, XContentType.JSON, randomBoolean());
+        assertEquals("{"
+                    + "\"cluster_uuid\":\"_cluster\","
+                    + "\"timestamp\":\"2017-08-07T12:03:22.133Z\","
+                    + "\"type\":\"shards\","
+                    + "\"source_node\":{"
+                        + "\"uuid\":\"_uuid\","
+                        + "\"host\":\"_host\","
+                        + "\"transport_address\":\"_addr\","
+                        + "\"ip\":\"_ip\","
+                        + "\"name\":\"_name\","
+                        + "\"timestamp\":\"2017-08-31T08:46:30.855Z\""
+                    + "},"
+                    + "\"state_uuid\":\"_state_uuid\","
+                    + "\"shard\":{"
+                        + "\"state\":\"INITIALIZING\","
+                        + "\"primary\":true,"
+                        + "\"node\":\"_index_uuid\","
+                        + "\"relocating_node\":\"_node_uuid\","
+                        + "\"shard\":1,"
+                        + "\"index\":\"_index\""
+                        + "}"
+                    + "}" , xContent.utf8ToString());
+    }
+}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/BaseFilteredMonitoringDocTestCase.java

@@ -0,0 +1,145 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.monitoring.exporter;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import static java.util.Collections.emptySet;
+import static org.elasticsearch.xpack.monitoring.exporter.FilteredMonitoringDoc.COMMON_XCONTENT_FILTERS;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+
+/**
+ * Tests {@link FilteredMonitoringDoc}
+ */
+public abstract class BaseFilteredMonitoringDocTestCase<F extends FilteredMonitoringDoc> extends BaseMonitoringDocTestCase<F> {
+
+    @Override
+    protected final void assertMonitoringDoc(final F document) {
+        assertFilteredMonitoringDoc(document);
+        assertXContentFilters(document);
+    }
+
+    /**
+     * Asserts that the specific fields of a {@link FilteredMonitoringDoc} have
+     * the expected values.
+     */
+    protected abstract void assertFilteredMonitoringDoc(F document);
+
+    /**
+     * Returns the expected list of XContent filters for the {@link FilteredMonitoringDoc}
+     */
+    protected abstract Set<String> getExpectedXContentFilters();
+
+    /**
+     * Asserts that the XContent filters of a {@link FilteredMonitoringDoc} contains
+     * the common filters and the expected custom ones.
+     */
+    private void assertXContentFilters(final F document) {
+        final Set<String> expectedFilters = Sets.union(COMMON_XCONTENT_FILTERS, getExpectedXContentFilters());
+        final Set<String> actualFilters = document.getFilters();
+
+        expectedFilters.forEach(filter -> assertThat(actualFilters, hasItem(filter)));
+        assertThat(actualFilters.size(), equalTo(expectedFilters.size()));
+    }
+
+    public void testConstructorFiltersMustNotBeNull() {
+        expectThrows(NullPointerException.class, () -> new TestFilteredMonitoringDoc(cluster, timestamp, node, system, type, id, null));
+    }
+
+    public void testConstructorFiltersMustNotBeEmpty() {
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> new TestFilteredMonitoringDoc(cluster, timestamp, node, system, type, id, emptySet()));
+
+        assertThat(e.getMessage(), equalTo("xContentFilters must not be empty"));
+    }
+
+    public void testFilteredMonitoringDocToXContent() throws IOException {
+        final Set<String> filters = new HashSet<>(5);
+        filters.add("_type.field_1");
+        filters.add("_type.field_3");
+        filters.add("_type.field_5.sub_*");
+
+        final MonitoringDoc.Node node =
+                new MonitoringDoc.Node("_uuid", "_host", "_addr", "_ip", "_name", 1504169190855L);
+        final TestFilteredMonitoringDoc document =
+                new TestFilteredMonitoringDoc("_cluster", 1502266739402L, node, MonitoredSystem.ES, "_type", "_id", filters);
+
+        final BytesReference xContent = XContentHelper.toXContent(document, XContentType.JSON, false);
+        assertEquals("{"
+                     + "\"cluster_uuid\":\"_cluster\","
+                     + "\"timestamp\":\"2017-08-09T08:18:59.402Z\","
+                     + "\"type\":\"_type\","
+                     + "\"source_node\":{"
+                       + "\"uuid\":\"_uuid\","
+                       + "\"host\":\"_host\","
+                       + "\"transport_address\":\"_addr\","
+                       + "\"ip\":\"_ip\","
+                       + "\"name\":\"_name\","
+                       + "\"timestamp\":\"2017-08-31T08:46:30.855Z\""
+                     + "},"
+                     + "\"_type\":{"
+                       + "\"field_1\":1,"
+                         + "\"field_3\":{"
+                           + "\"sub_field_3\":3"
+                         + "},"
+                         + "\"field_5\":["
+                           + "{\"sub_field_5\":5}"
+                         + "]"
+                      + "}"
+                    + "}", xContent.utf8ToString());
+    }
+
+    class TestFilteredMonitoringDoc extends FilteredMonitoringDoc {
+
+        TestFilteredMonitoringDoc(final String cluster,
+                                  final long timestamp,
+                                  final Node node,
+                                  final MonitoredSystem system,
+                                  final String type,
+                                  final String id,
+                                  final Set<String> xContentFilters) {
+            super(cluster, timestamp, node, system, type, id, xContentFilters);
+        }
+
+        @Override
+        protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject(getType());
+            {
+                builder.field("field_1", 1);
+                builder.field("field_2", 2);
+
+                builder.startObject("field_3");
+                builder.field("sub_field_3", 3);
+                builder.endObject();
+
+                builder.field("field_4", 4);
+
+                builder.startArray("field_5");
+                {
+                    builder.startObject();
+                    builder.field("sub_field_5", 5);
+                    builder.endObject();
+
+                    builder.startObject();
+                    builder.field("other_field_5", 5);
+                    builder.endObject();
+                }
+                builder.endArray();
+            }
+            builder.endObject();
+        }
+    }
+}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/BaseMonitoringDocTestCase.java

@@ -0,0 +1,286 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.monitoring.exporter;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+import org.elasticsearch.xpack.monitoring.MonitoringTestUtils;
+import org.elasticsearch.xpack.monitoring.collector.shards.ShardMonitoringDoc;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.emptyList;
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+/**
+ * Base tests of all {@link MonitoringDoc}
+ */
+public abstract class BaseMonitoringDocTestCase<T extends MonitoringDoc> extends ESTestCase {
+
+    protected String cluster;
+    protected long timestamp;
+    protected MonitoringDoc.Node node;
+    protected MonitoredSystem system;
+    protected String type;
+    protected String id;
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        cluster = UUIDs.randomBase64UUID();
+        timestamp = frequently() ? randomNonNegativeLong() : 0L;
+        node = frequently() ? MonitoringTestUtils.randomMonitoringNode(random()) : null;
+        system = randomFrom(MonitoredSystem.values());
+        type = randomAlphaOfLength(5);
+        id = randomBoolean() ? randomAlphaOfLength(10) : null;
+    }
+
+    /**
+     * Creates the {@link MonitoringDoc} to test. Returned value must be deterministic,
+     * ie multiple calls with the same parameters within the same test must return
+     * identical objects.
+     */
+    protected abstract T createMonitoringDoc(String cluster,
+                                             long timestamp,
+                                             @Nullable MonitoringDoc.Node node,
+                                             MonitoredSystem system,
+                                             String type,
+                                             @Nullable String id);
+
+    /**
+     * Assert that two {@link MonitoringDoc} are equal. By default, it
+     * uses {@link MonitoringDoc#equals(Object)} and {@link MonitoringDoc#hashCode()} methods
+     * and also checks XContent equality.
+     */
+    private void assertMonitoringDocEquals(T expected, T actual) throws IOException {
+        assertEquals(expected, actual);
+        assertEquals(expected.hashCode(), actual.hashCode());
+
+        final boolean human = randomBoolean();
+        final XContentType xContentType = randomFrom(XContentType.values());
+        assertToXContentEquivalent(toXContent(expected, xContentType, human), toXContent(actual, xContentType, human), xContentType);
+    }
+
+    public final void testCreateMonitoringDoc() throws IOException {
+        final int nbIterations = randomIntBetween(3, 20);
+        for (int i = 0; i < nbIterations; i++) {
+            final T document1 = createMonitoringDoc(cluster, timestamp, node, system, type, id);
+            final T document2 = createMonitoringDoc(cluster, timestamp, node, system, type, id);
+
+            assertNotSame(document1, document2);
+            assertMonitoringDocEquals(document1, document2);
+        }
+    }
+
+    public final void testConstructorClusterMustNotBeNull() {
+        expectThrows(NullPointerException.class, () -> createMonitoringDoc(null, timestamp, node, system, type, id));
+    }
+
+    public final void testConstructor() {
+        final T document = createMonitoringDoc(cluster, timestamp, node, system, type, id);
+
+        assertThat(document.getCluster(), equalTo(cluster));
+        assertThat(document.getTimestamp(), equalTo(timestamp));
+        assertThat(document.getNode(), equalTo(node));
+
+        assertMonitoringDoc(document);
+    }
+
+    /**
+     * Asserts that the specific fields of a {@link MonitoringDoc} have
+     * the expected values.
+     */
+    protected abstract void assertMonitoringDoc(T document);
+
+    public abstract void testToXContent() throws IOException;
+
+    /**
+     * Test that {@link MonitoringDoc} rendered using {@link ToXContent#toXContent(XContentBuilder, ToXContent.Params)}
+     * contain a common set of fields.
+     */
+    public final void testToXContentContainsCommonFields() throws IOException {
+        final XContentType xContentType = randomFrom(XContentType.values());
+        final T document = createMonitoringDoc(cluster, timestamp, node, system, type, id);
+
+        final BytesReference bytes = XContentHelper.toXContent(document, xContentType, false);
+        try (XContentParser parser = xContentType.xContent().createParser(NamedXContentRegistry.EMPTY, bytes)) {
+            final Map<String, ?> map = parser.map();
+
+            assertThat(map.get("cluster_uuid"), equalTo(cluster));
+            assertThat(map.get("timestamp"), equalTo(MonitoringDoc.toUTC(timestamp)));
+            assertThat(map.get("type"), equalTo(document.getType()));
+
+            if (document.getType().equals(ShardMonitoringDoc.TYPE)) {
+                assertThat(map.get("shard"), notNullValue());
+            } else {
+                assertThat(map.get(document.getType()), notNullValue());
+            }
+
+            @SuppressWarnings("unchecked")
+            final Map<String, ?> sourceNode = (Map<String, ?>) map.get("source_node");
+            if (node == null) {
+                assertThat(sourceNode, nullValue());
+            } else {
+                assertThat(sourceNode.get("uuid"), equalTo(node.getUUID()));
+                assertThat(sourceNode.get("transport_address"), equalTo(node.getTransportAddress()));
+                assertThat(sourceNode.get("ip"), equalTo(node.getIp()));
+                assertThat(sourceNode.get("host"), equalTo(node.getHost()));
+                assertThat(sourceNode.get("name"), equalTo(node.getName()));
+                assertThat(sourceNode.get("timestamp"), equalTo(MonitoringDoc.toUTC(node.getTimestamp())));
+            }
+        }
+    }
+
+    public void testToUTC() {
+        final long timestamp = System.currentTimeMillis();
+        final String expected = new DateTime(timestamp, DateTimeZone.UTC).toString();
+
+        assertEquals(expected, MonitoringDoc.toUTC(timestamp));
+    }
+
+    public void testMonitoringNodeConstructor() {
+        final String id = randomAlphaOfLength(5);
+        final String name = randomAlphaOfLengthBetween(3, 10);
+        final TransportAddress fakeTransportAddress = buildNewFakeTransportAddress();
+        final String host = fakeTransportAddress.address().getHostString();
+        final String transportAddress = fakeTransportAddress.toString();
+        final String ip = fakeTransportAddress.getAddress();
+        final long timestamp = randomNonNegativeLong();
+
+        final MonitoringDoc.Node node = new MonitoringDoc.Node(id, host, transportAddress, ip, name, timestamp);
+
+        assertThat(node.getUUID(), equalTo(id));
+        assertThat(node.getHost(), equalTo(host));
+        assertThat(node.getTransportAddress(), equalTo(transportAddress));
+        assertThat(node.getIp(), equalTo(ip));
+        assertThat(node.getName(), equalTo(name));
+        assertThat(node.getTimestamp(), equalTo(timestamp));
+    }
+
+    public void testMonitoringNodeToXContent() throws IOException {
+        final MonitoringDoc.Node node = new MonitoringDoc.Node("_uuid", "_host", "_addr", "_ip", "_name", 1504169190855L);
+
+        final BytesReference xContent = XContentHelper.toXContent(node, XContentType.JSON, randomBoolean());
+        assertEquals("{"
+                    + "\"uuid\":\"_uuid\","
+                    + "\"host\":\"_host\","
+                    + "\"transport_address\":\"_addr\","
+                    + "\"ip\":\"_ip\","
+                    + "\"name\":\"_name\","
+                    + "\"timestamp\":\"2017-08-31T08:46:30.855Z\""
+                + "}" , xContent.utf8ToString());
+    }
+
+    public void testMonitoringNodeEqualsAndHashcode() {
+        final EqualsHashCodeTestUtils.CopyFunction<MonitoringDoc.Node> copy = node -> new MonitoringDoc.Node(node.getUUID(), node.getHost(),
+                node.getTransportAddress(), node.getIp(), node.getName(), node.getTimestamp());
+
+        final List<EqualsHashCodeTestUtils.MutateFunction<MonitoringDoc.Node>> mutations = new ArrayList<>();
+        mutations.add(n -> {
+            String id;
+            do {
+                id = UUIDs.randomBase64UUID();
+            } while (id.equals(n.getUUID()));
+            return new MonitoringDoc.Node(id, n.getHost(), n.getTransportAddress(), n.getIp(), n.getName(), n.getTimestamp());
+        });
+        mutations.add(n -> {
+            String host;
+            do {
+                host = randomAlphaOfLength(10);
+            } while (host.equals(n.getHost()));
+            return new MonitoringDoc.Node(n.getUUID(), host, n.getTransportAddress(), n.getIp(), n.getName(), n.getTimestamp());
+        });
+        mutations.add(n -> {
+            String transportAddress;
+            do {
+                transportAddress = randomAlphaOfLength(10);
+            } while (transportAddress.equals(n.getTransportAddress()));
+            return new MonitoringDoc.Node(n.getUUID(), n.getHost(), transportAddress, n.getIp(), n.getName(), n.getTimestamp());
+        });
+        mutations.add(n -> {
+            String ip;
+            do {
+                ip = randomAlphaOfLength(10);
+            } while (ip.equals(n.getIp()));
+            return new MonitoringDoc.Node(n.getUUID(), n.getHost(), n.getTransportAddress(), ip, n.getName(), n.getTimestamp());
+        });
+        mutations.add(n -> {
+            String name;
+            do {
+                name = randomAlphaOfLengthBetween(3, 10);
+            } while (name.equals(n.getName()));
+            return new MonitoringDoc.Node(n.getUUID(), n.getHost(), n.getTransportAddress(), n.getIp(), name, n.getTimestamp());
+        });
+        mutations.add(n -> {
+            long timestamp;
+            do {
+                timestamp = randomBoolean() ? randomNonNegativeLong() : 0L;
+            } while (timestamp == n.getTimestamp());
+            return new MonitoringDoc.Node(n.getUUID(), n.getHost(), n.getTransportAddress(), n.getIp(), n.getName(), timestamp);
+        });
+
+        final MonitoringDoc.Node sourceNode = MonitoringTestUtils.randomMonitoringNode(random());
+
+        checkEqualsAndHashCode(sourceNode, copy, randomFrom(mutations));
+    }
+
+    public void testMonitoringNodeSerialization() throws IOException {
+        final NamedWriteableRegistry registry = new NamedWriteableRegistry(emptyList());
+
+        final MonitoringDoc.Node original = MonitoringTestUtils.randomMonitoringNode(random());
+        final MonitoringDoc.Node deserialized = copyWriteable(original, registry, MonitoringDoc.Node::new);
+
+        assertEquals(deserialized, original);
+        assertEquals(deserialized.hashCode(), original.hashCode());
+        assertNotSame(deserialized, original);
+    }
+
+    public void testMonitoringNodeBwcSerialization() throws IOException {
+        final Version version = randomVersionBetween(random(), Version.V_5_0_0, Version.V_6_1_0);
+
+        final byte[] data = Base64.getDecoder()
+                                  .decode("AQVFSWJKdgEDdFFOAQV3cGtMagEFa2xqeWEBBVZTamF2AwVrZXkjMgEyBWtleSMxATEFa2V5IzABMAAAAAAAAA==");
+        try (StreamInput in = StreamInput.wrap(data)) {
+            in.setVersion(version);
+
+            final MonitoringDoc.Node node = new MonitoringDoc.Node(in);
+            assertEquals("EIbJv", node.getUUID());
+            assertEquals("VSjav", node.getName());
+            assertEquals("tQN", node.getHost());
+            assertEquals("wpkLj", node.getTransportAddress());
+            assertEquals("kljya", node.getIp());
+            assertEquals(0L, node.getTimestamp());
+        }
+    }
+}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/BytesReferenceMonitoringDocTests.java

@@ -0,0 +1,160 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.monitoring.exporter;
+
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.elasticsearch.test.RandomObjects;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+import org.elasticsearch.xpack.monitoring.MonitoringTestUtils;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
+import static org.elasticsearch.xpack.monitoring.MonitoredSystem.KIBANA;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * Tests {@link BytesReferenceMonitoringDoc}
+ */
+public class BytesReferenceMonitoringDocTests extends BaseMonitoringDocTestCase<BytesReferenceMonitoringDoc> {
+
+    private XContentType xContentType;
+    private BytesReference source;
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        xContentType = randomFrom(XContentType.values());
+        source = RandomObjects.randomSource(random(), xContentType);
+    }
+
+    @Override
+    protected BytesReferenceMonitoringDoc createMonitoringDoc(final String cluster, final long timestamp, final MonitoringDoc.Node node,
+                                                              final MonitoredSystem system, final String type, final String id) {
+        return new BytesReferenceMonitoringDoc(cluster, timestamp, node, system, type, id, xContentType, source);
+    }
+
+    @Override
+    protected void assertMonitoringDoc(final BytesReferenceMonitoringDoc document) {
+        assertThat(document.getSystem(), equalTo(system));
+        assertThat(document.getType(), equalTo(type));
+        assertThat(document.getId(), equalTo(id));
+
+        assertThat(document.getXContentType(), equalTo(xContentType));
+        assertThat(document.getSource(), equalTo(source));
+    }
+
+    public void testConstructorMonitoredSystemMustNotBeNull() {
+        expectThrows(NullPointerException.class,
+                () -> new BytesReferenceMonitoringDoc(cluster, timestamp, node, null, type, id, xContentType, source));
+    }
+
+    public void testConstructorTypeMustNotBeNull() {
+        expectThrows(NullPointerException.class,
+                () -> new BytesReferenceMonitoringDoc(cluster, timestamp, node, system, null, id, xContentType, source));
+    }
+
+    public void testConstructorXContentTypeMustNotBeNull() {
+        expectThrows(NullPointerException.class,
+                () -> new BytesReferenceMonitoringDoc(cluster, timestamp, node, system, type, id, null, source));
+    }
+
+    public void testConstructorSourceMustNotBeNull() {
+        expectThrows(NullPointerException.class,
+                () -> new BytesReferenceMonitoringDoc(cluster, timestamp, node, system, type, id, xContentType, null));
+    }
+
+    @Override
+    public void testToXContent() throws IOException {
+        final XContentBuilder builder = XContentBuilder.builder(xContentType.xContent());
+        builder.startObject();
+        builder.field("field", "value");
+        builder.endObject();
+
+        final MonitoringDoc.Node node =
+                new MonitoringDoc.Node("_uuid", "_host", "_addr", "_ip", "_name", 1504169190855L);
+        final BytesReferenceMonitoringDoc document =
+                new BytesReferenceMonitoringDoc("_cluster", 1502266739402L, node, KIBANA, "_type", "_id", xContentType, builder.bytes());
+
+        final BytesReference xContent = XContentHelper.toXContent(document, XContentType.JSON, false);
+        assertEquals("{"
+                     + "\"cluster_uuid\":\"_cluster\","
+                     + "\"timestamp\":\"2017-08-09T08:18:59.402Z\","
+                     + "\"type\":\"_type\","
+                     + "\"source_node\":{"
+                       + "\"uuid\":\"_uuid\","
+                       + "\"host\":\"_host\","
+                       + "\"transport_address\":\"_addr\","
+                       + "\"ip\":\"_ip\","
+                       + "\"name\":\"_name\","
+                       + "\"timestamp\":\"2017-08-31T08:46:30.855Z\""
+                     + "},"
+                     + "\"_type\":{"
+                        + "\"field\":\"value\""
+                     + "}"
+                    + "}", xContent.utf8ToString());
+    }
+
+    public void testEqualsAndHashcode() {
+        final EqualsHashCodeTestUtils.CopyFunction<MonitoringDoc> copy = doc ->
+                createMonitoringDoc(doc.getCluster(), doc.getTimestamp(), doc.getNode(), doc.getSystem(), doc.getType(), doc.getId());
+
+        final List<EqualsHashCodeTestUtils.MutateFunction<MonitoringDoc>> mutations = new ArrayList<>();
+        mutations.add(doc -> {
+            String cluster;
+            do {
+                cluster = UUIDs.randomBase64UUID();
+            } while (cluster.equals(doc.getCluster()));
+            return createMonitoringDoc(cluster, doc.getTimestamp(), doc.getNode(), doc.getSystem(), doc.getType(), doc.getId());
+        });
+        mutations.add(doc -> {
+            long timestamp;
+            do {
+                timestamp = randomNonNegativeLong();
+            } while (timestamp == doc.getTimestamp());
+            return createMonitoringDoc(doc.getCluster(), timestamp, doc.getNode(), doc.getSystem(), doc.getType(), doc.getId());
+        });
+        mutations.add(doc -> {
+            MonitoringDoc.Node node;
+            do {
+                node = MonitoringTestUtils.randomMonitoringNode(random());
+            } while (node.equals(doc.getNode()));
+            return createMonitoringDoc(doc.getCluster(), doc.getTimestamp(), node, doc.getSystem(), doc.getType(), doc.getId());
+        });
+        mutations.add(doc -> {
+            MonitoredSystem system;
+            do {
+                system = randomFrom(MonitoredSystem.values());
+            } while (system == doc.getSystem());
+            return createMonitoringDoc(doc.getCluster(), doc.getTimestamp(), doc.getNode(), system, doc.getType(), doc.getId());
+        });
+        mutations.add(doc -> {
+            String type;
+            do {
+                type = randomAlphaOfLength(5);
+            } while (type.equals(doc.getType()));
+            return createMonitoringDoc(doc.getCluster(), doc.getTimestamp(), doc.getNode(), doc.getSystem(), type, doc.getId());
+        });
+        mutations.add(doc -> {
+            String id;
+            do {
+                id = randomAlphaOfLength(10);
+            } while (id.equals(doc.getId()));
+            return createMonitoringDoc(doc.getCluster(), doc.getTimestamp(), doc.getNode(), doc.getSystem(), doc.getType(), id);
+        });
+
+        checkEqualsAndHashCode(createMonitoringDoc(cluster, timestamp, node, system, type, id), copy, randomFrom(mutations));
+    }
+}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/ExportersTests.java

@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.monitoring.exporter;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
@@ -15,16 +14,19 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.monitoring.MonitoringSettings;
-import org.elasticsearch.xpack.monitoring.exporter.local.LocalExporter;
+import org.elasticsearch.xpack.monitoring.action.MonitoringBulkDoc;
 import org.elasticsearch.xpack.monitoring.cleaner.CleanerService;
+import org.elasticsearch.xpack.monitoring.exporter.local.LocalExporter;
 import org.elasticsearch.xpack.security.InternalClient;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -265,9 +267,8 @@ public class ExportersTests extends ESTestCase {
                 protected void doRun() throws Exception {
                     List<MonitoringDoc> docs = new ArrayList<>();
                     for (int n = 0; n < threadDocs; n++) {
-                        docs.add(new MonitoringDoc(MonitoredSystem.ES.getSystem(),
-                                Version.CURRENT.toString(), null, null, null, 0L,
-                                (MonitoringDoc.Node) null));
+                        docs.add(new TestMonitoringDoc(randomAlphaOfLength(5), randomNonNegativeLong(), null, MonitoredSystem.ES,
+                                                       randomAlphaOfLength(5), null, String.valueOf(n)));
                     }
                     barrier.await(10, TimeUnit.SECONDS);
                     exporters.export(docs, ActionListener.wrap(
@@ -389,4 +390,19 @@ public class ExportersTests extends ESTestCase {
             return count.get();
         }
     }
+
+    static class TestMonitoringDoc extends MonitoringDoc {
+
+        private final String value;
+
+        TestMonitoringDoc(String cluster, long timestamp, Node node, MonitoredSystem system, String type, String id, String value) {
+            super(cluster, timestamp, node, system, type, id);
+            this.value = value;
+        }
+
+        @Override
+        protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.field("test", value);
+        }
+    }
 }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/MonitoringDocTests.java

@@ -1,65 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.exporter;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import static org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc.Node.fromDiscoveryNode;
-
-public class MonitoringDocTests extends ESTestCase {
-
-    public void testEmptyIdBecomesNull() {
-        final String id = randomFrom("", null, randomAlphaOfLength(5));
-        final MonitoringDoc doc =
-                new MonitoringDoc("monitoringId", "monitoringVersion",
-                                  "type", id, "clusterUUID",
-                                  0, (DiscoveryNode)null);
-
-        if (Strings.isNullOrEmpty(id)) {
-            assertNull(doc.getId());
-        } else {
-            assertSame(id, doc.getId());
-        }
-    }
-
-    public void testFromDiscoveryNode() {
-        assertEquals(null, fromDiscoveryNode(null));
-
-        String nodeId = randomAlphaOfLength(5);
-        TransportAddress address = buildNewFakeTransportAddress();
-        Version version = randomFrom(Version.V_5_0_1, Version.V_5_3_0, Version.CURRENT);
-
-        String name = randomBoolean() ? randomAlphaOfLength(5) : "";
-        Map<String, String> attributes = new HashMap<>();
-        if (randomBoolean()) {
-            int nbAttrs = randomIntBetween(1, 5);
-            for (int i = 0; i < nbAttrs; i++) {
-                attributes.put("attr_" + String.valueOf(i), String.valueOf(i));
-            }
-        }
-        Set<DiscoveryNode.Role> roles = new HashSet<>();
-        if (randomBoolean()) {
-            roles.addAll(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values())));
-        }
-        final MonitoringDoc.Node expectedNode = new MonitoringDoc.Node(nodeId,
-                address.address().getHostString(), address.toString(),
-                address.getAddress(), name, attributes);
-
-        DiscoveryNode discoveryNode =
-                new DiscoveryNode(name, nodeId, address, attributes, roles, version);
-        assertEquals(expectedNode, fromDiscoveryNode(discoveryNode));
-    }
-}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/MonitoringTemplateUtilsTests.java

@@ -7,6 +7,11 @@ package org.elasticsearch.xpack.monitoring.exporter;
 
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
 
 import java.io.IOException;
 
@@ -14,6 +19,7 @@ import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtil
 import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.OLD_TEMPLATE_IDS;
 import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.OLD_TEMPLATE_VERSION;
 import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.TEMPLATE_VERSION;
+import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.indexName;
 import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.oldTemplateName;
 import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.pipelineName;
 import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.templateName;
@@ -84,4 +90,27 @@ public class MonitoringTemplateUtilsTests extends ESTestCase {
         assertThat(json, containsString("\"version\":" + LAST_UPDATED_VERSION));
     }
 
+    public void testIndexName() {
+        final long timestamp = new DateTime(2017, 8, 3, 13, 47, 58, DateTimeZone.UTC).getMillis();
+
+        DateTimeFormatter formatter = DateTimeFormat.forPattern("YYYY.MM.dd").withZoneUTC();
+        assertThat(indexName(formatter, MonitoredSystem.ES, timestamp),
+                equalTo(".monitoring-es-" + TEMPLATE_VERSION + "-2017.08.03"));
+        assertThat(indexName(formatter, MonitoredSystem.BEATS, timestamp),
+                equalTo(".monitoring-beats-" + TEMPLATE_VERSION + "-2017.08.03"));
+        assertThat(indexName(formatter, MonitoredSystem.KIBANA, timestamp),
+                equalTo(".monitoring-kibana-" + TEMPLATE_VERSION + "-2017.08.03"));
+        assertThat(indexName(formatter, MonitoredSystem.LOGSTASH, timestamp),
+                equalTo(".monitoring-logstash-" + TEMPLATE_VERSION + "-2017.08.03"));
+
+        formatter = DateTimeFormat.forPattern("YYYY-dd-MM-HH.mm.ss").withZoneUTC();
+        assertThat(indexName(formatter, MonitoredSystem.ES, timestamp),
+                equalTo(".monitoring-es-" + TEMPLATE_VERSION + "-2017-03-08-13.47.58"));
+        assertThat(indexName(formatter, MonitoredSystem.BEATS, timestamp),
+                equalTo(".monitoring-beats-" + TEMPLATE_VERSION + "-2017-03-08-13.47.58"));
+        assertThat(indexName(formatter, MonitoredSystem.KIBANA, timestamp),
+                equalTo(".monitoring-kibana-" + TEMPLATE_VERSION + "-2017-03-08-13.47.58"));
+        assertThat(indexName(formatter, MonitoredSystem.LOGSTASH, timestamp),
+                equalTo(".monitoring-logstash-" + TEMPLATE_VERSION + "-2017-03-08-13.47.58"));
+    }
 }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java

@@ -5,9 +5,6 @@
  */
 package org.elasticsearch.xpack.monitoring.exporter.http;
 
-import java.util.Arrays;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteRequest;
@@ -16,7 +13,6 @@ import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -33,14 +29,13 @@ import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.http.MockRequest;
 import org.elasticsearch.test.http.MockResponse;
 import org.elasticsearch.test.http.MockWebServer;
-import org.elasticsearch.xpack.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.monitoring.MonitoringSettings;
+import org.elasticsearch.xpack.monitoring.MonitoringTestUtils;
 import org.elasticsearch.xpack.monitoring.collector.indices.IndexRecoveryMonitoringDoc;
 import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
 import org.elasticsearch.xpack.monitoring.exporter.Exporter;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.monitoring.resolver.ResolversRegistry;
 import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
 import org.elasticsearch.xpack.ssl.SSLService;
 import org.joda.time.format.DateTimeFormat;
@@ -50,18 +45,20 @@ import org.junit.Before;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.emptySet;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.LAST_UPDATED_VERSION;
 import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.TEMPLATE_VERSION;
+import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.indexName;
 import static org.elasticsearch.xpack.monitoring.exporter.http.ClusterAlertHttpResource.CLUSTER_ALERT_VERSION_PARAMETERS;
 import static org.elasticsearch.xpack.monitoring.exporter.http.PublishableHttpResource.FILTER_PATH_RESOURCE_VERSION;
 import static org.elasticsearch.xpack.monitoring.exporter.http.WatcherExistsHttpResource.WATCHER_CHECK_PARAMETERS;
@@ -349,8 +346,7 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
                                remoteClusterAllowsWatcher, currentLicenseAllowsWatcher, watcherAlreadyExists);
         MockRequest recordedRequest = assertBulk(webServer);
 
-        @SuppressWarnings("unchecked")
-        String indexName = new ResolversRegistry(Settings.EMPTY).getResolver(doc).index(doc);
+        String indexName = indexName(DateTimeFormat.forPattern("YYYY.MM.dd").withZoneUTC(), doc.getSystem(), doc.getTimestamp());
 
         byte[] bytes = recordedRequest.getBody().getBytes(StandardCharsets.UTF_8);
         Map<String, Object> data = XContentHelper.convertToMap(new BytesArray(bytes), false, XContentType.JSON).v2();
@@ -607,13 +603,11 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
     }
 
     private MonitoringDoc newRandomMonitoringDoc() {
-        String monitoringId = MonitoredSystem.ES.getSystem();
-        String monitoringVersion = Version.CURRENT.toString();
         String clusterUUID = internalCluster().getClusterName();
         long timestamp = System.currentTimeMillis();
-        DiscoveryNode sourceNode = new DiscoveryNode("id", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
+        MonitoringDoc.Node sourceNode = MonitoringTestUtils.randomMonitoringNode(random());
 
-        return new IndexRecoveryMonitoringDoc(monitoringId, monitoringVersion, clusterUUID, timestamp, sourceNode, new RecoveryResponse());
+        return new IndexRecoveryMonitoringDoc(clusterUUID, timestamp, sourceNode, new RecoveryResponse());
     }
 
     private List<MonitoringDoc> newRandomMonitoringDocs(int nb) {

plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java

@@ -23,8 +23,6 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
 import org.elasticsearch.xpack.monitoring.exporter.Exporter;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
-import org.elasticsearch.xpack.monitoring.resolver.ResolversRegistry;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -71,7 +69,6 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
 
     private final RestClient client = mock(RestClient.class);
     private final Response versionResponse = mock(Response.class);
-    private final ResolversRegistry registry = new ResolversRegistry(Settings.EMPTY);
     private final List<String> templateNames = new ArrayList<>(EXPECTED_TEMPLATES);
     private final List<String> pipelineNames = new ArrayList<>(EXPECTED_PIPELINES);
     private final List<String> watchNames = new ArrayList<>(EXPECTED_WATCHES);
@@ -80,21 +77,12 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
 
     private final MultiHttpResource resources =
             HttpExporter.createResources(
-                    new Exporter.Config("_http", "http", Settings.EMPTY, exporterSettings, clusterService, licenseState), registry);
+                    new Exporter.Config("_http", "http", Settings.EMPTY, exporterSettings, clusterService, licenseState));
 
     @Before
     public void setupResources() {
         templateNames.addAll(Arrays.stream(TEMPLATE_IDS).map(MonitoringTemplateUtils::templateName).collect(Collectors.toList()));
 
-        // TODO: when resolvers are removed, all templates managed by this loop should be included in the TEMPLATE_IDS above
-        for (final MonitoringIndexNameResolver resolver : registry) {
-            final String templateName = resolver.templateName();
-
-            if (templateNames.contains(templateName) == false) {
-                templateNames.add(templateName);
-            }
-        }
-
         if (createOldTemplates) {
             templateNames.addAll(
                     Arrays.stream(OLD_TEMPLATE_IDS).map(MonitoringTemplateUtils::oldTemplateName).collect(Collectors.toList()));
@@ -569,8 +557,7 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
 
         final MultiHttpResource resources =
                 HttpExporter.createResources(
-                        new Exporter.Config("_http", "http", Settings.EMPTY, exporterSettings, clusterService, licenseState),
-                        new ResolversRegistry(Settings.EMPTY));
+                        new Exporter.Config("_http", "http", Settings.EMPTY, exporterSettings, clusterService, licenseState));
 
         final int successfulGetTemplates = randomIntBetween(0, EXPECTED_TEMPLATES);
         final int unsuccessfulGetTemplates = EXPECTED_TEMPLATES - successfulGetTemplates;

plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterTests.java

@@ -24,9 +24,7 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
 import org.elasticsearch.xpack.monitoring.exporter.Exporter.Config;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.monitoring.resolver.ResolversRegistry;
 import org.elasticsearch.xpack.ssl.SSLService;
-
 import org.junit.Before;
 import org.mockito.InOrder;
 
@@ -319,7 +317,7 @@ public class HttpExporterTests extends ESTestCase {
 
         final Config config = createConfig(builder.build());
 
-        final MultiHttpResource multiResource = HttpExporter.createResources(config, new ResolversRegistry(config.settings()));
+        final MultiHttpResource multiResource = HttpExporter.createResources(config);
 
         final List<HttpResource> resources = multiResource.getResources();
         final int version = (int)resources.stream().filter((resource) -> resource instanceof VersionHttpResource).count();
@@ -405,10 +403,9 @@ public class HttpExporterTests extends ESTestCase {
         final RestClient client = mock(RestClient.class);
         final Sniffer sniffer = randomFrom(mock(Sniffer.class), null);
         final NodeFailureListener listener = mock(NodeFailureListener.class);
-        final ResolversRegistry resolvers = mock(ResolversRegistry.class);
         final HttpResource resource = new MockHttpResource(exporterName(), true, PublishableHttpResource.CheckResponse.ERROR, false);
 
-        try (HttpExporter exporter = new HttpExporter(config, client, sniffer, threadContext, listener, resolvers, resource)) {
+        try (HttpExporter exporter = new HttpExporter(config, client, sniffer, threadContext, listener, resource)) {
             verify(listener).setResource(resource);
 
             assertThat(exporter.openBulk(), nullValue());
@@ -420,11 +417,10 @@ public class HttpExporterTests extends ESTestCase {
         final RestClient client = mock(RestClient.class);
         final Sniffer sniffer = randomFrom(mock(Sniffer.class), null);
         final NodeFailureListener listener = mock(NodeFailureListener.class);
-        final ResolversRegistry resolvers = mock(ResolversRegistry.class);
         // sometimes dirty to start with and sometimes not; but always succeeds on checkAndPublish
         final HttpResource resource = new MockHttpResource(exporterName(), randomBoolean());
 
-        try (HttpExporter exporter = new HttpExporter(config, client, sniffer, threadContext, listener, resolvers, resource)) {
+        try (HttpExporter exporter = new HttpExporter(config, client, sniffer, threadContext, listener, resource)) {
             verify(listener).setResource(resource);
 
             final HttpExportBulk bulk = exporter.openBulk();
@@ -438,7 +434,6 @@ public class HttpExporterTests extends ESTestCase {
         final RestClient client = mock(RestClient.class);
         final Sniffer sniffer = randomFrom(mock(Sniffer.class), null);
         final NodeFailureListener listener = mock(NodeFailureListener.class);
-        final ResolversRegistry resolvers = mock(ResolversRegistry.class);
         final MultiHttpResource resource = mock(MultiHttpResource.class);
 
         if (sniffer != null && rarely()) {
@@ -449,7 +444,7 @@ public class HttpExporterTests extends ESTestCase {
             doThrow(randomFrom(new IOException("expected"), new RuntimeException("expected"))).when(client).close();
         }
 
-        new HttpExporter(config, client, sniffer, threadContext, listener, resolvers, resource).close();
+        new HttpExporter(config, client, sniffer, threadContext, listener, resource).close();
 
         // order matters; sniffer must close first
         if (sniffer != null) {

plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java

@@ -27,9 +27,9 @@ import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xpack.XPackClient;
 import org.elasticsearch.xpack.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.monitoring.MonitoringSettings;
+import org.elasticsearch.xpack.monitoring.MonitoringTestUtils;
 import org.elasticsearch.xpack.monitoring.action.MonitoringBulkDoc;
 import org.elasticsearch.xpack.monitoring.action.MonitoringBulkRequestBuilder;
-import org.elasticsearch.xpack.monitoring.action.MonitoringIndex;
 import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
 import org.elasticsearch.xpack.watcher.client.WatcherClient;
@@ -104,7 +104,7 @@ public class LocalExporterIntegTests extends LocalExporterIntegTestCase {
                 final int nbDocs = randomIntBetween(1, 20);
                 List<MonitoringBulkDoc> monitoringDocs = new ArrayList<>(nbDocs);
                 for (int i = 0; i < nbDocs; i++) {
-                    monitoringDocs.add(createMonitoringBulkDoc(String.valueOf(i)));
+                    monitoringDocs.add(createMonitoringBulkDoc());
                 }
 
                 assertBusy(() -> {
@@ -228,9 +228,10 @@ public class LocalExporterIntegTests extends LocalExporterIntegTestCase {
     private void checkMonitoringTemplates() {
         final Set<String> templates = new HashSet<>();
         templates.add(".monitoring-alerts");
-        for (MonitoredSystem system : MonitoredSystem.values()) {
-            templates.add(String.join("-", ".monitoring", system.getSystem()));
-        }
+        templates.add(".monitoring-es");
+        templates.add(".monitoring-kibana");
+        templates.add(".monitoring-logstash");
+        templates.add(".monitoring-beats");
 
         GetIndexTemplatesResponse response =
                 client().admin().indices().prepareGetTemplates(".monitoring-*").get();
@@ -316,12 +317,9 @@ public class LocalExporterIntegTests extends LocalExporterIntegTestCase {
                 expectedSystem = MonitoredSystem.fromSystem((String) docSource.get("expected_system"));
             }
 
-            Set<String> expectedIndex = new HashSet<>();
-
             String dateTime = dateFormatter.print(dateParser.parseDateTime(timestamp));
-            expectedIndex.add(".monitoring-" + expectedSystem.getSystem() + "-" + TEMPLATE_VERSION + "-" + dateTime);
-
-            assertTrue("Expected " + expectedIndex + " but got " + hit.getIndex(), expectedIndex.contains(hit.getIndex()));
+            final String expectedIndex = ".monitoring-" + expectedSystem.getSystem() + "-" + TEMPLATE_VERSION + "-" + dateTime;
+            assertEquals("Expected " + expectedIndex + " but got " + hit.getIndex(), expectedIndex, hit.getIndex());
 
             @SuppressWarnings("unchecked")
             Map<String, Object> sourceNode = (Map<String, Object>) source.get("source_node");
@@ -331,17 +329,15 @@ public class LocalExporterIntegTests extends LocalExporterIntegTestCase {
         }
     }
 
-    private static MonitoringBulkDoc createMonitoringBulkDoc(String id) throws IOException {
-        String monitoringId = randomFrom(BEATS, KIBANA, LOGSTASH).getSystem();
-        String monitoringVersion = TEMPLATE_VERSION;
-        XContentType xContentType = randomFrom(XContentType.values());
-
-        BytesReference source;
+    private static MonitoringBulkDoc createMonitoringBulkDoc() throws IOException {
+        final MonitoredSystem system = randomFrom(BEATS, KIBANA, LOGSTASH);
+        final XContentType xContentType = randomFrom(XContentType.values());
+        final BytesReference source;
 
         try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
             builder.startObject();
             {
-                builder.field("expected_system", monitoringId);
+                builder.field("expected_system", system.getSystem());
                 final int nbFields = randomIntBetween(1, 3);
                 for (int i = 0; i < nbFields; i++) {
                     builder.field("field_" + i, i);
@@ -351,7 +347,7 @@ public class LocalExporterIntegTests extends LocalExporterIntegTestCase {
             source = builder.bytes();
         }
 
-        return new MonitoringBulkDoc(monitoringId, monitoringVersion, MonitoringIndex.TIMESTAMPED, "doc", id, source, xContentType);
+        return MonitoringTestUtils.randomMonitoringBulkDoc(random(), xContentType, source, system, "doc");
     }
 
 }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java

@@ -9,7 +9,6 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.MapBuilder;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.ingest.PipelineConfiguration;
@@ -48,8 +47,6 @@ public class LocalExporterResourceIntegTests extends LocalExporterIntegTestCase
 
     private final boolean useClusterAlerts = enableWatcher() && randomBoolean();
     private final MonitoredSystem system = randomFrom(MonitoredSystem.values());
-    private final MockTimestampedIndexNameResolver resolver =
-            new MockTimestampedIndexNameResolver(system, Settings.EMPTY, MonitoringTemplateUtils.TEMPLATE_VERSION);
 
     @Override
     protected boolean useClusterAlerts() {
@@ -125,9 +122,10 @@ public class LocalExporterResourceIntegTests extends LocalExporterIntegTestCase
     }
 
     private void putTemplate(final Integer version) throws Exception {
-        final BytesReference source = generateTemplateSource(resolver.templateName(), version);
+        final String templateName = MonitoringTemplateUtils.templateName(system.getSystem());
+        final BytesReference source = generateTemplateSource(templateName, version);
 
-        assertAcked(client().admin().indices().preparePutTemplate(resolver.templateName()).setSource(source, XContentType.JSON).get());
+        assertAcked(client().admin().indices().preparePutTemplate(templateName).setSource(source, XContentType.JSON).get());
     }
 
     private void putPipelines(final Integer version) throws Exception {
@@ -272,7 +270,7 @@ public class LocalExporterResourceIntegTests extends LocalExporterIntegTestCase
     }
 
     private void assertTemplateNotUpdated() {
-        final String name = resolver.templateName();
+        final String name = MonitoringTemplateUtils.templateName(system.getSystem());
 
         for (IndexTemplateMetaData template : client().admin().indices().prepareGetTemplates(name).get().getIndexTemplates()) {
             final String docMapping = template.getMappings().get("doc").toString();

plugin/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java

@@ -0,0 +1,655 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.monitoring.integration;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpStatus;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.nio.entity.NStringEntity;
+import org.apache.lucene.util.Constants;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.CheckedRunnable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.license.License;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.xpack.monitoring.MonitoredSystem;
+import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStatsMonitoringDoc;
+import org.elasticsearch.xpack.monitoring.collector.indices.IndexRecoveryMonitoringDoc;
+import org.elasticsearch.xpack.monitoring.collector.indices.IndexStatsMonitoringDoc;
+import org.elasticsearch.xpack.monitoring.collector.indices.IndicesStatsMonitoringDoc;
+import org.elasticsearch.xpack.monitoring.collector.node.NodeStatsMonitoringDoc;
+import org.elasticsearch.xpack.monitoring.collector.shards.ShardMonitoringDoc;
+import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
+import org.elasticsearch.xpack.monitoring.rest.action.RestMonitoringBulkAction;
+import org.hamcrest.Matcher;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.ISODateTimeFormat;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.common.xcontent.XContentHelper.toXContent;
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue;
+import static org.elasticsearch.test.SecuritySettingsSource.TEST_PASSWORD_SECURE_STRING;
+import static org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStatsMonitoringDoc.hash;
+import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.TEMPLATE_VERSION;
+import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.isEmptyOrNullString;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+
+public class MonitoringIT extends ESRestTestCase {
+
+    private static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("x_pack_rest_user", TEST_PASSWORD_SECURE_STRING);
+
+    @Override
+    protected Settings restClientSettings() {
+        return Settings.builder()
+                       .put(super.restClientSettings())
+                       .put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE)
+                       .build();
+    }
+
+    private HttpEntity createBulkEntity() {
+        final StringBuilder bulk = new StringBuilder();
+        bulk.append("{\"index\":{\"_type\":\"test\"}}\n");
+        bulk.append("{\"foo\":{\"bar\":0}}\n");
+        bulk.append("{\"index\":{\"_type\":\"test\"}}\n");
+        bulk.append("{\"foo\":{\"bar\":1}}\n");
+        bulk.append("{\"index\":{\"_type\":\"test\"}}\n");
+        bulk.append("{\"foo\":{\"bar\":2}}\n");
+        bulk.append("\n");
+        return new NStringEntity(bulk.toString(), ContentType.APPLICATION_JSON);
+    }
+
+    public void testMonitoringBulkApiWithMissingSystemId() throws IOException {
+        final Map<String, String> parameters = parameters(null, TEMPLATE_VERSION, "10s");
+
+        assertBadRequest(parameters, createBulkEntity(), containsString("no [system_id] for monitoring bulk request"));
+    }
+
+    public void testMonitoringBulkApiWithMissingSystemApiVersion() throws IOException {
+        final Map<String, String> parameters = parameters(randomSystemId(), null, "10s");
+
+        assertBadRequest(parameters, createBulkEntity(), containsString("no [system_api_version] for monitoring bulk request"));
+    }
+
+    public void testMonitoringBulkApiWithMissingInterval() throws IOException {
+        final Map<String, String> parameters = parameters(randomSystemId(), TEMPLATE_VERSION, null);
+
+        assertBadRequest(parameters, createBulkEntity(), containsString("no [interval] for monitoring bulk request"));
+    }
+
+    public void testMonitoringBulkApiWithWrongInterval() throws IOException {
+        final Map<String, String> parameters = parameters(randomSystemId(), TEMPLATE_VERSION, "null");
+
+        assertBadRequest(parameters, createBulkEntity(), containsString("failed to parse setting [interval] with value [null]"));
+    }
+
+    public void testMonitoringBulkApiWithMissingContent() throws IOException {
+        final Map<String, String> parameters = parameters(randomSystemId(), TEMPLATE_VERSION, "30s");
+
+        assertBadRequest(parameters, null, containsString("no body content for monitoring bulk request"));
+    }
+
+    public void testMonitoringBulkApiWithUnsupportedSystemVersion() throws IOException {
+        final String systemId = randomSystemId();
+        final String systemApiVersion = randomFrom(TEMPLATE_VERSION, MonitoringTemplateUtils.OLD_TEMPLATE_VERSION);
+
+        Map<String, String> parameters = parameters(MonitoredSystem.UNKNOWN.getSystem(), systemApiVersion, "30s");
+        assertBadRequest(parameters, createBulkEntity(),
+                containsString("system_api_version ["+ systemApiVersion + "] is not supported by system_id [unknown]"));
+
+        parameters = parameters(systemId, "0", "30s");
+        assertBadRequest(parameters, createBulkEntity(),
+                containsString("system_api_version [0] is not supported by system_id [" + systemId + "]"));
+    }
+
+    /**
+     * Monitoring Bulk API test:
+     *
+     * This test uses the Monitoring Bulk API to index document as an external application like Kibana would do. It
+     * then ensure that the documents were correctly indexed and have the expected information.
+     */
+    @SuppressWarnings("unchecked")
+    public void testMonitoringBulk() throws Exception {
+        whenExportersAreReady(() -> {
+            final MonitoredSystem system = randomSystem();
+            final String interval = randomIntBetween(1, 20) + "s";
+
+            // Use Monitoring Bulk API to index 3 documents
+            Response bulkResponse = client().performRequest("POST", "/_xpack/monitoring/_bulk",
+                                                            parameters(system.getSystem(), TEMPLATE_VERSION, interval), createBulkEntity());
+
+            assertThat(bulkResponse.getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_OK));
+            assertThat(toMap(bulkResponse.getEntity()).get("errors"), equalTo(false));
+
+            final String indexPrefix = ".monitoring-" + system.getSystem() + "-" + TEMPLATE_VERSION;
+
+            // Wait for the monitoring index to be created
+            awaitRestApi("GET", "/" + indexPrefix + "-*/_search", singletonMap("size", "0"), null,
+                    resp -> {
+                        Number hitsTotal = (Number) extractValue("hits.total", resp);
+                        return hitsTotal != null && hitsTotal.intValue() >= 3;
+                    }, "Exception when waiting for monitoring bulk documents to be indexed");
+
+            Response searchResponse = client().performRequest("GET", "/" + indexPrefix + "-*/_search");
+            final Map<String, Object> results = toMap(searchResponse.getEntity());
+
+            final Map<String, Object> hits = (Map<String, Object>) results.get("hits");
+            final List<Map<String, Object>> searchHits = (List<Map<String,Object>>) hits.get("hits");
+            assertEquals(3, searchHits.size());
+
+            assertEquals("Monitoring documents must have the same timestamp",
+                    1, searchHits.stream().map(map -> extractValue("_source.timestamp", map)).distinct().count());
+
+            assertEquals("Monitoring documents must have the same source_node timestamp",
+                    1, searchHits.stream().map(map -> extractValue("_source.source_node.timestamp", map)).distinct().count());
+
+            for (Map<String,Object> searchHit : searchHits) {
+                assertMonitoringDoc(searchHit, system, "test");
+            }
+        });
+    }
+
+    /**
+     * Monitoring Service test:
+     *
+     * This test waits for the monitoring service to collect monitoring documents and then checks that all expected documents
+     * have been indexed with the expected information.
+     */
+    @SuppressWarnings("unchecked")
+    public void testMonitoringService() throws Exception {
+        final HttpEntity document = new StringEntity("{\"field\":\"value\"}", ContentType.APPLICATION_JSON);
+        assertThat(client().performRequest("POST", "/books/book/0", singletonMap("refresh", "true"), document)
+                           .getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_CREATED));
+
+        whenExportersAreReady(() -> {
+            final Response searchResponse = client().performRequest("GET", "/.monitoring-es-*/_search", singletonMap("size", "100"));
+            assertThat(searchResponse.getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_OK));
+
+            final Map<String, Object> results = toMap(searchResponse.getEntity());
+
+            final Map<String, Object> hits = (Map<String, Object>) results.get("hits");
+            assertThat("Expecting a minimum number of 6 docs, one per collector", (Integer) hits.get("total"), greaterThanOrEqualTo(6));
+
+            final List<Map<String, Object>> searchHits = (List<Map<String,Object>>) hits.get("hits");
+
+            for (Map<String,Object> searchHit : searchHits) {
+                final String type = (String) extractValue("_source.type", searchHit);
+
+                if (ClusterStatsMonitoringDoc.TYPE.equals(type)) {
+                    assertClusterStatsMonitoringDoc(searchHit);
+                } else if (IndexRecoveryMonitoringDoc.TYPE.equals(type)) {
+                    assertIndexRecoveryMonitoringDoc(searchHit);
+                } else if (IndicesStatsMonitoringDoc.TYPE.equals(type)) {
+                    assertIndicesStatsMonitoringDoc(searchHit);
+                } else if (IndexStatsMonitoringDoc.TYPE.equals(type)) {
+                    assertIndexStatsMonitoringDoc(searchHit);
+                } else if (NodeStatsMonitoringDoc.TYPE.equals(type)) {
+                    assertNodeStatsMonitoringDoc(searchHit);
+                } else if (ShardMonitoringDoc.TYPE.equals(type)) {
+                    assertShardMonitoringDoc(searchHit);
+                } else {
+                    fail("Monitoring document of type [" + type + "] is not supported by this test");
+                }
+            }
+        });
+    }
+
+    /**
+     * Asserts that the monitoring document (provided as a Map) contains the common information that
+     * all monitoring documents must have
+     */
+    @SuppressWarnings("unchecked")
+    private static void assertMonitoringDoc(final Map<String, Object> document,
+                                            final MonitoredSystem expectedSystem,
+                                            final String expectedType) throws Exception {
+        assertEquals(5, document.size());
+
+        final String index = (String) document.get("_index");
+        assertThat(index, containsString(".monitoring-" + expectedSystem.getSystem() + "-" + TEMPLATE_VERSION + "-"));
+        assertThat(document.get("_type"), equalTo("doc"));
+        assertThat((String) document.get("_id"), not(isEmptyOrNullString()));
+
+        final Map<String, Object> source = (Map<String, Object>) document.get("_source");
+        assertThat(source, notNullValue());
+        assertThat((String) source.get("cluster_uuid"), not(isEmptyOrNullString()));
+        assertThat(source.get("type"), equalTo(expectedType));
+
+        final String timestamp = (String) source.get("timestamp");
+        assertThat(timestamp, not(isEmptyOrNullString()));
+
+        assertThat(index, equalTo(MonitoringTemplateUtils.indexName(DateTimeFormat.forPattern("YYYY.MM.dd").withZoneUTC(),
+                                                                    expectedSystem,
+                                                                    ISODateTimeFormat.dateTime().parseMillis(timestamp))));
+
+        final Map<String, Object> sourceNode = (Map<String, Object>) source.get("source_node");
+        if (sourceNode != null) {
+            assertMonitoringDocSourceNode(sourceNode);
+        }
+    }
+
+    /**
+     * Asserts that the source_node information (provided as a Map) of a monitoring document correspond to
+     * the current local node information
+     */
+    @SuppressWarnings("unchecked")
+    private static void assertMonitoringDocSourceNode(final Map<String, Object> sourceNode) throws Exception {
+        assertEquals(6, sourceNode.size());
+
+        Map<String, String> filterPath = singletonMap("filter_path", "nodes.*.name,nodes.*.transport_address,nodes.*.host,nodes.*.ip");
+        final Response nodesResponse = client().performRequest("GET", "/_nodes", filterPath);
+
+        final Map<String, Object> nodes = (Map<String, Object>) toMap(nodesResponse.getEntity()).get("nodes");
+        assertEquals(1, nodes.size());
+
+        final String nodeId = nodes.keySet().iterator().next();
+
+        @SuppressWarnings("unchecked")
+        final Map<String, Object> node = (Map<String, Object>) nodes.get(nodeId);
+
+        assertThat(sourceNode.get("uuid"), equalTo(nodeId));
+        assertThat(sourceNode.get("host"), equalTo(node.get("host")));
+        assertThat(sourceNode.get("transport_address"),equalTo(node.get("transport_address")));
+        assertThat(sourceNode.get("ip"), equalTo(node.get("ip")));
+        assertThat(sourceNode.get("name"), equalTo(node.get("name")));
+        assertThat((String) sourceNode.get("timestamp"), not(isEmptyOrNullString()));
+    }
+
+    /**
+     * Assert that a {@link ClusterStatsMonitoringDoc} contains the expected information
+     */
+    @SuppressWarnings("unchecked")
+    private static void assertClusterStatsMonitoringDoc(final Map<String, Object> document) throws Exception {
+        assertMonitoringDoc(document, MonitoredSystem.ES, ClusterStatsMonitoringDoc.TYPE);
+
+        final Map<String, Object> source = (Map<String, Object>) document.get("_source");
+        assertEquals(10, source.size());
+
+        assertThat((String) source.get("cluster_name"), not(isEmptyOrNullString()));
+        assertThat(source.get("version"), equalTo(Version.CURRENT.toString()));
+
+        final Map<String, Object> license = (Map<String, Object>) source.get("license");
+        assertThat(license, notNullValue());
+        assertThat((String) license.get(License.Fields.ISSUER), not(isEmptyOrNullString()));
+        assertThat((String) license.get(License.Fields.ISSUED_TO), not(isEmptyOrNullString()));
+        assertThat((Long) license.get(License.Fields.ISSUE_DATE_IN_MILLIS), greaterThan(0L));
+        assertThat((Integer) license.get(License.Fields.MAX_NODES), greaterThan(0));
+
+        String uid = (String) license.get("uid");
+        assertThat(uid, not(isEmptyOrNullString()));
+
+        String type = (String) license.get("type");
+        assertThat(type, not(isEmptyOrNullString()));
+
+        String status = (String) license.get(License.Fields.STATUS);
+        assertThat(status, not(isEmptyOrNullString()));
+
+        Long expiryDate = (Long) license.get(License.Fields.EXPIRY_DATE_IN_MILLIS);
+        assertThat(expiryDate, greaterThan(0L));
+
+        // We basically recompute the hash here
+        assertThat("Hash key should be the same",
+                license.get("hkey"), equalTo(hash(status, uid, type, String.valueOf(expiryDate), (String) source.get("cluster_uuid"))));
+
+        final Map<String, Object> clusterStats = (Map<String, Object>) source.get("cluster_stats");
+        assertThat(clusterStats, notNullValue());
+        assertThat(clusterStats.size(), equalTo(4));
+
+        final Map<String, Object> stackStats = (Map<String, Object>) source.get("stack_stats");
+        assertThat(stackStats, notNullValue());
+        assertThat(stackStats.size(), equalTo(1));
+
+        final Map<String, Object> xpackStats = (Map<String, Object>) stackStats.get("xpack");
+        assertThat(xpackStats, notNullValue());
+        assertThat("X-Pack stats must have at least monitoring, but others may be hidden", xpackStats.size(), greaterThanOrEqualTo(1));
+
+        final Map<String, Object> monitoring = (Map<String, Object>) xpackStats.get("monitoring");
+        // we don't make any assumptions about what's in it, only that it's there
+        assertThat(monitoring, notNullValue());
+
+        final Map<String, Object> clusterState = (Map<String, Object>) source.get("cluster_state");
+        assertThat(clusterState, notNullValue());
+        assertThat(clusterState.size(), equalTo(5));
+        assertThat(clusterState.remove("status"), notNullValue());
+        assertThat(clusterState.remove("version"), notNullValue());
+        assertThat(clusterState.remove("state_uuid"), notNullValue());
+        assertThat(clusterState.remove("master_node"), notNullValue());
+        assertThat(clusterState.remove("nodes"), notNullValue());
+        assertThat(clusterState.isEmpty(), is(true));
+    }
+
+    /**
+     * Assert that a {@link IndexRecoveryMonitoringDoc} contains the expected information
+     */
+    @SuppressWarnings("unchecked")
+    private static void assertIndexRecoveryMonitoringDoc(final Map<String, Object> document) throws Exception {
+        assertMonitoringDoc(document, MonitoredSystem.ES, IndexRecoveryMonitoringDoc.TYPE);
+
+        final Map<String, Object> source = (Map<String, Object>) document.get("_source");
+        assertEquals(5, source.size());
+
+        final Map<String, Object> indexRecovery = (Map<String, Object>) source.get(IndexRecoveryMonitoringDoc.TYPE);
+        assertEquals(1, indexRecovery.size());
+
+        final List<Object> shards = (List<Object>) indexRecovery.get("shards");
+        assertThat(shards, notNullValue());
+    }
+
+    /**
+     * Assert that a {@link IndicesStatsMonitoringDoc} contains the expected information
+     */
+    @SuppressWarnings("unchecked")
+    private static void assertIndicesStatsMonitoringDoc(final Map<String, Object> document) throws Exception {
+        assertMonitoringDoc(document, MonitoredSystem.ES, IndicesStatsMonitoringDoc.TYPE);
+
+        final Map<String, Object> source = (Map<String, Object>) document.get("_source");
+        assertEquals(5, source.size());
+
+        final Map<String, Object> indicesStats = (Map<String, Object>) source.get(IndicesStatsMonitoringDoc.TYPE);
+        assertEquals(1, indicesStats.size());
+
+        IndicesStatsMonitoringDoc.XCONTENT_FILTERS.forEach(filter ->
+                assertThat(filter + " must not be null in the monitoring document", extractValue(filter, source), notNullValue()));
+    }
+
+    /**
+     * Assert that a {@link IndexStatsMonitoringDoc} contains the expected information
+     */
+    @SuppressWarnings("unchecked")
+    private static void assertIndexStatsMonitoringDoc(final Map<String, Object> document) throws Exception {
+        assertMonitoringDoc(document, MonitoredSystem.ES, IndexStatsMonitoringDoc.TYPE);
+
+        final Map<String, Object> source = (Map<String, Object>) document.get("_source");
+        assertEquals(5, source.size());
+
+        final Map<String, Object> indexStats = (Map<String, Object>) source.get(IndexStatsMonitoringDoc.TYPE);
+        assertEquals(3, indexStats.size());
+        assertThat(indexStats.get("primaries"), notNullValue());
+        assertThat(indexStats.get("total"), notNullValue());
+        assertThat((String) indexStats.get("index"), not(isEmptyOrNullString()));
+
+        IndexStatsMonitoringDoc.XCONTENT_FILTERS.forEach(filter ->
+                assertThat(filter + " must not be null in the monitoring document", extractValue(filter, source), notNullValue()));
+    }
+
+    /**
+     * Assert that a {@link NodeStatsMonitoringDoc} contains the expected information
+     */
+    @SuppressWarnings("unchecked")
+    private static void assertNodeStatsMonitoringDoc(final Map<String, Object> document) throws Exception {
+        assertMonitoringDoc(document, MonitoredSystem.ES, NodeStatsMonitoringDoc.TYPE);
+
+        final Map<String, Object> source = (Map<String, Object>) document.get("_source");
+        assertEquals(5, source.size());
+
+        final Map<String, Object> nodeStats = (Map<String, Object>) source.get(NodeStatsMonitoringDoc.TYPE);
+        assertEquals(9, nodeStats.size());
+
+        NodeStatsMonitoringDoc.XCONTENT_FILTERS.forEach(filter -> {
+            if (Constants.WINDOWS && filter.startsWith("node_stats.os.cpu.load_average")) {
+                // load average is unavailable on Windows
+                return;
+            }
+
+            // fs and cgroup stats are only reported on Linux, but it's acceptable for _node/stats to report them as null if the OS is
+            //  misconfigured or not reporting them for some reason (e.g., older kernel)
+            if (filter.startsWith("node_stats.fs") || filter.startsWith("node_stats.os.cgroup")) {
+                return;
+            }
+
+            // load average is unavailable on macOS for 5m and 15m (but we get 1m), but it's also possible on Linux too
+            if ("node_stats.os.cpu.load_average.5m".equals(filter) || "node_stats.os.cpu.load_average.15m".equals(filter)) {
+                return;
+            }
+
+            assertThat(filter + " must not be null in the monitoring document", extractValue(filter, source), notNullValue());
+        });
+    }
+
+    /**
+     * Assert that a {@link ShardMonitoringDoc} contains the expected information
+     */
+    @SuppressWarnings("unchecked")
+    private static void assertShardMonitoringDoc(final Map<String, Object> document) throws Exception {
+        assertMonitoringDoc(document, MonitoredSystem.ES, ShardMonitoringDoc.TYPE);
+
+        final Map<String, Object> source = (Map<String, Object>) document.get("_source");
+        assertEquals(6, source.size());
+        assertThat(source.get("state_uuid"), notNullValue());
+
+        final Map<String, Object> shard = (Map<String, Object>) source.get("shard");
+        assertEquals(6, shard.size());
+
+        final String currentNodeId = (String) shard.get("node");
+        if (Strings.hasLength(currentNodeId)) {
+            assertThat(source.get("source_node"), notNullValue());
+        } else {
+            assertThat(source.get("source_node"), nullValue());
+        }
+
+        ShardMonitoringDoc.XCONTENT_FILTERS.forEach(filter -> {
+            if (filter.equals("shard.relocating_node")) {
+                // Shard's relocating node is null most of the time in this test, we only check that the field is here
+                assertTrue(filter + " must exist in the monitoring document", shard.containsKey("relocating_node"));
+                return;
+            }
+            if (filter.equals("shard.node")) {
+                // Current node is null for replicas in this test, we only check that the field is here
+                assertTrue(filter + " must exist in the monitoring document", shard.containsKey("node"));
+                return;
+            }
+            assertThat(filter + " must not be null in the monitoring document", extractValue(filter, source), notNullValue());
+        });
+    }
+
+    /**
+     * Executes the given {@link Runnable} once the monitoring exporters are ready and functional. Ensure that
+     * the exporters and the monitoring service are shut down after the runnable has been executed.
+     */
+    private static void whenExportersAreReady(final CheckedRunnable<Exception> runnable) throws Exception {
+        try {
+            enableMonitoring();
+            runnable.run();
+        } finally {
+            disableMonitoring();
+        }
+    }
+
+    /**
+     * Enable the monitoring service and the Local exporter, waiting for some monitoring documents
+     * to be indexed before it returns.
+     */
+    public static void enableMonitoring() throws Exception {
+        final Map<String, Object> exporters = callRestApi("GET", "/_xpack/usage?filter_path=monitoring.enabled_exporters", 200);
+        assertThat("List of enabled exporters must be empty before enabling monitoring",
+                    XContentMapValues.extractRawValues("monitoring.enabled_exporters", exporters), hasSize(0));
+
+        final Settings settings = Settings.builder()
+                .put("transient.xpack.monitoring.collection.interval", "3s")
+                .put("transient.xpack.monitoring.exporters._local.enabled", true)
+                .build();
+
+        final HttpEntity entity =
+                new StringEntity(toXContent(settings, XContentType.JSON, false).utf8ToString(), ContentType.APPLICATION_JSON);
+
+        awaitRestApi("PUT", "/_cluster/settings", emptyMap(), entity,
+                response -> {
+                    Boolean acknowledged = (Boolean) response.get("acknowledged");
+                    return acknowledged != null && acknowledged;
+                },"Exception when enabling monitoring");
+
+        awaitRestApi("HEAD", "/.monitoring-es-*", singletonMap("allow_no_indices", "false"), null,
+                response -> true,
+                "Exception when waiting for monitoring-es-* index to be created");
+
+        awaitRestApi("GET", "/.monitoring-es-*/_search", emptyMap(), null,
+                response -> {
+                    Number hitsTotal = (Number) XContentMapValues.extractRawValues("hits.total", response).get(0);
+                    return hitsTotal != null && hitsTotal.intValue() > 0;
+                },"Exception when waiting for monitoring documents to be indexed");
+    }
+
+    /**
+     * Disable the monitoring service and the Local exporter, waiting for the monitoring indices to
+     * be deleted before it returns.
+     */
+    @SuppressWarnings("unchecked")
+    public static void disableMonitoring() throws Exception {
+        final Settings settings = Settings.builder()
+                .put("transient.xpack.monitoring.collection.interval", (String) null)
+                .put("transient.xpack.monitoring.exporters._local.enabled", (String) null)
+                .build();
+
+        final HttpEntity entity =
+                new StringEntity(toXContent(settings, XContentType.JSON, false).utf8ToString(), ContentType.APPLICATION_JSON);
+
+        awaitRestApi("PUT", "/_cluster/settings", emptyMap(), entity,
+                response -> {
+                    Boolean acknowledged = (Boolean) response.get("acknowledged");
+                    return acknowledged != null && acknowledged;
+                },"Exception when disabling monitoring");
+
+        awaitBusy(() -> {
+            try {
+                Map<String, Object> response = callRestApi("GET", "/_xpack/usage?filter_path=monitoring.enabled_exporters", 200);
+                final List<?> exporters = XContentMapValues.extractRawValues("monitoring.enabled_exporters", response);
+
+                if (exporters.isEmpty() == false) {
+                    return false;
+                }
+                response = callRestApi("GET", "/_nodes/_local/stats/thread_pool?filter_path=nodes.*.thread_pool.bulk.active", 200);
+
+                final Map<String, Object> nodes = (Map<String, Object>) response.get("nodes");
+                final Map<String, Object> threadPool = (Map<String, Object>) nodes.values().iterator().next();
+                final Number activeBulks = (Number) extractValue("bulk.active", threadPool);
+                return activeBulks != null && activeBulks.longValue() == 0L;
+            } catch (Exception e) {
+                throw new ElasticsearchException("Failed to delete monitoring indices:", e);
+            }
+        });
+    }
+
+    /**
+     * Executes a request using {@link org.elasticsearch.client.RestClient}, waiting for it to succeed.
+     */
+    private static void awaitRestApi(final String method, final String endpoint, final Map<String, String> params, final HttpEntity entity,
+                              final CheckedFunction<Map<String, Object>, Boolean, IOException> success,
+                              final String error) throws Exception {
+
+        final AtomicReference<IOException> exceptionHolder = new AtomicReference<>();
+        awaitBusy(() -> {
+            try {
+                final Response response = client().performRequest(method, endpoint, params, entity);
+                if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+                    exceptionHolder.set(null);
+                    final Map<String, Object> map = ("HEAD".equals(method) == false) ? toMap(response.getEntity()) : null;
+                    return success.apply(map);
+                }
+            } catch (IOException e) {
+                exceptionHolder.set(e);
+            }
+            return false;
+        });
+
+        IOException exception = exceptionHolder.get();
+        if (exception != null) {
+            throw new IllegalStateException(error, exception);
+        }
+    }
+
+    /**
+     * Executes a request using {@link org.elasticsearch.client.RestClient} in a synchronous manner, asserts that the response code
+     * is equal to {@code expectedCode} and then returns the {@link Response} as a {@link Map}.
+     */
+    private static Map<String, Object> callRestApi(final String method, final String endpoint, final int expectedCode) throws Exception {
+        final Response response = client().performRequest(method, endpoint);
+        assertEquals("Unexpected HTTP response code", expectedCode, response.getStatusLine().getStatusCode());
+
+        return toMap(response.getEntity());
+    }
+
+    /**
+     * Returns the {@link HttpEntity} content as a {@link Map} object.
+     */
+    private static Map<String, Object> toMap(final HttpEntity httpEntity) throws IOException {
+        final String contentType = httpEntity.getContentType().getValue();
+        final XContentType xContentType = XContentType.fromMediaTypeOrFormat(contentType);
+        if (xContentType == null) {
+            throw new IOException("Content-type not supported [" + contentType + "]");
+        }
+        return XContentHelper.convertToMap(xContentType.xContent(), httpEntity.getContent(), false);
+    }
+
+    /**
+     * Execute a Monitoring Bulk request and checks that it returns a 400 error with a given message.
+     *
+     * @param parameters the request parameters
+     * @param httpEntity the request body (can be null)
+     * @param matcher a {@link Matcher} to match the message against
+     */
+    private static void assertBadRequest(final Map<String, String> parameters, final HttpEntity httpEntity, final Matcher<String> matcher) {
+        ResponseException responseException = expectThrows(ResponseException.class, () ->
+                client().performRequest("POST", "/_xpack/monitoring/_bulk", parameters, httpEntity));
+
+        assertThat(responseException.getMessage(), matcher);
+        assertThat(responseException.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));
+    }
+
+    /**
+     * Builds a map of parameters for the Monitoring Bulk API
+     */
+    private static Map<String, String> parameters(final String systemId, final String systemApiVersion, final String interval) {
+        final Map<String, String> parameters = new HashMap<>();
+        if (systemId != null) {
+            parameters.put(RestMonitoringBulkAction.MONITORING_ID, systemId);
+        }
+        if (systemApiVersion != null) {
+            parameters.put(RestMonitoringBulkAction.MONITORING_VERSION, systemApiVersion);
+        }
+        if (interval != null) {
+            parameters.put(RestMonitoringBulkAction.INTERVAL, interval);
+        }
+        return parameters;
+    }
+
+    /**
+     * Returns a {@link String} representing a {@link MonitoredSystem} supported by the Monitoring Bulk API
+     */
+    private static String randomSystemId() {
+        return randomSystem().getSystem();
+    }
+
+    /**
+     * Returns a {@link MonitoredSystem} supported by the Monitoring Bulk API
+     */
+    private static MonitoredSystem randomSystem() {
+        return randomFrom(MonitoredSystem.LOGSTASH, MonitoredSystem.KIBANA, MonitoredSystem.BEATS);
+    }
+}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/MonitoringIndexNameResolverTestCase.java

@@ -1,185 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver;
-
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.common.xcontent.support.XContentMapValues;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import static org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils.TEMPLATE_VERSION;
-import static org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver.DELIMITER;
-import static org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver.Fields.CLUSTER_UUID;
-import static org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver.Fields.SOURCE_NODE;
-import static org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver.Fields.TIMESTAMP;
-import static org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver.Fields.TYPE;
-import static org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver.PREFIX;
-import static org.hamcrest.Matchers.allOf;
-import static org.hamcrest.Matchers.anyOf;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.hasItems;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.isEmptyOrNullString;
-import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
-import static org.hamcrest.Matchers.startsWith;
-import static org.hamcrest.collection.IsEmptyCollection.empty;
-
-public abstract class MonitoringIndexNameResolverTestCase<M extends MonitoringDoc, R extends MonitoringIndexNameResolver<M>>
-        extends ESTestCase {
-
-    protected final ResolversRegistry resolversRegistry = new ResolversRegistry(Settings.EMPTY);
-
-    /**
-     * @return the {@link MonitoringIndexNameResolver} to test
-     */
-    protected R newResolver() {
-        return newResolver(newMonitoringDoc());
-    }
-
-    /**
-     * @return the {@link MonitoringIndexNameResolver} to test
-     */
-    @SuppressWarnings("unchecked")
-    protected R newResolver(M doc) {
-        return (R) resolversRegistry.getResolver(doc);
-    }
-
-    /**
-     * @return a new randomly created {@link MonitoringDoc} instance to use in tests
-     */
-    protected abstract M newMonitoringDoc();
-
-    public void testMonitoringDoc() {
-        MonitoringDoc doc = newMonitoringDoc();
-        assertThat(doc.getMonitoringId(), equalTo(MonitoredSystem.fromSystem(doc.getMonitoringId()).getSystem()));
-        assertThat(doc.getMonitoringVersion(), not(isEmptyOrNullString()));
-        assertThat(doc.getClusterUUID(), not(isEmptyOrNullString()));
-        assertThat(doc.getTimestamp(), greaterThan(0L));
-    }
-
-    protected boolean checkResolvedIndex() {
-        return true;
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testIndex() {
-        if (checkResolvedIndex()) {
-            assertThat(newResolver().index(newMonitoringDoc()), not(isEmptyOrNullString()));
-        } else {
-            assertThat(newResolver().index(newMonitoringDoc()), is(nullValue()));
-        }
-    }
-
-    protected boolean checkFilters() {
-        return true;
-    }
-
-    public void testFilters() {
-        if (checkFilters()) {
-            assertThat(newResolver().filters(), allOf(not(empty()), notNullValue()));
-        } else {
-            assertThat(newResolver().filters(), anyOf(empty(), nullValue()));
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testSource() throws IOException {
-        MonitoringIndexNameResolver resolver = newResolver();
-        final XContentType xContentType = randomFrom(XContentType.values());
-        BytesReference source = resolver.source(newMonitoringDoc(), xContentType);
-
-        assertNotNull(source);
-        assertThat(source.length(), greaterThan(0));
-        assertSource(source, resolver.filters(), xContentType);
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testResolver() {
-        MonitoringIndexNameResolver resolver = newResolver();
-        assertThat(resolver, notNullValue());
-
-        if (resolver instanceof MonitoringIndexNameResolver.Timestamped) {
-            MonitoringIndexNameResolver.Timestamped timestamped = (MonitoringIndexNameResolver.Timestamped) resolver;
-            assertThat(resolver.index(newMonitoringDoc()),
-                    startsWith(PREFIX + DELIMITER + timestamped.getId() + DELIMITER + TEMPLATE_VERSION + DELIMITER));
-        }
-    }
-
-    protected void assertSource(BytesReference source, Set<String> fields, XContentType xContentType) {
-        Map<String, Object> sourceFields = XContentHelper.convertToMap(source, false, xContentType).v2();
-        assertNotNull(sourceFields);
-
-        String[] commons = new String[]{
-                CLUSTER_UUID,
-                TIMESTAMP,
-                TYPE,
-                SOURCE_NODE,
-        };
-        assertThat("source must contain default fields", sourceFields.keySet(), hasItems(commons));
-
-        assertThat("type is set improperly", sourceFields.get(TYPE), equalTo(newMonitoringDoc().getType()));
-
-        if (fields != null && fields.isEmpty() == false) {
-            for (String field : fields) {
-                assertSourceField(field, sourceFields);
-            }
-
-            // Checks that no extra fields are present
-            Set<String> extra = new HashSet<>();
-            for (String key : flattenMapKeys(new HashSet<>(), null, sourceFields)) {
-                boolean found = false;
-                for (String field : fields) {
-                    if (key.equalsIgnoreCase(field) || key.startsWith(field + ".")) {
-                        found = true;
-                        break;
-                    }
-                }
-                if (!found) {
-                    extra.add(key);
-                }
-            }
-            assertThat("found extra field(s) " + extra, extra.size(), equalTo(0));
-        }
-    }
-
-    protected void assertSourceField(String field, Map<String, Object> sourceFields) {
-        assertNotNull("source must contain field [" + field + "] with a non-null value",
-                XContentMapValues.extractValue(field, sourceFields));
-    }
-
-    protected static String randomMonitoringId() {
-        return randomFrom(MonitoredSystem.values()).getSystem();
-    }
-
-    @SuppressWarnings("unchecked")
-    private static Set<String> flattenMapKeys(final Set<String> keys, String path, Map<String, Object> map) {
-        if (map == null || map.isEmpty()) {
-            return keys;
-        }
-
-        for (Map.Entry<String, Object> entry : map.entrySet()) {
-            String p = path != null ? path + "." + entry.getKey() : entry.getKey();
-            if (entry.getValue() instanceof Map) {
-                flattenMapKeys(keys, p, (Map)entry.getValue());
-            } else {
-                keys.add(p);
-            }
-        }
-        return keys;
-    }
-}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/TimestampedResolverTests.java

@@ -1,78 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.joda.time.format.DateTimeFormat;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.emptySet;
-import static org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver.DELIMITER;
-import static org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver.PREFIX;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.notNullValue;
-
-public class TimestampedResolverTests extends MonitoringIndexNameResolverTestCase {
-
-    private MonitoredSystem randomId = randomFrom(MonitoredSystem.values());
-
-    @Override
-    protected MonitoringIndexNameResolver<MonitoringDoc> newResolver() {
-        return newTimestampedResolver(randomId, Settings.EMPTY);
-    }
-
-    @Override
-    protected MonitoringDoc newMonitoringDoc() {
-        MonitoringDoc doc = new MonitoringDoc(randomMonitoringId(), randomAlphaOfLength(2),
-                null, null, randomAlphaOfLength(5), 1437580442979L,  // "2015-07-22T15:54:02.979Z"
-                new DiscoveryNode("id", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT));
-        return doc;
-    }
-
-    @Override
-    protected boolean checkFilters() {
-        return false;
-    }
-
-    public void testTimestampedResolver() {
-        final MonitoringDoc doc = newMonitoringDoc();
-
-        for (String format : Arrays.asList("YYYY", "YYYY.MM", "YYYY.MM.dd", "YYYY.MM.dd-HH", "YYYY.MM.dd-HH.mm", "YYYY.MM.dd-HH.mm.SS")) {
-            Settings settings = Settings.EMPTY;
-            if (format != null) {
-                settings = Settings.builder()
-                            .put(MonitoringIndexNameResolver.Timestamped.INDEX_NAME_TIME_FORMAT_SETTING.getKey(), format)
-                            .build();
-            }
-
-            MonitoringIndexNameResolver.Timestamped resolver = newTimestampedResolver(randomId, settings);
-            assertThat(resolver, notNullValue());
-            assertThat(resolver.getId(), equalTo(randomId.getSystem()));
-            assertThat(resolver.index(doc),
-                    equalTo(PREFIX + DELIMITER + resolver.getId() + DELIMITER + String.valueOf(MonitoringTemplateUtils.TEMPLATE_VERSION)
-                            + DELIMITER + DateTimeFormat.forPattern(format).withZoneUTC().print(1437580442979L)));
-        }
-    }
-
-    private MonitoringIndexNameResolver.Timestamped<MonitoringDoc> newTimestampedResolver(MonitoredSystem id, Settings settings) {
-        return new MonitoringIndexNameResolver.Timestamped<MonitoringDoc>(id, settings) {
-            @Override
-            protected void buildXContent(MonitoringDoc document, XContentBuilder builder, ToXContent.Params params) throws IOException {
-                // noop
-            }
-        };
-    }
-}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/bulk/MonitoringBulkTimestampedResolverTests.java

@@ -1,58 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.bulk;
-
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.xpack.monitoring.MonitoredSystem;
-import org.elasticsearch.xpack.monitoring.action.MonitoringBulkDoc;
-import org.elasticsearch.xpack.monitoring.action.MonitoringBulkDocTests;
-import org.elasticsearch.xpack.monitoring.action.MonitoringIndex;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolverTestCase;
-
-import static org.hamcrest.Matchers.equalTo;
-
-/**
- * Tests {@link MonitoringBulkTimestampedResolver}.
- */
-public class MonitoringBulkTimestampedResolverTests
-        extends MonitoringIndexNameResolverTestCase<MonitoringBulkDoc, MonitoringBulkTimestampedResolver> {
-
-    @Override
-    protected MonitoringBulkDoc newMonitoringDoc() {
-        MonitoringBulkDoc doc = new MonitoringBulkDoc(MonitoredSystem.KIBANA.getSystem(),
-                MonitoringTemplateUtils.TEMPLATE_VERSION, MonitoringIndex.TIMESTAMPED,
-                "kibana_stats",
-                randomBoolean() ? randomAlphaOfLength(35) : null,
-                randomAlphaOfLength(5), 1437580442979L,
-                MonitoringBulkDocTests.newRandomSourceNode(),
-                new BytesArray("{\"field1\" : \"value1\"}"), XContentType.JSON);
-        return doc;
-    }
-
-    @Override
-    protected boolean checkFilters() {
-        return false;
-    }
-
-    public void testMonitoringBulkResolver() throws Exception {
-        MonitoringBulkDoc doc = newMonitoringDoc();
-
-        MonitoringBulkTimestampedResolver resolver = newResolver(doc);
-        assertThat(resolver.index(doc), equalTo(".monitoring-kibana-" + MonitoringTemplateUtils.TEMPLATE_VERSION + "-2015.07.22"));
-
-        assertSource(resolver.source(doc, XContentType.JSON),
-                Sets.newHashSet(
-                        "cluster_uuid",
-                        "timestamp",
-                        "type",
-                        "source_node",
-                        "kibana_stats",
-                        "kibana_stats.field1"), XContentType.JSON);
-    }
-}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterStatsResolverTests.java

@@ -1,106 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.cluster;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.license.License;
-import org.elasticsearch.xpack.monitoring.MonitoringFeatureSet;
-import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStatsMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolverTestCase;
-
-import java.util.Collections;
-import java.util.UUID;
-
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.emptySet;
-import static org.hamcrest.Matchers.startsWith;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class ClusterStatsResolverTests extends MonitoringIndexNameResolverTestCase<ClusterStatsMonitoringDoc, ClusterStatsResolver> {
-
-    @Override
-    protected ClusterStatsMonitoringDoc newMonitoringDoc() {
-        try {
-            License.Builder licenseBuilder = License.builder()
-                    .uid(UUID.randomUUID().toString())
-                    .type("trial")
-                    .issuer("elasticsearch")
-                    .issuedTo("customer")
-                    .maxNodes(5)
-                    .issueDate(1437580442979L)
-                    .expiryDate(1437580442979L + TimeValue.timeValueHours(2).getMillis());
-
-            final String nodeUuid = "the-master-nodes-uuid";
-            final ClusterName clusterName = ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY);
-            final DiscoveryNodes nodes = mock(DiscoveryNodes.class);
-            final DiscoveryNode node =
-                    new DiscoveryNode(nodeUuid, new TransportAddress(TransportAddress.META_ADDRESS, 9300), Version.CURRENT);
-
-            when(nodes.getMasterNodeId()).thenReturn(nodeUuid);
-            when(nodes.iterator()).thenReturn(Collections.singleton(node).iterator());
-
-            return new ClusterStatsMonitoringDoc(randomMonitoringId(),
-                    randomAlphaOfLength(2), randomAlphaOfLength(5),
-                    Math.abs(randomLong()),
-                    new DiscoveryNode("id", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
-                    randomAlphaOfLength(5),
-                    randomFrom(Version.V_5_0_0, Version.CURRENT).toString(),
-                    licenseBuilder.build(),
-                    Collections.singletonList(new MonitoringFeatureSet.Usage(randomBoolean(), randomBoolean(), emptyMap())),
-                    new ClusterStatsResponse(
-                            Math.abs(randomLong()),
-                            clusterName,
-                            Collections.emptyList(),
-                            Collections.emptyList()),
-                    new ClusterState(clusterName, randomLong(), "a-real-state-uuid", null, null, nodes, null, null, false),
-                    randomFrom(ClusterHealthStatus.values())
-                );
-        } catch (Exception e) {
-            throw new IllegalStateException("Failed to generated random ClusterStatsMonitoringDoc", e);
-        }
-    }
-
-    @Override
-    protected boolean checkFilters() {
-        return false;
-    }
-
-    public void testClusterInfoResolver() throws Exception {
-        ClusterStatsMonitoringDoc doc = newMonitoringDoc();
-
-        ClusterStatsResolver resolver = newResolver();
-        assertThat(resolver.index(doc), startsWith(".monitoring-es-" + MonitoringTemplateUtils.TEMPLATE_VERSION));
-        assertSource(resolver.source(doc, XContentType.JSON),
-                     Sets.newHashSet(
-                         "cluster_uuid",
-                         "timestamp",
-                         "type",
-                         "source_node",
-                         "cluster_name",
-                         "version",
-                         "license",
-                         "cluster_stats",
-                         "cluster_state.status",
-                         "cluster_state.version",
-                         "cluster_state.state_uuid",
-                         "cluster_state.master_node",
-                         "cluster_state.nodes",
-                         "stack_stats.xpack"), XContentType.JSON);
-    }
-}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterStatsTests.java

@@ -1,154 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.cluster;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.license.License;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.sort.SortOrder;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.xpack.monitoring.MonitoringSettings;
-import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStatsMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
-import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
-import org.junit.After;
-import org.junit.Before;
-
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.isEmptyOrNullString;
-import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.notNullValue;
-
-@ClusterScope(scope = TEST)
-public class ClusterStatsTests extends MonitoringIntegTestCase {
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder()
-                .put(super.nodeSettings(nodeOrdinal))
-                .put(MonitoringSettings.INTERVAL.getKey(), "-1")
-                .build();
-    }
-
-    @Before
-    public void init() throws Exception {
-        updateMonitoringInterval(3L, TimeUnit.SECONDS);
-    }
-
-    @After
-    public void cleanup() throws Exception {
-        disableMonitoringInterval();
-        wipeMonitoringIndices();
-    }
-
-    @SuppressWarnings("unchecked")
-    public void testClusterStats() throws Exception {
-        ensureGreen();
-
-        final String clusterUUID = client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID();
-        assertTrue(Strings.hasText(clusterUUID));
-
-        // waiting for cluster stats collector to collect data
-        awaitMonitoringDocsCountOnPrimary(greaterThanOrEqualTo(1L), ClusterStatsMonitoringDoc.TYPE);
-
-        refresh();
-
-        // retrieving cluster stats document
-        final SearchResponse response =
-                client().prepareSearch(".monitoring-es-*")
-                        .setQuery(QueryBuilders.termQuery("type", ClusterStatsMonitoringDoc.TYPE))
-                        .addSort("timestamp", SortOrder.DESC)
-                        .setSize(1)
-                        .setPreference("_primary").get();
-        assertTrue("cluster_stats document does not exist", response.getHits().getTotalHits() != 0);
-
-        final SearchHit hit = response.getHits().getAt(0);
-
-        assertTrue("_source is disabled", hit.hasSource());
-
-        Map<String, Object> source = hit.getSourceAsMap();
-
-        assertThat(source.get(MonitoringIndexNameResolver.Fields.CLUSTER_UUID), notNullValue());
-        assertThat(source.get(MonitoringIndexNameResolver.Fields.TIMESTAMP), notNullValue());
-        assertThat(source.get(MonitoringIndexNameResolver.Fields.SOURCE_NODE), notNullValue());
-        assertThat(source.get("cluster_name"), equalTo(cluster().getClusterName()));
-        assertThat(source.get("version"), equalTo(Version.CURRENT.toString()));
-
-        Object licenseObj = source.get("license");
-        assertThat(licenseObj, instanceOf(Map.class));
-        Map license = (Map) licenseObj;
-
-        assertThat(license, instanceOf(Map.class));
-
-        String uid = (String) license.get("uid");
-        assertThat(uid, not(isEmptyOrNullString()));
-
-        String type = (String) license.get("type");
-        assertThat(type, not(isEmptyOrNullString()));
-
-        String status = (String) license.get(License.Fields.STATUS);
-        assertThat(status, not(isEmptyOrNullString()));
-
-        Long expiryDate = (Long) license.get(License.Fields.EXPIRY_DATE_IN_MILLIS);
-        assertThat(expiryDate, greaterThan(0L));
-
-        // We basically recompute the hash here
-        String hkey = (String) license.get("hkey");
-        String recalculated = ClusterStatsResolver.hash(status, uid, type, String.valueOf(expiryDate), clusterUUID);
-        assertThat(hkey, equalTo(recalculated));
-
-        assertThat((String) license.get(License.Fields.ISSUER), not(isEmptyOrNullString()));
-        assertThat((String) license.get(License.Fields.ISSUED_TO), not(isEmptyOrNullString()));
-        assertThat((Long) license.get(License.Fields.ISSUE_DATE_IN_MILLIS), greaterThan(0L));
-        assertThat((Integer) license.get(License.Fields.MAX_NODES), greaterThan(0));
-
-        Object clusterStats = source.get("cluster_stats");
-        assertNotNull(clusterStats);
-        assertThat(clusterStats, instanceOf(Map.class));
-        assertThat(((Map) clusterStats).size(), greaterThan(0));
-
-        Object stackStats = source.get("stack_stats");
-        assertNotNull(stackStats);
-        assertThat(stackStats, instanceOf(Map.class));
-        assertThat(((Map) stackStats).size(), equalTo(1));
-
-        Object xpack = ((Map)stackStats).get("xpack");
-        assertNotNull(xpack);
-        assertThat(xpack, instanceOf(Map.class));
-        // it must have at least monitoring, but others may be hidden
-        assertThat(((Map) xpack).size(), greaterThanOrEqualTo(1));
-
-        Object monitoring = ((Map)xpack).get("monitoring");
-        assertNotNull(monitoring);
-        // we don't make any assumptions about what's in it, only that it's there
-        assertThat(monitoring, instanceOf(Map.class));
-
-        Object clusterState = source.get("cluster_state");
-        assertNotNull(clusterState);
-        assertThat(clusterState, instanceOf(Map.class));
-
-        Map<String, Object> clusterStateMap = (Map<String, Object>)clusterState;
-
-        assertThat(clusterStateMap.keySet(), hasSize(5));
-        assertThat(clusterStateMap.remove("status"), notNullValue());
-        assertThat(clusterStateMap.remove("version"), notNullValue());
-        assertThat(clusterStateMap.remove("state_uuid"), notNullValue());
-        assertThat(clusterStateMap.remove("master_node"), notNullValue());
-        assertThat(clusterStateMap.remove("nodes"), notNullValue());
-    }
-}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/indices/IndexRecoveryResolverTests.java

@@ -1,68 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.indices;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RecoverySource;
-import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.ShardRoutingState;
-import org.elasticsearch.cluster.routing.TestShardRouting;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.indices.recovery.RecoveryState;
-import org.elasticsearch.xpack.monitoring.collector.indices.IndexRecoveryMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolverTestCase;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.emptySet;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
-
-public class IndexRecoveryResolverTests extends MonitoringIndexNameResolverTestCase<IndexRecoveryMonitoringDoc, IndexRecoveryResolver> {
-
-    @Override
-    protected IndexRecoveryMonitoringDoc newMonitoringDoc() {
-        DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
-        Map<String, java.util.List<RecoveryState>> shardRecoveryStates = new HashMap<>();
-        ShardRouting shardRouting = TestShardRouting.newShardRouting(new ShardId("test", "uuid", 0), localNode.getId(), true,
-                ShardRoutingState.INITIALIZING, RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE);
-        shardRecoveryStates.put("test", Collections.singletonList(new RecoveryState(shardRouting, localNode, null)));
-
-        IndexRecoveryMonitoringDoc doc = new IndexRecoveryMonitoringDoc(randomMonitoringId(),
-                randomAlphaOfLength(2), randomAlphaOfLength(5), 1437580442979L,
-                new DiscoveryNode("id", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
-                new RecoveryResponse(10, 10, 0, false, shardRecoveryStates, Collections.emptyList()));
-        return doc;
-    }
-
-    @Override
-    protected boolean checkFilters() {
-        return false;
-    }
-
-    public void testIndexRecoveryResolver() throws Exception {
-        IndexRecoveryMonitoringDoc doc = newMonitoringDoc();
-
-        IndexRecoveryResolver resolver = newResolver();
-        assertThat(resolver.index(doc), equalTo(".monitoring-es-" + MonitoringTemplateUtils.TEMPLATE_VERSION + "-2015.07.22"));
-
-        assertSource(resolver.source(doc, XContentType.JSON),
-                Sets.newHashSet(
-                        "cluster_uuid",
-                        "timestamp",
-                        "type",
-                        "source_node",
-                        "index_recovery"), XContentType.JSON);
-    }
-}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/indices/IndexRecoveryTests.java

@@ -1,127 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.indices;
-
-import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.TermQueryBuilder;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.xpack.monitoring.MonitoringSettings;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
-import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
-import org.junit.After;
-
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.is;
-
-@ClusterScope(scope = TEST)
-public class IndexRecoveryTests extends MonitoringIntegTestCase {
-
-    private static final String INDEX_PREFIX = "test-index-recovery-";
-    private final TermQueryBuilder indexRecoveryType = QueryBuilders.termQuery("type", IndexRecoveryResolver.TYPE);
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder()
-                .put(super.nodeSettings(nodeOrdinal))
-                .put(MonitoringSettings.INTERVAL.getKey(), "-1")
-                .put(MonitoringSettings.INDICES.getKey(), INDEX_PREFIX + "*")
-                .put("xpack.monitoring.exporters.default_local.type", "local")
-                .build();
-    }
-
-    @After
-    public void cleanup() throws Exception {
-        disableMonitoringInterval();
-        wipeMonitoringIndices();
-    }
-
-    public void testIndexRecovery() throws Exception {
-        for (int i = 0; i < randomIntBetween(1, 10); i++) {
-            client().prepareIndex(INDEX_PREFIX + i, "foo").setSource("field1", "value1").get();
-        }
-
-        assertBusy(() -> {
-            flush();
-            refresh();
-
-            RecoveryResponse recoveries = client().admin().indices().prepareRecoveries().get();
-            assertThat(recoveries.hasRecoveries(), is(true));
-        });
-
-        updateMonitoringInterval(3L, TimeUnit.SECONDS);
-        waitForMonitoringIndices();
-
-        awaitMonitoringDocsCountOnPrimary(greaterThan(0L), IndexRecoveryResolver.TYPE);
-
-        String clusterUUID = client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID();
-        assertTrue(Strings.hasText(clusterUUID));
-
-        SearchResponse response =
-                client().prepareSearch()
-                        .setQuery(indexRecoveryType)
-                        .setPreference("_primary")
-                        .get();
-        assertThat(response.getHits().getTotalHits(), greaterThan(0L));
-
-        String[] filters = {
-                MonitoringIndexNameResolver.Fields.CLUSTER_UUID,
-                MonitoringIndexNameResolver.Fields.TIMESTAMP,
-                MonitoringIndexNameResolver.Fields.TYPE,
-                MonitoringIndexNameResolver.Fields.SOURCE_NODE,
-                IndexRecoveryResolver.Fields.INDEX_RECOVERY,
-                IndexRecoveryResolver.Fields.INDEX_RECOVERY + "." + IndexRecoveryResolver.Fields.SHARDS,
-        };
-
-        for (SearchHit searchHit : response.getHits().getHits()) {
-            Map<String, Object> fields = searchHit.getSourceAsMap();
-            for (String filter : filters) {
-                assertContains(filter, fields);
-            }
-        }
-
-        refresh();
-
-        // ensure these fields are not indexed (searchable)
-        String[] fields = {
-                "index_recovery.shards.primary",
-                "index_recovery.shards.id",
-                "index_recovery.shards.stage",
-                "index_recovery.shards.index_name",
-                "index_recovery.shards.source.host",
-                "index_recovery.shards.source.name",
-        };
-
-        for (String field : fields) {
-            response =
-                    client().prepareSearch()
-                            .setQuery(checkForFieldQuery(field))
-                            .setPreference("_primary")
-                            .setSize(0)
-                            .get();
-            assertHitCount(response, 0L);
-        }
-    }
-
-    private BoolQueryBuilder checkForFieldQuery(final String field) {
-        final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
-
-        boolQuery.must().add(indexRecoveryType);
-        boolQuery.must().add(QueryBuilders.existsQuery(field));
-
-        return boolQuery;
-    }
-}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/indices/IndexStatsResolverTests.java

@@ -1,93 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.indices;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.indices.stats.CommonStats;
-import org.elasticsearch.action.admin.indices.stats.IndexStats;
-import org.elasticsearch.action.admin.indices.stats.ShardStats;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RecoverySource;
-import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.UnassignedInfo;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.cache.query.QueryCacheStats;
-import org.elasticsearch.index.cache.request.RequestCacheStats;
-import org.elasticsearch.index.engine.SegmentsStats;
-import org.elasticsearch.index.fielddata.FieldDataStats;
-import org.elasticsearch.index.merge.MergeStats;
-import org.elasticsearch.index.refresh.RefreshStats;
-import org.elasticsearch.index.search.stats.SearchStats;
-import org.elasticsearch.index.shard.DocsStats;
-import org.elasticsearch.index.shard.IndexingStats;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.shard.ShardPath;
-import org.elasticsearch.index.store.StoreStats;
-import org.elasticsearch.xpack.monitoring.collector.indices.IndexStatsMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolverTestCase;
-
-import java.nio.file.Path;
-import java.util.UUID;
-
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.emptySet;
-import static org.hamcrest.Matchers.equalTo;
-
-public class IndexStatsResolverTests extends MonitoringIndexNameResolverTestCase<IndexStatsMonitoringDoc, IndexStatsResolver> {
-
-    @Override
-    protected IndexStatsMonitoringDoc newMonitoringDoc() {
-        IndexStatsMonitoringDoc doc = new IndexStatsMonitoringDoc(randomMonitoringId(),
-                randomAlphaOfLength(2), randomAlphaOfLength(5), 1437580442979L,
-                new DiscoveryNode("id", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
-                randomIndexStats());
-        return doc;
-    }
-
-    public void testIndexStatsResolver() throws Exception {
-        IndexStatsMonitoringDoc doc = newMonitoringDoc();
-
-        IndexStatsResolver resolver = newResolver();
-        assertThat(resolver.index(doc), equalTo(".monitoring-es-" + MonitoringTemplateUtils.TEMPLATE_VERSION + "-2015.07.22"));
-
-        assertSource(resolver.source(doc, XContentType.JSON),
-                Sets.newHashSet(
-                        "cluster_uuid",
-                        "timestamp",
-                        "type",
-                        "source_node",
-                        "index_stats"), XContentType.JSON);
-    }
-
-    /**
-     * @return a random {@link IndexStats} object.
-     */
-    private IndexStats randomIndexStats() {
-        Index index = new Index("test-" + randomIntBetween(0, 5), UUID.randomUUID().toString());
-        ShardId shardId = new ShardId(index, 0);
-        Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("0");
-        ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE,
-                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
-        shardRouting = shardRouting.initialize("node-0", null, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
-        shardRouting = shardRouting.moveToStarted();
-        CommonStats stats = new CommonStats();
-        stats.fieldData = new FieldDataStats();
-        stats.queryCache = new QueryCacheStats();
-        stats.requestCache = new RequestCacheStats();
-        stats.docs = new DocsStats();
-        stats.store = new StoreStats();
-        stats.indexing = new IndexingStats();
-        stats.search = new SearchStats();
-        stats.segments = new SegmentsStats();
-        stats.merge = new MergeStats();
-        stats.refresh = new RefreshStats();
-        ShardStats shardStats = new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null);
-        return new IndexStats(index.getName(), new ShardStats[]{shardStats});
-    }
-}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/indices/IndexStatsTests.java

@@ -1,97 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.indices;
-
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.elasticsearch.xpack.monitoring.MonitoringSettings;
-import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
-import org.junit.After;
-
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static org.hamcrest.Matchers.greaterThan;
-
-@ClusterScope(scope = Scope.TEST, numClientNodes = 0)
-public class IndexStatsTests extends MonitoringIntegTestCase {
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder()
-                .put(super.nodeSettings(nodeOrdinal))
-                .put(MonitoringSettings.INTERVAL.getKey(), "-1")
-                .put("xpack.monitoring.exporters.default_local.type", "local")
-                .build();
-    }
-
-    @After
-    public void cleanup() throws Exception {
-        disableMonitoringInterval();
-        wipeMonitoringIndices();
-    }
-
-    @AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/1396")
-    public void testIndexStats() throws Exception {
-        logger.debug("--> creating some indices for future index stats");
-        final int nbIndices = randomIntBetween(1, 5);
-        String[] indices = new String[nbIndices];
-        for (int i = 0; i < nbIndices; i++) {
-            indices[i] = "stat" + i;
-            createIndex(indices[i]);
-        }
-
-        final long[] nbDocsPerIndex = new long[nbIndices];
-        for (int i = 0; i < nbIndices; i++) {
-            nbDocsPerIndex[i] = randomIntBetween(1, 50);
-            for (int j = 0; j < nbDocsPerIndex[i]; j++) {
-                client().prepareIndex("stat" + i, "type1").setSource("num", i).get();
-            }
-        }
-
-        flush();
-        refresh();
-
-        updateMonitoringInterval(3L, TimeUnit.SECONDS);
-        waitForMonitoringIndices();
-
-        awaitMonitoringDocsCountOnPrimary(greaterThan(0L), IndexStatsResolver.TYPE);
-
-        logger.debug("--> wait for index stats collector to collect stat for each index");
-        assertBusy(() -> {
-            flush(indices);
-            refresh();
-            for (int i = 0; i < nbIndices; i++) {
-                SearchResponse count = client().prepareSearch()
-                        .setSize(0)
-                        .setTypes(IndexStatsResolver.TYPE)
-                        .setPreference("_primary")
-                        .setQuery(QueryBuilders.termQuery("index_stats.index", indices[i]))
-                        .get();
-                assertThat(count.getHits().getTotalHits(), greaterThan(0L));
-            }
-        });
-
-        logger.debug("--> searching for monitoring documents of type [{}]", IndexStatsResolver.TYPE);
-        SearchResponse response = client().prepareSearch().setTypes(IndexStatsResolver.TYPE).setPreference("_primary").get();
-        assertThat(response.getHits().getTotalHits(), greaterThan(0L));
-
-        logger.debug("--> checking that every document contains the expected fields");
-        for (SearchHit searchHit : response.getHits().getHits()) {
-            Map<String, Object> fields = searchHit.getSourceAsMap();
-
-            for (String filter : IndexStatsResolver.FILTERS) {
-                assertContains(filter, fields);
-            }
-        }
-
-        logger.debug("--> index stats successfully collected");
-    }
-}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/indices/IndicesStatsResolverTests.java

@@ -1,96 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.indices;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.indices.stats.CommonStats;
-import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
-import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponseTestUtils;
-import org.elasticsearch.action.admin.indices.stats.ShardStats;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RecoverySource;
-import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.UnassignedInfo;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.cache.query.QueryCacheStats;
-import org.elasticsearch.index.engine.SegmentsStats;
-import org.elasticsearch.index.fielddata.FieldDataStats;
-import org.elasticsearch.index.merge.MergeStats;
-import org.elasticsearch.index.refresh.RefreshStats;
-import org.elasticsearch.index.search.stats.SearchStats;
-import org.elasticsearch.index.shard.DocsStats;
-import org.elasticsearch.index.shard.IndexingStats;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.shard.ShardPath;
-import org.elasticsearch.index.store.StoreStats;
-import org.elasticsearch.xpack.monitoring.collector.indices.IndicesStatsMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolverTestCase;
-
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-
-import static java.util.Collections.emptyList;
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.emptySet;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
-
-public class IndicesStatsResolverTests extends MonitoringIndexNameResolverTestCase<IndicesStatsMonitoringDoc, IndicesStatsResolver> {
-
-    @Override
-    protected IndicesStatsMonitoringDoc newMonitoringDoc() {
-        IndicesStatsMonitoringDoc doc = new IndicesStatsMonitoringDoc(randomMonitoringId(),
-                randomAlphaOfLength(2), randomAlphaOfLength(5), 1437580442979L,
-                new DiscoveryNode("id", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
-                randomIndicesStats());
-        return doc;
-    }
-
-    public void testIndicesStatsResolver() throws Exception {
-        IndicesStatsMonitoringDoc doc = newMonitoringDoc();
-
-        IndicesStatsResolver resolver = newResolver();
-        assertThat(resolver.index(doc), equalTo(".monitoring-es-" + MonitoringTemplateUtils.TEMPLATE_VERSION + "-2015.07.22"));
-
-        assertSource(resolver.source(doc, XContentType.JSON),
-                Sets.newHashSet("cluster_uuid", "timestamp", "type", "source_node", "indices_stats"), XContentType.JSON);
-    }
-
-    /**
-     * @return a random {@link IndicesStatsResponse} object.
-     */
-    private IndicesStatsResponse randomIndicesStats() {
-        Index index = new Index("test", UUID.randomUUID().toString());
-
-        List<ShardStats> shardStats = new ArrayList<>();
-        for (int i=0; i < randomIntBetween(2, 5); i++) {
-            ShardId shardId = new ShardId(index, i);
-            Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve(String.valueOf(i));
-            ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE,
-                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
-            shardRouting = shardRouting.initialize("node-0", null, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
-            shardRouting = shardRouting.moveToStarted();
-            CommonStats stats = new CommonStats();
-            stats.fieldData = new FieldDataStats();
-            stats.queryCache = new QueryCacheStats();
-            stats.docs = new DocsStats();
-            stats.store = new StoreStats();
-            stats.indexing = new IndexingStats();
-            stats.search = new SearchStats();
-            stats.segments = new SegmentsStats();
-            stats.merge = new MergeStats();
-            stats.refresh = new RefreshStats();
-            shardStats.add(new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null));
-        }
-        return IndicesStatsResponseTestUtils.newIndicesStatsResponse(shardStats.toArray(new ShardStats[shardStats.size()]),
-                shardStats.size(), shardStats.size(), 0, emptyList());
-    }
-}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/indices/IndicesStatsTests.java

@@ -1,93 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.indices;
-
-import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.elasticsearch.xpack.monitoring.MonitoringSettings;
-import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
-import org.junit.After;
-
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static org.hamcrest.Matchers.greaterThan;
-
-@ClusterScope(scope = Scope.TEST, numClientNodes = 0)
-public class IndicesStatsTests extends MonitoringIntegTestCase {
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder()
-                .put(super.nodeSettings(nodeOrdinal))
-                .put(MonitoringSettings.INTERVAL.getKey(), "-1")
-                .put("xpack.monitoring.exporters.default_local.type", "local")
-                .build();
-    }
-
-    @After
-    public void cleanup() throws Exception {
-        disableMonitoringInterval();
-        wipeMonitoringIndices();
-    }
-
-    public void testIndicesStats() throws Exception {
-        logger.debug("--> creating some indices for future indices stats");
-        final int nbIndices = randomIntBetween(1, 5);
-        for (int i = 0; i < nbIndices; i++) {
-            createIndex("stat-" + i);
-        }
-
-        final long[] nbDocsPerIndex = new long[nbIndices];
-        for (int i = 0; i < nbIndices; i++) {
-            nbDocsPerIndex[i] = randomIntBetween(1, 50);
-            for (int j = 0; j < nbDocsPerIndex[i]; j++) {
-                client().prepareIndex("stat-" + i, "type1").setSource("num", i).get();
-            }
-        }
-
-        logger.debug("--> wait for indices stats collector to collect stats for all primaries shards");
-        assertBusy(() -> {
-            flush();
-            refresh();
-
-            for (int i = 0; i < nbIndices; i++) {
-                IndicesStatsResponse indicesStats = client().admin().indices().prepareStats().get();
-                assertThat(indicesStats.getPrimaries().getDocs().getCount(), greaterThan(0L));
-            }
-        });
-
-        updateMonitoringInterval(3L, TimeUnit.SECONDS);
-        waitForMonitoringIndices();
-
-        logger.debug("--> wait for indices stats collector to collect global stat");
-        awaitMonitoringDocsCountOnPrimary(greaterThan(0L), IndicesStatsResolver.TYPE);
-
-        logger.debug("--> searching for monitoring documents of type [{}]", IndicesStatsResolver.TYPE);
-        SearchResponse response =
-                client().prepareSearch()
-                        .setQuery(QueryBuilders.termQuery("type", IndicesStatsResolver.TYPE))
-                        .setPreference("_primary")
-                        .get();
-        assertThat(response.getHits().getTotalHits(), greaterThan(0L));
-
-        logger.debug("--> checking that every document contains the expected fields");
-        for (SearchHit searchHit : response.getHits().getHits()) {
-            Map<String, Object> fields = searchHit.getSourceAsMap();
-
-            for (String filter : IndicesStatsResolver.FILTERS) {
-                assertContains(filter, fields);
-            }
-        }
-
-        logger.debug("--> indices stats successfully collected");
-    }
-}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/ml/JobStatsResolverTests.java

@@ -1,56 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.ml;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.xpack.ml.action.GetJobsStatsAction.Response.JobStats;
-import org.elasticsearch.xpack.ml.job.config.JobState;
-import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
-import org.elasticsearch.xpack.monitoring.collector.ml.JobStatsMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolverTestCase;
-
-import static org.hamcrest.Matchers.startsWith;
-
-public class JobStatsResolverTests extends MonitoringIndexNameResolverTestCase<JobStatsMonitoringDoc, JobStatsResolver> {
-
-    @Override
-    protected JobStatsMonitoringDoc newMonitoringDoc() {
-        final JobStats jobStats =
-                new JobStats("fake-job1", new DataCounts("fake-job1"),
-                             null, JobState.OPENED, null, null, null);
-
-        return new JobStatsMonitoringDoc(randomAlphaOfLength(5),
-                                         Math.abs(randomLong()),
-                                         new DiscoveryNode("id", buildNewFakeTransportAddress(), Version.CURRENT),
-                                         jobStats);
-    }
-
-    @Override
-    protected boolean checkFilters() {
-        return false;
-    }
-
-    public void testClusterInfoResolver() throws Exception {
-        JobStatsMonitoringDoc doc = newMonitoringDoc();
-        JobStatsResolver resolver = newResolver();
-
-        assertThat(resolver.index(doc), startsWith(".monitoring-es-" + MonitoringTemplateUtils.TEMPLATE_VERSION));
-
-        assertSource(resolver.source(doc, XContentType.JSON),
-                     Sets.newHashSet(
-                         "cluster_uuid",
-                         "timestamp",
-                         "type",
-                         "source_node",
-                         "job_stats"),
-                     XContentType.JSON);
-    }
-
-}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/node/NodeStatsResolverTests.java

@@ -1,175 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.node;
-
-import org.apache.lucene.util.Constants;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
-import org.elasticsearch.action.admin.indices.stats.CommonStats;
-import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
-import org.elasticsearch.action.admin.indices.stats.ShardStats;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RecoverySource;
-import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.UnassignedInfo;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.cache.query.QueryCacheStats;
-import org.elasticsearch.index.cache.request.RequestCacheStats;
-import org.elasticsearch.index.engine.SegmentsStats;
-import org.elasticsearch.index.fielddata.FieldDataStats;
-import org.elasticsearch.index.search.stats.SearchStats;
-import org.elasticsearch.index.shard.DocsStats;
-import org.elasticsearch.index.shard.IndexingStats;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.shard.ShardPath;
-import org.elasticsearch.index.store.StoreStats;
-import org.elasticsearch.indices.NodeIndicesStats;
-import org.elasticsearch.monitor.fs.FsInfo;
-import org.elasticsearch.monitor.jvm.JvmStats;
-import org.elasticsearch.monitor.os.OsProbe;
-import org.elasticsearch.monitor.process.ProcessProbe;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.threadpool.ThreadPoolStats;
-import org.elasticsearch.xpack.monitoring.collector.node.NodeStatsMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolverTestCase;
-import org.elasticsearch.xpack.watcher.execution.InternalWatchExecutor;
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.emptySet;
-import static org.hamcrest.Matchers.equalTo;
-
-public class NodeStatsResolverTests extends MonitoringIndexNameResolverTestCase<NodeStatsMonitoringDoc, NodeStatsResolver> {
-
-    @Override
-    protected NodeStatsMonitoringDoc newMonitoringDoc() {
-        NodeStatsMonitoringDoc doc = new NodeStatsMonitoringDoc(randomMonitoringId(),
-                randomAlphaOfLength(2), randomAlphaOfLength(5),
-                new DiscoveryNode("id", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
-                randomBoolean(), randomNodeStats(), randomBoolean());
-        return doc;
-    }
-
-    @Override
-    protected void assertSourceField(String field, Map<String, Object> sourceFields) {
-        // Assertions on node stats fields that are not reported on Windows platforms
-        if (Constants.WINDOWS) {
-            if (field.startsWith("node_stats.os.cpu.load_average")) {
-                return;
-            }
-        }
-
-        // we only report IoStats on Linux
-        if (Constants.LINUX == false) {
-            if (field.startsWith("node_stats.fs.io_stats")) {
-                return;
-            }
-        }
-
-        // node_stats.fs.data.spins can be null and it's only reported on Linux
-        if (field.startsWith("node_stats.fs.data.spins")) {
-            return;
-        }
-
-        // cgroups can be null, and it's only reported on Linux
-        if (field.startsWith("node_stats.os.cgroup")) {
-            return;
-        }
-
-        // load average is unavailable on macOS for 5m and 15m (but we get 1m), but it's also possible on Linux too
-        if ("node_stats.os.cpu.load_average.5m".equals(field) || "node_stats.os.cpu.load_average.15m".equals(field)) {
-            return;
-        }
-
-        super.assertSourceField(field, sourceFields);
-    }
-
-    public void testNodeStatsResolver() throws IOException {
-        NodeStatsMonitoringDoc doc = newMonitoringDoc();
-
-        NodeStatsResolver resolver = newResolver();
-        assertThat(resolver.index(doc), equalTo(".monitoring-es-" + MonitoringTemplateUtils.TEMPLATE_VERSION + "-2015.07.22"));
-
-        assertSource(resolver.source(doc, XContentType.JSON),
-                Sets.newHashSet(
-                        "cluster_uuid",
-                        "timestamp",
-                        "type",
-                        "source_node",
-                        "node_stats"), XContentType.JSON);
-    }
-
-    /**
-     * @return a random {@link NodeStats} object.
-     */
-    private NodeStats randomNodeStats() {
-        Index index = new Index("test-" + randomIntBetween(0, 5), UUID.randomUUID().toString());
-        ShardId shardId = new ShardId(index, 0);
-        Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("0");
-        ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE,
-                new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
-        shardRouting = shardRouting.initialize("node-0", null, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
-        shardRouting = shardRouting.moveToStarted();
-        CommonStats stats = new CommonStats();
-        stats.fieldData = new FieldDataStats();
-        stats.queryCache = new QueryCacheStats();
-        stats.requestCache = new RequestCacheStats();
-        stats.docs = new DocsStats();
-        stats.store = new StoreStats();
-        stats.indexing = new IndexingStats();
-        stats.search = new SearchStats();
-        stats.segments = new SegmentsStats();
-        ShardStats shardStats = new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null);
-        FsInfo.Path[] pathInfo = new FsInfo.Path[]{
-                new FsInfo.Path("/test", "/dev/sda", 10, -8, 0),
-        };
-        Map<Index, List<IndexShardStats>> statsByShard = new HashMap<>();
-        statsByShard.put(index, Collections.singletonList(new IndexShardStats(shardId, new ShardStats[]{shardStats})));
-        List<ThreadPoolStats.Stats> threadPoolStats = Arrays.asList(
-                new ThreadPoolStats.Stats(ThreadPool.Names.BULK, 0, 0, 0, 0, 0, 0),
-                new ThreadPoolStats.Stats(ThreadPool.Names.GENERIC, 0, 0, 0, 0, 0, 0),
-                new ThreadPoolStats.Stats(ThreadPool.Names.GET, 0, 0, 0, 0, 0, 0),
-                new ThreadPoolStats.Stats(ThreadPool.Names.INDEX, 0, 0, 0, 0, 0, 0),
-                new ThreadPoolStats.Stats(ThreadPool.Names.MANAGEMENT, 0, 0, 0, 0, 0, 0),
-                new ThreadPoolStats.Stats(ThreadPool.Names.SEARCH, 0, 0, 0, 0, 0, 0),
-                new ThreadPoolStats.Stats(InternalWatchExecutor.THREAD_POOL_NAME, 0, 0, 0, 0, 0, 0)
-        );
-        return new NodeStats(new DiscoveryNode("node_0", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
-                1437580442979L,
-                new NodeIndicesStats(new CommonStats(), statsByShard), OsProbe.getInstance().osStats(),
-                ProcessProbe.getInstance().processStats(), JvmStats.jvmStats(),
-                new ThreadPoolStats(threadPoolStats),
-                new FsInfo(0, randomIoStats(), pathInfo), null, null, null, null, null, null);
-    }
-
-    @Nullable
-    private FsInfo.IoStats randomIoStats() {
-        if (Constants.LINUX) {
-            final int stats = randomIntBetween(1, 3);
-            final FsInfo.DeviceStats[] devices = new FsInfo.DeviceStats[stats];
-
-            for (int i = 0; i < devices.length; ++i) {
-                devices[i] = new FsInfo.DeviceStats(253, 0, "dm-" + i, 287734, 7185242, 8398869, 118857776, null);
-            }
-
-            return new FsInfo.IoStats(devices);
-        }
-
-        return null;
-    }
-}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/node/NodeStatsTests.java

@@ -1,118 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.node;
-
-import org.apache.lucene.util.Constants;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.elasticsearch.xpack.monitoring.MonitoringSettings;
-import org.elasticsearch.xpack.monitoring.exporter.local.LocalExporter;
-import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
-import org.junit.After;
-
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import static org.hamcrest.Matchers.greaterThan;
-
-// numClientNodes is set to 0 because Client nodes don't have Filesystem stats
-@ClusterScope(scope = Scope.TEST, numClientNodes = 0, transportClientRatio = 0.0)
-public class NodeStatsTests extends MonitoringIntegTestCase {
-
-    private static Boolean WATCHER_ENABLED = null;
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder()
-                .put(super.nodeSettings(nodeOrdinal))
-                .put(MonitoringSettings.INTERVAL.getKey(), "-1")
-                .put("xpack.monitoring.exporters.default_local.type", LocalExporter.TYPE)
-                .build();
-    }
-
-    @After
-    public void cleanup() throws Exception {
-        disableMonitoringInterval();
-        wipeMonitoringIndices();
-    }
-
-    public void testNodeStats() throws Exception {
-        final int numDocs = between(50, 150);
-        for (int i = 0; i < numDocs; i++) {
-            client().prepareIndex("test", "foo").setSource("value", randomInt()).get();
-        }
-
-        flush();
-        refresh();
-
-        updateMonitoringInterval(3L, TimeUnit.SECONDS);
-        waitForMonitoringIndices();
-
-        awaitMonitoringDocsCountOnPrimary(greaterThan(0L), NodeStatsResolver.TYPE);
-
-        SearchResponse response =
-                client().prepareSearch()
-                        .setQuery(QueryBuilders.termQuery("type", NodeStatsResolver.TYPE))
-                        .setPreference("_primary")
-                        .get();
-        assertThat(response.getHits().getTotalHits(), greaterThan(0L));
-
-        for (SearchHit searchHit : response.getHits().getHits()) {
-            Map<String, Object> fields = searchHit.getSourceAsMap();
-
-            for (String filter : nodeStatsFilters()) {
-                if (Constants.WINDOWS) {
-                    // load average is unavailable on Windows
-                    if (filter.startsWith("node_stats.os.cpu.load_average")) {
-                        continue;
-                    }
-                }
-
-                // fs and cgroup stats are only reported on Linux, but it's acceptable for _node/stats to report them as null if the OS is
-                //  misconfigured or not reporting them for some reason (e.g., older kernel)
-                if (filter.startsWith("node_stats.fs") || filter.startsWith("node_stats.os.cgroup")) {
-                    continue;
-                }
-
-                // load average is unavailable on macOS for 5m and 15m (but we get 1m), but it's also possible on Linux too
-                if ("node_stats.os.cpu.load_average.5m".equals(filter) || "node_stats.os.cpu.load_average.15m".equals(filter)) {
-                    continue;
-                }
-
-                assertContains(filter, fields);
-            }
-        }
-    }
-
-    /**
-     * Optionally exclude {@link NodeStatsResolver#FILTERS} that require Watcher to be enabled.
-     *
-     * @return Never {@code null} or empty.
-     */
-    private Set<String> nodeStatsFilters() {
-        if (enableWatcher()) {
-            return NodeStatsResolver.FILTERS;
-        }
-
-        return NodeStatsResolver.FILTERS.stream().filter(s -> s.contains("watcher") == false).collect(Collectors.toSet());
-    }
-
-    @Override
-    protected boolean enableWatcher() {
-        if (WATCHER_ENABLED == null) {
-            WATCHER_ENABLED = randomBoolean();
-        }
-        // currently this is the only Monitoring test that expects Watcher to be enabled.
-        // Once this becomes the default, then this should be removed.
-        return WATCHER_ENABLED;
-    }
-}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/shards/ShardsResolverTests.java

@@ -1,66 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.shards;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.ShardRoutingState;
-import org.elasticsearch.cluster.routing.TestShardRouting;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.xpack.monitoring.collector.shards.ShardMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolverTestCase;
-
-import java.util.UUID;
-
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.emptySet;
-import static org.hamcrest.Matchers.equalTo;
-
-public class ShardsResolverTests extends MonitoringIndexNameResolverTestCase<ShardMonitoringDoc, ShardsResolver> {
-
-    @Override
-    protected ShardMonitoringDoc newMonitoringDoc() {
-
-        ShardRouting shardRouting = TestShardRouting.newShardRouting(
-                new ShardId(new Index(randomAlphaOfLength(5), UUID.randomUUID().toString()), randomIntBetween(0, 5)),
-                null, randomBoolean(), ShardRoutingState.UNASSIGNED);
-        shardRouting = shardRouting.initialize("node-0", null, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
-        shardRouting = shardRouting.moveToStarted();
-        shardRouting = shardRouting.relocate("node-1", ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
-
-        ShardMonitoringDoc doc = new ShardMonitoringDoc(randomMonitoringId(),
-                randomAlphaOfLength(2), randomAlphaOfLength(5), 1437580442979L,
-                new DiscoveryNode("id", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT),
-                shardRouting,
-                UUID.randomUUID().toString());
-        return doc;
-    }
-
-    public void testShardsResolver() throws Exception {
-        ShardMonitoringDoc doc = newMonitoringDoc();
-
-        ShardsResolver resolver = newResolver();
-        assertThat(resolver.index(doc), equalTo(".monitoring-es-" + MonitoringTemplateUtils.TEMPLATE_VERSION + "-2015.07.22"));
-        assertSource(resolver.source(doc, XContentType.JSON),
-                Sets.newHashSet(
-                        "cluster_uuid",
-                        "timestamp",
-                        "type",
-                        "source_node",
-                        "state_uuid",
-                        "shard.state",
-                        "shard.primary",
-                        "shard.node",
-                        "shard.relocating_node",
-                        "shard.shard",
-                        "shard.index"), XContentType.JSON);
-    }
-}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/shards/ShardsTests.java

@@ -1,127 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.shards;
-
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
-import org.apache.lucene.util.LuceneTestCase.BadApple;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.aggregations.Aggregation;
-import org.elasticsearch.search.aggregations.AggregationBuilders;
-import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.elasticsearch.xpack.monitoring.MonitoringSettings;
-import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
-import org.junit.After;
-
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.instanceOf;
-
-@AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/498")
-//test is just too slow, please fix it to not be sleep-based
-@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007")
-@ClusterScope(scope = Scope.TEST)
-public class ShardsTests extends MonitoringIntegTestCase {
-
-    private static final String INDEX_PREFIX = "test-shards-";
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder()
-                .put(super.nodeSettings(nodeOrdinal))
-                .put(MonitoringSettings.INTERVAL.getKey(), "-1")
-                .put(MonitoringSettings.INDICES.getKey(), INDEX_PREFIX + "*")
-                .put("xpack.monitoring.exporters.default_local.type", "local")
-                .build();
-    }
-
-    @After
-    public void cleanup() throws Exception {
-        disableMonitoringInterval();
-        wipeMonitoringIndices();
-    }
-
-    public void testShards() throws Exception {
-        logger.debug("--> creating some indices so that shards collector reports data");
-        for (int i = 0; i < randomIntBetween(1, 5); i++) {
-            client().prepareIndex(INDEX_PREFIX + i, "foo").setRefreshPolicy(IMMEDIATE).setSource("field1", "value1").get();
-        }
-
-        flush();
-        refresh();
-
-        updateMonitoringInterval(1L, TimeUnit.SECONDS);
-        waitForMonitoringIndices();
-
-        awaitMonitoringDocsCountOnPrimary(greaterThan(0L), ShardsResolver.TYPE);
-
-        logger.debug("--> searching for monitoring documents of type [{}]", ShardsResolver.TYPE);
-        SearchResponse response = client().prepareSearch().setTypes(ShardsResolver.TYPE).setPreference("_primary").get();
-        assertThat(response.getHits().getTotalHits(), greaterThan(0L));
-
-        logger.debug("--> checking that every document contains the expected fields");
-        for (SearchHit searchHit : response.getHits().getHits()) {
-            Map<String, Object> fields = searchHit.getSourceAsMap();
-
-            for (String filter : ShardsResolver.FILTERS) {
-                assertContains(filter, fields);
-            }
-        }
-
-        logger.debug("--> shards successfully collected");
-    }
-
-    /**
-     * This test uses a terms aggregation to check that the "not_analyzed"
-     * fields of the "shards" document type are indeed not analyzed
-     */
-    public void testNotAnalyzedFields() throws Exception {
-        final String indexName = INDEX_PREFIX + randomInt();
-        assertAcked(prepareCreate(indexName)
-                .setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0));
-
-        updateMonitoringInterval(1L, TimeUnit.SECONDS);
-        waitForMonitoringIndices();
-
-        awaitMonitoringDocsCountOnPrimary(greaterThan(0L), ShardsResolver.TYPE);
-
-        SearchRequestBuilder searchRequestBuilder = client()
-                .prepareSearch()
-                .setTypes(ShardsResolver.TYPE)
-                .setPreference("_primary")
-                .setQuery(QueryBuilders.termQuery("shard.index", indexName));
-
-        String[] notAnalyzedFields = {"state_uuid", "shard.state", "shard.index", "shard.node"};
-        for (String field : notAnalyzedFields) {
-            searchRequestBuilder.addAggregation(AggregationBuilders.terms("agg_" + field.replace('.', '_')).field(field));
-        }
-
-        SearchResponse response = searchRequestBuilder.get();
-        assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(1L));
-
-        for (Aggregation aggregation : response.getAggregations()) {
-            assertThat(aggregation, instanceOf(StringTerms.class));
-            if (aggregation.getName().equals("agg_state_uuid")) {
-                // there is a chance that multiple documents are indexed and the cluster state uuid differs...
-                assertThat(((StringTerms) aggregation).getBuckets().size(), greaterThanOrEqualTo(1));
-            } else {
-                assertThat(((StringTerms) aggregation).getBuckets().size(), equalTo(1));
-            }
-        }
-    }
-}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java

@@ -20,8 +20,6 @@ import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.CountDown;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -35,15 +33,11 @@ import org.elasticsearch.xpack.XPackClient;
 import org.elasticsearch.xpack.XPackPlugin;
 import org.elasticsearch.xpack.XPackSettings;
 import org.elasticsearch.xpack.ml.MachineLearning;
-import org.elasticsearch.xpack.monitoring.MonitoredSystem;
 import org.elasticsearch.xpack.monitoring.MonitoringService;
 import org.elasticsearch.xpack.monitoring.MonitoringSettings;
 import org.elasticsearch.xpack.monitoring.client.MonitoringClient;
 import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
 import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
-import org.elasticsearch.xpack.monitoring.resolver.ResolversRegistry;
 import org.elasticsearch.xpack.security.Security;
 import org.elasticsearch.xpack.security.authc.file.FileRealm;
 import org.elasticsearch.xpack.security.authc.support.Hasher;
@@ -68,7 +62,6 @@ import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
@@ -79,7 +72,8 @@ import static org.hamcrest.Matchers.lessThan;
 
 public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
 
-    public static final String MONITORING_INDICES_PREFIX = MonitoringIndexNameResolver.PREFIX + MonitoringIndexNameResolver.DELIMITER;
+    protected static final String MONITORING_INDICES_PREFIX = ".monitoring-";
+    protected static final String ALL_MONITORING_INDICES = MONITORING_INDICES_PREFIX + "*";
 
     /**
      * Per test run this is enabled or disabled.
@@ -238,7 +232,7 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
         CountDown retries = new CountDown(3);
         assertBusy(() -> {
             try {
-                boolean exist = client().admin().indices().prepareExists(MONITORING_INDICES_PREFIX + "*")
+                boolean exist = client().admin().indices().prepareExists(ALL_MONITORING_INDICES)
                         .get().isExists();
                 if (exist) {
                     deleteMonitoringIndices();
@@ -253,7 +247,7 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
     }
 
     protected void deleteMonitoringIndices() {
-        assertAcked(client().admin().indices().prepareDelete(MONITORING_INDICES_PREFIX + "*"));
+        assertAcked(client().admin().indices().prepareDelete(ALL_MONITORING_INDICES));
     }
 
     protected void awaitMonitoringDocsCountOnPrimary(Matcher<Long> matcher, String... types) throws Exception {
@@ -265,8 +259,8 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
     }
 
     protected void assertMonitoringDocsCountOnPrimary(Matcher<Long> matcher, String... types) {
-        flushAndRefresh(MONITORING_INDICES_PREFIX + "*");
-        long count = client().prepareSearch(MONITORING_INDICES_PREFIX + "*").setSize(0)
+        flushAndRefresh(ALL_MONITORING_INDICES);
+        long count = client().prepareSearch(ALL_MONITORING_INDICES).setSize(0)
                 .setQuery(QueryBuilders.termsQuery("type", types))
                 .setPreference("_primary").get().getHits().getTotalHits();
         logger.trace("--> searched for [{}] documents on primary, found [{}]", Strings.arrayToCommaDelimitedString(types), count);
@@ -274,36 +268,15 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
     }
 
     protected List<Tuple<String, String>> monitoringTemplates() {
-        final List<Tuple<String, String>> templates = Arrays.stream(MonitoringTemplateUtils.TEMPLATE_IDS)
-                .map(id -> new Tuple<>(MonitoringTemplateUtils.templateName(id), MonitoringTemplateUtils.loadTemplate(id)))
-                .collect(Collectors.toList());
-
-        // TODO: remove this when we remove resolvers
-        templates.addAll(StreamSupport.stream(new ResolversRegistry(Settings.EMPTY).spliterator(), false)
-                .map((resolver) -> new Tuple<>(resolver.templateName(), resolver.template()))
-                .distinct()
-                .collect(Collectors.toList()));
-
-        return templates;
+        return Arrays.stream(MonitoringTemplateUtils.TEMPLATE_IDS)
+                    .map(id -> new Tuple<>(MonitoringTemplateUtils.templateName(id), MonitoringTemplateUtils.loadTemplate(id)))
+                    .collect(Collectors.toList());
     }
 
     protected List<String> monitoringTemplateNames() {
-        final List<String> templateNames = Arrays.stream(MonitoringTemplateUtils.TEMPLATE_IDS)
-                .map(MonitoringTemplateUtils::templateName)
-                .collect(Collectors.toList());
-
-        // TODO: remove this when we remove resolvers
-        final ResolversRegistry registry = new ResolversRegistry(Settings.EMPTY);
-
-        for (final MonitoringIndexNameResolver resolver : registry) {
-            final String templateName = resolver.templateName();
-
-            if (templateNames.contains(templateName) == false) {
-                templateNames.add(templateName);
-            }
-        }
-
-        return templateNames;
+        return Arrays.stream(MonitoringTemplateUtils.TEMPLATE_IDS)
+                    .map(MonitoringTemplateUtils::templateName)
+                    .collect(Collectors.toList());
     }
 
     private Tuple<String, String> monitoringPipeline(final String pipelineId) {
@@ -356,25 +329,21 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
     }
 
     protected void waitForMonitoringIndices() throws Exception {
-        awaitIndexExists(MONITORING_INDICES_PREFIX + "*");
+        awaitIndexExists(ALL_MONITORING_INDICES);
         assertBusy(this::ensureMonitoringIndicesYellow);
     }
 
-    protected void awaitIndexExists(final String index) throws Exception {
+    private void awaitIndexExists(final String index) throws Exception {
         assertBusy(() -> {
             assertIndicesExists(index);
         }, 30, TimeUnit.SECONDS);
     }
 
-    protected void assertIndicesExists(String... indices) {
+    private void assertIndicesExists(String... indices) {
         logger.trace("checking if index exists [{}]", Strings.arrayToCommaDelimitedString(indices));
         assertThat(client().admin().indices().prepareExists(indices).get().isExists(), is(true));
     }
 
-    protected void updateClusterSettings(Settings.Builder settings) {
-        updateClusterSettings(settings.build());
-    }
-
     protected void updateClusterSettings(Settings settings) {
         assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(settings));
     }
@@ -438,18 +407,6 @@ public abstract class MonitoringIntegTestCase extends ESIntegTestCase {
                 Settings.builder().put(MonitoringSettings.INTERVAL.getKey(), value, timeUnit)));
     }
 
-    protected class MockTimestampedIndexNameResolver extends MonitoringIndexNameResolver.Timestamped<MonitoringDoc> {
-
-        public MockTimestampedIndexNameResolver(MonitoredSystem system, Settings settings, String version) {
-            super(system, settings, version);
-        }
-
-        @Override
-        protected void buildXContent(MonitoringDoc document, XContentBuilder builder, ToXContent.Params params) throws IOException {
-            throw new UnsupportedOperationException("MockTimestampedIndexNameResolver does not support resolving building XContent");
-        }
-    }
-
     /** security related settings */
 
     public static class SecuritySettings {