Marvel: ignore IndexNotFoundException for _all index when Shield is enabled

When Shield and Marvel are installed together and no indices exist in the cluster yet, Shield returns an IndexNotFoundException. This commit ignores and logs at DEBUG level any IndexNotFoundException iff a) Shield is enabled and 2) marvel.agent.indices setting is empty.

Closes elastic/elasticsearch#887

Original commit: elastic/x-pack-elasticsearch@5d227d775d

marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexRecoveryCollector.java

@@ -9,15 +9,19 @@ import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.marvel.agent.collector.AbstractCollector;
 import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.MarvelLicensee;
+import org.elasticsearch.marvel.shield.MarvelShieldIntegration;
 import org.elasticsearch.marvel.shield.SecuredClient;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -36,7 +40,7 @@ public class IndexRecoveryCollector extends AbstractCollector<IndexRecoveryColle
     private final Client client;
 
     @Inject
-    public IndexRecoveryCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings,  MarvelLicensee marvelLicensee,
+    public IndexRecoveryCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, MarvelLicensee marvelLicensee,
                                   SecuredClient client) {
         super(settings, NAME, clusterService, marvelSettings, marvelLicensee);
         this.client = client;
@@ -50,15 +54,22 @@ public class IndexRecoveryCollector extends AbstractCollector<IndexRecoveryColle
     @Override
     protected Collection<MarvelDoc> doCollect() throws Exception {
         List<MarvelDoc> results = new ArrayList<>(1);
+        try {
+            RecoveryResponse recoveryResponse = client.admin().indices().prepareRecoveries()
+                    .setIndices(marvelSettings.indices())
+                    .setIndicesOptions(IndicesOptions.lenientExpandOpen())
+                    .setActiveOnly(marvelSettings.recoveryActiveOnly())
+                    .get(marvelSettings.recoveryTimeout());
 
-        RecoveryResponse recoveryResponse = client.admin().indices().prepareRecoveries()
+            if (recoveryResponse.hasRecoveries()) {
-                .setIndices(marvelSettings.indices())
+                results.add(new IndexRecoveryMarvelDoc(clusterUUID(), TYPE, System.currentTimeMillis(), recoveryResponse));
-                .setIndicesOptions(IndicesOptions.lenientExpandOpen())
+            }
-                .setActiveOnly(marvelSettings.recoveryActiveOnly())
+        } catch (IndexNotFoundException e) {
-                .get(marvelSettings.recoveryTimeout());
+            if (MarvelShieldIntegration.enabled(settings) && IndexNameExpressionResolver.isAllIndices(Arrays.asList(marvelSettings.indices()))) {
-
+                logger.debug("collector [{}] - unable to collect data for missing index [{}]", name(), e.getIndex());
-        if (recoveryResponse.hasRecoveries()) {
+            } else {
-            results.add(new IndexRecoveryMarvelDoc(clusterUUID(), TYPE, System.currentTimeMillis(), recoveryResponse));
+                throw e;
+            }
         }
         return Collections.unmodifiableCollection(results);
     }

marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollector.java

@@ -10,18 +10,18 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.marvel.agent.collector.AbstractCollector;
 import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.MarvelLicensee;
+import org.elasticsearch.marvel.shield.MarvelShieldIntegration;
 import org.elasticsearch.marvel.shield.SecuredClient;
 
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
 
 /**
  * Collector for indices statistics.
@@ -37,7 +37,7 @@ public class IndexStatsCollector extends AbstractCollector<IndexStatsCollector>
     private final Client client;
 
     @Inject
-    public IndexStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings,  MarvelLicensee marvelLicensee,
+    public IndexStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, MarvelLicensee marvelLicensee,
                                SecuredClient client) {
         super(settings, NAME, clusterService, marvelSettings, marvelLicensee);
         this.client = client;
@@ -51,17 +51,24 @@ public class IndexStatsCollector extends AbstractCollector<IndexStatsCollector>
     @Override
     protected Collection<MarvelDoc> doCollect() throws Exception {
         List<MarvelDoc> results = new ArrayList<>(1);
+        try {
+            IndicesStatsResponse indicesStats = client.admin().indices().prepareStats()
+                    .setRefresh(true)
+                    .setIndices(marvelSettings.indices())
+                    .setIndicesOptions(IndicesOptions.lenientExpandOpen())
+                    .get(marvelSettings.indexStatsTimeout());
 
-        IndicesStatsResponse indicesStats = client.admin().indices().prepareStats()
+            long timestamp = System.currentTimeMillis();
-                .setRefresh(true)
+            String clusterUUID = clusterUUID();
-                .setIndices(marvelSettings.indices())
+            for (IndexStats indexStats : indicesStats.getIndices().values()) {
-                .setIndicesOptions(IndicesOptions.lenientExpandOpen())
+                results.add(new IndexStatsMarvelDoc(clusterUUID, TYPE, timestamp, indexStats));
-                .get(marvelSettings.indexStatsTimeout());
+            }
-
+        } catch (IndexNotFoundException e) {
-        long timestamp = System.currentTimeMillis();
+            if (MarvelShieldIntegration.enabled(settings) && IndexNameExpressionResolver.isAllIndices(Arrays.asList(marvelSettings.indices()))) {
-        String clusterUUID = clusterUUID();
+                logger.debug("collector [{}] - unable to collect data for missing index [{}]", name(), e.getIndex());
-        for (IndexStats indexStats : indicesStats.getIndices().values()) {
+            } else {
-            results.add(new IndexStatsMarvelDoc(clusterUUID, TYPE, timestamp, indexStats));
+                throw e;
+            }
         }
         return Collections.unmodifiableCollection(results);
     }

marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsCollector.java

@@ -6,16 +6,21 @@
 package org.elasticsearch.marvel.agent.collector.indices;
 
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.marvel.agent.collector.AbstractCollector;
 import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.MarvelLicensee;
+import org.elasticsearch.marvel.shield.MarvelShieldIntegration;
 import org.elasticsearch.marvel.shield.SecuredClient;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 
@@ -45,11 +50,20 @@ public class IndicesStatsCollector extends AbstractCollector<IndicesStatsCollect
 
     @Override
     protected Collection<MarvelDoc> doCollect() throws Exception {
-        IndicesStatsResponse indicesStats = client.admin().indices().prepareStats()
+        try {
-                .setRefresh(true)
+            IndicesStatsResponse indicesStats = client.admin().indices().prepareStats()
-                .get(marvelSettings.indicesStatsTimeout());
+                    .setIndices(marvelSettings.indices())
+                    .setIndicesOptions(IndicesOptions.lenientExpandOpen())
+                    .setRefresh(true)
+                    .get(marvelSettings.indicesStatsTimeout());
 
-        MarvelDoc result = new IndicesStatsMarvelDoc(clusterUUID(), TYPE, System.currentTimeMillis(), indicesStats);
+            return Collections.singletonList(new IndicesStatsMarvelDoc(clusterUUID(), TYPE, System.currentTimeMillis(), indicesStats));
-        return Collections.singletonList(result);
+        } catch (IndexNotFoundException e) {
+            if (MarvelShieldIntegration.enabled(settings) && IndexNameExpressionResolver.isAllIndices(Arrays.asList(marvelSettings.indices()))) {
+                logger.debug("collector [{}] - unable to collect data for missing index [{}]", name(), e.getIndex());
+                return Collections.emptyList();
+            }
+            throw e;
+        }
     }
 }

marvel/src/main/java/org/elasticsearch/marvel/agent/collector/shards/ShardsCollector.java

@@ -7,18 +7,19 @@ package org.elasticsearch.marvel.agent.collector.shards;
 
 import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.marvel.agent.collector.AbstractCollector;
 import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.MarvelLicensee;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -72,7 +73,7 @@ public class ShardsCollector extends AbstractCollector<ShardsCollector> {
 
     private boolean match(String indexName) {
         String[] indices = marvelSettings.indices();
-        return CollectionUtils.isEmpty(indices) || Regex.simpleMatch(indices, indexName);
+        return IndexNameExpressionResolver.isAllIndices(Arrays.asList(marvelSettings.indices())) || Regex.simpleMatch(indices, indexName);
     }
 
     /**

marvel/src/test/java/org/elasticsearch/marvel/agent/collector/AbstractCollectorTestCase.java

@@ -93,7 +93,7 @@ public class AbstractCollectorTestCase extends MarvelIntegTestCase {
                 .issuer("test")
                 .maxNodes(Integer.MAX_VALUE)
                 .signature("_signature")
-                .type("basic")
+                .type("trial")
                 .uid(String.valueOf(RandomizedTest.systemPropertyAsInt(SysGlobals.CHILDVM_SYSPROP_JVM_ID, 0)) + System.identityHashCode(AbstractCollectorTestCase.class))
                 .build();
     }

marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexRecoveryCollectorTests.java

@@ -7,8 +7,10 @@ package org.elasticsearch.marvel.agent.collector.indices;
 
 import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
 import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.marvel.agent.collector.AbstractCollectorTestCase;
 import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
@@ -25,16 +27,11 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF
 import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.*;
-import static org.hamcrest.Matchers.empty;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.is;
 
 @ClusterScope(numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
 public class IndexRecoveryCollectorTests extends AbstractCollectorTestCase {
+
     private final boolean activeOnly = false;
     private final String indexName = "test";
 
@@ -118,10 +115,12 @@ public class IndexRecoveryCollectorTests extends AbstractCollectorTestCase {
         }
     }
 
-    public void testIndexRecoveryCollectorWithLicensing() {
+    public void testIndexRecoveryCollectorWithLicensing() throws Exception {
+        List<String> nodesIds = internalCluster().startNodesAsync(randomIntBetween(2, 5)).get();
+        waitForNoBlocksOnNodes();
+
         try {
-            String[] nodes = internalCluster().getNodeNames();
+            for (String node : nodesIds) {
-            for (String node : nodes) {
                 logger.debug("--> creating a new instance of the collector");
                 IndexRecoveryCollector collector = newIndexRecoveryCollector(node);
                 assertNotNull(collector);
@@ -156,6 +155,39 @@ public class IndexRecoveryCollectorTests extends AbstractCollectorTestCase {
         }
     }
 
+    public void testEmptyCluster() throws Exception {
+        final String node = internalCluster().startNode(settingsBuilder().put(MarvelSettings.INDICES, Strings.EMPTY_ARRAY));
+        waitForNoBlocksOnNode(node);
+
+        try {
+            assertThat(newIndexRecoveryCollector(node).doCollect(), hasSize(0));
+        } catch (IndexNotFoundException e) {
+            fail("IndexNotFoundException has been thrown but it should have been swallowed by the collector");
+        }
+    }
+
+    public void testEmptyClusterAllIndices() throws Exception {
+        final String node = internalCluster().startNode(settingsBuilder().put(MarvelSettings.INDICES, MetaData.ALL));
+        waitForNoBlocksOnNode(node);
+
+        try {
+            assertThat(newIndexRecoveryCollector(node).doCollect(), hasSize(0));
+        } catch (IndexNotFoundException e) {
+            fail("IndexNotFoundException has been thrown but it should have been swallowed by the collector");
+        }
+    }
+
+    public void testEmptyClusterMissingIndex() throws Exception {
+        final String node = internalCluster().startNode(settingsBuilder().put(MarvelSettings.INDICES, "unknown"));
+        waitForNoBlocksOnNode(node);
+
+        try {
+            assertThat(newIndexRecoveryCollector(node).doCollect(), hasSize(0));
+        } catch (IndexNotFoundException e) {
+            fail("IndexNotFoundException has been thrown but it should have been swallowed by the collector");
+        }
+    }
+
     private IndexRecoveryCollector newIndexRecoveryCollector(String nodeId) {
         if (!Strings.hasText(nodeId)) {
             nodeId = randomFrom(internalCluster().getNodeNames());

marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollectorTests.java

@@ -7,39 +7,71 @@ package org.elasticsearch.marvel.agent.collector.indices;
 
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
 import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.marvel.agent.collector.AbstractCollectorTestCase;
 import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.MarvelLicensee;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.junit.Before;
 
 import java.util.Collection;
 import java.util.Iterator;
+import java.util.List;
 
+import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.*;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.is;
 
-@ClusterScope(numClientNodes = 0)
+@ClusterScope(numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
 public class IndexStatsCollectorTests extends AbstractCollectorTestCase {
+
     @Override
     protected int numberOfReplicas() {
         return 0;
     }
 
-    @Before
+    public void testEmptyCluster() throws Exception {
-    public void beforeIndexStatsCollectorTests() throws Exception {
+        final String node = internalCluster().startNode();
-        waitForNoBlocksOnNodes();
+        waitForNoBlocksOnNode(node);
+
+        try {
+            assertThat(newIndexStatsCollector(node).doCollect(), hasSize(0));
+        } catch (IndexNotFoundException e) {
+            fail("IndexNotFoundException has been thrown but it should have been swallowed by the collector");
+        }
+    }
+
+    public void testEmptyClusterAllIndices() throws Exception {
+        final String node = internalCluster().startNode(settingsBuilder().put(MarvelSettings.INDICES, MetaData.ALL));
+        waitForNoBlocksOnNode(node);
+
+        try {
+            assertThat(newIndexStatsCollector(node).doCollect(), hasSize(0));
+        } catch (IndexNotFoundException e) {
+            fail("IndexNotFoundException has been thrown but it should have been swallowed by the collector");
+        }
+    }
+
+    public void testEmptyClusterMissingIndex() throws Exception {
+        final String node = internalCluster().startNode(settingsBuilder().put(MarvelSettings.INDICES, "unknown"));
+        waitForNoBlocksOnNode(node);
+
+        try {
+            assertThat(newIndexStatsCollector(node).doCollect(), hasSize(0));
+        } catch (IndexNotFoundException e) {
+            fail("IndexNotFoundException has been thrown but it should have been swallowed by the collector");
+        }
     }
 
     public void testIndexStatsCollectorOneIndex() throws Exception {
+        final String node = internalCluster().startNode();
+        waitForNoBlocksOnNode(node);
+
         final String indexName = "one-index";
+        createIndex(indexName);
+        securedEnsureGreen(indexName);
 
         final int nbDocs = randomIntBetween(1, 20);
         for (int i = 0; i < nbDocs; i++) {
@@ -48,7 +80,6 @@ public class IndexStatsCollectorTests extends AbstractCollectorTestCase {
 
         securedFlush();
         securedRefresh();
-        securedEnsureGreen(indexName);
 
         assertHitCount(client().prepareSearch().setSize(0).get(), nbDocs);
 
@@ -77,20 +108,26 @@ public class IndexStatsCollectorTests extends AbstractCollectorTestCase {
     }
 
     public void testIndexStatsCollectorMultipleIndices() throws Exception {
+        final String node = internalCluster().startNode();
+        waitForNoBlocksOnNode(node);
+
         final String indexPrefix = "multi-indices-";
         final int nbIndices = randomIntBetween(1, 5);
         int[] docsPerIndex = new int[nbIndices];
 
         for (int i = 0; i < nbIndices; i++) {
+            String index = indexPrefix + i;
+            createIndex(index);
+            securedEnsureGreen(index);
+
             docsPerIndex[i] = randomIntBetween(1, 20);
             for (int j = 0; j < docsPerIndex[i]; j++) {
-                client().prepareIndex(indexPrefix + i, "test").setSource("num", i).get();
+                client().prepareIndex(index, "test").setSource("num", i).get();
             }
         }
 
         securedFlush();
         securedRefresh();
-        securedEnsureGreen(indexPrefix + "*");
 
         for (int i = 0; i < nbIndices; i++) {
             assertHitCount(client().prepareSearch(indexPrefix + i).setSize(0).get(), docsPerIndex[i]);
@@ -134,7 +171,10 @@ public class IndexStatsCollectorTests extends AbstractCollectorTestCase {
         }
     }
 
-    public void testIndexStatsCollectorWithLicensing() {
+    public void testIndexStatsCollectorWithLicensing() throws Exception {
+        List<String> nodesIds = internalCluster().startNodesAsync(randomIntBetween(2, 5)).get();
+        waitForNoBlocksOnNodes();
+
         try {
             final int nbDocs = randomIntBetween(1, 20);
             for (int i = 0; i < nbDocs; i++) {
@@ -145,8 +185,7 @@ public class IndexStatsCollectorTests extends AbstractCollectorTestCase {
             securedRefresh();
             securedEnsureGreen("test");
 
-            String[] nodes = internalCluster().getNodeNames();
+            for (String node : nodesIds) {
-            for (String node : nodes) {
                 logger.debug("--> creating a new instance of the collector");
                 IndexStatsCollector collector = newIndexStatsCollector(node);
                 assertNotNull(collector);

marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsCollectorTests.java

@@ -0,0 +1,205 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.marvel.agent.collector.indices;
+
+import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.marvel.agent.collector.AbstractCollectorTestCase;
+import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
+import org.elasticsearch.marvel.agent.settings.MarvelSettings;
+import org.elasticsearch.marvel.license.MarvelLicensee;
+import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+
+import java.util.Collection;
+import java.util.List;
+
+import static org.elasticsearch.common.settings.Settings.settingsBuilder;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.*;
+
+
+@ClusterScope(numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
+public class IndicesStatsCollectorTests extends AbstractCollectorTestCase {
+
+    @Override
+    protected int numberOfReplicas() {
+        return 0;
+    }
+
+    public void testEmptyCluster() throws Exception {
+        final String node = internalCluster().startNode();
+        waitForNoBlocksOnNode(node);
+
+        try {
+            assertThat(newIndicesStatsCollector(node).doCollect(), hasSize(shieldEnabled ? 0 : 1));
+        } catch (IndexNotFoundException e) {
+            fail("IndexNotFoundException has been thrown but it should have been swallowed by the collector");
+        }
+    }
+
+    public void testEmptyClusterAllIndices() throws Exception {
+        final String node = internalCluster().startNode(settingsBuilder().put(MarvelSettings.INDICES, MetaData.ALL));
+        waitForNoBlocksOnNode(node);
+
+        try {
+            assertThat(newIndicesStatsCollector(node).doCollect(), hasSize(shieldEnabled ? 0 : 1));
+        } catch (IndexNotFoundException e) {
+            fail("IndexNotFoundException has been thrown but it should have been swallowed by the collector");
+        }
+    }
+
+    public void testEmptyClusterMissingIndex() throws Exception {
+        final String node = internalCluster().startNode(settingsBuilder().put(MarvelSettings.INDICES, "unknown"));
+        waitForNoBlocksOnNode(node);
+
+        try {
+            assertThat(newIndicesStatsCollector(node).doCollect(), hasSize(1));
+        } catch (IndexNotFoundException e) {
+            fail("IndexNotFoundException has been thrown but it should have been swallowed by the collector");
+        }
+    }
+
+    public void testIndicesStatsCollectorOneIndex() throws Exception {
+        final String node = internalCluster().startNode();
+        waitForNoBlocksOnNode(node);
+
+        final String indexName = "one-index";
+        createIndex(indexName);
+        securedEnsureGreen(indexName);
+
+        final int nbDocs = randomIntBetween(1, 20);
+        for (int i = 0; i < nbDocs; i++) {
+            client().prepareIndex(indexName, "test").setSource("num", i).get();
+        }
+
+        securedFlush();
+        securedRefresh();
+
+        assertHitCount(client().prepareSearch().setSize(0).get(), nbDocs);
+
+        Collection<MarvelDoc> results = newIndicesStatsCollector().doCollect();
+        assertThat(results, hasSize(1));
+
+        MarvelDoc marvelDoc = results.iterator().next();
+        assertThat(marvelDoc, instanceOf(IndicesStatsMarvelDoc.class));
+
+        IndicesStatsMarvelDoc indicesStatsMarvelDoc = (IndicesStatsMarvelDoc) marvelDoc;
+        IndicesStatsResponse indicesStats = indicesStatsMarvelDoc.getIndicesStats();
+        assertNotNull(indicesStats);
+        assertThat(indicesStats.getIndices().keySet(), hasSize(1));
+
+        IndexStats indexStats = indicesStats.getIndex(indexName);
+        assertThat(indexStats.getShards(), arrayWithSize(getNumShards(indexName).totalNumShards));
+    }
+
+    public void testIndicesStatsCollectorMultipleIndices() throws Exception {
+        final String node = internalCluster().startNode();
+        waitForNoBlocksOnNode(node);
+
+        final String indexPrefix = "multi-indices-";
+        final int nbIndices = randomIntBetween(1, 5);
+        int[] docsPerIndex = new int[nbIndices];
+
+        for (int i = 0; i < nbIndices; i++) {
+            String index = indexPrefix + i;
+            createIndex(index);
+            securedEnsureGreen(index);
+
+            docsPerIndex[i] = randomIntBetween(1, 20);
+            for (int j = 0; j < docsPerIndex[i]; j++) {
+                client().prepareIndex(index, "test").setSource("num", i).get();
+            }
+        }
+
+        securedFlush();
+        securedRefresh();
+
+        for (int i = 0; i < nbIndices; i++) {
+            assertHitCount(client().prepareSearch(indexPrefix + i).setSize(0).get(), docsPerIndex[i]);
+        }
+
+        Collection<MarvelDoc> results = newIndicesStatsCollector().doCollect();
+        assertThat(results, hasSize(1));
+
+        MarvelDoc marvelDoc = results.iterator().next();
+        assertThat(marvelDoc, instanceOf(IndicesStatsMarvelDoc.class));
+
+        IndicesStatsMarvelDoc indicesStatsMarvelDoc = (IndicesStatsMarvelDoc) marvelDoc;
+        IndicesStatsResponse indicesStats = indicesStatsMarvelDoc.getIndicesStats();
+        assertNotNull(indicesStats);
+        assertThat(indicesStats.getIndices().keySet(), hasSize(nbIndices));
+    }
+
+    public void testIndicesStatsCollectorWithLicensing() throws Exception {
+        List<String> nodesIds = internalCluster().startNodesAsync(randomIntBetween(2, 5)).get();
+        waitForNoBlocksOnNodes();
+
+        try {
+            final int nbDocs = randomIntBetween(1, 20);
+            for (int i = 0; i < nbDocs; i++) {
+                client().prepareIndex("test", "test").setSource("num", i).get();
+            }
+
+            securedFlush();
+            securedRefresh();
+            securedEnsureGreen("test");
+
+            for (String node : nodesIds) {
+                logger.debug("--> creating a new instance of the collector");
+                IndicesStatsCollector collector = newIndicesStatsCollector(node);
+                assertNotNull(collector);
+
+                logger.debug("--> enabling license and checks that the collector can collect data if node is master");
+                enableLicense();
+                if (node.equals(internalCluster().getMasterName())) {
+                    assertCanCollect(collector);
+                } else {
+                    assertCannotCollect(collector);
+                }
+
+                logger.debug("--> starting graceful period and checks that the collector can still collect data if node is master");
+                beginGracefulPeriod();
+                if (node.equals(internalCluster().getMasterName())) {
+                    assertCanCollect(collector);
+                } else {
+                    assertCannotCollect(collector);
+                }
+
+                logger.debug("--> ending graceful period and checks that the collector cannot collect data");
+                endGracefulPeriod();
+                assertCannotCollect(collector);
+
+                logger.debug("--> disabling license and checks that the collector cannot collect data");
+                disableLicense();
+                assertCannotCollect(collector);
+            }
+        } finally {
+            // Ensure license is enabled before finishing the test
+            enableLicense();
+        }
+    }
+
+    private IndicesStatsCollector newIndicesStatsCollector() {
+        // This collector runs on master node only
+        return newIndicesStatsCollector(internalCluster().getMasterName());
+    }
+
+    private IndicesStatsCollector newIndicesStatsCollector(String nodeId) {
+        if (!Strings.hasText(nodeId)) {
+            nodeId = randomFrom(internalCluster().getNodeNames());
+        }
+        return new IndicesStatsCollector(internalCluster().getInstance(Settings.class, nodeId),
+                internalCluster().getInstance(ClusterService.class, nodeId),
+                internalCluster().getInstance(MarvelSettings.class, nodeId),
+                internalCluster().getInstance(MarvelLicensee.class, nodeId),
+                securedClient(nodeId));
+    }
+}

marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java

@@ -51,10 +51,13 @@ import static org.hamcrest.Matchers.*;
  */
 public abstract class MarvelIntegTestCase extends ESIntegTestCase {
 
-    protected Boolean shieldEnabled = enableShield();
+    protected static Boolean shieldEnabled;
 
     @Override
     protected TestCluster buildTestCluster(Scope scope, long seed) throws IOException {
+        if (shieldEnabled == null) {
+            shieldEnabled = enableShield();
+        }
         logger.info("--> shield {}", shieldEnabled ? "enabled" : "disabled");
         return super.buildTestCluster(scope, seed);
     }
@@ -105,7 +108,9 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
      * Override and returns {@code false} to force running without shield
      */
     protected boolean enableShield() {
-        return randomBoolean();
+        boolean r = randomBoolean();
+        logger.info("--> shield is{}", r);
+        return r;
     }
 
     protected void stopCollection() {