Marvel: Fix integration tests in org.elasticsearch.marvel.agent.renderer package

Closes elastic/elasticsearch#470

Original commit: elastic/x-pack-elasticsearch@db07ac416d

marvel/src/main/java/org/elasticsearch/marvel/agent/AgentService.java

@@ -82,7 +82,6 @@ public class AgentService extends AbstractLifecycleComponent<AgentService> imple
             workerThread = new Thread(exportingWorker, EsExecutors.threadName(settings, "marvel.exporters"));
             workerThread.setDaemon(true);
             workerThread.start();
-
         }
     }
 
@@ -154,13 +153,10 @@ public class AgentService extends AbstractLifecycleComponent<AgentService> imple
 
         @Override
         public void run() {
-            boolean firstRun = true;
-
             while (!closed) {
                 // sleep first to allow node to complete initialization before collecting the first start
                 try {
-                    long interval = (firstRun && (marvelSettings.startUpDelay() != null)) ? marvelSettings.startUpDelay().millis() : samplingInterval;
+                    Thread.sleep(samplingInterval);
-                    Thread.sleep(interval);
 
                     if (closed) {
                         continue;
@@ -198,8 +194,6 @@ public class AgentService extends AbstractLifecycleComponent<AgentService> imple
                     Thread.currentThread().interrupt();
                 } catch (Throwable t) {
                     logger.error("background thread had an uncaught exception", t);
-                } finally {
-                    firstRun = false;
                 }
             }
             logger.debug("worker shutdown");

marvel/src/main/java/org/elasticsearch/marvel/agent/settings/MarvelSettings.java

@@ -30,7 +30,6 @@ public class MarvelSettings extends AbstractComponent implements NodeSettingsSer
     public static final TimeValue MAX_LICENSE_GRACE_PERIOD = TimeValue.timeValueHours(7 * 24);
 
     public static final String INTERVAL                     = PREFIX + "interval";
-    public static final String STARTUP_DELAY                = PREFIX + "startup.delay";
     public static final String INDEX_STATS_TIMEOUT          = PREFIX + "index.stats.timeout";
     public static final String INDICES_STATS_TIMEOUT        = PREFIX + "indices.stats.timeout";
     public static final String INDICES                      = PREFIX + "indices";
@@ -60,8 +59,6 @@ public class MarvelSettings extends AbstractComponent implements NodeSettingsSer
         Map<String, MarvelSetting> map = new HashMap<>();
         map.put(INTERVAL, timeSetting(INTERVAL, TimeValue.timeValueSeconds(10),
                 "Sampling interval between two collections (default to 10s)"));
-        map.put(STARTUP_DELAY, timeSetting(STARTUP_DELAY, null,
-                "Waiting time before the agent start to collect data (default to sampling interval)"));
         map.put(INDEX_STATS_TIMEOUT, timeoutSetting(INDEX_STATS_TIMEOUT, TimeValue.timeValueMinutes(10),
                 "Timeout value when collecting index statistics (default to 10m)"));
         map.put(INDICES_STATS_TIMEOUT, timeoutSetting(INDICES_STATS_TIMEOUT, TimeValue.timeValueMinutes(10),
@@ -146,10 +143,6 @@ public class MarvelSettings extends AbstractComponent implements NodeSettingsSer
         return getSettingValue(INTERVAL);
     }
 
-    public TimeValue startUpDelay() {
-        return getSettingValue(STARTUP_DELAY);
-    }
-
     public TimeValue indexStatsTimeout() {
         return getSettingValue(INDEX_STATS_TIMEOUT);
     }

marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollectorTests.java

@@ -5,42 +5,38 @@
  */
 package org.elasticsearch.marvel.agent.collector.indices;
 
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
-import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.cluster.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.marvel.agent.collector.AbstractCollectorTestCase;
 import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.MarvelLicensee;
-import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.junit.Before;
 import org.junit.Test;
 
 import java.util.Collection;
 import java.util.Iterator;
-import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.*;
 
-@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/470")
+@ClusterScope(numClientNodes = 0)
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, randomDynamicTemplates = false, transportClientRatio = 0.0, numDataNodes = 1, numClientNodes = 0)
 public class IndexStatsCollectorTests extends AbstractCollectorTestCase {
 
-    @Test
+    @Override
-    public void testIndexStatsCollectorNoIndices() throws Exception {
+    protected int numberOfReplicas() {
-        waitForNoBlocksOnNodes();
+        return 0;
+    }
 
-        Collection<MarvelDoc> results = newIndexStatsCollector().doCollect();
+    @Before
-        assertThat(results, is(empty()));
+    public void beforeIndexStatsCollectorTests() throws Exception {
+        waitForNoBlocksOnNodes();
     }
 
     @Test
     public void testIndexStatsCollectorOneIndex() throws Exception {
-        waitForNoBlocksOnNodes();
-
         final String indexName = "one-index";
 
         final int nbDocs = randomIntBetween(1, 20);
@@ -48,30 +44,14 @@ public class IndexStatsCollectorTests extends AbstractCollectorTestCase {
             client().prepareIndex(indexName, "test").setSource("num", i).get();
         }
 
-        waitForRelocation();
+        securedFlush();
-        ensureGreen(indexName);
+        securedRefresh();
-        refresh();
+        securedEnsureGreen(indexName);
 
         assertHitCount(client().prepareCount().get(), nbDocs);
 
-        logger.debug("--> wait for index stats to report data about indices");
+        Collection<MarvelDoc> results = newIndexStatsCollector().doCollect();
-        assertBusy(new Runnable() {
+        assertThat(results, hasSize(1));
-            @Override
-            public void run() {
-                IndicesStatsResponse response = client(internalCluster().getMasterName()).admin().indices().prepareStats().setRefresh(true).get();
-                assertNotNull(response.getIndices());
-                assertThat(response.getIndices().size(), greaterThan(0));
-            }
-        }, 30L, TimeUnit.SECONDS);
-
-        Collection<MarvelDoc> results = assertBusy(new Callable<Collection<MarvelDoc>>() {
-            @Override
-            public Collection<MarvelDoc> call() throws Exception {
-                Collection<MarvelDoc> results = newIndexStatsCollector().doCollect();
-                assertThat(results, hasSize(1));
-                return results;
-            }
-        }, 30L, TimeUnit.SECONDS);
 
         MarvelDoc marvelDoc = results.iterator().next();
         assertNotNull(marvelDoc);
@@ -96,8 +76,6 @@ public class IndexStatsCollectorTests extends AbstractCollectorTestCase {
 
     @Test
     public void testIndexStatsCollectorMultipleIndices() throws Exception {
-        waitForNoBlocksOnNodes();
-
         final String indexPrefix = "multi-indices-";
         final int nbIndices = randomIntBetween(1, 5);
         int[] docsPerIndex = new int[nbIndices];
@@ -109,34 +87,18 @@ public class IndexStatsCollectorTests extends AbstractCollectorTestCase {
             }
         }
 
-        String clusterUUID = client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID();
+        securedFlush();
-        client().admin().indices().prepareRefresh().get();
+        securedRefresh();
+        securedEnsureGreen(indexPrefix + "*");
+
         for (int i = 0; i < nbIndices; i++) {
             assertHitCount(client().prepareCount(indexPrefix + i).get(), docsPerIndex[i]);
         }
 
-        waitForRelocation();
+        String clusterUUID = client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID();
-        ensureGreen();
-        refresh();
 
-        logger.debug("--> wait for index stats to report data about indices");
+        Collection<MarvelDoc> results = newIndexStatsCollector().doCollect();
-        assertBusy(new Runnable() {
+        assertThat(results, hasSize(nbIndices));
-            @Override
-            public void run() {
-                IndicesStatsResponse response = client().admin().indices().prepareStats().setRefresh(true).get();
-                assertNotNull(response.getIndices());
-                assertThat(response.getIndices().size(), greaterThan(0));
-            }
-        }, 30L, TimeUnit.SECONDS);
-
-        Collection<MarvelDoc> results = assertBusy(new Callable<Collection<MarvelDoc>>() {
-            @Override
-            public Collection<MarvelDoc> call() throws Exception {
-                Collection<MarvelDoc> results = newIndexStatsCollector().doCollect();
-                assertThat(results, hasSize(nbIndices));
-                return results;
-            }
-        }, 30L, TimeUnit.SECONDS);
 
         for (int i = 0; i < nbIndices; i++) {
             String indexName = indexPrefix + i;

marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterInfoTests.java

@@ -14,47 +14,59 @@ import org.elasticsearch.license.core.License;
 import org.elasticsearch.marvel.agent.collector.cluster.ClusterInfoCollector;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.test.MarvelIntegTestCase;
+import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
 import java.util.Map;
-import java.util.concurrent.Callable;
 import java.util.concurrent.TimeUnit;
 
+import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.*;
 
-public class ClusterInfoIT extends MarvelIntegTestCase {
+@ClusterScope(scope = TEST)
+public class ClusterInfoTests extends MarvelIntegTestCase {
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put(MarvelSettings.INTERVAL, "3s")
+                .put(MarvelSettings.INTERVAL, "-1")
                 .put(MarvelSettings.COLLECTORS, ClusterInfoCollector.NAME)
                 .build();
     }
 
+    @Before
+    public void init() throws Exception {
+        updateMarvelInterval(3L, TimeUnit.SECONDS);
+    }
+
+    @After
+    public void cleanup() throws Exception {
+        updateMarvelInterval(-1, TimeUnit.SECONDS);
+        wipeMarvelIndices();
+    }
+
     @Test
     public void testClusterInfo() throws Exception {
-        ensureGreen();
+        securedEnsureGreen();
 
         final String clusterUUID = client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID();
         assertTrue(Strings.hasText(clusterUUID));
 
-        logger.debug("--> waiting for cluster info collector to collect data (ie, the trial marvel license)");
+        logger.debug("--> waiting for the marvel data index to be created (it should have been created by the LicenseCollector)");
-        GetResponse response = assertBusy(new Callable<GetResponse>() {
+        awaitIndexExists(MarvelSettings.MARVEL_DATA_INDEX_NAME);
-            @Override
-            public GetResponse call() throws Exception {
-                // Checks if the marvel data index exists (it should have been created by the LicenseCollector)
-                assertTrue(MarvelSettings.MARVEL_DATA_INDEX_NAME + " index does not exist", client().admin().indices().prepareExists(MarvelSettings.MARVEL_DATA_INDEX_NAME).get().isExists());
-                ensureYellow(MarvelSettings.MARVEL_DATA_INDEX_NAME);
 
-                client().admin().indices().prepareRefresh(MarvelSettings.MARVEL_DATA_INDEX_NAME).get();
+        logger.debug("--> waiting for cluster info collector to collect data");
-                GetResponse response = client().prepareGet(MarvelSettings.MARVEL_DATA_INDEX_NAME, ClusterInfoCollector.TYPE, clusterUUID).get();
+        awaitMarvelDocsCount(equalTo(1L), ClusterInfoCollector.TYPE);
-                assertTrue(MarvelSettings.MARVEL_DATA_INDEX_NAME + " document does not exist", response.isExists());
+
-                return response;
+        logger.debug("--> retrieving cluster info document");
-            }
+        GetResponse response = client().prepareGet(MarvelSettings.MARVEL_DATA_INDEX_NAME, ClusterInfoCollector.TYPE, clusterUUID).get();
-        }, 30L, TimeUnit.SECONDS);
+        assertTrue(MarvelSettings.MARVEL_DATA_INDEX_NAME + " document does not exist", response.isExists());
+
+        logger.debug("--> checking that the document contains all required information");
 
         logger.debug("--> checking that the document contains license information");
         assertThat(response.getIndex(), equalTo(MarvelSettings.MARVEL_DATA_INDEX_NAME));
@@ -98,8 +110,12 @@ public class ClusterInfoIT extends MarvelIntegTestCase {
         assertThat(clusterStats, instanceOf(Map.class));
         assertThat(((Map) clusterStats).size(), greaterThan(0));
 
+        assertMarvelTemplateInstalled();
+
         logger.debug("--> check that the cluster_info is not indexed");
-        refresh();
+        securedFlush();
+        securedRefresh();
+
         assertHitCount(client().prepareCount()
                 .setIndices(MarvelSettings.MARVEL_DATA_INDEX_NAME)
                 .setTypes(ClusterInfoCollector.TYPE)
@@ -109,6 +125,5 @@ public class ClusterInfoIT extends MarvelIntegTestCase {
                                 .should(QueryBuilders.matchQuery(License.XFields.STATUS.underscore().toString(), License.Status.EXPIRED.label()))
                                 .minimumNumberShouldMatch(1)
                 ).get(), 0L);
-
     }
 }

marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateTests.java

@@ -10,41 +10,54 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateCollector;
-import org.elasticsearch.marvel.agent.exporter.MarvelTemplateUtils;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.test.MarvelIntegTestCase;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.elasticsearch.test.ESIntegTestCase.Scope;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.greaterThan;
 
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
+@ClusterScope(scope = Scope.TEST)
-
+public class ClusterStateTests extends MarvelIntegTestCase {
-@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/730")
-public class ClusterStateIT extends MarvelIntegTestCase {
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put(MarvelSettings.INTERVAL, "3s")
+                .put(MarvelSettings.INTERVAL, "-1")
                 .put(MarvelSettings.COLLECTORS, ClusterStateCollector.NAME)
+                .put("marvel.agent.exporters.default_local.type", "local")
+                .put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
                 .build();
     }
 
+    @Before
+    public void init() throws Exception {
+        updateMarvelInterval(3L, TimeUnit.SECONDS);
+        waitForMarvelIndices();
+    }
+
+    @After
+    public void cleanup() throws Exception {
+        updateMarvelInterval(-1, TimeUnit.SECONDS);
+        wipeMarvelIndices();
+    }
+
     @Test
     public void testClusterState() throws Exception {
-        ensureGreen();
+        logger.debug("--> waiting for documents to be collected");
-
         awaitMarvelDocsCount(greaterThan(0L), ClusterStateCollector.TYPE);
 
         logger.debug("--> searching for marvel documents of type [{}]", ClusterStateCollector.TYPE);
         SearchResponse response = client().prepareSearch().setTypes(ClusterStateCollector.TYPE).get();
-
         assertThat(response.getHits().getTotalHits(), greaterThan(0L));
 
         logger.debug("--> checking that every document contains the expected fields");
@@ -63,22 +76,11 @@ public class ClusterStateIT extends MarvelIntegTestCase {
     /**
      * This test should fail if the mapping for the 'nodes' attribute
      * in the 'cluster_state' document is NOT set to 'enable: false'
-     *
-     * See
      */
     @Test
     public void testNoNodesIndexing() throws Exception {
-        ensureGreen();
+        logger.debug("--> waiting for documents to be collected");
-
-        logger.debug("--> forcing marvel's index template update");
-        assertAcked(client().admin().indices().preparePutTemplate("marvel").setSource(MarvelTemplateUtils.loadDefaultTemplate()).execute().actionGet());
-
-        logger.debug("--> deleting all marvel indices");
-        deleteMarvelIndices();
-
-        logger.debug("--> checking for template existence");
         awaitMarvelDocsCount(greaterThan(0L), ClusterStateCollector.TYPE);
-        assertMarvelTemplateInstalled();
 
         logger.debug("--> searching for marvel documents of type [{}]", ClusterStateCollector.TYPE);
         SearchResponse response = client().prepareSearch().setTypes(ClusterStateCollector.TYPE).get();

marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStatsTests.java

@@ -13,34 +13,40 @@ import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.test.MarvelIntegTestCase;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.elasticsearch.test.ESIntegTestCase.Scope;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
 import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
 import static org.hamcrest.Matchers.greaterThan;
 
-@ClusterScope(scope = SUITE, numClientNodes = 0)
+@ClusterScope(scope = Scope.TEST, numClientNodes = 0)
 public class ClusterStatsTests extends MarvelIntegTestCase {
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put(MarvelSettings.INTERVAL, "3s")
+                .put(MarvelSettings.INTERVAL, "-1")
                 .put(MarvelSettings.COLLECTORS, ClusterStatsCollector.NAME)
+                .put("marvel.agent.exporters.default_local.type", "local")
+                .put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
                 .build();
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/729")
+    @After
+    public void cleanup() throws Exception {
+        updateMarvelInterval(-1, TimeUnit.SECONDS);
+        wipeMarvelIndices();
+    }
+
     @Test
     public void testClusterStats() throws Exception {
-
-        // lets wait with the collection until all the shards started
-        stopCollection();
-
         logger.debug("--> creating some indices so that every data nodes will at least a shard");
         ClusterStatsNodes.Counts counts = client().admin().cluster().prepareClusterStats().get().getNodesStats().getCounts();
         assertThat(counts.getTotal(), greaterThan(0));
@@ -58,7 +64,7 @@ public class ClusterStatsTests extends MarvelIntegTestCase {
         securedEnsureGreen();
 
         // ok.. we'll start collecting now...
-        startCollection();
+        updateMarvelInterval(3L, TimeUnit.SECONDS);
 
         awaitMarvelTemplateInstalled();
 

marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndexRecoveryTests.java

@@ -5,43 +5,73 @@
  */
 package org.elasticsearch.marvel.agent.renderer.indices;
 
+import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.marvel.agent.collector.indices.IndexRecoveryCollector;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.test.MarvelIntegTestCase;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.junit.After;
 import org.junit.Test;
 
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
+import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
 
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
+@ClusterScope(scope = TEST)
+public class IndexRecoveryTests extends MarvelIntegTestCase {
 
-@AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/730")
+    private static final String INDEX_PREFIX = "test-index-recovery-";
-public class IndexRecoveryIT extends MarvelIntegTestCase {
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put(MarvelSettings.INTERVAL, "3s")
+                .put(MarvelSettings.INTERVAL, "-1")
+                .put(MarvelSettings.INDICES, INDEX_PREFIX + "*")
                 .put(MarvelSettings.COLLECTORS, IndexRecoveryCollector.NAME)
+                .put("marvel.agent.exporters.default_local.type", "local")
+                .put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
                 .build();
     }
 
+    @After
+    public void cleanup() throws Exception {
+        updateMarvelInterval(-1, TimeUnit.SECONDS);
+        wipeMarvelIndices();
+    }
+
     @Test
     public void testIndexRecovery() throws Exception {
         logger.debug("--> creating some indices so that index recovery collector reports data");
-        for (int i = 0; i < randomIntBetween(1, 5); i++) {
+        for (int i = 0; i < randomIntBetween(1, 10); i++) {
-            client().prepareIndex("test-" + i, "foo").setRefresh(true).setSource("field1", "value1").get();
+            client().prepareIndex(INDEX_PREFIX + i, "foo").setSource("field1", "value1").get();
         }
 
+        logger.debug("--> wait for index recovery collector to collect data");
+        assertBusy(new Runnable() {
+            @Override
+            public void run() {
+                securedFlush();
+                securedRefresh();
+
+                RecoveryResponse recoveries = client().admin().indices().prepareRecoveries().get();
+                assertThat(recoveries.hasRecoveries(), is(true));
+            }
+        });
+
+        updateMarvelInterval(3L, TimeUnit.SECONDS);
+        waitForMarvelIndices();
+
         awaitMarvelDocsCount(greaterThan(0L), IndexRecoveryCollector.TYPE);
 
         logger.debug("--> searching for marvel documents of type [{}]", IndexRecoveryCollector.TYPE);
-        SearchResponse response = client().prepareSearch().setTypes(IndexRecoveryCollector.TYPE).get();
+        SearchResponse response = client().prepareSearch(MarvelSettings.MARVEL_INDICES_PREFIX + "*").setTypes(IndexRecoveryCollector.TYPE).get();
         assertThat(response.getHits().getTotalHits(), greaterThan(0L));
 
         logger.debug("--> checking that every document contains the expected fields");

marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndexStatsTests.java

@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.marvel.agent.renderer.indices;
 
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.action.count.CountResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
@@ -14,24 +13,36 @@ import org.elasticsearch.marvel.agent.collector.indices.IndexStatsCollector;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.test.MarvelIntegTestCase;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.elasticsearch.test.ESIntegTestCase.Scope;
+import org.junit.After;
 import org.junit.Test;
 
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.Matchers.greaterThan;
 
-@LuceneTestCase.AwaitsFix(bugUrl = "https://internal-build.elastic.co/job/es_xplugins_master_medium/3319/testReport/junit/org.elasticsearch.marvel.agent.renderer.indices/IndexStatsIT/testIndexStats/")
+@ClusterScope(scope = Scope.TEST, numClientNodes = 0)
-public class IndexStatsIT extends MarvelIntegTestCase {
+public class IndexStatsTests extends MarvelIntegTestCase {
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put(MarvelSettings.INTERVAL, "3s")
+                .put(MarvelSettings.INTERVAL, "-1")
                 .put(MarvelSettings.COLLECTORS, IndexStatsCollector.NAME)
+                .put("marvel.agent.exporters.default_local.type", "local")
+                .put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
                 .build();
     }
 
+    @After
+    public void cleanup() throws Exception {
+        updateMarvelInterval(-1, TimeUnit.SECONDS);
+        wipeMarvelIndices();
+    }
+
     @Test
     public void testIndexStats() throws Exception {
         logger.debug("--> creating some indices for future index stats");
@@ -50,7 +61,11 @@ public class IndexStatsIT extends MarvelIntegTestCase {
             }
         }
 
-        ensureGreen();
+        securedFlush();
+        securedRefresh();
+
+        updateMarvelInterval(3L, TimeUnit.SECONDS);
+        waitForMarvelIndices();
 
         awaitMarvelDocsCount(greaterThan(0L), IndexStatsCollector.TYPE);
 

marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndicesStatsTests.java

@@ -5,63 +5,79 @@
  */
 package org.elasticsearch.marvel.agent.renderer.indices;
 
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
-import org.elasticsearch.action.count.CountResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.marvel.agent.collector.indices.IndicesStatsCollector;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.test.MarvelIntegTestCase;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.elasticsearch.test.ESIntegTestCase.Scope;
+import org.junit.After;
 import org.junit.Test;
 
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.Matchers.greaterThan;
 
-@AwaitsFix(bugUrl="https://github.com/elastic/x-plugins/issues/729")
+@ClusterScope(scope = Scope.TEST, numClientNodes = 0)
-public class IndicesStatsIT extends MarvelIntegTestCase {
+public class IndicesStatsTests extends MarvelIntegTestCase {
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put(MarvelSettings.INTERVAL, "3s")
+                .put(MarvelSettings.INTERVAL, "-1")
                 .put(MarvelSettings.COLLECTORS, IndicesStatsCollector.NAME)
+                .put("marvel.agent.exporters.default_local.type", "local")
+                .put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
                 .build();
     }
 
+    @After
+    public void cleanup() throws Exception {
+        updateMarvelInterval(-1, TimeUnit.SECONDS);
+        wipeMarvelIndices();
+    }
+
     @Test
     public void testIndicesStats() throws Exception {
         logger.debug("--> creating some indices for future indices stats");
         final int nbIndices = randomIntBetween(1, 5);
         for (int i = 0; i < nbIndices; i++) {
-            createIndex("stat" + i);
+            createIndex("stat-" + i);
         }
 
         final long[] nbDocsPerIndex = new long[nbIndices];
         for (int i = 0; i < nbIndices; i++) {
             nbDocsPerIndex[i] = randomIntBetween(1, 50);
             for (int j = 0; j < nbDocsPerIndex[i]; j++) {
-                client().prepareIndex("stat" + i, "type1").setSource("num", i).get();
+                client().prepareIndex("stat-" + i, "type1").setSource("num", i).get();
             }
         }
 
-        awaitMarvelDocsCount(greaterThan(0L), IndicesStatsCollector.TYPE);
+        logger.debug("--> wait for indices stats collector to collect stats for all primaries shards");
-
-        logger.debug("--> wait for indicesx stats collector to collect global stat");
         assertBusy(new Runnable() {
             @Override
             public void run() {
+                securedFlush();
+                securedRefresh();
+
                 for (int i = 0; i < nbIndices; i++) {
-                    CountResponse count = client().prepareCount()
+                    IndicesStatsResponse indicesStats = client().admin().indices().prepareStats().get();
-                            .setTypes(IndicesStatsCollector.TYPE)
+                    assertThat(indicesStats.getPrimaries().getDocs().getCount(), greaterThan(0L));
-                            .get();
-                    assertThat(count.getCount(), greaterThan(0L));
                 }
             }
         });
 
+        updateMarvelInterval(3L, TimeUnit.SECONDS);
+        waitForMarvelIndices();
+
+        logger.debug("--> wait for indices stats collector to collect global stat");
+        awaitMarvelDocsCount(greaterThan(0L), IndicesStatsCollector.TYPE);
+
         logger.debug("--> searching for marvel documents of type [{}]", IndicesStatsCollector.TYPE);
         SearchResponse response = client().prepareSearch().setTypes(IndicesStatsCollector.TYPE).get();
         assertThat(response.getHits().getTotalHits(), greaterThan(0L));

marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsTests.java

@@ -8,26 +8,41 @@ package org.elasticsearch.marvel.agent.renderer.node;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.marvel.agent.collector.node.NodeStatsCollector;
+import org.elasticsearch.marvel.agent.exporter.local.LocalExporter;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.test.MarvelIntegTestCase;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.elasticsearch.test.ESIntegTestCase.Scope;
+import org.junit.After;
 import org.junit.Test;
 
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.Matchers.greaterThan;
 
-public class NodeStatsIT extends MarvelIntegTestCase {
+@ClusterScope(scope = Scope.TEST)
+public class NodeStatsTests extends MarvelIntegTestCase {
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put(MarvelSettings.INTERVAL, "3s")
+                .put(MarvelSettings.INTERVAL, "-1")
                 .put(MarvelSettings.COLLECTORS, NodeStatsCollector.NAME)
+                .put("marvel.agent.exporters.default_local.type", LocalExporter.TYPE)
+                .put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
                 .build();
     }
 
+    @After
+    public void cleanup() throws Exception {
+        updateMarvelInterval(-1, TimeUnit.SECONDS);
+        wipeMarvelIndices();
+    }
+
     @Test
     public void testNodeStats() throws Exception {
         logger.debug("--> creating some indices for future node stats");
@@ -36,6 +51,12 @@ public class NodeStatsIT extends MarvelIntegTestCase {
             client().prepareIndex("test", "foo").setSource("value", randomInt()).get();
         }
 
+        securedFlush();
+        securedRefresh();
+
+        updateMarvelInterval(3L, TimeUnit.SECONDS);
+        waitForMarvelIndices();
+
         awaitMarvelDocsCount(greaterThan(0L), NodeStatsCollector.TYPE);
 
         logger.debug("--> searching for marvel documents of type [{}]", NodeStatsCollector.TYPE);

marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/shards/ShardsTests.java

@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.marvel.agent.renderer.shards;
 
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -18,15 +17,19 @@ import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
+import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.elasticsearch.test.ESIntegTestCase.Scope;
+import org.junit.After;
 import org.junit.Test;
 
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.*;
 
-@AwaitsFix(bugUrl="https://github.com/elastic/x-plugins/issues/729")
+@ClusterScope(scope = Scope.TEST)
-public class ShardsIT extends MarvelIntegTestCase {
+public class ShardsTests extends MarvelIntegTestCase {
 
     private static final String INDEX_PREFIX = "test-shards-";
 
@@ -34,12 +37,20 @@ public class ShardsIT extends MarvelIntegTestCase {
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put(MarvelSettings.INTERVAL, "3s")
+                .put(MarvelSettings.INTERVAL, "-1")
                 .put(MarvelSettings.COLLECTORS, ShardsCollector.NAME)
                 .put(MarvelSettings.INDICES, INDEX_PREFIX + "*")
+                .put("marvel.agent.exporters.default_local.type", "local")
+                .put("marvel.agent.exporters.default_local.template.settings.index.number_of_replicas", 0)
                 .build();
     }
 
+    @After
+    public void cleanup() throws Exception {
+        updateMarvelInterval(-1, TimeUnit.SECONDS);
+        wipeMarvelIndices();
+    }
+
     @Test
     public void testShards() throws Exception {
         logger.debug("--> creating some indices so that shards collector reports data");
@@ -47,6 +58,12 @@ public class ShardsIT extends MarvelIntegTestCase {
             client().prepareIndex(INDEX_PREFIX + i, "foo").setRefresh(true).setSource("field1", "value1").get();
         }
 
+        securedFlush();
+        securedRefresh();
+
+        updateMarvelInterval(3L, TimeUnit.SECONDS);
+        waitForMarvelIndices();
+
         awaitMarvelDocsCount(greaterThan(0L), ShardsCollector.TYPE);
 
         logger.debug("--> searching for marvel documents of type [{}]", ShardsCollector.TYPE);
@@ -75,6 +92,9 @@ public class ShardsIT extends MarvelIntegTestCase {
         final String indexName = INDEX_PREFIX + randomInt();
         assertAcked(prepareCreate(indexName).setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0));
 
+        updateMarvelInterval(3L, TimeUnit.SECONDS);
+        waitForMarvelIndices();
+
         awaitMarvelDocsCount(greaterThan(0L), ShardsCollector.TYPE);
 
         SearchRequestBuilder searchRequestBuilder = client()

marvel/src/test/java/org/elasticsearch/marvel/agent/settings/MarvelSettingsTests.java

@@ -19,7 +19,6 @@ import static org.hamcrest.Matchers.equalTo;
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 1)
 public class MarvelSettingsTests extends MarvelIntegTestCase {
 
-    private final TimeValue startUp = newRandomTimeValue();
     private final TimeValue interval = newRandomTimeValue();
     private final TimeValue indexStatsTimeout = newRandomTimeValue();
     private final String[] indices = randomStringArray();
@@ -41,7 +40,6 @@ public class MarvelSettingsTests extends MarvelIntegTestCase {
 
     private Settings marvelSettings() {
         return Settings.builder()
-                .put(MarvelSettings.STARTUP_DELAY, startUp)
                 .put(MarvelSettings.INTERVAL, interval)
                 .put(MarvelSettings.INDEX_STATS_TIMEOUT, indexStatsTimeout)
                 .putArray(MarvelSettings.INDICES, indices)
@@ -57,7 +55,6 @@ public class MarvelSettingsTests extends MarvelIntegTestCase {
     public void testMarvelSettings() throws Exception {
         logger.info("--> testing marvel settings service initialization");
         for (final MarvelSettings marvelSettings : internalCluster().getInstances(MarvelSettings.class)) {
-            assertThat(marvelSettings.startUpDelay().millis(), equalTo(startUp.millis()));
             assertThat(marvelSettings.interval().millis(), equalTo(interval.millis()));
             assertThat(marvelSettings.indexStatsTimeout().millis(), equalTo(indexStatsTimeout.millis()));
             assertArrayEquals(marvelSettings.indices(), indices);

marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java

@@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.CountDown;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.cache.IndexCacheModule;
 import org.elasticsearch.license.plugin.LicensePlugin;
@@ -119,6 +120,26 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
         }
     }
 
+    protected void wipeMarvelIndices() throws Exception {
+        CountDown retries = new CountDown(3);
+        assertBusy(new Runnable() {
+            @Override
+            public void run() {
+                try {
+                    boolean exist = client().admin().indices().prepareExists(".marvel-es-*").get().isExists();
+                    if (exist) {
+                        deleteMarvelIndices();
+                    } else {
+                        retries.countDown();
+                    }
+                } catch (IndexNotFoundException e) {
+                    retries.countDown();
+                }
+                assertThat(retries.isCountedDown(), is(true));
+            }
+        });
+    }
+
     protected void deleteMarvelIndices() {
         if (shieldEnabled) {
             try {
@@ -223,6 +244,17 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
         fail("marvel template could not be found");
     }
 
+    protected void waitForMarvelIndices() throws Exception {
+        awaitIndexExists(MarvelSettings.MARVEL_DATA_INDEX_NAME);
+        awaitIndexExists(MarvelSettings.MARVEL_INDICES_PREFIX + "*");
+        assertBusy(new Runnable() {
+            @Override
+            public void run() {
+                ensureMarvelIndicesGreen();
+            }
+        });
+    }
+
     protected void awaitIndexExists(final String... indices) throws Exception {
         assertBusy(new Runnable() {
             @Override
@@ -313,6 +345,10 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
         }
     }
 
+    protected void updateMarvelInterval(long value, TimeUnit timeUnit) {
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(MarvelSettings.INTERVAL, value, timeUnit)));
+    }
+
     /** Shield related settings */
 
     public static class ShieldSettings {
@@ -340,7 +376,7 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
 
         public static final String ROLES =
                 "test:\n" + // a user for the test infra.
-                "  cluster: cluster:monitor/nodes/info, cluster:monitor/nodes/stats, cluster:monitor/state, cluster:monitor/health, cluster:monitor/stats, cluster:admin/settings/update, cluster:admin/repository/delete, cluster:monitor/nodes/liveness, indices:admin/template/get, indices:admin/template/put, indices:admin/template/delete\n" +
+                "  cluster: cluster:monitor/nodes/info, cluster:monitor/nodes/stats, cluster:monitor/state, cluster:monitor/health, cluster:monitor/stats, cluster:monitor/task, cluster:admin/settings/update, cluster:admin/repository/delete, cluster:monitor/nodes/liveness, indices:admin/template/get, indices:admin/template/put, indices:admin/template/delete\n" +
                 "  indices:\n" +
                 "    '*': all\n" +
                 "\n" +