Marvel: various tests fixes

Two regressions have been introduced in elastic/x-pack@156d9e4d5b: marvel index templates should not be deleted between tests and checking for marvel indices existence should not fail with IndexNotFoundException when the indices are not yet created and Shield enabled.

closes elastic/elasticsearch#1396 elastic/elasticsearch#1394 elastic/elasticsearch#1382

In MultiNodesStatsTests.java, multiple nodes are started in async: the first node may collect marvel data multiple times when the last one just started. So we should not check for exact 1 doc per node but at least 1 doc per node.

closes elastic/elasticsearch#1370

In HttpExporterTemplateTests.java, we must compare a long count with a long value.

Original commit: elastic/x-pack-elasticsearch@732fef995a

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/cluster/ClusterStatsCollectorTests.java

@@ -132,7 +132,7 @@ public class ClusterStatsCollectorTests extends AbstractCollectorTestCase {
         if (classes != null) {
             assertThat(results.size(), equalTo(classes.length));
             for (Class<?> cl : classes) {
-                assertThat(results.stream().filter(o -> cl.isInstance(o)).count(), equalTo(1));
+                assertThat(results.stream().filter(o -> cl.isInstance(o)).count(), equalTo(1L));
             }
         }
     }

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/http/HttpExporterTemplateTests.java

@@ -93,15 +93,8 @@ public class HttpExporterTemplateTests extends AbstractExporterTemplateTestCase
     }
 
     @Override
-    protected void awaitIndexExists(String... indices) throws Exception {
+    protected void awaitIndexExists(String index) throws Exception {
-        assertBusy(new Runnable() {
+        assertBusy(() -> assertThat("could not find index " + index, dispatcher.hasIndex(index), is(true)), 10, TimeUnit.SECONDS);
-            @Override
-            public void run() {
-                for (String index : indices) {
-                    assertThat("could not find index " + index, dispatcher.hasIndex(index), is(true));
-                }
-            }
-        }, 10, TimeUnit.SECONDS);
     }
 
     class MockServerDispatcher extends Dispatcher {

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/node/MultiNodesStatsTests.java

@@ -7,6 +7,7 @@ package org.elasticsearch.marvel.agent.renderer.node;
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.marvel.agent.collector.node.NodeStatsCollector;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.test.MarvelIntegTestCase;
@@ -24,6 +25,7 @@ import java.util.concurrent.TimeUnit;
 import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.instanceOf;
 
 @ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
@@ -73,23 +75,22 @@ public class MultiNodesStatsTests extends MarvelIntegTestCase {
 
         final int nbNodes = nodes;
         logger.debug("--> waiting for {} nodes to be available", nbNodes);
-        assertBusy(new Runnable() {
+        assertBusy(() -> {
-            @Override
-            public void run() {
             assertThat(cluster().size(), equalTo(nbNodes));
             assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(nbNodes)).get());
-            }
         });
 
         updateMarvelInterval(3L, TimeUnit.SECONDS);
         waitForMarvelIndices();
 
-        assertBusy(new Runnable() {
+        logger.debug("--> checking that every node correctly reported its own node stats");
-            @Override
+        assertBusy(() -> {
-            public void run() {
+            String indices = MarvelSettings.MARVEL_INDICES_PREFIX + "*";
-                securedFlush();
+            securedFlush(indices);
+            securedRefresh();
 
-                SearchResponse response = client().prepareSearch()
+            try {
+                SearchResponse response = client().prepareSearch(indices)
                         .setTypes(NodeStatsCollector.TYPE)
                         .setSize(0)
                         .addAggregation(AggregationBuilders.terms("nodes_ids").field("node_stats.node_id"))
@@ -101,9 +102,14 @@ public class MultiNodesStatsTests extends MarvelIntegTestCase {
 
                     for (String nodeName : internalCluster().getNodeNames()) {
                         StringTerms.Bucket bucket = (StringTerms.Bucket) ((StringTerms) aggregation).getBucketByKey(internalCluster().clusterService(nodeName).localNode().getId());
-                        assertThat(bucket.getDocCount(), equalTo(1L));
+                        // At least 1 doc must exist per node, but it can be more than 1
+                        // because the first node may have already collected many node stats documents
+                        // whereas the last node just started to collect node stats.
+                        assertThat(bucket.getDocCount(), greaterThanOrEqualTo(1L));
                     }
                 }
+            } catch (IndexNotFoundException e) {
+                fail("Caught unexpected IndexNotFoundException");
             }
         });
     }

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/test/MarvelIntegTestCase.java

@@ -50,7 +50,6 @@ import java.util.function.Function;
 
 import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
@@ -130,6 +129,14 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
         return client -> (client instanceof NodeClient) ? client.filterWithHeader(headers) : client;
     }
 
+    @Override
+    protected Set<String> excludeTemplates() {
+        Set<String> templates = new HashSet<>();
+        templates.add(MarvelTemplateUtils.indexTemplateName());
+        templates.add(MarvelTemplateUtils.dataTemplateName());
+        return templates;
+    }
+
     /**
      * Override and returns {@code false} to force running without shield
      */
@@ -182,15 +189,7 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
     }
 
     protected void awaitMarvelDocsCount(Matcher<Long> matcher, String... types) throws Exception {
-        securedFlush();
+        assertBusy(() -> assertMarvelDocsCount(matcher, types), 30, TimeUnit.SECONDS);
-        securedRefresh();
-
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertMarvelDocsCount(matcher, types);
-            }
-        }, 30, TimeUnit.SECONDS);
     }
 
     protected void ensureMarvelIndicesYellow() {
@@ -208,9 +207,8 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
     protected void assertMarvelDocsCount(Matcher<Long> matcher, String... types) {
         String indices = MarvelSettings.MARVEL_INDICES_PREFIX + "*";
         try {
-            assertNoFailures(client().admin().indices().prepareRefresh(indices).get());
+            securedFlushAndRefresh(indices);
-            long count = client().prepareSearch(indices).setSize(0)
+            long count = client().prepareSearch(indices).setSize(0).setTypes(types).get().getHits().totalHits();
-                    .setTypes(types).get().getHits().totalHits();
             logger.trace("--> searched for [{}] documents, found [{}]", Strings.arrayToCommaDelimitedString(types), count);
             assertThat(count, matcher);
         } catch (IndexNotFoundException e) {
@@ -242,22 +240,21 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
     }
 
     protected void waitForMarvelIndices() throws Exception {
-        String currentVersion = String.valueOf(MarvelTemplateUtils.TEMPLATE_VERSION);
+        awaitIndexExists(MarvelSettings.MARVEL_INDICES_PREFIX + "*");
-        awaitIndexExists(MarvelSettings.MARVEL_DATA_INDEX_PREFIX + currentVersion);
+        assertBusy(this::ensureMarvelIndicesYellow);
-        awaitIndexExists(MarvelSettings.MARVEL_INDICES_PREFIX + currentVersion + "-*");
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                ensureMarvelIndicesYellow();
-            }
-        });
     }
 
-    protected void awaitIndexExists(final String... indices) throws Exception {
+    protected void awaitIndexExists(final String index) throws Exception {
-        assertBusy(new Runnable() {
+        assertBusy(() -> {
-            @Override
+                try {
-            public void run() {
+                    assertIndicesExists(index);
-                assertIndicesExists(indices);
+                } catch (IndexNotFoundException e) {
+                    if (shieldEnabled) {
+                        // with shield we might get that if wildcards were resolved to no indices
+                        fail("IndexNotFoundException when checking for existence of index [" + index + "]");
+                    } else {
+                        throw e;
+                    }
                 }
         }, 30, TimeUnit.SECONDS);
     }
@@ -299,6 +296,18 @@ public abstract class MarvelIntegTestCase extends ESIntegTestCase {
         }
     }
 
+    protected void securedFlushAndRefresh(String... indices) {
+        if (shieldEnabled) {
+            try {
+                flushAndRefresh(indices);
+            } catch (IndexNotFoundException e) {
+                // with shield we might get that if wildcards were resolved to no indices
+            }
+        } else {
+            flushAndRefresh(indices);
+        }
+    }
+
     protected void securedEnsureGreen(String... indices) {
         if (shieldEnabled) {
             try {