Split monitoring smoke tests into separate smoke tests

There is a race condition between the smoke tests that get run because of the teardown conditions of
REST tests. By splitting them, we can avoid the unrealistic scenario/race condition.

Original commit: elastic/x-pack-elasticsearch@f95ae0e595

elasticsearch/qa/build.gradle

@@ -1,5 +1,5 @@
 subprojects {
-  project.afterEvaluate {
+  plugins.withType(org.elasticsearch.gradle.test.RestTestPlugin) {
     // someone figure out what the x-plugins logic should be
     project.licenseHeaders.enabled = false
   }

elasticsearch/qa/smoke-test-monitoring-with-shield/build.gradle

@@ -1,41 +1,50 @@
-apply plugin: 'elasticsearch.rest-test'
+subprojects {
+  apply plugin: 'elasticsearch.rest-test'
 
-dependencies {
+  dependencies {
-  testCompile project(path: ':x-plugins:elasticsearch:x-pack', configuration: 'runtime')
+    testCompile project(path: ':x-plugins:elasticsearch:x-pack', configuration: 'runtime')
-}
+  }
 
-// bring in monitoring rest test suite
+  // bring in monitoring rest test suite
-task copyMonitoringRestTests(type: Copy) {
+  task copyMonitoringRestTests(type: Copy) {
-  into project.sourceSets.test.output.resourcesDir
+    into project.sourceSets.test.output.resourcesDir
-  from project(':x-plugins:elasticsearch:x-pack').sourceSets.test.resources.srcDirs
+    from project(':x-plugins:elasticsearch:x-pack').sourceSets.test.resources.srcDirs
-  include 'rest-api-spec/test/monitoring/**'
+    include 'rest-api-spec/test/monitoring/**'
-}
+  }
 
-integTest {
+  integTest {
-  dependsOn copyMonitoringRestTests
+    dependsOn copyMonitoringRestTests
 
-  cluster {
+    cluster {
-    systemProperty 'es.logger.level', 'TRACE'
+      systemProperty 'es.logger.level', 'TRACE'
-    plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack')
+      plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack')
-    setting 'xpack.monitoring.agent.interval', '3s'
+      setting 'xpack.monitoring.agent.interval', '3s'
-    extraConfigFile 'x-pack/roles.yml', 'roles.yml'
+      extraConfigFile 'x-pack/roles.yml', '../roles.yml'
-    setupCommand 'setupTestAdminUser',
+      setupCommand 'setupTestAdminUser',
-                 'bin/x-pack/users', 'useradd', 'test_admin', '-p', 'changeme', '-r', 'superuser'
+                   'bin/x-pack/users', 'useradd', 'test_admin', '-p', 'changeme', '-r', 'superuser'
-    setupCommand 'setupMonitoredSystemUser',
+      setupCommand 'setupMonitoredSystemUser',
-                 'bin/x-pack/users', 'useradd', 'monitored_system', '-p', 'changeme', '-r', 'monitored_system,required_for_test'
+                   'bin/x-pack/users', 'useradd', 'monitoring_system', '-p', 'changeme', '-r', 'monitoring_system,monitoring_without_bulk'
-    setupCommand 'setupPowerlessUser',
+      setupCommand 'setupPowerlessUser',
-                 'bin/x-pack/users', 'useradd', 'no_monitored_system', '-p', 'changeme', '-r', 'required_for_test'
+                   'bin/x-pack/users', 'useradd', 'not_monitoring_system', '-p', 'changeme', '-r', 'monitoring_without_bulk'
 
-    waitCondition = { node, ant ->
+      waitCondition = { node, ant ->
-      File tmpFile = new File(node.cwd, 'wait.success')
+        File tmpFile = new File(node.cwd, 'wait.success')
-      ant.get(src: "http://${node.httpUri()}",
+        ant.get(src: "http://${node.httpUri()}",
-              dest: tmpFile.toString(),
+                dest: tmpFile.toString(),
-              username: 'test_admin',
+                username: 'test_admin',
-              password: 'changeme',
+                password: 'changeme',
-              ignoreerrors: true,
+                ignoreerrors: true,
-              retries: 10)
+                retries: 10)
-      return tmpFile.exists()
+        return tmpFile.exists()
+      }
     }
   }
 }
 
+/**
+ * Allow {@code integTest} to be invoked on this project to run both Monitoring+Security smoke tests.
+ */
+task integTest {
+  dependsOn subprojects.integTest
+}
+

elasticsearch/qa/smoke-test-monitoring-with-shield/insufficient-rights/build.gradle

@@ -0,0 +1 @@
+apply plugin: 'elasticsearch.rest-test'

elasticsearch/qa/smoke-test-monitoring-with-shield/insufficient-rights/src/test/java/org/elasticsearch/smoketest/MonitoringWithShieldInsufficientRoleIT.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.smoketest;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.shield.authc.support.SecuredString;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.RestTestCandidate;
+import org.elasticsearch.test.rest.parser.RestTestParseException;
+
+import java.io.IOException;
+
+import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+import static org.hamcrest.Matchers.containsString;
+
+public class MonitoringWithShieldInsufficientRoleIT extends ESRestTestCase {
+
+    public MonitoringWithShieldInsufficientRoleIT(@Name("yaml") RestTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws IOException, RestTestParseException {
+        return ESRestTestCase.createParameters(0, 1);
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("not_monitoring_system", new SecuredString("changeme".toCharArray()));
+        return Settings.builder()
+                .put(ThreadContext.PREFIX + ".Authorization", token)
+                .build();
+    }
+
+    @Override
+    protected Settings restAdminSettings() {
+        String token = basicAuthHeaderValue("test_admin", new SecuredString("changeme".toCharArray()));
+        return Settings.builder()
+                .put(ThreadContext.PREFIX + ".Authorization", token)
+                .build();
+    }
+
+    @Override
+    public void test() throws IOException {
+        try {
+            super.test();
+            fail("should have failed because of missing role");
+        } catch(AssertionError ae) {
+            assertThat(ae.getMessage(), containsString("action [cluster:admin/xpack/monitoring/bulk]"));
+            assertThat(ae.getMessage(), containsString("returned [403 Forbidden]"));
+            assertThat(ae.getMessage(), containsString("is unauthorized for user [not_monitoring_system]"));
+        }
+    }
+
+}

elasticsearch/qa/smoke-test-monitoring-with-shield/roles.yml

@@ -1,15 +1,7 @@
-admin:
+monitoring_system:
-  cluster:
-    - all
-  indices:
-    - names: '*'
-      privileges:
-        - all
-
-monitored_system:
   cluster: [ 'cluster:admin/xpack/monitoring/bulk' ]
 
-required_for_test:
+monitoring_without_bulk:
   cluster: [ 'monitor' ]
   indices:
     - names: '.monitoring-*'

elasticsearch/qa/smoke-test-monitoring-with-shield/src/test/java/org/elasticsearch/smoketest/MonitoringWithShieldInsufficientRoleIT.java

@@ -1,36 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.smoketest;
-
-import com.carrotsearch.randomizedtesting.annotations.Name;
-import org.elasticsearch.test.rest.RestTestCandidate;
-
-import java.io.IOException;
-
-import static org.hamcrest.Matchers.containsString;
-
-public class MonitoringWithShieldInsufficientRoleIT extends MonitoringWithShieldIT {
-
-    public MonitoringWithShieldInsufficientRoleIT(@Name("yaml") RestTestCandidate testCandidate) {
-        super(testCandidate);
-    }
-
-    public void test() throws IOException {
-        try {
-            super.test();
-            fail("should have failed because of missing role");
-        } catch(AssertionError ae) {
-            assertThat(ae.getMessage(), containsString("action [cluster:admin/xpack/monitoring/bulk]"));
-            assertThat(ae.getMessage(), containsString("returned [403 Forbidden]"));
-            assertThat(ae.getMessage(), containsString("is unauthorized for user [no_monitored_system]"));
-        }
-    }
-
-    @Override
-    protected String[] getCredentials() {
-        return new String[]{"no_monitored_system", "changeme"};
-    }
-}

elasticsearch/qa/smoke-test-monitoring-with-shield/sufficient-rights/build.gradle

@@ -0,0 +1 @@
+apply plugin: 'elasticsearch.rest-test'

elasticsearch/qa/smoke-test-monitoring-with-shield/sufficient-rights/src/test/java/org/elasticsearch/smoketest/MonitoringWithShieldIT.java

@@ -20,9 +20,6 @@ import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basic
 
 public class MonitoringWithShieldIT extends ESRestTestCase {
 
-    private final static String TEST_ADMIN_USERNAME = "test_admin";
-    private final static String TEST_ADMIN_PASSWORD = "changeme";
-
     public MonitoringWithShieldIT(@Name("yaml") RestTestCandidate testCandidate) {
         super(testCandidate);
     }
@@ -32,14 +29,9 @@ public class MonitoringWithShieldIT extends ESRestTestCase {
         return ESRestTestCase.createParameters(0, 1);
     }
 
-    protected String[] getCredentials() {
-        return new String[]{"monitored_system", "changeme"};
-    }
-
     @Override
     protected Settings restClientSettings() {
-        String[] creds = getCredentials();
+        String token = basicAuthHeaderValue("monitoring_system", new SecuredString("changeme".toCharArray()));
-        String token = basicAuthHeaderValue(creds[0], new SecuredString(creds[1].toCharArray()));
         return Settings.builder()
                 .put(ThreadContext.PREFIX + ".Authorization", token)
                 .build();
@@ -47,7 +39,7 @@ public class MonitoringWithShieldIT extends ESRestTestCase {
 
     @Override
     protected Settings restAdminSettings() {
-        String token = basicAuthHeaderValue(TEST_ADMIN_USERNAME, new SecuredString(TEST_ADMIN_PASSWORD.toCharArray()));
+        String token = basicAuthHeaderValue("test_admin", new SecuredString("changeme".toCharArray()));
         return Settings.builder()
                 .put(ThreadContext.PREFIX + ".Authorization", token)
                 .build();