Merge pull request elastic/elasticsearch#1034 from rmuir/smoke_test_plugins_ssl_qa

re-enable smoke-test-plugins with ssl

Original commit: elastic/x-pack-elasticsearch@9f76ecfd6b

qa/smoke-test-plugins-ssl/build.gradle

@@ -0,0 +1,88 @@
+apply plugin: 'elasticsearch.rest-test'
+
+dependencies {
+  testCompile project(path: ':x-plugins:shield', configuration: 'runtime')
+}
+
+// ssl setup, it reuses the ssl-setup.xml from ant, for now.
+
+// location of target keystore
+File keystore = new File(project.buildDir, 'keystore/test-node.jks')
+
+// we touch keystore because otherwise it fails, extraConfigFile does not exist
+// this tricks some broken compile-time check into just moving along: we nuke this stuff before we actually generate
+keystore.parentFile.mkdirs()
+keystore.createNewFile()
+
+// add keystore to test classpath: it expects it there
+sourceSets.test.resources.srcDir(keystore.parentFile)
+
+configurations {
+  antcontrib {
+    description = 'ant-contrib'
+    transitive = false
+  }
+}
+
+dependencies {
+  antcontrib "ant-contrib:ant-contrib:1.0b3"
+}
+
+// this loop must be outside of a configuration closure, otherwise it may get executed multiple times
+for (Project subproj : project.rootProject.subprojects) {
+  if (subproj.path.startsWith(':plugins:')) {
+    // need to get a non-decorated project object, so must re-lookup the project by path
+    integTest.clusterConfig.plugin(subproj.name, project(subproj.path))
+  }
+}
+
+// we should be able to taskdef, but gradle has *the worst* classloader management
+// so just do a hack, jam ant-contrib directly into gradle's ant's classloader
+ClassLoader antClassLoader = org.apache.tools.ant.Project.class.classLoader
+configurations.antcontrib.each { File f ->
+  antClassLoader.addURL(f.toURI().toURL())
+}
+
+// suck in ssl-setup.xml, defining matching tasks in gradle
+ant.property(name: 'integ.scratch', location: project.buildDir)
+ant.property(name: 'keystore.path', keystore)
+ant.importBuild 'ssl-setup.xml'
+
+// clean all intermediate/keystore files before regenerating it
+task cleanKeystore(type: Delete) {
+  delete new File(project.buildDir, 'keystore'), 
+         new File(project.buildDir, 'cert'), 
+         new File(project.buildDir, 'ca')
+}
+
+// wipe and regenerate keystore so its available as a test dep
+processTestResources.dependsOn('cleanKeystore')
+processTestResources.dependsOn('generate-keystore')
+
+integTest {
+  cluster {
+    // TODO: use some variable here for port number
+    systemProperty 'es.marvel.agent.exporter.es.hosts', 'https://marvel_export:changeme@localhost:9400'
+    systemProperty 'es.marvel.agent.exporter.es.ssl.truststore.path', 'test-node.jks'
+    systemProperty 'es.marvel.agent.exporter.es.ssl.truststore.password', 'keypass'
+    systemProperty 'es.shield.transport.ssl', 'true'
+    systemProperty 'es.shield.http.ssl', 'true'
+    systemProperty 'es.shield.ssl.keystore.path', 'test-node.jks'
+    systemProperty 'es.shield.ssl.keystore.password', 'keypass'
+    plugin 'licence', project(':x-plugins:license:plugin')
+    plugin 'shield', project(':x-plugins:shield')
+    plugin 'watcher', project(':x-plugins:watcher')
+    plugin 'marvel-agent', project(':x-plugins:marvel')
+
+    // copy keystore into config/
+    extraConfigFile 'test-node.jks', keystore
+    setupCommand 'setupTestUser',
+                 'bin/shield/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
+    setupCommand 'setupMarvelUser',
+                 'bin/shield/esusers', 'useradd', 'marvel_export', '-p', 'changeme', '-r', 'marvel_agent'
+    waitCondition = { node, ant ->
+      // we just return true, doing an https check is tricky here
+      return true
+    }
+  }
+} 

qa/smoke-test-plugins-ssl/integration-tests.xml

@@ -1,90 +0,0 @@
-<?xml version="1.0"?>
-<project name="smoke-test-x-plugins-ssl"
-         xmlns:ac="antlib:net.sf.antcontrib">
-
-  <import file="${elasticsearch.integ.antfile.default}"/>
-
-  <property name="home" location="${integ.scratch}/elasticsearch-${elasticsearch.version}"/>
-
-  <!-- redefined to work with auth -->
-  <macrodef name="waitfor-elasticsearch">
-      <attribute name="port"/>
-      <attribute name="timeoutproperty"/>
-    <sequential>
-      <echo>Waiting for elasticsearch to become available on port @{port}...</echo>
-      <waitfor maxwait="30" maxwaitunit="second"
-               checkevery="500" checkeveryunit="millisecond"
-               timeoutproperty="@{timeoutproperty}">
-        <socket server="127.0.0.1" port="@{port}"/>
-      </waitfor>
-    </sequential>
-  </macrodef>
-
-  <target name="start-external-cluster-with-plugins" depends="setup-workspace">
-    <ac:for list="${xplugins.list}" param="xplugin.name">
-      <sequential>
-        <fail message="Expected @{xplugin.name}-${version}.zip as a dependency, but could not be found in ${integ.deps}/plugins}">
-          <condition>
-            <not>
-              <available file="${integ.deps}/plugins/@{xplugin.name}-${elasticsearch.version}.zip" />
-            </not>
-          </condition>
-        </fail>
-      </sequential>
-    </ac:for>
-
-    <ac:for param="file">
-      <path>
-        <fileset dir="${integ.deps}/plugins"/>
-      </path>
-      <sequential>
-        <local name="plugin.name"/>
-        <convert-plugin-name file="@{file}" outputproperty="plugin.name"/>
-        <install-plugin name="${plugin.name}" file="@{file}"/>
-      </sequential>
-    </ac:for>
-
-    <echo>Setting up Shield auth</echo>
-    <run-script script="${home}/bin/shield/esusers">
-      <nested>
-        <arg value="useradd"/>
-        <arg value="test_user"/>
-        <arg value="-p"/>
-        <arg value="changeme"/>
-        <arg value="-r"/>
-        <arg value="admin"/>
-      </nested>
-    </run-script>
-    <run-script script="${home}/bin/shield/esusers"> 
-      <nested>
-        <arg value="useradd"/>
-        <arg value="marvel_export"/>
-        <arg value="-p"/>
-        <arg value="changeme"/>
-        <arg value="-r"/>
-        <arg value="marvel_agent"/>
-      </nested>
-    </run-script>
-    <copy file="${keystore.path}" todir="${home}/config/"/> <!-- move to a dir that the security manager will be happy with -->
-
-    <startup-elasticsearch>
-      <additional-args>
-        <arg value="-Des.marvel.agent.exporter.es.hosts=https://marvel_export:changeme@localhost:${integ.http.port}"/>
-        <arg value="-Des.marvel.agent.exporter.es.ssl.truststore.path=${home}/config/test-node.jks"/>
-        <arg value="-Des.marvel.agent.exporter.es.ssl.truststore.password=keypass"/>
-        <arg value="-Des.shield.transport.ssl=true"/>
-        <arg value="-Des.shield.http.ssl=true"/>
-        <arg value="-Des.shield.ssl.keystore.path=${home}/config/test-node.jks"/>
-        <arg value="-Des.shield.ssl.keystore.password=keypass"/>
-        <arg value="-Des.discovery.zen.ping.multicast.enabled=false"/>
-      </additional-args>
-    </startup-elasticsearch>
-
-    <echo>Checking we can connect with basic auth on port ${integ.http.port}...</echo>
-    <local name="temp.file"/>
-    <tempfile property="temp.file" destdir="${java.io.tmpdir}"/>
-    <!-- TODO: how to make the get task take a non-default keystore? -->
-    <!--<get src="https://127.0.0.1:${integ.http.port}" dest="${temp.file}" 
-         username="test_user" password="changeme" verbose="true" retries="10"/>-->
-  </target>
-</project>

qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsSslIT.java

@@ -11,11 +11,14 @@ import java.io.IOException;
 import java.net.URISyntaxException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Collections;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.client.support.Headers;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.shield.ShieldPlugin;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.test.rest.ESRestTestCase;

qa/smoke-test-plugins-ssl/src/test/resources/rest-api-spec/test/smoke_test_plugins_ssl/10_basic.yaml

@@ -11,6 +11,6 @@
         nodes.info: {}
 
     - match:  { nodes.$master.plugins.15.name: license }
-    - match:  { nodes.$master.plugins.18.name: marvel-agent }
+    - match:  { nodes.$master.plugins.19.name: marvel-agent }
-    - match:  { nodes.$master.plugins.21.name: shield }
+    - match:  { nodes.$master.plugins.22.name: shield }
-    - match:  { nodes.$master.plugins.24.name: watcher }
+    - match:  { nodes.$master.plugins.25.name: watcher }