Marvel: Add integration test for Marvel+Shield with SSL

closes elastic/elasticsearch#1467

Original commit: elastic/x-pack-elasticsearch@9dd6bf9629

elasticsearch/qa/smoke-test-plugins-ssl/build.gradle

@@ -7,34 +7,134 @@ dependencies {
   testCompile project(path: ':x-plugins:elasticsearch:x-pack', configuration: 'runtime')
 }
 
-// location of keystore and files to generate it
+// needed to be consistent with ssl host checking
-File keystore = new File(project.buildDir, 'keystore/test-node.jks')
+String san = getSubjectAlternativeNameString()
 
-// generate the keystore
+// location of generated keystores and certificates
-task createKey(type: LoggedExec) {
+File keystoreDir = new File(project.buildDir, 'keystore')
+
+// Generate the node's keystore
+File nodeKeystore = new File(keystoreDir, 'test-node.jks')
+task createNodeKeyStore(type: LoggedExec) {
   doFirst {
-    project.delete(keystore.parentFile)
+    if (nodeKeystore.parentFile.exists() == false) {
-    keystore.parentFile.mkdirs()
+      nodeKeystore.parentFile.mkdirs()
+    }
+    if (nodeKeystore.exists()) {
+      delete nodeKeystore
+    }
   }
-  // needed to be consistent with ssl host checking
-  String san = getSubjectAlternativeNameString()
   executable = 'keytool'
   standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
   args '-genkey',
-       '-alias', 'test-node',
+          '-alias', 'test-node',
-       '-keystore', keystore,
+          '-keystore', nodeKeystore,
-       '-keyalg', 'RSA',
+          '-keyalg', 'RSA',
-       '-keysize', '2048',
+          '-keysize', '2048',
-       '-validity', '712',
+          '-validity', '712',
-       '-dname', 'CN=smoke-test-plugins-ssl',
+          '-dname', 'CN=smoke-test-plugins-ssl',
-       '-keypass', 'keypass',
+          '-keypass', 'keypass',
-       '-storepass', 'keypass',
+          '-storepass', 'keypass',
-       '-ext', san
+          '-ext', san
 }
 
-// add keystore to test classpath: it expects it there
+// Generate the client's keystore
-sourceSets.test.resources.srcDir(keystore.parentFile)
+File clientKeyStore = new File(keystoreDir, 'test-client.jks')
-processTestResources.dependsOn(createKey)
+task createClientKeyStore(type: LoggedExec) {
+  doFirst {
+    if (clientKeyStore.parentFile.exists() == false) {
+      clientKeyStore.parentFile.mkdirs()
+    }
+    if (clientKeyStore.exists()) {
+      delete clientKeyStore
+    }
+  }
+  executable = 'keytool'
+  standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
+  args '-genkey',
+          '-alias', 'test-client',
+          '-keystore', clientKeyStore,
+          '-keyalg', 'RSA',
+          '-keysize', '2048',
+          '-validity', '712',
+          '-dname', 'CN=smoke-test-plugins-ssl',
+          '-keypass', 'keypass',
+          '-storepass', 'keypass',
+          '-ext', san
+}
+
+// Export the node's certificate
+File nodeCertificate = new File(keystoreDir, 'test-node.cert')
+task exportNodeCertificate(type: LoggedExec) {
+  doFirst {
+    if (nodeCertificate.parentFile.exists() == false) {
+      nodeCertificate.parentFile.mkdirs()
+    }
+    if (nodeCertificate.exists()) {
+      delete nodeCertificate
+    }
+  }
+  executable = 'keytool'
+  args '-export',
+          '-alias', 'test-node',
+          '-keystore', nodeKeystore,
+          '-storepass', 'keypass',
+          '-file', nodeCertificate
+}
+
+// Import the node certificate in the client's keystore
+task importNodeCertificateInClientKeyStore(type: LoggedExec) {
+  dependsOn exportNodeCertificate
+  executable = 'keytool'
+  args '-import',
+          '-alias', 'test-node',
+          '-keystore', clientKeyStore,
+          '-storepass', 'keypass',
+          '-file', nodeCertificate,
+          '-noprompt'
+}
+
+// Export the client's certificate
+File clientCertificate = new File(keystoreDir, 'test-client.cert')
+task exportClientCertificate(type: LoggedExec) {
+  doFirst {
+    if (clientCertificate.parentFile.exists() == false) {
+      clientCertificate.parentFile.mkdirs()
+    }
+    if (clientCertificate.exists()) {
+      delete clientCertificate
+    }
+  }
+  executable = 'keytool'
+  args '-export',
+          '-alias', 'test-client',
+          '-keystore', clientKeyStore,
+          '-storepass', 'keypass',
+          '-file', clientCertificate
+}
+
+// Import the client certificate in the node's keystore
+task importClientCertificateInNodeKeyStore(type: LoggedExec) {
+  dependsOn exportClientCertificate
+  executable = 'keytool'
+  args '-import',
+          '-alias', 'test-client',
+          '-keystore', nodeKeystore,
+          '-storepass', 'keypass',
+          '-file', clientCertificate,
+          '-noprompt'
+}
+
+forbiddenPatterns {
+  exclude '**/*.cert'
+}
+
+// Add keystores to test classpath: it expects it there
+sourceSets.test.resources.srcDir(keystoreDir)
+processTestResources.dependsOn(
+        createNodeKeyStore, createClientKeyStore,
+        importNodeCertificateInClientKeyStore, importClientCertificateInNodeKeyStore
+)
 
 ext.pluginsCount = 1 // we install xpack explicitly
 project.rootProject.subprojects.findAll { it.path.startsWith(':plugins:') }.each { subproj ->
@@ -45,25 +145,46 @@ project.rootProject.subprojects.findAll { it.path.startsWith(':plugins:') }.each
 
 integTest {
   cluster {
-    systemProperty 'es.xpack.monitoring.agent.exporters.es.type', 'http'
+    systemProperty 'es.xpack.monitoring.agent.interval', '3s'
-    systemProperty 'es.xpack.monitoring.agent.exporters.es.enabled', 'false'
+    systemProperty 'es.xpack.monitoring.agent.exporters._http.type', 'http'
-    systemProperty 'es.xpack.monitoring.agent.exporters.es.ssl.truststore.path', keystore.name
+    systemProperty 'es.xpack.monitoring.agent.exporters._http.enabled', 'false'
-    systemProperty 'es.xpack.monitoring.agent.exporters.es.ssl.truststore.password', 'keypass'
+    systemProperty 'es.xpack.monitoring.agent.exporters._http.ssl.truststore.path', clientKeyStore.name
+    systemProperty 'es.xpack.monitoring.agent.exporters._http.ssl.truststore.password', 'keypass'
+    systemProperty 'es.xpack.monitoring.agent.exporters._http.auth.username', 'monitoring_agent'
+    systemProperty 'es.xpack.monitoring.agent.exporters._http.auth.password', 'changeme'
+
     systemProperty 'es.shield.transport.ssl', 'true'
     systemProperty 'es.shield.http.ssl', 'true'
-    systemProperty 'es.shield.ssl.keystore.path', keystore.name
+    systemProperty 'es.shield.ssl.keystore.path', nodeKeystore.name
     systemProperty 'es.shield.ssl.keystore.password', 'keypass'
+
     plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack')
 
-    // copy keystore into config/
+    // copy keystores into config/
-    extraConfigFile keystore.name, keystore
+    extraConfigFile nodeKeystore.name, nodeKeystore
+    extraConfigFile clientKeyStore.name, clientKeyStore
+
     setupCommand 'setupTestUser',
                  'bin/xpack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
     setupCommand 'setupMarvelUser',
-                 'bin/xpack/esusers', 'useradd', 'marvel_export', '-p', 'changeme', '-r', 'marvel_agent'
+                 'bin/xpack/esusers', 'useradd', 'monitoring_agent', '-p', 'changeme', '-r', 'remote_monitoring_agent'
+
     waitCondition = { node, ant ->
-      // we just return true, doing an https check is tricky here
+      // HTTPS check is tricky to do, so we wait for the log file to indicate that the node is started
-      return true
+      String waitForNodeStartProp = "waitForNodeStart${name}"
+      ant.waitfor(maxwait: '10', maxwaitunit: 'second', checkevery: '100', checkeveryunit: 'millisecond',
+              timeoutproperty: waitForNodeStartProp) {
+        and {
+          resourcecontains(resource: "${node.startLog.toString()}", substring: "bound_addresses {${node.httpUri()}")
+          resourcecontains(resource: "${node.startLog.toString()}", substring: 'started')
+        }
+      }
+
+      if (ant.project.getProperty(waitForNodeStartProp)) {
+        println "Timed out when looking for bound_addresses in log file ${node.startLog.toString()}"
+        return false;
+      }
+      return true;
     }
   }
 }

elasticsearch/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithShieldIT.java

@@ -0,0 +1,134 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.smoketest;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
+import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.shield.transport.netty.ShieldNettyTransport;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xpack.XPackPlugin;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Collections;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+
+/**
+ * This test checks that a Monitoring's HTTP exporter correctly exports to a monitoring cluster
+ * protected by Shield with HTTPS/SSL.
+ *
+ * It sets up a cluster with Monitoring and Shield configured with SSL. Once started,
+ * an HTTP exporter is activated and it exports data locally over HTTPS/SSL. The test
+ * then uses a transport client to check that the data have been correctly received and
+ * indexed in the cluster.
+ */
+public class SmokeTestMonitoringWithShieldIT extends ESIntegTestCase {
+
+    private static final String USER = "test_user";
+    private static final String PASS = "changeme";
+    private static final String KEYSTORE_PASS = "keypass";
+    
+    private static final String MONITORING_PATTERN = ".monitoring-*";
+
+    @Override
+    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
+        return Collections.singletonList(XPackPlugin.class);
+    }
+
+    @Override
+    protected Settings externalClusterClientSettings() {
+        return Settings.builder()
+                .put("shield.user", USER + ":" + PASS)
+                .put(ShieldNettyTransport.TRANSPORT_SSL_SETTING, true)
+                .put("shield.ssl.keystore.path", clientKeyStore)
+                .put("shield.ssl.keystore.password", KEYSTORE_PASS)
+                .build();
+    }
+
+    @Before
+    public void enableExporter() throws Exception {
+        InetSocketAddress httpAddress = randomFrom(httpAddresses());
+        URI uri = new URI("https", null, httpAddress.getHostString(), httpAddress.getPort(), "/", null, null);
+
+        Settings exporterSettings = Settings.builder()
+                .put("xpack.monitoring.agent.exporters._http.enabled", true)
+                .put("xpack.monitoring.agent.exporters._http.host", uri.toString())
+                .build();
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(exporterSettings));
+    }
+
+    @After
+    public void disableExporter() {
+        Settings exporterSettings = Settings.builder()
+                .putNull("xpack.monitoring.agent.exporters._http.enabled")
+                .putNull("xpack.monitoring.agent.exporters._http.host")
+                .build();
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(exporterSettings));
+    }
+
+    public void testHTTPExporterWithSSL() throws Exception {
+        // Checks that the monitoring index templates have been installed
+        assertBusy(() -> {
+            GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates(MONITORING_PATTERN).get();
+            assertThat(response.getIndexTemplates().size(), equalTo(2));
+        });
+
+        // Checks that the HTTP exporter has successfully exported some data
+        assertBusy(() -> {
+            try {
+                assertThat(client().prepareSearch(MONITORING_PATTERN).setSize(0).get().getHits().getTotalHits(), greaterThan(0L));
+            } catch (Exception e) {
+                fail("exception when checking for monitoring documents: " + e.getMessage());
+            }
+        });
+    }
+
+    private InetSocketAddress[] httpAddresses() {
+        NodeInfo[] nodes = client().admin().cluster().prepareNodesInfo().clear().setHttp(true).get().getNodes();
+        assertThat(nodes.length, greaterThan(0));
+
+        InetSocketAddress[] httpAddresses = new InetSocketAddress[nodes.length];
+        for (int i = 0; i < nodes.length; i++) {
+            httpAddresses[i] = ((InetSocketTransportAddress) nodes[i].getHttp().address().publishAddress()).address();
+        }
+        return httpAddresses;
+    }
+
+    static Path clientKeyStore;
+
+    @BeforeClass
+    public static void loadKeyStore() {
+        try {
+            clientKeyStore = PathUtils.get(SmokeTestMonitoringWithShieldIT.class.getResource("/test-client.jks").toURI());
+        } catch (URISyntaxException e) {
+            throw new ElasticsearchException("exception while reading the store", e);
+        }
+        if (!Files.exists(clientKeyStore)) {
+            throw new IllegalStateException("Keystore file [" + clientKeyStore + "] does not exist.");
+        }
+    }
+
+    @AfterClass
+    public static void clearClientKeyStore() {
+        clientKeyStore = null;
+    }
+}

elasticsearch/x-pack/shield/config/xpack/roles.yml

@@ -55,7 +55,7 @@ logstash:
 # Monitoring user role. Assign to monitoring users.
 monitoring_user:
   indices:
-    '.monitoring-es-*':
+    '.monitoring-*':
       privileges: read
     '.kibana':
       privileges: indices:admin/exists, indices:admin/mappings/fields/get, indices:admin/validate/query, indices:data/read/get, indices:data/read/mget, indices:data/read/search
@@ -65,7 +65,7 @@ monitoring_user:
 remote_monitoring_agent:
   cluster: indices:admin/template/put, indices:admin/template/get
   indices:
-    '.monitoring-es-*':
+    '.monitoring-*':
       privileges: all
 
 # Allows all operations required to manage ingest pipelines

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/ShieldTemplateService.java

@@ -54,7 +54,7 @@ public class ShieldTemplateService extends AbstractComponent implements ClusterS
             ByteArrayOutputStream out = new ByteArrayOutputStream();
             Streams.copy(is, out);
             final byte[] template = out.toByteArray();
-            logger.info("--> putting the shield index template");
+            logger.debug("putting the shield index template");
             PutIndexTemplateRequest putTemplateRequest = client.admin().indices()
                     .preparePutTemplate(SECURITY_TEMPLATE_NAME).setSource(template).request();
             PutIndexTemplateResponse templateResponse = client.admin().indices().putTemplate(putTemplateRequest).get();