honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Marvel: Add integration test for Marvel+Shield with SSL 

closes elastic/elasticsearch#1467

Original commit: elastic/x-pack-elasticsearch@9dd6bf9629
This commit is contained in:
Tanguy Leroux 2016-02-08 14:33:27 +01:00
parent a8e52eb520
commit 66e49a0546
4 changed files with 288 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

165
elasticsearch/qa/smoke-test-plugins-ssl/build.gradle
View File

@ -7,22 +7,28 @@ dependencies {
testCompile project(path: ':x-plugins:elasticsearch:x-pack', configuration: 'runtime')
}
// location of keystore and files to generate it
File keystore = new File(project.buildDir, 'keystore/test-node.jks')
// needed to be consistent with ssl host checking
String san = getSubjectAlternativeNameString()
// generate the keystore
task createKey(type: LoggedExec) {
// location of generated keystores and certificates
File keystoreDir = new File(project.buildDir, 'keystore')
// Generate the node's keystore
File nodeKeystore = new File(keystoreDir, 'test-node.jks')
task createNodeKeyStore(type: LoggedExec) {
doFirst {
project.delete(keystore.parentFile)
keystore.parentFile.mkdirs()
if (nodeKeystore.parentFile.exists() == false) {
nodeKeystore.parentFile.mkdirs()
}
if (nodeKeystore.exists()) {
delete nodeKeystore
}
}
// needed to be consistent with ssl host checking
String san = getSubjectAlternativeNameString()
executable = 'keytool'
standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
args '-genkey',
'-alias', 'test-node',
'-keystore', keystore,
'-keystore', nodeKeystore,
'-keyalg', 'RSA',
'-keysize', '2048',
'-validity', '712',
@ -32,9 +38,103 @@ task createKey(type: LoggedExec) {
'-ext', san
}
// add keystore to test classpath: it expects it there
sourceSets.test.resources.srcDir(keystore.parentFile)
processTestResources.dependsOn(createKey)
// Generate the client's keystore
File clientKeyStore = new File(keystoreDir, 'test-client.jks')
task createClientKeyStore(type: LoggedExec) {
doFirst {
if (clientKeyStore.parentFile.exists() == false) {
clientKeyStore.parentFile.mkdirs()
}
if (clientKeyStore.exists()) {
delete clientKeyStore
}
}
executable = 'keytool'
standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
args '-genkey',
'-alias', 'test-client',
'-keystore', clientKeyStore,
'-keyalg', 'RSA',
'-keysize', '2048',
'-validity', '712',
'-dname', 'CN=smoke-test-plugins-ssl',
'-keypass', 'keypass',
'-storepass', 'keypass',
'-ext', san
}
// Export the node's certificate
File nodeCertificate = new File(keystoreDir, 'test-node.cert')
task exportNodeCertificate(type: LoggedExec) {
doFirst {
if (nodeCertificate.parentFile.exists() == false) {
nodeCertificate.parentFile.mkdirs()
}
if (nodeCertificate.exists()) {
delete nodeCertificate
}
}
executable = 'keytool'
args '-export',
'-alias', 'test-node',
'-keystore', nodeKeystore,
'-storepass', 'keypass',
'-file', nodeCertificate
}
// Import the node certificate in the client's keystore
task importNodeCertificateInClientKeyStore(type: LoggedExec) {
dependsOn exportNodeCertificate
executable = 'keytool'
args '-import',
'-alias', 'test-node',
'-keystore', clientKeyStore,
'-storepass', 'keypass',
'-file', nodeCertificate,
'-noprompt'
}
// Export the client's certificate
File clientCertificate = new File(keystoreDir, 'test-client.cert')
task exportClientCertificate(type: LoggedExec) {
doFirst {
if (clientCertificate.parentFile.exists() == false) {
clientCertificate.parentFile.mkdirs()
}
if (clientCertificate.exists()) {
delete clientCertificate
}
}
executable = 'keytool'
args '-export',
'-alias', 'test-client',
'-keystore', clientKeyStore,
'-storepass', 'keypass',
'-file', clientCertificate
}
// Import the client certificate in the node's keystore
task importClientCertificateInNodeKeyStore(type: LoggedExec) {
dependsOn exportClientCertificate
executable = 'keytool'
args '-import',
'-alias', 'test-client',
'-keystore', nodeKeystore,
'-storepass', 'keypass',
'-file', clientCertificate,
'-noprompt'
}
forbiddenPatterns {
exclude '**/*.cert'
}
// Add keystores to test classpath: it expects it there
sourceSets.test.resources.srcDir(keystoreDir)
processTestResources.dependsOn(
createNodeKeyStore, createClientKeyStore,
importNodeCertificateInClientKeyStore, importClientCertificateInNodeKeyStore
)
ext.pluginsCount = 1 // we install xpack explicitly
project.rootProject.subprojects.findAll { it.path.startsWith(':plugins:') }.each { subproj ->
@ -45,25 +145,46 @@ project.rootProject.subprojects.findAll { it.path.startsWith(':plugins:') }.each
integTest {
cluster {
systemProperty 'es.xpack.monitoring.agent.exporters.es.type', 'http'
systemProperty 'es.xpack.monitoring.agent.exporters.es.enabled', 'false'
systemProperty 'es.xpack.monitoring.agent.exporters.es.ssl.truststore.path', keystore.name
systemProperty 'es.xpack.monitoring.agent.exporters.es.ssl.truststore.password', 'keypass'
systemProperty 'es.xpack.monitoring.agent.interval', '3s'
systemProperty 'es.xpack.monitoring.agent.exporters._http.type', 'http'
systemProperty 'es.xpack.monitoring.agent.exporters._http.enabled', 'false'
systemProperty 'es.xpack.monitoring.agent.exporters._http.ssl.truststore.path', clientKeyStore.name
systemProperty 'es.xpack.monitoring.agent.exporters._http.ssl.truststore.password', 'keypass'
systemProperty 'es.xpack.monitoring.agent.exporters._http.auth.username', 'monitoring_agent'
systemProperty 'es.xpack.monitoring.agent.exporters._http.auth.password', 'changeme'
systemProperty 'es.shield.transport.ssl', 'true'
systemProperty 'es.shield.http.ssl', 'true'
systemProperty 'es.shield.ssl.keystore.path', keystore.name
systemProperty 'es.shield.ssl.keystore.path', nodeKeystore.name
systemProperty 'es.shield.ssl.keystore.password', 'keypass'
plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack')
// copy keystore into config/
extraConfigFile keystore.name, keystore
// copy keystores into config/
extraConfigFile nodeKeystore.name, nodeKeystore
extraConfigFile clientKeyStore.name, clientKeyStore
setupCommand 'setupTestUser',
'bin/xpack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
setupCommand 'setupMarvelUser',
'bin/xpack/esusers', 'useradd', 'marvel_export', '-p', 'changeme', '-r', 'marvel_agent'
'bin/xpack/esusers', 'useradd', 'monitoring_agent', '-p', 'changeme', '-r', 'remote_monitoring_agent'
waitCondition = { node, ant ->
// we just return true, doing an https check is tricky here
return true
// HTTPS check is tricky to do, so we wait for the log file to indicate that the node is started
String waitForNodeStartProp = "waitForNodeStart${name}"
ant.waitfor(maxwait: '10', maxwaitunit: 'second', checkevery: '100', checkeveryunit: 'millisecond',
timeoutproperty: waitForNodeStartProp) {
and {
resourcecontains(resource: "${node.startLog.toString()}", substring: "bound_addresses {${node.httpUri()}")
resourcecontains(resource: "${node.startLog.toString()}", substring: 'started')
}
}
if (ant.project.getProperty(waitForNodeStartProp)) {
println "Timed out when looking for bound_addresses in log file ${node.startLog.toString()}"
return false;
}
return true;
}
}
}

134
elasticsearch/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithShieldIT.java Normal file
View File

@ -0,0 +1,134 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.smoketest;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
import org.elasticsearch.common.io.PathUtils;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.shield.transport.netty.ShieldNettyTransport;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.xpack.XPackPlugin;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collection;
import java.util.Collections;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
/**
* This test checks that a Monitoring's HTTP exporter correctly exports to a monitoring cluster
* protected by Shield with HTTPS/SSL.
*
* It sets up a cluster with Monitoring and Shield configured with SSL. Once started,
* an HTTP exporter is activated and it exports data locally over HTTPS/SSL. The test
* then uses a transport client to check that the data have been correctly received and
* indexed in the cluster.
*/
public class SmokeTestMonitoringWithShieldIT extends ESIntegTestCase {
private static final String USER = "test_user";
private static final String PASS = "changeme";
private static final String KEYSTORE_PASS = "keypass";
private static final String MONITORING_PATTERN = ".monitoring-*";
@Override
protected Collection<Class<? extends Plugin>> transportClientPlugins() {
return Collections.singletonList(XPackPlugin.class);
}
@Override
protected Settings externalClusterClientSettings() {
return Settings.builder()
.put("shield.user", USER + ":" + PASS)
.put(ShieldNettyTransport.TRANSPORT_SSL_SETTING, true)
.put("shield.ssl.keystore.path", clientKeyStore)
.put("shield.ssl.keystore.password", KEYSTORE_PASS)
.build();
}
@Before
public void enableExporter() throws Exception {
InetSocketAddress httpAddress = randomFrom(httpAddresses());
URI uri = new URI("https", null, httpAddress.getHostString(), httpAddress.getPort(), "/", null, null);
Settings exporterSettings = Settings.builder()
.put("xpack.monitoring.agent.exporters._http.enabled", true)
.put("xpack.monitoring.agent.exporters._http.host", uri.toString())
.build();
assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(exporterSettings));
}
@After
public void disableExporter() {
Settings exporterSettings = Settings.builder()
.putNull("xpack.monitoring.agent.exporters._http.enabled")
.putNull("xpack.monitoring.agent.exporters._http.host")
.build();
assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(exporterSettings));
}
public void testHTTPExporterWithSSL() throws Exception {
// Checks that the monitoring index templates have been installed
assertBusy(() -> {
GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates(MONITORING_PATTERN).get();
assertThat(response.getIndexTemplates().size(), equalTo(2));
});
// Checks that the HTTP exporter has successfully exported some data
assertBusy(() -> {
try {
assertThat(client().prepareSearch(MONITORING_PATTERN).setSize(0).get().getHits().getTotalHits(), greaterThan(0L));
} catch (Exception e) {
fail("exception when checking for monitoring documents: " + e.getMessage());
}
});
}
private InetSocketAddress[] httpAddresses() {
NodeInfo[] nodes = client().admin().cluster().prepareNodesInfo().clear().setHttp(true).get().getNodes();
assertThat(nodes.length, greaterThan(0));
InetSocketAddress[] httpAddresses = new InetSocketAddress[nodes.length];
for (int i = 0; i < nodes.length; i++) {
httpAddresses[i] = ((InetSocketTransportAddress) nodes[i].getHttp().address().publishAddress()).address();
}
return httpAddresses;
}
static Path clientKeyStore;
@BeforeClass
public static void loadKeyStore() {
try {
clientKeyStore = PathUtils.get(SmokeTestMonitoringWithShieldIT.class.getResource("/test-client.jks").toURI());
} catch (URISyntaxException e) {
throw new ElasticsearchException("exception while reading the store", e);
}
if (!Files.exists(clientKeyStore)) {
throw new IllegalStateException("Keystore file [" + clientKeyStore + "] does not exist.");
}
}
@AfterClass
public static void clearClientKeyStore() {
clientKeyStore = null;
}
}

4
elasticsearch/x-pack/shield/config/xpack/roles.yml
View File

@ -55,7 +55,7 @@ logstash:
# Monitoring user role. Assign to monitoring users.
monitoring_user:
indices:
'.monitoring-es-*':
'.monitoring-*':
privileges: read
'.kibana':
privileges: indices:admin/exists, indices:admin/mappings/fields/get, indices:admin/validate/query, indices:data/read/get, indices:data/read/mget, indices:data/read/search
@ -65,7 +65,7 @@ monitoring_user:
remote_monitoring_agent:
cluster: indices:admin/template/put, indices:admin/template/get
indices:
'.monitoring-es-*':
'.monitoring-*':
privileges: all
# Allows all operations required to manage ingest pipelines

2
elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/ShieldTemplateService.java
View File

@ -54,7 +54,7 @@ public class ShieldTemplateService extends AbstractComponent implements ClusterS
ByteArrayOutputStream out = new ByteArrayOutputStream();
Streams.copy(is, out);
final byte[] template = out.toByteArray();
logger.info("--> putting the shield index template");
logger.debug("putting the shield index template");
PutIndexTemplateRequest putTemplateRequest = client.admin().indices()
.preparePutTemplate(SECURITY_TEMPLATE_NAME).setSource(template).request();
PutIndexTemplateResponse templateResponse = client.admin().indices().putTemplate(putTemplateRequest).get();