From 706a8fd38db637edac2a74eca7f1b5108853d59a Mon Sep 17 00:00:00 2001 From: javanna Date: Mon, 6 Oct 2014 16:56:57 +0200 Subject: [PATCH] [TEST] move integration tests to use the global cluster and run against multiple nodes Every test class was previously running against its own SUITE cluster composed of a single node due to misconfiguration. Also there were some repetitions and bugs in the settings: first of all unicast wasn't properly configured, also the plugin wasn't registered properly in the transport client, thus the "shield.user" settings wasn't properly converted into the basic auth header. For the very same reason the settings used for authc wasn't randomized for transport client. Extracted out all the needed configuration to the `ShieldSettingsSource` class, that takes care of the unicast configuration, loading of the plugin and all of the configuration files and parameters. Used the global cluster whenever possible, that has the following characteristics: - unicast discovery - ssl configured and enabled at the transport level - ssl configured but disabled at the http level (REST tests use the same cluster and don't support SSL at this time) - single user configured with an allow_all role - auditing enabled or not is randomized - the setting used to do basic auth is randomized between reuest.headers.Authorization and our own shield.user for both node and transport client Test classes that need to override defaults settings can do so by declaring scope=SUITE and overriding the nodeSettings method. Also roles, users and users_roles have specialized methods to be overridden that just return the content of the whole file if it differs from the default. Note that given that ssl is properly configured although disabled for http, tests that need it on can just enable it without any additional configuration. Closes elastic/elasticsearch#31 Original commit: elastic/x-pack-elasticsearch@fa6f1624976595a333cf3b0cba5ed779b254d339 --- .../MultipleIndicesPermissionsTests.java | 68 +++-- .../PermissionPrecedenceTests.java | 143 +++++----- .../integration/ScrollIdSigningTests.java | 35 +-- .../shield/ShieldPluginTests.java | 21 +- .../CachingUsernamePasswordRealmTests.java | 2 +- .../IndicesResolverIntegrationTests.java | 12 +- .../shield/test/ShieldIntegrationTest.java | 205 --------------- .../n2n/IpFilteringIntegrationTests.java | 14 +- .../transport/ssl/SslIntegrationTests.java | 101 +------- .../transport/ssl/SslMultiPortTests.java | 61 ++--- .../test/ShieldIntegrationTest.java | 245 ++++++++++++++++++ .../elasticsearch/test/ShieldRestTests.java | 202 +++------------ .../test/ShieldSettingsSource.java | 212 +++++++++++++++ .../ClusterDiscoveryConfiguration.java | 145 +++++++++++ .../transport/KnownActionsTests.java | 2 +- .../org/elasticsearch/transport/handlers | 7 +- 16 files changed, 806 insertions(+), 669 deletions(-) delete mode 100644 src/test/java/org/elasticsearch/shield/test/ShieldIntegrationTest.java create mode 100644 src/test/java/org/elasticsearch/test/ShieldIntegrationTest.java create mode 100644 src/test/java/org/elasticsearch/test/ShieldSettingsSource.java create mode 100644 src/test/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java diff --git a/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java b/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java index 096c3447c3a..32b7d792808 100644 --- a/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java +++ b/src/test/java/org/elasticsearch/integration/MultipleIndicesPermissionsTests.java @@ -12,62 +12,54 @@ import org.elasticsearch.client.Client; import org.elasticsearch.shield.authc.support.SecuredStringTests; import org.elasticsearch.shield.authc.support.UsernamePasswordToken; import org.elasticsearch.shield.authz.AuthorizationException; -import org.elasticsearch.shield.test.ShieldIntegrationTest; +import org.elasticsearch.test.ShieldIntegrationTest; +import org.elasticsearch.test.ShieldSettingsSource; import org.junit.Test; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.indicesQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER; +import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; +import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.is; -/** - * - */ +@ClusterScope(scope = Scope.SUITE) public class MultipleIndicesPermissionsTests extends ShieldIntegrationTest { - public static final String ROLES = - DEFAULT_ROLE + ":\n" + - " cluster: all\n" + - " indices:\n" + - " '*': manage\n" + - " '/.*/': write\n" + - " 'test': read\n" + - " 'test1': read\n" + - "\n" + - "role_a:\n" + - " indices:\n" + - " 'a': all\n" + - "\n" + - "role_b:\n" + - " indices:\n" + - " 'b': all\n"; - - public static final String USERS = - CONFIG_STANDARD_USER + - "user_a:{plain}passwd\n" + - "user_b:{plain}passwd\n"; - - public static final String USERS_ROLES = - CONFIG_STANDARD_USER_ROLES + - "role_a:user_a,user_b\n" + - "role_b:user_b\n"; - @Override - protected String configRole() { - return ROLES; + protected String configRoles() { + return ShieldSettingsSource.DEFAULT_ROLE + ":\n" + + " cluster: all\n" + + " indices:\n" + + " '*': manage\n" + + " '/.*/': write\n" + + " 'test': read\n" + + " 'test1': read\n" + + "\n" + + "role_a:\n" + + " indices:\n" + + " 'a': all\n" + + "\n" + + "role_b:\n" + + " indices:\n" + + " 'b': all\n"; } @Override protected String configUsers() { - return USERS; + return ShieldSettingsSource.CONFIG_STANDARD_USER + + "user_a:{plain}passwd\n" + + "user_ab:{plain}passwd\n"; } @Override protected String configUsersRoles() { - return USERS_ROLES; + return ShieldSettingsSource.CONFIG_STANDARD_USER_ROLES + + "role_a:user_a,user_ab\n" + + "role_b:user_ab\n"; } @Test @@ -191,14 +183,14 @@ public class MultipleIndicesPermissionsTests extends ShieldIntegrationTest { } response = client.prepareSearch("b") - .putHeader(BASIC_AUTH_HEADER, userHeader("user_b", "passwd")) + .putHeader(BASIC_AUTH_HEADER, userHeader("user_ab", "passwd")) .get(); assertNoFailures(response); assertHitCount(response, 1); indices = randomBoolean() ? new String[] { "a", "b" } : new String[] { "b", "a" }; response = client.prepareSearch(indices) - .putHeader(BASIC_AUTH_HEADER, userHeader("user_b", "passwd")) + .putHeader(BASIC_AUTH_HEADER, userHeader("user_ab", "passwd")) .get(); assertNoFailures(response); assertHitCount(response, 2); @@ -208,7 +200,7 @@ public class MultipleIndicesPermissionsTests extends ShieldIntegrationTest { new String[] { "*" } : new String[] {}; response = client.prepareSearch(indices) - .putHeader(BASIC_AUTH_HEADER, userHeader("user_b", "passwd")) + .putHeader(BASIC_AUTH_HEADER, userHeader("user_ab", "passwd")) .get(); assertNoFailures(response); assertHitCount(response, 2); diff --git a/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java b/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java index 7ed865f9202..81f35e94690 100644 --- a/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java +++ b/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java @@ -7,130 +7,121 @@ package org.elasticsearch.integration; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse; -import org.elasticsearch.client.transport.TransportClient; -import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.shield.authc.support.SecuredString; import org.elasticsearch.shield.authc.support.SecuredStringTests; +import org.elasticsearch.shield.authc.support.UsernamePasswordToken; import org.elasticsearch.shield.authz.AuthorizationException; -import org.elasticsearch.shield.test.ShieldIntegrationTest; +import org.elasticsearch.test.ShieldIntegrationTest; import org.junit.Test; import java.util.List; import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; +import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.hasSize; /** * This test makes sure that if an action is a cluster action (according to our - * internal categorization in shield, then we apply the cluster priv checks and don't + * internal categorization in shield), then we apply the cluster priv checks and don't * fallback on the indices privs at all. In particular, this is useful when we want to treat * actions that are normally categorized as index actions as cluster actions - for example, * index template actions. */ +@ClusterScope(scope = Scope.SUITE) public class PermissionPrecedenceTests extends ShieldIntegrationTest { - static final String ROLES = - "admin:\n" + - " cluster: all\n" + - " indices:\n" + - " '*': all\n" + - "\n" + - "transport_client:\n" + - " cluster:\n" + - " - cluster:monitor/nodes/info\n" + - " - cluster:monitor/state\n" + - "\n" + - "user:\n" + - " indices:\n" + - " 'test_*': all\n"; - - static final String USERS = - "admin:{plain}test123\n" + - "client:{plain}test123\n" + - "user:{plain}test123\n"; - - static final String USERS_ROLES = - "admin:admin\n" + - "transport_client:client\n" + - "user:user\n"; - @Override - protected String configRole() { - return ROLES; + protected String configRoles() { + return "admin:\n" + + " cluster: all\n" + + " indices:\n" + + " '*': all\n" + + "\n" + + "transport_client:\n" + + " cluster:\n" + + " - cluster:monitor/nodes/info\n" + + " - cluster:monitor/state\n" + + "\n" + + "user:\n" + + " indices:\n" + + " 'test_*': all\n"; } @Override protected String configUsers() { - return USERS; + return "admin:{plain}test123\n" + + "client:{plain}test123\n" + + "user:{plain}test123\n"; } @Override protected String configUsersRoles() { - return USERS_ROLES; + return "admin:admin\n" + + "transport_client:client\n" + + "user:user\n"; } @Override - protected String getClientUsername() { + protected String nodeClientUsername() { return "admin"; } @Override - protected SecuredString getClientPassword() { - return SecuredStringTests.build("test123"); + protected SecuredString nodeClientPassword() { + return new SecuredString("test123".toCharArray()); + } + + @Override + protected String transportClientUsername() { + return "admin"; + } + + @Override + protected SecuredString transportClientPassword() { + return new SecuredString("test123".toCharArray()); } @Test - public void testDifferetCombinationsOfIndices() throws Exception { + public void testDifferentCombinationsOfIndices() throws Exception { - ClusterService clusterService = internalCluster().getInstance(ClusterService.class); - TransportAddress address = clusterService.localNode().address(); + Client client = internalCluster().transportClient(); - try (TransportClient client = new TransportClient(ImmutableSettings.builder() - .put("shield.user", "client:test123") - .put("cluster.name", internalCluster().getClusterName()) - .put("node.mode", "network") - .put(getSSLSettingsForStore("/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.jks", "testclient"))) - .addTransportAddress(address)) { + // first lets try with "admin"... all should work + PutIndexTemplateResponse putResponse = client.admin().indices().preparePutTemplate("template1") + .setTemplate("test_*") + .putHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, basicAuthHeaderValue(transportClientUsername(), transportClientPassword())) + .get(); + assertAcked(putResponse); - // first lets try with "admin"... all should work + GetIndexTemplatesResponse getResponse = client.admin().indices().prepareGetTemplates("template1") + .get(); + List templates = getResponse.getIndexTemplates(); + assertThat(templates, hasSize(1)); - PutIndexTemplateResponse putResponse = client.admin().indices().preparePutTemplate("template1") + // now lets try with "user" + + try { + client.admin().indices().preparePutTemplate("template1") .setTemplate("test_*") - .putHeader("Authorization", basicAuthHeaderValue("admin", SecuredStringTests.build("test123"))) + .putHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, basicAuthHeaderValue("user", transportClientPassword())) .get(); - assertAcked(putResponse); + fail("expected an authorization exception as template APIs should require cluster ALL permission"); + } catch (AuthorizationException ae) { + // expected; + } - GetIndexTemplatesResponse getResponse = client.admin().indices().prepareGetTemplates("template1") - .putHeader("Authorization", basicAuthHeaderValue("admin", SecuredStringTests.build("test123"))) + try { + client.admin().indices().prepareGetTemplates("template1") + .putHeader("Authorization", basicAuthHeaderValue("user", SecuredStringTests.build("test123"))) .get(); - List templates = getResponse.getIndexTemplates(); - assertThat(templates, hasSize(1)); - - // now lets try with "user" - - try { - client.admin().indices().preparePutTemplate("template1") - .setTemplate("test_*") - .putHeader("Authorization", basicAuthHeaderValue("user", SecuredStringTests.build("test123"))) - .get(); - fail("expected an authorization exception as template APIs should require cluster ALL permission"); - } catch (AuthorizationException ae) { - // expected; - } - - try { - client.admin().indices().prepareGetTemplates("template1") - .putHeader("Authorization", basicAuthHeaderValue("user", SecuredStringTests.build("test123"))) - .get(); - fail("expected an authorization exception as template APIs should require cluster ALL permission"); - } catch (AuthorizationException ae) { - // expected - } + fail("expected an authorization exception as template APIs should require cluster ALL permission"); + } catch (AuthorizationException ae) { + // expected } } } diff --git a/src/test/java/org/elasticsearch/integration/ScrollIdSigningTests.java b/src/test/java/org/elasticsearch/integration/ScrollIdSigningTests.java index 065e8f3f435..c73ae4ed349 100644 --- a/src/test/java/org/elasticsearch/integration/ScrollIdSigningTests.java +++ b/src/test/java/org/elasticsearch/integration/ScrollIdSigningTests.java @@ -8,21 +8,16 @@ package org.elasticsearch.integration; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.shield.authz.AuthorizationException; -import org.elasticsearch.shield.signature.InternalSignatureService; import org.elasticsearch.shield.signature.SignatureService; -import org.elasticsearch.shield.test.ShieldIntegrationTest; +import org.elasticsearch.test.ShieldIntegrationTest; import org.junit.Before; import org.junit.Test; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.*; /** * @@ -31,14 +26,6 @@ public class ScrollIdSigningTests extends ShieldIntegrationTest { private SignatureService signatureService; - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return ImmutableSettings.builder() - .put(super.nodeSettings(nodeOrdinal)) - .put(InternalSignatureService.FILE_SETTING, writeFile(newFolder(), "system_key", generateKey())) - .build(); - } - @Before public void init() throws Exception { signatureService = internalCluster().getInstance(SignatureService.class); @@ -56,17 +43,14 @@ public class ScrollIdSigningTests extends ShieldIntegrationTest { .setScroll(TimeValue.timeValueMinutes(2)) .setSize(randomIntBetween(1, 10)).get(); - assertHitCount(response, docs.length); - int hits = response.getHits().hits().length; - + int hits = 0; try { - assertSigned(response.getScrollId()); while (true) { - response = client().prepareSearchScroll(response.getScrollId()) - .setScroll(TimeValue.timeValueMinutes(2)).get(); assertSigned(response.getScrollId()); assertHitCount(response, docs.length); hits += response.getHits().hits().length; + response = client().prepareSearchScroll(response.getScrollId()) + .setScroll(TimeValue.timeValueMinutes(2)).get(); if (response.getHits().getHits().length == 0) { break; } @@ -126,13 +110,4 @@ public class ScrollIdSigningTests extends ShieldIntegrationTest { private void assertSigned(String scrollId) { assertThat(signatureService.signed(scrollId), is(true)); } - - private static byte[] generateKey() { - try { - return InternalSignatureService.generateKey(); - } catch (Exception e) { - fail("failed to generate key"); - return null; - } - } } diff --git a/src/test/java/org/elasticsearch/shield/ShieldPluginTests.java b/src/test/java/org/elasticsearch/shield/ShieldPluginTests.java index 7fa6f4049b9..cac7e949a01 100644 --- a/src/test/java/org/elasticsearch/shield/ShieldPluginTests.java +++ b/src/test/java/org/elasticsearch/shield/ShieldPluginTests.java @@ -9,19 +9,17 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.http.HttpServerTransport; -import org.elasticsearch.node.internal.InternalNode; -import org.elasticsearch.shield.ShieldPlugin; import org.elasticsearch.shield.authc.support.SecuredString; -import org.elasticsearch.shield.test.ShieldIntegrationTest; +import org.elasticsearch.shield.authc.support.UsernamePasswordToken; +import org.elasticsearch.test.ShieldIntegrationTest; +import org.elasticsearch.test.ShieldSettingsSource; import org.elasticsearch.test.rest.client.http.HttpRequestBuilder; import org.elasticsearch.test.rest.client.http.HttpResponse; import org.junit.Test; import java.io.IOException; -import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; import static org.elasticsearch.rest.RestStatus.OK; import static org.elasticsearch.rest.RestStatus.UNAUTHORIZED; import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue; @@ -29,16 +27,6 @@ import static org.hamcrest.Matchers.*; public class ShieldPluginTests extends ShieldIntegrationTest { - @Override - protected Settings nodeSettings(int nodeOrdinal) { - return settingsBuilder() - .put(super.nodeSettings(nodeOrdinal)) - .put(InternalNode.HTTP_ENABLED, true) - .put("shield.http.ssl", false) - .put("shield.audit.enabled", false) - .build(); - } - @Test public void testThatPluginIsLoaded() throws IOException { logger.info("--> Getting nodes info"); @@ -56,7 +44,8 @@ public class ShieldPluginTests extends ShieldIntegrationTest { assertThat(response.getStatusCode(), is(UNAUTHORIZED.getStatus())); logger.info("Executing authorized request to /_shield infos"); - response = new HttpRequestBuilder(httpClient).httpTransport(httpServerTransport).method("GET").path("/_shield").addHeader("Authorization", basicAuthHeaderValue(DEFAULT_USER_NAME, new SecuredString(DEFAULT_PASSWORD.toCharArray()))).execute(); + response = new HttpRequestBuilder(httpClient).httpTransport(httpServerTransport).method("GET").path("/_shield").addHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, + basicAuthHeaderValue(ShieldSettingsSource.DEFAULT_USER_NAME, new SecuredString(ShieldSettingsSource.DEFAULT_PASSWORD.toCharArray()))).execute(); assertThat(response.getStatusCode(), is(OK.getStatus())); assertThat(response.getBody(), allOf(containsString("build_hash"), containsString("build_timestamp"), containsString("build_version"))); } diff --git a/src/test/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealmTests.java b/src/test/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealmTests.java index d3908c4bce7..924386f4d36 100644 --- a/src/test/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealmTests.java +++ b/src/test/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealmTests.java @@ -55,7 +55,7 @@ public class CachingUsernamePasswordRealmTests extends ElasticsearchTestCase { } @Test - public void testAutheticateContract() throws Exception { + public void testAuthenticateContract() throws Exception { Realm realm = new FailingAuthenticationRealm(ImmutableSettings.EMPTY); User user = realm.authenticate(new UsernamePasswordToken("user", SecuredStringTests.build("pass"))); assertThat(user , nullValue()); diff --git a/src/test/java/org/elasticsearch/shield/authz/indicesresolver/IndicesResolverIntegrationTests.java b/src/test/java/org/elasticsearch/shield/authz/indicesresolver/IndicesResolverIntegrationTests.java index d1e0287f30e..c3b39968301 100644 --- a/src/test/java/org/elasticsearch/shield/authz/indicesresolver/IndicesResolverIntegrationTests.java +++ b/src/test/java/org/elasticsearch/shield/authz/indicesresolver/IndicesResolverIntegrationTests.java @@ -14,19 +14,23 @@ import org.elasticsearch.client.Requests; import org.elasticsearch.indices.IndexMissingException; import org.elasticsearch.search.SearchHit; import org.elasticsearch.shield.authz.AuthorizationException; -import org.elasticsearch.shield.test.ShieldIntegrationTest; +import org.elasticsearch.test.ShieldIntegrationTest; +import org.elasticsearch.test.ShieldSettingsSource; import org.junit.Test; import java.util.ArrayList; import java.util.List; +import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; +import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; import static org.hamcrest.CoreMatchers.*; +@ClusterScope(scope = Scope.SUITE) public class IndicesResolverIntegrationTests extends ShieldIntegrationTest { @Override - protected String configRole() { - return DEFAULT_ROLE + ":\n" + + protected String configRoles() { + return ShieldSettingsSource.DEFAULT_ROLE + ":\n" + " cluster: ALL\n" + " indices:\n" + " '*': manage,write\n" + @@ -195,7 +199,7 @@ public class IndicesResolverIntegrationTests extends ShieldIntegrationTest { actionRequestBuilder.get(); fail("search should fail due to attempt to access non authorized indices"); } catch(AuthorizationException e) { - assertThat(e.getMessage(), containsString("is unauthorized for user [test_trans_client_user]")); + assertThat(e.getMessage(), containsString("is unauthorized for user [")); } } diff --git a/src/test/java/org/elasticsearch/shield/test/ShieldIntegrationTest.java b/src/test/java/org/elasticsearch/shield/test/ShieldIntegrationTest.java deleted file mode 100644 index 5e16857d3a5..00000000000 --- a/src/test/java/org/elasticsearch/shield/test/ShieldIntegrationTest.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.shield.test; - -import com.google.common.base.Charsets; -import com.google.common.net.InetAddresses; -import org.apache.lucene.util.AbstractRandomizedTest; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.io.Streams; -import org.elasticsearch.common.os.OsUtils; -import org.elasticsearch.common.settings.ImmutableSettings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.InetSocketTransportAddress; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.plugins.PluginsService; -import org.elasticsearch.shield.ShieldPlugin; -import org.elasticsearch.shield.authc.support.SecuredString; -import org.elasticsearch.shield.transport.netty.NettySecuredTransport; -import org.elasticsearch.test.ElasticsearchIntegrationTest; -import org.elasticsearch.transport.Transport; -import org.elasticsearch.transport.TransportModule; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; - -import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; -import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue; -import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; -import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; - - -@Ignore -@AbstractRandomizedTest.Integration -@ClusterScope(scope = Scope.SUITE, numDataNodes = 1, numClientNodes = 0) -public abstract class ShieldIntegrationTest extends ElasticsearchIntegrationTest { - - protected static final String DEFAULT_USER_NAME = "test_user"; - protected static final String DEFAULT_PASSWORD = "changeme"; - protected static final String DEFAULT_ROLE = "user"; - - protected static final String DEFAULT_TRANSPORT_CLIENT_ROLE = "trans_client_user"; - protected static final String DEFAULT_TRANSPORT_CLIENT_USER_NAME = "test_trans_client_user"; - - public static final String CONFIG_STANDARD_USER = - DEFAULT_USER_NAME + ":{plain}" + DEFAULT_PASSWORD + "\n" + - DEFAULT_TRANSPORT_CLIENT_USER_NAME + ":{plain}" + DEFAULT_PASSWORD + "\n"; - - public static final String CONFIG_STANDARD_USER_ROLES = - DEFAULT_ROLE + ":" + DEFAULT_USER_NAME + "," + DEFAULT_TRANSPORT_CLIENT_USER_NAME + "\n" + - DEFAULT_TRANSPORT_CLIENT_ROLE + ":" + DEFAULT_TRANSPORT_CLIENT_USER_NAME+ "\n"; - - public static final String CONFIG_ROLE_ALLOW_ALL = - DEFAULT_ROLE + ":\n" + - " cluster: ALL\n" + - " indices:\n" + - " '*': ALL\n" + - "transport_client:\n" + - " cluster:\n" + - " - cluster:monitor/nodes/info\n" + - " - cluster:monitor/state"; - - @ClassRule - public static TemporaryFolder tmpFolder = new TemporaryFolder(); - - @Override - protected Settings nodeSettings(int nodeOrdinal) { - File folder = newFolder(); - - ImmutableSettings.Builder builder = ImmutableSettings.builder() - .put("discovery.zen.ping.multicast.enabled", false) - .put("discovery.type", "zen") - .put("node.mode", "network") - .put("plugin.types", ShieldPlugin.class.getName()) - .put("shield.authc.esusers.files.users", writeFile(folder, "users", configUsers())) - .put("shield.authc.esusers.files.users_roles", writeFile(folder, "users_roles", configUsersRoles())) - .put("shield.authz.store.files.roles", writeFile(folder, "roles.yml", configRole())) - .put(getSSLSettingsForStore("/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.jks", "testnode")) - .put("shield.audit.enabled", true) - .put("plugins.load_classpath_plugins", false); - - setUser(builder); - - if (OsUtils.MAC) { - builder.put("network.host", randomBoolean() ? "127.0.0.1" : "::1"); - } - - return builder.build(); - } - - protected String configRole() { - return CONFIG_ROLE_ALLOW_ALL; - } - - protected String configUsers() { - return CONFIG_STANDARD_USER; - } - - protected String configUsersRoles() { - return CONFIG_STANDARD_USER_ROLES; - } - - @Override - protected Settings transportClientSettings() { - ImmutableSettings.Builder builder = ImmutableSettings.builder() - .put("shield.user", getClientUsername() + ":" + getClientPassword()) - .put(TransportModule.TRANSPORT_TYPE_KEY, NettySecuredTransport.class.getName()) - .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, false) - .put("plugin.types", ShieldPlugin.class.getName()) - .put("node.mode", "network") - .put(getSSLSettingsForStore("/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.jks", "testclient")); - - setUser(builder); - - return builder.build(); - } - - protected void setUser(ImmutableSettings.Builder settings) { - if (randomBoolean()) { - settings.put("request.headers.Authorization", basicAuthHeaderValue(getClientUsername(), getClientPassword())); - } else { - settings.put("shield.user", getClientUsername() + ":" + new String(getClientPassword().internalChars())); - } - } - - protected String writeFile(File folder, String name, String content) { - return writeFile(folder, name, content.getBytes(Charsets.UTF_8)); - } - - protected String writeFile(File folder, String name, byte[] content) { - Path file = folder.toPath().resolve(name); - try { - Streams.copy(content, file.toFile()); - } catch (IOException e) { - throw new ElasticsearchException("Error writing file in test", e); - } - return file.toFile().getAbsolutePath(); - } - - protected String getUnicastHostAddress() { - TransportAddress transportAddress = internalCluster().getDataNodeInstance(Transport.class).boundAddress().publishAddress(); - assertThat(transportAddress, instanceOf(InetSocketTransportAddress.class)); - InetSocketTransportAddress address = (InetSocketTransportAddress) transportAddress; - return InetAddresses.toAddrString(address.address().getAddress()) + ":" + address.address().getPort(); - } - - protected String getClientUsername() { - return DEFAULT_TRANSPORT_CLIENT_USER_NAME; - } - - protected SecuredString getClientPassword() { - return new SecuredString(DEFAULT_PASSWORD.toCharArray()); - } - - protected Settings getSSLSettingsForStore(String resourcePathToStore, String password) { - File store; - try { - store = new File(getClass().getResource(resourcePathToStore).toURI()); - assertThat(store.exists(), is(true)); - } catch (Exception e) { - throw new RuntimeException(e); - } - - ImmutableSettings.Builder builder = settingsBuilder() - .put("shield.transport.ssl", true) - .put("shield.ssl.keystore.path", store.getPath()) - .put("shield.ssl.keystore.password", password) - .put("shield.http.ssl", true); - - if (randomBoolean()) { - builder.put("shield.ssl.truststore.path", store.getPath()) - .put("shield.ssl.truststore.password", password); - } - - return builder.build(); - } - - protected File newFolder() { - try { - return tmpFolder.newFolder(); - } catch (IOException ioe) { - logger.error("could not create temporary folder", ioe); - fail("could not create temporary folder"); - return null; - } - } - - protected void assertGreenClusterState(Client client) { - ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth().get(); - assertNoTimeout(clusterHealthResponse); - assertThat(clusterHealthResponse.getStatus(), is(ClusterHealthStatus.GREEN)); - } -} diff --git a/src/test/java/org/elasticsearch/shield/transport/n2n/IpFilteringIntegrationTests.java b/src/test/java/org/elasticsearch/shield/transport/n2n/IpFilteringIntegrationTests.java index bd0e6588b49..72371d91477 100644 --- a/src/test/java/org/elasticsearch/shield/transport/n2n/IpFilteringIntegrationTests.java +++ b/src/test/java/org/elasticsearch/shield/transport/n2n/IpFilteringIntegrationTests.java @@ -6,12 +6,14 @@ package org.elasticsearch.shield.transport.n2n; import com.google.common.base.Charsets; +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.node.internal.InternalNode; -import org.elasticsearch.shield.test.ShieldIntegrationTest; +import org.elasticsearch.test.ShieldIntegrationTest; import org.elasticsearch.transport.Transport; import org.junit.Test; @@ -21,23 +23,21 @@ import java.net.InetSocketAddress; import java.net.Socket; import java.net.SocketException; -import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; -// no client nodes, no transport nodes, as they all get rejected on network connections +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elasticsearch/elasticsearch-shield/issues/378") +// no client nodes, no transport clients, as they all get rejected on network connections @ClusterScope(scope = Scope.SUITE, numDataNodes = 1, numClientNodes = 0, transportClientRatio = 0.0) public class IpFilteringIntegrationTests extends ShieldIntegrationTest { @Override protected Settings nodeSettings(int nodeOrdinal) { - return settingsBuilder() - .put(super.nodeSettings(nodeOrdinal)) + return ImmutableSettings.builder().put(super.nodeSettings(nodeOrdinal)) .put(InternalNode.HTTP_ENABLED, true) - .put("shield.transport.filter.deny", "_all") - .build(); + .put("shield.transport.filter.deny", "_all").build(); } @Test(expected = SocketException.class) diff --git a/src/test/java/org/elasticsearch/shield/transport/ssl/SslIntegrationTests.java b/src/test/java/org/elasticsearch/shield/transport/ssl/SslIntegrationTests.java index 190b380bad0..e6ecaabda7a 100644 --- a/src/test/java/org/elasticsearch/shield/transport/ssl/SslIntegrationTests.java +++ b/src/test/java/org/elasticsearch/shield/transport/ssl/SslIntegrationTests.java @@ -6,7 +6,6 @@ package org.elasticsearch.shield.transport.ssl; import com.google.common.base.Charsets; -import org.elasticsearch.client.Client; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.io.Streams; @@ -16,65 +15,38 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.http.HttpServerTransport; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeBuilder; import org.elasticsearch.node.internal.InternalNode; -import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.shield.authc.support.UsernamePasswordToken; -import org.elasticsearch.shield.test.ShieldIntegrationTest; +import org.elasticsearch.test.ShieldIntegrationTest; import org.elasticsearch.transport.Transport; import org.junit.Test; import javax.net.ssl.*; -import java.io.File; import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.net.URL; import java.util.Locale; import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; -import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue; +import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; +import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; import static org.hamcrest.Matchers.*; +@ClusterScope(scope = Scope.SUITE) public class SslIntegrationTests extends ShieldIntegrationTest { @Override protected Settings nodeSettings(int nodeOrdinal) { return ImmutableSettings.builder().put(super.nodeSettings(nodeOrdinal)) - .put(InternalNode.HTTP_ENABLED, true).build(); - } - - @Test - public void testThatInternallyCreatedTransportClientCanConnect() throws Exception { - Client transportClient = internalCluster().transportClient(); - assertGreenClusterState(transportClient); - } - - @Test - public void testThatProgrammaticallyCreatedTransportClientCanConnect() throws Exception { - Settings settings = settingsBuilder() - .put(transportClientSettings()) - .put("name", "programmatic_transport_client_") - .put("cluster.name", internalCluster().getClusterName()) - .build(); - - try (TransportClient transportClient = new TransportClient(settings, false)) { - TransportAddress transportAddress = internalCluster().getInstance(Transport.class).boundAddress().boundAddress(); - transportClient.addTransportAddress(transportAddress); - assertGreenClusterState(transportClient); - } - - try (TransportClient transportClient = new TransportClient(settings, true)) { - TransportAddress transportAddress = internalCluster().getInstance(Transport.class).boundAddress().boundAddress(); - transportClient.addTransportAddress(transportAddress); - assertGreenClusterState(transportClient); - } + .put(InternalNode.HTTP_ENABLED, true) + .put("shield.http.ssl", true).build(); } // no SSL exception as this is the exception is returned when connecting @Test(expected = NoNodeAvailableException.class) - public void testThatUnconfiguredCipchersAreRejected() { - try (TransportClient transportClient = new TransportClient(settingsBuilder() + public void testThatUnconfiguredCiphersAreRejected() { + try(TransportClient transportClient = new TransportClient(settingsBuilder() + .put(transportClientSettings()) .put("name", "programmatic_transport_client") .put("cluster.name", internalCluster().getClusterName()) @@ -88,59 +60,6 @@ public class SslIntegrationTests extends ShieldIntegrationTest { } } - @Test - public void testConnectNodeWorks() throws Exception { - File folder = newFolder(); - Settings settings = settingsBuilder() - .put("name", "programmatic_node") - .put("cluster.name", internalCluster().getClusterName()) - - .put("shield.authc.esusers.files.users", writeFile(folder, "users", configUsers())) - .put("shield.authc.esusers.files.users_roles", writeFile(folder, "users_roles", configUsersRoles())) - .put("shield.authz.store.files.roles", writeFile(folder, "roles.yml", configRole())) - - .put("request.headers.Authorization", basicAuthHeaderValue(getClientUsername(), getClientPassword())) - .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) - - .put(getSSLSettingsForStore("certs/simple/testclient.jks", "testclient")) - .build(); - - try (Node node = NodeBuilder.nodeBuilder().settings(settings).node()) { - try (Client client = node.client()) { - assertGreenClusterState(client); - } - } - } - - @Test - public void testConnectNodeClientWorks() throws Exception { - File folder = newFolder(); - Settings settings = settingsBuilder() - .put("name", "programmatic_node_client") - .put("cluster.name", internalCluster().getClusterName()) - .put("node.mode", "network") - - .put("shield.authc.esusers.files.users", writeFile(folder, "users", configUsers())) - .put("shield.authc.esusers.files.users_roles", writeFile(folder, "users_roles", configUsersRoles())) - .put("shield.authz.store.files.roles", writeFile(folder, "roles.yml", configRole())) - - .put("discovery.zen.ping.multicast.enabled", false) - .put("discovery.type", "zen") - .putArray("discovery.zen.ping.unicast.hosts", getUnicastHostAddress()) - - .put("request.headers.Authorization", basicAuthHeaderValue(getClientUsername(), getClientPassword())) - .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, true) - - .put(getSSLSettingsForStore("certs/simple/testclient.jks", "testclient")) - .build(); - - try (Node node = NodeBuilder.nodeBuilder().settings(settings).client(true).node()) { - try (Client client = node.client()) { - assertGreenClusterState(client); - } - } - } - @Test public void testThatConnectionToHTTPWorks() throws Exception { TrustManager[] trustAllCerts = new TrustManager[]{ @@ -177,7 +96,7 @@ public class SslIntegrationTests extends ShieldIntegrationTest { String url = String.format(Locale.ROOT, "https://%s:%s/", InetAddresses.toUriString(inetSocketTransportAddress.address().getAddress()), inetSocketTransportAddress.address().getPort()); HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection(); - connection.setRequestProperty("Authorization", UsernamePasswordToken.basicAuthHeaderValue(getClientUsername(), getClientPassword())); + connection.setRequestProperty(UsernamePasswordToken.BASIC_AUTH_HEADER, UsernamePasswordToken.basicAuthHeaderValue(nodeClientUsername(), nodeClientPassword())); connection.connect(); assertThat(connection.getResponseCode(), is(200)); diff --git a/src/test/java/org/elasticsearch/shield/transport/ssl/SslMultiPortTests.java b/src/test/java/org/elasticsearch/shield/transport/ssl/SslMultiPortTests.java index a1ded90407e..35014072665 100644 --- a/src/test/java/org/elasticsearch/shield/transport/ssl/SslMultiPortTests.java +++ b/src/test/java/org/elasticsearch/shield/transport/ssl/SslMultiPortTests.java @@ -11,27 +11,22 @@ import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.plugins.PluginsService; -import org.elasticsearch.shield.test.ShieldIntegrationTest; -import org.elasticsearch.shield.transport.netty.NettySecuredTransport; +import org.elasticsearch.test.ShieldIntegrationTest; +import org.elasticsearch.test.ShieldSettingsSource; import org.elasticsearch.transport.Transport; -import org.elasticsearch.transport.TransportModule; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import java.io.File; import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; -import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue; -import static org.hamcrest.Matchers.is; +import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; +import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; +import static org.hamcrest.CoreMatchers.is; -/** - * - */ +@ClusterScope(scope = Scope.SUITE) public class SslMultiPortTests extends ShieldIntegrationTest { - private ImmutableSettings.Builder builder; private static int randomClientPort; @BeforeClass @@ -39,18 +34,6 @@ public class SslMultiPortTests extends ShieldIntegrationTest { randomClientPort = randomIntBetween(49000, 65500); // ephemeral port } - @Before - public void setupBuilder() { - builder = settingsBuilder() - // we have to set the user with the Authorization header as shield.user only works when - // shield is installed on the client as a plugin - .put("request.headers.Authorization", basicAuthHeaderValue(getClientUsername(), getClientPassword())) - .put(TransportModule.TRANSPORT_TYPE_KEY, NettySecuredTransport.class.getName()) - .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, false) - .put("node.mode", "network") - .put("cluster.name", internalCluster().getClusterName()); - } - @Override protected Settings nodeSettings(int nodeOrdinal) { String randomClientPortRange = randomClientPort + "-" + (randomClientPort+100); @@ -65,33 +48,31 @@ public class SslMultiPortTests extends ShieldIntegrationTest { return settingsBuilder() .put(super.nodeSettings(nodeOrdinal)) - // settings for default key profile - .put(getSSLSettingsForStore("/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.jks", "testnode")) // client set up here .put("transport.profiles.client.port", randomClientPortRange) .put("transport.profiles.client.bind_host", "localhost") // make sure this is "localhost", no matter if ipv4 or ipv6, but be consistent .put("transport.profiles.client.shield.truststore.path", store.getAbsolutePath()) // settings for client truststore .put("transport.profiles.client.shield.truststore.password", "testnode-client-profile") - .put("shield.audit.enabled", false ) - .build(); } + private TransportClient createTransportClient(Settings additionalSettings) { + Settings settings = ImmutableSettings.builder().put(transportClientSettings()) + .put("name", "programmatic_transport_client") + .put("cluster.name", internalCluster().getClusterName()) + .put(additionalSettings) + .build(); + return new TransportClient(settings, false); + } + @Test public void testThatStandardTransportClientCanConnectToDefaultProfile() throws Exception { - builder.put(getSSLSettingsForStore("/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.jks", "testclient")); - try (TransportClient transportClient = new TransportClient(builder, false)) { - TransportAddress transportAddress = internalCluster().getInstance(Transport.class).boundAddress().boundAddress(); - transportClient.addTransportAddress(transportAddress); - assertGreenClusterState(transportClient); - - } + assertGreenClusterState(internalCluster().transportClient()); } @Test(expected = NoNodeAvailableException.class) public void testThatStandardTransportClientCannotConnectToClientProfile() throws Exception { - builder.put(getSSLSettingsForStore("/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.jks", "testclient")); - try (TransportClient transportClient = new TransportClient(builder, false)) { + try(TransportClient transportClient = createTransportClient(ImmutableSettings.EMPTY)) { transportClient.addTransportAddress(new InetSocketTransportAddress("localhost", randomClientPort)); transportClient.admin().cluster().prepareHealth().get(); } @@ -99,8 +80,8 @@ public class SslMultiPortTests extends ShieldIntegrationTest { @Test public void testThatProfileTransportClientCanConnectToClientProfile() throws Exception { - builder.put(getSSLSettingsForStore("/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.jks", "testclient-client-profile")); - try (TransportClient transportClient = new TransportClient(builder, false)) { + Settings settings = ShieldSettingsSource.getSSLSettingsForStore("/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.jks", "testclient-client-profile"); + try (TransportClient transportClient = createTransportClient(settings)) { transportClient.addTransportAddress(new InetSocketTransportAddress("localhost", randomClientPort)); assertGreenClusterState(transportClient); } @@ -108,8 +89,8 @@ public class SslMultiPortTests extends ShieldIntegrationTest { @Test(expected = NoNodeAvailableException.class) public void testThatProfileTransportClientCannotConnectToDefaultProfile() throws Exception { - builder.put(getSSLSettingsForStore("/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.jks", "testclient-client-profile")); - try (TransportClient transportClient = new TransportClient(builder, false)) { + Settings settings = ShieldSettingsSource.getSSLSettingsForStore("/org/elasticsearch/shield/transport/ssl/certs/simple/testclient-client-profile.jks", "testclient-client-profile"); + try (TransportClient transportClient = createTransportClient(settings)) { TransportAddress transportAddress = internalCluster().getInstance(Transport.class).boundAddress().boundAddress(); transportClient.addTransportAddress(transportAddress); transportClient.admin().cluster().prepareHealth().get(); diff --git a/src/test/java/org/elasticsearch/test/ShieldIntegrationTest.java b/src/test/java/org/elasticsearch/test/ShieldIntegrationTest.java new file mode 100644 index 00000000000..533250a1857 --- /dev/null +++ b/src/test/java/org/elasticsearch/test/ShieldIntegrationTest.java @@ -0,0 +1,245 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.test; + +import com.carrotsearch.randomizedtesting.LifecycleScope; +import org.apache.lucene.util.AbstractRandomizedTest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.shield.ShieldPlugin; +import org.elasticsearch.shield.authc.support.SecuredString; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.rules.ExternalResource; + +import java.io.File; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; + +/** + * Base class to run tests against a cluster with shield installed. + * The default {@link org.elasticsearch.test.ElasticsearchIntegrationTest.Scope} is {@link org.elasticsearch.test.ElasticsearchIntegrationTest.Scope#GLOBAL}, + * meaning that all subclasses that don't specify a different scope will share the same cluster with shield installed. + * @see org.elasticsearch.test.ShieldSettingsSource + */ +@Ignore +@AbstractRandomizedTest.Integration +public abstract class ShieldIntegrationTest extends ElasticsearchIntegrationTest { + + private static ShieldSettingsSource SHIELD_DEFAULT_SETTINGS; + + //UnicastZen requires the number of nodes in a cluster to generate the unicast configuration. + //The number of nodes is randomized though, but we can predict what the maximum number of nodes will be + //and configure them all in unicast.hosts + private static int maxNumberOfNodes() { + ClusterScope clusterScope = ShieldIntegrationTest.class.getAnnotation(ClusterScope.class); + if (clusterScope == null || clusterScope.scope() == Scope.GLOBAL) { + return InternalTestCluster.DEFAULT_MAX_NUM_DATA_NODES + InternalTestCluster.DEFAULT_MAX_NUM_CLIENT_NODES; + } else { + if (clusterScope.numClientNodes() < 0) { + return clusterScope.maxNumDataNodes() + InternalTestCluster.DEFAULT_MAX_NUM_CLIENT_NODES; + } else { + return clusterScope.maxNumDataNodes() + clusterScope.numClientNodes(); + } + } + } + + private static ClusterScope getAnnotation(Class clazz) { + if (clazz == Object.class || clazz == ShieldIntegrationTest.class) { + return null; + } + ClusterScope annotation = clazz.getAnnotation(ClusterScope.class); + if (annotation != null) { + return annotation; + } + return getAnnotation(clazz.getSuperclass()); + } + + private Scope getCurrentClusterScope() { + return getCurrentClusterScope(this.getClass()); + } + + private static Scope getCurrentClusterScope(Class clazz) { + ClusterScope annotation = getAnnotation(clazz); + return annotation == null ? Scope.GLOBAL : annotation.scope(); + } + + /** + * Settings used when the {@link org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope} is set to + * {@link org.elasticsearch.test.ElasticsearchIntegrationTest.Scope#SUITE} or {@link org.elasticsearch.test.ElasticsearchIntegrationTest.Scope#TEST} + * so that some of the configuration parameters can be overridden through test instance methods, similarly + * to how {@link #nodeSettings(int)} and {@link #transportClientSettings()} work. + */ + private CustomShieldSettingsSource customShieldSettingsSource = null; + + @BeforeClass + public static void initDefaultSettings() { + if (SHIELD_DEFAULT_SETTINGS == null) { + SHIELD_DEFAULT_SETTINGS = new ShieldSettingsSource(maxNumberOfNodes(), globalTempDir(), Scope.GLOBAL); + } + } + + @Rule + //Rules are the only way to have something run before the before (final) method inherited from ElasticsearchIntegrationTest + public ExternalResource externalResource = new ExternalResource() { + @Override + protected void before() throws Throwable { + Scope currentClusterScope = getCurrentClusterScope(); + switch(currentClusterScope) { + case GLOBAL: + if (InternalTestCluster.DEFAULT_SETTINGS_SOURCE == SettingsSource.EMPTY) { + InternalTestCluster.DEFAULT_SETTINGS_SOURCE = SHIELD_DEFAULT_SETTINGS; + } + break; + case SUITE: + if (customShieldSettingsSource == null) { + customShieldSettingsSource = new CustomShieldSettingsSource(newTempDir(LifecycleScope.SUITE), currentClusterScope); + } + break; + case TEST: + customShieldSettingsSource = new CustomShieldSettingsSource(newTempDir(LifecycleScope.TEST), currentClusterScope); + break; + } + } + }; + + @Before + //before methods from the superclass are run before this, which means that the current cluster is ready to go + public void assertShieldIsInstalled() { + NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().clear().setPlugins(true).get(); + for (NodeInfo nodeInfo : nodeInfos) { + assertThat(ShieldPlugin.NAME + " should be the only installed plugin, found the following ones: " + nodeInfo.getPlugins().getInfos(), nodeInfo.getPlugins().getInfos().size(), equalTo(1)); + assertThat(ShieldPlugin.NAME + " should be the only installed plugin, found the following ones: " + nodeInfo.getPlugins().getInfos(), nodeInfo.getPlugins().getInfos().get(0).getName(), equalTo(ShieldPlugin.NAME)); + } + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return ImmutableSettings.builder().put(super.nodeSettings(nodeOrdinal)) + .put(customShieldSettingsSource.node(nodeOrdinal)) + .build(); + } + + @Override + protected Settings transportClientSettings() { + return ImmutableSettings.builder().put(super.transportClientSettings()) + .put(customShieldSettingsSource.transportClient()) + .build(); + } + + /** + * Allows to override the users config file when the {@link org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope} is set to + * {@link org.elasticsearch.test.ElasticsearchIntegrationTest.Scope#SUITE} or {@link org.elasticsearch.test.ElasticsearchIntegrationTest.Scope#TEST} + */ + protected String configUsers() { + return SHIELD_DEFAULT_SETTINGS.configUsers(); + } + + /** + * Allows to override the users_roles config file when the {@link org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope} is set to + * {@link org.elasticsearch.test.ElasticsearchIntegrationTest.Scope#SUITE} or {@link org.elasticsearch.test.ElasticsearchIntegrationTest.Scope#TEST} + */ + protected String configUsersRoles() { + return SHIELD_DEFAULT_SETTINGS.configUsersRoles(); + } + + /** + * Allows to override the roles config file when the {@link org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope} is set to + * {@link org.elasticsearch.test.ElasticsearchIntegrationTest.Scope#SUITE} or {@link org.elasticsearch.test.ElasticsearchIntegrationTest.Scope#TEST} + */ + protected String configRoles() { + return SHIELD_DEFAULT_SETTINGS.configRoles(); + } + + /** + * Allows to override the node client username (used while sending requests to the test cluster) when the {@link org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope} is set to + * {@link org.elasticsearch.test.ElasticsearchIntegrationTest.Scope#SUITE} or {@link org.elasticsearch.test.ElasticsearchIntegrationTest.Scope#TEST} + */ + protected String nodeClientUsername() { + return SHIELD_DEFAULT_SETTINGS.nodeClientUsername(); + } + + /** + * Allows to override the node client password (used while sending requests to the test cluster) when the {@link org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope} is set to + * {@link org.elasticsearch.test.ElasticsearchIntegrationTest.Scope#SUITE} or {@link org.elasticsearch.test.ElasticsearchIntegrationTest.Scope#TEST} + */ + protected SecuredString nodeClientPassword() { + return SHIELD_DEFAULT_SETTINGS.nodeClientPassword(); + } + + /** + * Allows to override the transport client username (used while sending requests to the test cluster) when the {@link org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope} is set to + * {@link org.elasticsearch.test.ElasticsearchIntegrationTest.Scope#SUITE} or {@link org.elasticsearch.test.ElasticsearchIntegrationTest.Scope#TEST} + */ + protected String transportClientUsername() { + return SHIELD_DEFAULT_SETTINGS.transportClientUsername(); + } + + /** + * Allows to override the transport client password (used while sending requests to the test cluster) when the {@link org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope} is set to + * {@link org.elasticsearch.test.ElasticsearchIntegrationTest.Scope#SUITE} or {@link org.elasticsearch.test.ElasticsearchIntegrationTest.Scope#TEST} + */ + protected SecuredString transportClientPassword() { + return SHIELD_DEFAULT_SETTINGS.transportClientPassword(); + } + + private class CustomShieldSettingsSource extends ShieldSettingsSource { + private CustomShieldSettingsSource(File configDir, Scope scope) { + super(maxNumberOfNodes(), configDir, scope); + } + + @Override + protected String configUsers() { + return ShieldIntegrationTest.this.configUsers(); + } + + @Override + protected String configUsersRoles() { + return ShieldIntegrationTest.this.configUsersRoles(); + } + + @Override + protected String configRoles() { + return ShieldIntegrationTest.this.configRoles(); + } + + @Override + protected String nodeClientUsername() { + return ShieldIntegrationTest.this.nodeClientUsername(); + } + + @Override + protected SecuredString nodeClientPassword() { + return ShieldIntegrationTest.this.nodeClientPassword(); + } + + + @Override + protected String transportClientUsername() { + return ShieldIntegrationTest.this.transportClientUsername(); + } + + @Override + protected SecuredString transportClientPassword() { + return ShieldIntegrationTest.this.transportClientPassword(); + } + } + + protected void assertGreenClusterState(Client client) { + ClusterHealthResponse clusterHealthResponse = client.admin().cluster().prepareHealth().get(); + assertNoTimeout(clusterHealthResponse); + assertThat(clusterHealthResponse.getStatus(), is(ClusterHealthStatus.GREEN)); + } +} diff --git a/src/test/java/org/elasticsearch/test/ShieldRestTests.java b/src/test/java/org/elasticsearch/test/ShieldRestTests.java index 10549db0eab..9cee7bf72c1 100644 --- a/src/test/java/org/elasticsearch/test/ShieldRestTests.java +++ b/src/test/java/org/elasticsearch/test/ShieldRestTests.java @@ -5,14 +5,10 @@ */ package org.elasticsearch.test; -import com.carrotsearch.randomizedtesting.RandomizedTest; -import com.carrotsearch.randomizedtesting.SysGlobals; import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.TestGroup; -import com.google.common.base.Charsets; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.common.io.Streams; -import org.elasticsearch.common.os.OsUtils; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.apache.lucene.util.AbstractRandomizedTest; +import org.elasticsearch.client.support.Headers; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.PluginsService; @@ -20,178 +16,66 @@ import org.elasticsearch.shield.authc.support.SecuredStringTests; import org.elasticsearch.shield.signature.InternalSignatureService; import org.elasticsearch.shield.ShieldPlugin; import org.elasticsearch.shield.transport.netty.NettySecuredTransport; +import org.elasticsearch.shield.authc.support.SecuredString; +import org.elasticsearch.shield.authc.support.UsernamePasswordToken; import org.elasticsearch.test.rest.ElasticsearchRestTests; import org.elasticsearch.test.rest.RestTestCandidate; -import org.elasticsearch.transport.TransportModule; +import org.elasticsearch.test.rest.client.RestException; +import org.elasticsearch.test.rest.parser.RestTestParseException; import org.junit.AfterClass; -import org.junit.ClassRule; -import org.junit.rules.TemporaryFolder; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; -import java.io.File; import java.io.IOException; -import java.nio.file.Path; import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue; -import static org.hamcrest.Matchers.is; /** - * + * Allows to run Elasticsearch REST tests against a cluster with shield installed. + * Subclasses {@link org.elasticsearch.test.ShieldIntegrationTest} that contains all the needed code to override the global + * cluster settings and make sure shield is properly installed and configured. + * Delegates all of the tests to {@link org.elasticsearch.test.rest.ElasticsearchRestTests}. */ -public class ShieldRestTests extends ElasticsearchRestTests { +@AbstractRandomizedTest.Rest +@ElasticsearchIntegrationTest.ClusterScope(randomDynamicTemplates = false) +public class ShieldRestTests extends ShieldIntegrationTest { - public static final int CHILD_JVM_ID = Integer.parseInt(System.getProperty(SysGlobals.CHILDVM_SYSPROP_JVM_ID, "0")); - public static final int BASE_PORT = 33000 + CHILD_JVM_ID * 100; - public static final String BASE_PORT_RANGE = BASE_PORT + "-" + (BASE_PORT+10) ; - - protected static final boolean ENABLE_TRANSPORT_SSL = true; - protected static final boolean SHIELD_AUDIT_ENABLED = false; - - protected static final String DEFAULT_USER_NAME = "test_user"; - protected static final String DEFAULT_PASSWORD = "changeme"; - protected static final String DEFAULT_ROLE = "user"; - - public static final String CONFIG_STANDARD_USER = DEFAULT_USER_NAME + ":{plain}" + DEFAULT_PASSWORD + "\n"; - public static final String CONFIG_STANDARD_USER_ROLES = DEFAULT_ROLE + ":" + DEFAULT_USER_NAME+ "\n"; - public static final String CONFIG_ROLE_ALLOW_ALL = - DEFAULT_ROLE + ":\n" + - " cluster: ALL\n" + - " indices:\n" + - " '*': ALL\n"; - - static { - - TestGroup testGroup = Rest.class.getAnnotation(TestGroup.class); - String sysProperty = TestGroup.Utilities.getSysProperty(Rest.class); - boolean enabled; - try { - enabled = RandomizedTest.systemPropertyAsBoolean(sysProperty, testGroup.enabled()); - } catch (IllegalArgumentException e) { - // Ignore malformed system property, disable the group if malformed though. - enabled = false; - } - - //customize the global cluster only if rest tests are enabled - //not perfect but good enough as REST tests are supposed to be run only separately on CI - if (enabled) { - final byte[] key; - try { - key = InternalSignatureService.generateKey(); - } catch (Exception e) { - throw new RuntimeException(e); - } - - InternalTestCluster.DEFAULT_SETTINGS_SOURCE = new SettingsSource() { - - @Override - public Settings node(int nodeOrdinal) { - File store; - try { - store = new File(getClass().getResource("/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.jks").toURI()); - assertThat(store.exists(), is(true)); - } catch (Exception e) { - throw new ElasticsearchException("Error reading test node cert", e); - } - String password = "testnode"; - - File folder = createFolder(); - - String keyFile = writeFile(folder, "system_key", key); - - ImmutableSettings.Builder builder = ImmutableSettings.builder() - .put(InternalSignatureService.FILE_SETTING, keyFile) - .put("request.headers.Authorization", basicAuthHeaderValue(DEFAULT_USER_NAME, SecuredStringTests.build(DEFAULT_PASSWORD))) - .put("discovery.zen.ping.multicast.enabled", false) - .put("discovery.type", "zen") - .put("node.mode", "network") - .put("plugin.types", ShieldPlugin.class.getName()) - .put("shield.authc.esusers.files.users", createFile(folder, "users", CONFIG_STANDARD_USER)) - .put("shield.authc.esusers.files.users_roles", createFile(folder, "users_roles", CONFIG_STANDARD_USER_ROLES)) - .put("shield.authz.store.files.roles", createFile(folder, "roles.yml", CONFIG_ROLE_ALLOW_ALL)) - .put("shield.transport.ssl", ENABLE_TRANSPORT_SSL) - .put("shield.ssl.keystore.path", store.getPath()) - .put("shield.ssl.keystore.password", password) - .put("shield.ssl.truststore.path", store.getPath()) - .put("shield.ssl.truststore.password", password) - .put("shield.http.ssl", false) - .put("transport.tcp.port", BASE_PORT_RANGE) - .putArray("discovery.zen.ping.unicast.hosts", "127.0.0.1:" + BASE_PORT, "127.0.0.1:" + (BASE_PORT + 1), "127.0.0.1:" + (BASE_PORT + 2), "127.0.0.1:" + (BASE_PORT + 3)) - .put("shield.audit.enabled", SHIELD_AUDIT_ENABLED); - - builder.put("network.host", "127.0.0.1"); - if (OsUtils.MAC) { - builder.put("network.host", randomBoolean() ? "127.0.0.1" : "::1"); - } - - return builder.build(); - } - - @Override - public Settings transportClient() { - File store; - String password = "testclient"; - try { - store = new File(getClass().getResource("/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.jks").toURI()); - assertThat(store.exists(), is(true)); - } catch (Exception e) { - throw new ElasticsearchException("Error reading test client cert", e); - } - - return ImmutableSettings.builder() - .put("request.headers.Authorization", basicAuthHeaderValue(DEFAULT_USER_NAME, SecuredStringTests.build(DEFAULT_PASSWORD))) - .put(TransportModule.TRANSPORT_TYPE_KEY, NettySecuredTransport.class.getName()) - .put("plugins." + PluginsService.LOAD_PLUGIN_FROM_CLASSPATH, false) - .put("node.mode", "network") - .put("shield.transport.ssl", ENABLE_TRANSPORT_SSL) - .put("shield.ssl.keystore.path", store.getPath()) - .put("shield.ssl.keystore.password", password) - .put("shield.ssl.truststore.path", store.getPath()) - .put("shield.ssl.truststore.password", password) - .put("cluster.name", internalCluster().getClusterName()) - .build(); - } - }; - } - } - - @ClassRule - public static TemporaryFolder tmpFolder = new TemporaryFolder(); - - @AfterClass - public static void cleanup() { - tmpFolder = null; - } + private final ElasticsearchRestTests delegate; public ShieldRestTests(@Name("yaml") RestTestCandidate testCandidate) { - super(testCandidate); + delegate = new ElasticsearchRestTests(testCandidate) { + @Override + protected Settings restClientSettings() { + return ImmutableSettings.builder() + .put(Headers.PREFIX + "." + UsernamePasswordToken.BASIC_AUTH_HEADER, basicAuthHeaderValue(ShieldSettingsSource.DEFAULT_USER_NAME, + new SecuredString(ShieldSettingsSource.DEFAULT_PASSWORD.toCharArray()))).build(); + } + }; } - @Override - protected Settings restClientSettings() { - return ImmutableSettings.builder() - .put("request.headers.Authorization", basicAuthHeaderValue(DEFAULT_USER_NAME, SecuredStringTests.build(DEFAULT_PASSWORD))).build(); + @ParametersFactory + public static Iterable parameters() throws IOException, RestTestParseException { + return ElasticsearchRestTests.parameters(); } - /* static helper methods for the global test class */ - static File createFolder() { - try { - return tmpFolder.newFolder(); - } catch (IOException ioe) { - fail("could not create temporary folder"); - return null; - } + @BeforeClass + public static void initExecutionContext() throws IOException, RestException { + ElasticsearchRestTests.initExecutionContext(); } - static String createFile(File folder, String name, String content) { - return writeFile(folder, name, content.getBytes(Charsets.UTF_8)); + @AfterClass + public static void close() { + ElasticsearchRestTests.close(); } - static String writeFile(File folder, String name, byte[] content) { - Path file = folder.toPath().resolve(name); - try { - Streams.copy(content, file.toFile()); - } catch (IOException e) { - throw new ElasticsearchException("Error writing file in test", e); - } - return file.toFile().getAbsolutePath(); + @Test + public void test() throws IOException { + delegate.test(); + } + + @Before + public void reset() throws IOException, RestException { + delegate.reset(); } } diff --git a/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java b/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java new file mode 100644 index 00000000000..481fbd8fe66 --- /dev/null +++ b/src/test/java/org/elasticsearch/test/ShieldSettingsSource.java @@ -0,0 +1,212 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.test; + +import com.carrotsearch.randomizedtesting.RandomizedTest; +import com.google.common.base.Charsets; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.client.support.Headers; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.os.OsUtils; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.shield.ShieldPlugin; +import org.elasticsearch.shield.authc.support.SecuredString; +import org.elasticsearch.shield.authc.support.UsernamePasswordToken; +import org.elasticsearch.shield.signature.InternalSignatureService; +import org.elasticsearch.test.discovery.ClusterDiscoveryConfiguration; + +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Path; + +import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; +import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue; + +/** + * {@link org.elasticsearch.test.SettingsSource} subclass that allows to set all needed settings for shield. + * Unicast discovery is configured through {@link org.elasticsearch.test.discovery.ClusterDiscoveryConfiguration.UnicastZen}, + * also shield is installed with all the needed configuration and files. + * To avoid conflicts, every cluster should have its own instance of this class as some configuration files need to be created. + */ +public class ShieldSettingsSource extends ClusterDiscoveryConfiguration.UnicastZen { + + public static final String DEFAULT_USER_NAME = "test_user"; + public static final String DEFAULT_PASSWORD = "changeme"; + public static final String DEFAULT_ROLE = "user"; + + public static final String DEFAULT_TRANSPORT_CLIENT_ROLE = "trans_client_user"; + public static final String DEFAULT_TRANSPORT_CLIENT_USER_NAME = "test_trans_client_user"; + + public static final String CONFIG_STANDARD_USER = + DEFAULT_USER_NAME + ":{plain}" + DEFAULT_PASSWORD + "\n" + + DEFAULT_TRANSPORT_CLIENT_USER_NAME + ":{plain}" + DEFAULT_PASSWORD + "\n"; + + public static final String CONFIG_STANDARD_USER_ROLES = + DEFAULT_ROLE + ":" + DEFAULT_USER_NAME + "," + DEFAULT_TRANSPORT_CLIENT_USER_NAME + "\n" + + DEFAULT_TRANSPORT_CLIENT_ROLE + ":" + DEFAULT_TRANSPORT_CLIENT_USER_NAME+ "\n"; + + public static final String CONFIG_ROLE_ALLOW_ALL = + DEFAULT_ROLE + ":\n" + + " cluster: ALL\n" + + " indices:\n" + + " '*': ALL\n" + + DEFAULT_TRANSPORT_CLIENT_ROLE + ":\n" + + " cluster:\n" + + " - cluster:monitor/nodes/info\n" + + " - cluster:monitor/state"; + + private final File parentFolder; + private final String subfolderPrefix; + private final byte[] systemKey; + + /** + * Creates a new {@link org.elasticsearch.test.SettingsSource} for the shield configuration. + * + * @param numOfNodes the number of nodes for proper unicast configuration (can be more than actually available) + * @param parentFolder the parent folder that will contain all of the configuration files that need to be created + * @param scope the scope of the test that is requiring an instance of ShieldSettingsSource + */ + public ShieldSettingsSource(int numOfNodes, File parentFolder, ElasticsearchIntegrationTest.Scope scope) { + super(numOfNodes, ImmutableSettings.builder() + .put("node.mode", "network") + .put("plugin.types", ShieldPlugin.class.getName()) + .put("plugins.load_classpath_plugins", false) + .build(), + scope); + this.systemKey = generateKey(); + this.parentFolder = parentFolder; + this.subfolderPrefix = scope.name(); + } + + @Override + public Settings node(int nodeOrdinal) { + File folder = createFolder(parentFolder, subfolderPrefix + "-" + nodeOrdinal); + ImmutableSettings.Builder builder = ImmutableSettings.builder().put(super.node(nodeOrdinal)) + .put("shield.audit.enabled", RandomizedTest.randomBoolean()) + .put(InternalSignatureService.FILE_SETTING, writeFile(folder, "system_key", systemKey)) + .put("shield.authc.esusers.files.users", writeFile(folder, "users", configUsers())) + .put("shield.authc.esusers.files.users_roles", writeFile(folder, "users_roles", configUsersRoles())) + .put("shield.authz.store.files.roles", writeFile(folder, "roles.yml", configRoles())) + .put(getNodeSSLSettings()); + + if (OsUtils.MAC) { + builder.put("network.host", RandomizedTest.randomBoolean() ? "127.0.0.1" : "::1"); + } + + setUser(builder, nodeClientUsername(), nodeClientPassword()); + + return builder.build(); + } + + protected String configUsers() { + return CONFIG_STANDARD_USER; + } + + protected String configUsersRoles() { + return CONFIG_STANDARD_USER_ROLES; + } + + protected String configRoles() { + return CONFIG_ROLE_ALLOW_ALL; + } + + protected String nodeClientUsername() { + return DEFAULT_USER_NAME; + } + + protected SecuredString nodeClientPassword() { + return new SecuredString(DEFAULT_PASSWORD.toCharArray()); + } + + protected String transportClientUsername() { + return DEFAULT_TRANSPORT_CLIENT_USER_NAME; + } + + protected SecuredString transportClientPassword() { + return new SecuredString(DEFAULT_PASSWORD.toCharArray()); + } + + @Override + public Settings transportClient() { + ImmutableSettings.Builder builder = ImmutableSettings.builder().put(super.transportClient()) + .put(getClientSSLSettings()); + setUser(builder, transportClientUsername(), transportClientPassword()); + return builder.build(); + } + + private void setUser(ImmutableSettings.Builder builder, String username, SecuredString password) { + if (RandomizedTest.randomBoolean()) { + builder.put(Headers.PREFIX + "." + UsernamePasswordToken.BASIC_AUTH_HEADER, basicAuthHeaderValue(username, password)); + } else { + builder.put("shield.user", username + ":" + new String(password.internalChars())); + } + } + + private static File createFolder(File parent, String name) { + File createdFolder = new File(parent, name); + if (!createdFolder.mkdir()) { + throw new RuntimeException("Could not create temporary folder: " + createdFolder.getAbsolutePath()); + } + return createdFolder; + } + + private static String writeFile(File folder, String name, byte[] content) { + Path file = folder.toPath().resolve(name); + try { + Streams.copy(content, file.toFile()); + } catch (IOException e) { + throw new ElasticsearchException("Error writing file in test", e); + } + return file.toFile().getAbsolutePath(); + } + + private static String writeFile(File folder, String name, String content) { + return writeFile(folder, name, content.getBytes(Charsets.UTF_8)); + } + + private static byte[] generateKey() { + try { + return InternalSignatureService.generateKey(); + } catch (Exception e) { + throw new ElasticsearchException("exception while generating the system key", e); + } + } + + private static Settings getNodeSSLSettings() { + return getSSLSettingsForStore("/org/elasticsearch/shield/transport/ssl/certs/simple/testnode.jks", "testnode"); + } + + private static Settings getClientSSLSettings() { + return getSSLSettingsForStore("/org/elasticsearch/shield/transport/ssl/certs/simple/testclient.jks", "testclient"); + } + + public static Settings getSSLSettingsForStore(String resourcePathToStore, String password) { + File store; + try { + store = new File(ShieldSettingsSource.class.getResource(resourcePathToStore).toURI()); + } catch (URISyntaxException e) { + throw new ElasticsearchException("exception while reading the store", e); + } + + if (!store.exists()) { + throw new ElasticsearchException("store path doesn't exist"); + } + + ImmutableSettings.Builder builder = settingsBuilder() + .put("shield.ssl.keystore.path", store.getPath()) + .put("shield.ssl.keystore.password", password) + .put("shield.transport.ssl", true) + .put("shield.http.ssl", false); + + if (RandomizedTest.randomBoolean()) { + builder.put("shield.ssl.truststore.path", store.getPath()) + .put("shield.ssl.truststore.password", password); + } + return builder.build(); + } +} diff --git a/src/test/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java b/src/test/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java new file mode 100644 index 00000000000..00325bfeace --- /dev/null +++ b/src/test/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.test.discovery; + +import com.carrotsearch.randomizedtesting.RandomizedTest; +import com.google.common.primitives.Ints; +import org.elasticsearch.Version; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.test.SettingsSource; +import org.elasticsearch.transport.local.LocalTransport; + +import java.util.HashSet; +import java.util.Set; + +public class ClusterDiscoveryConfiguration extends SettingsSource { + + static { + //see https://github.com/elasticsearch/elasticsearch/pull/8634 + assert Version.CURRENT.onOrBefore(Version.V_1_4_0) : "Remove this class as the required fixes should have been released with core"; + } + + public static Settings DEFAULT_SETTINGS = ImmutableSettings.settingsBuilder() + .put("gateway.type", "local") + .put("discovery.type", "zen") + .build(); + + final int numOfNodes; + + final Settings baseSettings; + + public ClusterDiscoveryConfiguration(int numOfNodes) { + this(numOfNodes, ImmutableSettings.EMPTY); + } + + public ClusterDiscoveryConfiguration(int numOfNodes, Settings extraSettings) { + this.numOfNodes = numOfNodes; + this.baseSettings = ImmutableSettings.builder().put(DEFAULT_SETTINGS).put(extraSettings).build(); + } + + @Override + public Settings node(int nodeOrdinal) { + return baseSettings; + } + + @Override + public Settings transportClient() { + return baseSettings; + } + + public static class UnicastZen extends ClusterDiscoveryConfiguration { + + private final int[] unicastHostOrdinals; + private final int basePort; + + public UnicastZen(int numOfNodes, ElasticsearchIntegrationTest.Scope scope) { + this(numOfNodes, numOfNodes, scope); + } + + public UnicastZen(int numOfNodes, Settings extraSettings, ElasticsearchIntegrationTest.Scope scope) { + this(numOfNodes, numOfNodes, extraSettings, scope); + } + + public UnicastZen(int numOfNodes, int numOfUnicastHosts, ElasticsearchIntegrationTest.Scope scope) { + this(numOfNodes, numOfUnicastHosts, ImmutableSettings.EMPTY, scope); + } + + public UnicastZen(int numOfNodes, int numOfUnicastHosts, Settings extraSettings, ElasticsearchIntegrationTest.Scope scope) { + super(numOfNodes, extraSettings); + if (numOfUnicastHosts == numOfNodes) { + unicastHostOrdinals = new int[numOfNodes]; + for (int i = 0; i < numOfNodes; i++) { + unicastHostOrdinals[i] = i; + } + } else { + Set ordinals = new HashSet<>(numOfUnicastHosts); + while (ordinals.size() != numOfUnicastHosts) { + ordinals.add(RandomizedTest.randomInt(numOfNodes - 1)); + } + unicastHostOrdinals = Ints.toArray(ordinals); + } + this.basePort = calcBasePort(scope); + } + + public UnicastZen(int numOfNodes, int[] unicastHostOrdinals, ElasticsearchIntegrationTest.Scope scope) { + this(numOfNodes, ImmutableSettings.EMPTY, unicastHostOrdinals, scope); + } + + public UnicastZen(int numOfNodes, Settings extraSettings, int[] unicastHostOrdinals, ElasticsearchIntegrationTest.Scope scope) { + super(numOfNodes, extraSettings); + this.unicastHostOrdinals = unicastHostOrdinals; + this.basePort = calcBasePort(scope); + } + + private static int calcBasePort(ElasticsearchIntegrationTest.Scope scope) { + // note that this has properly co-exist with the port logic at InternalTestCluster's constructor + return 30000 + + 1000 * (ElasticsearchIntegrationTest.CHILD_JVM_ID % 60) + // up to 30 jvms + //up to 100 nodes per cluster + 100 * scopeId(scope); + } + + private static int scopeId(ElasticsearchIntegrationTest.Scope scope) { + switch(scope) { + case GLOBAL: + //we reserve a special base port for global clusters, as they stick around + //the assumption is that no counter is needed as there's only one global cluster per jvm + return 0; + default: + //ports can be reused as suite or test clusters are never run concurrently + //prevent conflicts between jvms by never going above 9 + return RandomizedTest.randomIntBetween(1, 9); + } + } + + @Override + public Settings node(int nodeOrdinal) { + ImmutableSettings.Builder builder = ImmutableSettings.builder() + .put("discovery.zen.ping.multicast.enabled", false); + + String[] unicastHosts = new String[unicastHostOrdinals.length]; + String mode = baseSettings.get("node.mode", InternalTestCluster.NODE_MODE); + if (mode.equals("local")) { + builder.put(LocalTransport.TRANSPORT_LOCAL_ADDRESS, "node_" + nodeOrdinal); + for (int i = 0; i < unicastHosts.length; i++) { + unicastHosts[i] = "node_" + unicastHostOrdinals[i]; + } + } else { + // we need to pin the node port & host so we'd know where to point things + builder.put("transport.tcp.port", basePort + nodeOrdinal); + builder.put("transport.host", "localhost"); + for (int i = 0; i < unicastHosts.length; i++) { + unicastHosts[i] = "localhost:" + (basePort + unicastHostOrdinals[i]); + } + } + builder.putArray("discovery.zen.ping.unicast.hosts", unicastHosts); + return builder.put(super.node(nodeOrdinal)).build(); + } + } +} diff --git a/src/test/java/org/elasticsearch/transport/KnownActionsTests.java b/src/test/java/org/elasticsearch/transport/KnownActionsTests.java index be5f29b78ae..92f25affb71 100644 --- a/src/test/java/org/elasticsearch/transport/KnownActionsTests.java +++ b/src/test/java/org/elasticsearch/transport/KnownActionsTests.java @@ -11,7 +11,7 @@ import org.elasticsearch.ElasticsearchIllegalStateException; import org.elasticsearch.action.Action; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.util.Callback; -import org.elasticsearch.shield.test.ShieldIntegrationTest; +import org.elasticsearch.test.ShieldIntegrationTest; import org.junit.BeforeClass; import org.junit.Test; diff --git a/src/test/resources/org/elasticsearch/transport/handlers b/src/test/resources/org/elasticsearch/transport/handlers index 305a7cfb4d7..0d3ac2ba143 100644 --- a/src/test/resources/org/elasticsearch/transport/handlers +++ b/src/test/resources/org/elasticsearch/transport/handlers @@ -68,6 +68,11 @@ internal:discovery/zen/publish internal:discovery/zen/rejoin internal:discovery/zen/unicast internal:discovery/zen/unicast_gte_1_4 +internal:gateway/local/allocate_dangled +internal:gateway/local/meta_state +internal:gateway/local/meta_state[n] +internal:gateway/local/started_shards +internal:gateway/local/started_shards[n] internal:index/shard/exists internal:index/shard/recovery/clean_files internal:index/shard/recovery/file_chunk @@ -77,4 +82,4 @@ internal:index/shard/recovery/prepare_translog internal:index/shard/recovery/start_recovery internal:index/shard/recovery/translog_ops internal:river/state/publish -internal:admin/repository/verify \ No newline at end of file +internal:admin/repository/verify