diff --git a/docs/en/settings/ml-settings.asciidoc b/docs/en/settings/ml-settings.asciidoc index d7d1a020255..dd01df8f8cd 100644 --- a/docs/en/settings/ml-settings.asciidoc +++ b/docs/en/settings/ml-settings.asciidoc @@ -12,8 +12,11 @@ Set to `true` (default) to enable {ml}. + + If set to `false` in `elasticsearch.yml`, the {ml} APIs are disabled. You also cannot open jobs, start {dfeeds}, or receive transport (internal) -communication requests related to {ml} APIs. The {ml} icon is also not visible -in all {kib} instances that connect to this {es} instance. + +communication requests related to {ml} APIs. It also affects all {kib} instances +that connect to this {es} instance; you do not need to disable {ml} in those +`kibana.yml` files. For more information about disabling {ml} in specific {kib} +instances, see +{kibana-ref}/ml-settings-kb.html[{kib} Machine Learning Settings]. + IMPORTANT: If you want to use {ml} features in your cluster, you must have `xpack.ml.enabled` set to `true` on all master-eligible nodes. This is the diff --git a/docs/en/settings/security-settings.asciidoc b/docs/en/settings/security-settings.asciidoc index cd21cd8182f..45f1021f604 100644 --- a/docs/en/settings/security-settings.asciidoc +++ b/docs/en/settings/security-settings.asciidoc @@ -13,8 +13,13 @@ and <>. [[general-security-settings]] ==== General Security Settings `xpack.security.enabled`:: -Set to `false` to disable {security}. -Configure in both `elasticsearch.yml` and `kibana.yml`. +Set to `true` (default) to enable {security}. + ++ +If set to `false` in `elasticsearch.yml`, {security} is disabled. It also +affects all {kib} instances that connect to this {es} instance; you do not +need to disable {security} in those `kibana.yml` files. For more information +about disabling {security} in specific {kib} instances, see +{kibana-ref}/security-settings-kb.html[{kib} Security Settings]. [float] [[password-security-settings]] diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java index 8de44e59cd5..2b5dfb48ca3 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java @@ -205,9 +205,6 @@ public class ElasticsearchMappings { .startObject(Result.IS_INTERIM.getPreferredName()) .field(TYPE, BOOLEAN) .endObject() - .startObject(Bucket.RECORD_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() .startObject(Bucket.EVENT_COUNT.getPreferredName()) .field(TYPE, LONG) .endObject() diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/Bucket.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/Bucket.java index 623b44ab4e6..1cda0abb4c8 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/Bucket.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/Bucket.java @@ -38,7 +38,6 @@ public class Bucket implements ToXContentObject, Writeable { public static final ParseField ANOMALY_SCORE = new ParseField("anomaly_score"); public static final ParseField INITIAL_ANOMALY_SCORE = new ParseField("initial_anomaly_score"); - public static final ParseField RECORD_COUNT = new ParseField("record_count"); public static final ParseField EVENT_COUNT = new ParseField("event_count"); public static final ParseField RECORDS = new ParseField("records"); public static final ParseField BUCKET_INFLUENCERS = new ParseField("bucket_influencers"); @@ -46,6 +45,9 @@ public class Bucket implements ToXContentObject, Writeable { public static final ParseField PROCESSING_TIME_MS = new ParseField("processing_time_ms"); public static final ParseField PARTITION_SCORES = new ParseField("partition_scores"); + // Only exists for backwards compatibility; no longer added to mappings + private static final ParseField RECORD_COUNT = new ParseField("record_count"); + // Used for QueryPage public static final ParseField RESULTS_FIELD = new ParseField("buckets"); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/ReservedFieldNames.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/ReservedFieldNames.java index 4a4091a3b6d..b4404e931f4 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/ReservedFieldNames.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/ReservedFieldNames.java @@ -77,7 +77,6 @@ public final class ReservedFieldNames { Bucket.ANOMALY_SCORE.getPreferredName(), Bucket.BUCKET_INFLUENCERS.getPreferredName(), Bucket.BUCKET_SPAN.getPreferredName(), - Bucket.RECORD_COUNT.getPreferredName(), Bucket.EVENT_COUNT.getPreferredName(), Bucket.INITIAL_ANOMALY_SCORE.getPreferredName(), Bucket.PROCESSING_TIME_MS.getPreferredName(), diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/utils/DomainSplitFunction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/utils/DomainSplitFunction.java index 94d07d27cac..293885fb87f 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/utils/DomainSplitFunction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/utils/DomainSplitFunction.java @@ -24,9 +24,10 @@ public final class DomainSplitFunction { Map paramsMap = new HashMap<>(); Map exact = new HashMap<>(2048); - try { - InputStream resource = - DomainSplitFunction.class.getClassLoader().getResourceAsStream("org/elasticsearch/xpack/ml/transforms/exact.properties"); + + String exactResourceName = "org/elasticsearch/xpack/ml/transforms/exact.properties"; + + try (InputStream resource = DomainSplitFunction.class.getClassLoader().getResourceAsStream(exactResourceName)) { List lines = Streams.readAllLines(resource); for (String line : lines) { String[] split = line.split("="); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/CommandLineHttpClient.java b/plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/CommandLineHttpClient.java index 47e18dfa285..b8bda5a36fa 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/CommandLineHttpClient.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/CommandLineHttpClient.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; @@ -26,14 +27,18 @@ import javax.net.ssl.HttpsURLConnection; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.UncheckedIOException; import java.net.HttpURLConnection; +import java.net.InetAddress; import java.net.URI; import java.net.URL; import java.nio.charset.StandardCharsets; import java.security.AccessController; import java.security.PrivilegedAction; +import java.util.Collections; import java.util.List; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_PORT; import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_PUBLISH_HOST; import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_PUBLISH_PORT; import static org.elasticsearch.xpack.security.Security.setting; @@ -103,12 +108,34 @@ public class CommandLineHttpClient { } } - public String getDefaultURL() { + String getDefaultURL() { final String scheme = XPackSettings.HTTP_SSL_ENABLED.get(settings) ? "https" : "http"; List httpPublishHost = SETTING_HTTP_PUBLISH_HOST.get(settings); - final String host = - (httpPublishHost.isEmpty() ? NetworkService.GLOBAL_NETWORK_PUBLISHHOST_SETTING.get(settings) : httpPublishHost).get(0); - final int port = SETTING_HTTP_PUBLISH_PORT.get(settings); - return scheme + "://" + host + ":" + port; + if (httpPublishHost.isEmpty()) { + httpPublishHost = NetworkService.GLOBAL_NETWORK_PUBLISHHOST_SETTING.get(settings); + } + + // we cannot do custom name resolution here... + NetworkService networkService = new NetworkService(Collections.emptyList()); + try { + InetAddress publishAddress = networkService.resolvePublishHostAddresses(httpPublishHost.toArray(Strings.EMPTY_ARRAY)); + int port = SETTING_HTTP_PUBLISH_PORT.get(settings); + if (port <= 0) { + int[] ports = SETTING_HTTP_PORT.get(settings).ports(); + if (ports.length > 0) { + port = ports[0]; + } + + // this sucks but a port can be specified with a value of 0, we'll never be able to connect to it so just default to + // what we know + if (port <= 0) { + throw new IllegalStateException("unable to determine http port from settings, please use the -u option to provide the" + + " url"); + } + } + return scheme + "://" + InetAddresses.toUriString(publishAddress) + ":" + port; + } catch (IOException e) { + throw new UncheckedIOException("failed to resolve default URL", e); + } } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordTool.java b/plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordTool.java index ea1c029cac1..e9a26d994c3 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordTool.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordTool.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.security.authc.esnative.tool; +import joptsimple.OptionParser; import joptsimple.OptionSet; import joptsimple.OptionSpec; import org.elasticsearch.cli.EnvironmentAwareCommand; @@ -77,6 +78,11 @@ public class SetupPasswordTool extends MultiCommand { exit(new SetupPasswordTool().main(args, Terminal.DEFAULT)); } + // Visible for testing + OptionParser getParser() { + return this.parser; + } + /** * This class sets the passwords using automatically generated random passwords. The passwords will be * printed to the console. diff --git a/plugin/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java b/plugin/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java index 1b63b8baa72..ede64fdfbec 100644 --- a/plugin/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java +++ b/plugin/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java @@ -31,7 +31,11 @@ import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; @@ -89,33 +93,6 @@ public abstract class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase ) }; - private static final RoleMappingEntry[] OLDAP_ROLE_MAPPING = new RoleMappingEntry[] { - new RoleMappingEntry( - "SHIELD: [ \"cn=SHIELD,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com\" ]", - "{ \"roles\": [\"SHIELD\"], \"enabled\":true, \"rules\":" + - " {\"field\": {\"groups\": \"cn=SHIELD,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com\" } } }" - ), - new RoleMappingEntry( - "Avengers: [ \"cn=Avengers,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com\" ]", - "{ \"roles\": [\"Avengers\"], \"enabled\":true, \"rules\":" + - " {\"field\": {\"groups\": \"cn=Avengers,ou=people,*\" } } }" - ), - new RoleMappingEntry( - "Gods: [ \"cn=Gods,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com\" ]", - "{ \"roles\" : [ \"Gods\" ], \"enabled\":true, \"rules\" : { \"any\": [" + - " {\"field\": {\"groups\": \"cn=Gods,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com\" } }," + - " {\"field\": {\"groups\": \"cn=Deities,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com\" } } " + - "] } }" - ), - new RoleMappingEntry( - "Philanthropists: [ \"cn=Philanthropists,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com\" ]", - "{ \"roles\" : [ \"Philanthropists\" ], \"enabled\":true, \"rules\" : { \"all\": [" + - " { \"field\": { \"groups\" : \"cn=Philanthropists,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com\" } }, " + - " { \"field\": { \"realm.name\" : \"external\" } } " + - "] } }" - ) - }; - protected static final String TESTNODE_KEYSTORE = "/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks"; protected static RealmConfig realmConfig; protected static List roleMappings; @@ -141,7 +118,42 @@ public abstract class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase Path store = getDataPath(TESTNODE_KEYSTORE); Settings.Builder builder = Settings.builder(); if (useGlobalSSL) { - builder.put(super.nodeSettings(nodeOrdinal).filter((s) -> s.startsWith("xpack.ssl.") == false)); + // don't use filter since it returns a prefixed secure setting instead of mock! + Settings settingsToAdd = super.nodeSettings(nodeOrdinal); + for (Map.Entry settingsEntry : settingsToAdd.getAsMap().entrySet()) { + if (settingsEntry.getKey().startsWith("xpack.ssl.") == false) { + builder.put(settingsEntry.getKey(), settingsEntry.getValue()); + } + } + MockSecureSettings mockSecureSettings = (MockSecureSettings) Settings.builder().put(settingsToAdd).getSecureSettings(); + if (mockSecureSettings != null) { + MockSecureSettings filteredSecureSettings = new MockSecureSettings(); + builder.setSecureSettings(filteredSecureSettings); + for (String secureSetting : mockSecureSettings.getSettingNames()) { + if (secureSetting.startsWith("xpack.ssl.") == false) { + SecureString secureString = mockSecureSettings.getString(secureSetting); + if (secureString == null) { + final byte[] fileBytes; + try (InputStream in = mockSecureSettings.getFile(secureSetting); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { + int numRead; + byte[] bytes = new byte[1024]; + while ((numRead = in.read(bytes)) != -1) { + byteArrayOutputStream.write(bytes, 0, numRead); + } + byteArrayOutputStream.flush(); + fileBytes = byteArrayOutputStream.toByteArray(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + filteredSecureSettings.setFile(secureSetting, fileBytes); + } else { + filteredSecureSettings.setString(secureSetting, new String(secureString.getChars())); + } + } + } + } addSslSettingsForStore(builder, store, "testnode"); } else { builder.put(super.nodeSettings(nodeOrdinal)); @@ -166,8 +178,7 @@ public abstract class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase return; } SecurityClient securityClient = securityClient(); - Map> futures - = new LinkedHashMap<>(content.size()); + Map> futures = new LinkedHashMap<>(content.size()); for (int i = 0; i < content.size(); i++) { final String name = "external_" + i; final PutRoleMappingRequestBuilder builder = securityClient.preparePutRoleMapping( @@ -393,17 +404,6 @@ public abstract class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".url", "ldaps://ad.test.elasticsearch.com:636") .putArray(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".user_dn_templates", "cn={0},CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com") - .build()), - - OLDAP(false, OLDAP_ROLE_MAPPING, - Settings.builder() - .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".type", LdapRealm.LDAP_TYPE) - .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".url", "ldaps://54.200.235.244:636") - .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".group_search.base_dn", - "ou=people, dc=oldap, dc=test, dc=elasticsearch, dc=com") - .put(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".group_search.scope", randomBoolean() ? SUB_TREE : ONE_LEVEL) - .putArray(XPACK_SECURITY_AUTHC_REALMS_EXTERNAL + ".user_dn_templates", - "uid={0},ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com") .build()); final boolean mapGroupsAsRoles; diff --git a/plugin/src/test/java/org/elasticsearch/integration/ldap/GroupMappingTests.java b/plugin/src/test/java/org/elasticsearch/integration/ldap/GroupMappingTests.java index 90df577fe8c..9badf115f64 100644 --- a/plugin/src/test/java/org/elasticsearch/integration/ldap/GroupMappingTests.java +++ b/plugin/src/test/java/org/elasticsearch/integration/ldap/GroupMappingTests.java @@ -5,8 +5,6 @@ */ package org.elasticsearch.integration.ldap; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.test.junit.annotations.Network; import java.io.IOException; @@ -16,7 +14,6 @@ import java.io.IOException; * The super class will provide appropriate group mappings via configGroupMappings() */ @Network -@AwaitsFix(bugUrl="https://github.com/elastic/x-pack-elasticsearch/issues/1823") public class GroupMappingTests extends AbstractAdLdapRealmTestCase { public void testAuthcAuthz() throws IOException { String avenger = realmConfig.loginWithCommonName ? "Natasha Romanoff" : "blackwidow"; diff --git a/plugin/src/test/java/org/elasticsearch/integration/ldap/MultiGroupMappingTests.java b/plugin/src/test/java/org/elasticsearch/integration/ldap/MultiGroupMappingTests.java index 439b2707568..f5cee6c0b9a 100644 --- a/plugin/src/test/java/org/elasticsearch/integration/ldap/MultiGroupMappingTests.java +++ b/plugin/src/test/java/org/elasticsearch/integration/ldap/MultiGroupMappingTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.integration.ldap; -import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.test.junit.annotations.Network; import org.junit.BeforeClass; @@ -16,7 +15,6 @@ import java.util.ArrayList; * This tests the mapping of multiple groups to a role in a file based role-mapping */ @Network -@AwaitsFix(bugUrl="https://github.com/elastic/x-pack-elasticsearch/issues/1823") public class MultiGroupMappingTests extends AbstractAdLdapRealmTestCase { @BeforeClass diff --git a/plugin/src/test/java/org/elasticsearch/integration/ldap/MultipleAdRealmTests.java b/plugin/src/test/java/org/elasticsearch/integration/ldap/MultipleAdRealmTests.java index 1eb9028d9f6..5592530ee69 100644 --- a/plugin/src/test/java/org/elasticsearch/integration/ldap/MultipleAdRealmTests.java +++ b/plugin/src/test/java/org/elasticsearch/integration/ldap/MultipleAdRealmTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.integration.ldap; -import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.junit.annotations.Network; @@ -23,7 +22,6 @@ import java.util.stream.Collectors; * just their userid (the AuthenticationService tries them in order) */ @Network -@AwaitsFix(bugUrl="https://github.com/elastic/x-pack-elasticsearch/issues/1823") public class MultipleAdRealmTests extends AbstractAdLdapRealmTestCase { private static RealmConfig secondaryRealmConfig; diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/ldap/GroupsResolverTestCase.java b/plugin/src/test/java/org/elasticsearch/xpack/security/authc/ldap/GroupsResolverTestCase.java index d10f26b9a48..a713b336608 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/ldap/GroupsResolverTestCase.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/authc/ldap/GroupsResolverTestCase.java @@ -32,7 +32,7 @@ public abstract class GroupsResolverTestCase extends ESTestCase { @Before public void setUpLdapConnection() throws Exception { - Path truststore = getDataPath("../ldap/support/ldaptrust.jks"); + Path truststore = getDataPath("/org/elasticsearch/xpack/security/authc/ldap/support/ldaptrust.jks"); this.ldapConnection = LdapTestUtils.openConnection(ldapUrl(), bindDN(), bindPassword(), truststore); } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactoryTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactoryTests.java index 2b52e6f0f0c..5400e6b7fdd 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactoryTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapUserSearchSessionFactoryTests.java @@ -32,15 +32,11 @@ import org.elasticsearch.test.junit.annotations.Network; import org.junit.Before; import java.nio.file.Path; -import java.text.MessageFormat; import java.util.List; -import java.util.Locale; import java.util.Map; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.isEmptyString; import static org.hamcrest.Matchers.notNullValue; @@ -385,7 +381,6 @@ public class LdapUserSearchSessionFactoryTests extends LdapTestCase { } @Network - @AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/1823") public void testUserSearchWithActiveDirectory() throws Exception { String groupSearchBase = "DC=ad,DC=test,DC=elasticsearch,DC=com"; String userSearchBase = "CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com"; @@ -397,7 +392,7 @@ public class LdapUserSearchSessionFactoryTests extends LdapTestCase { .put("user_search.base_dn", userSearchBase) .put("bind_dn", "ironman@ad.test.elasticsearch.com") .put("bind_password", ActiveDirectorySessionFactoryTests.PASSWORD) - .put("user_search.attribute", "cn") + .put("user_search.filter", "(cn={0})") .put("user_search.pool.enabled", randomBoolean()) .build(); Settings.Builder builder = Settings.builder() @@ -438,50 +433,6 @@ public class LdapUserSearchSessionFactoryTests extends LdapTestCase { } } - @Network - @AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/1823") - public void testUserSearchwithBindUserOpenLDAP() throws Exception { - String groupSearchBase = "ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; - String userSearchBase = "ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; - RealmConfig config = new RealmConfig("oldap-test", Settings.builder() - .put(LdapTestCase.buildLdapSettings(new String[] { OpenLdapTests.OPEN_LDAP_URL }, Strings.EMPTY_ARRAY, groupSearchBase, - LdapSearchScope.ONE_LEVEL)) - .put("user_search.base_dn", userSearchBase) - .put("bind_dn", "uid=blackwidow,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com") - .put("bind_password", OpenLdapTests.PASSWORD) - .put("user_search.pool.enabled", randomBoolean()) - .build(), globalSettings, new Environment(globalSettings), new ThreadContext(globalSettings)); - Settings.Builder builder = Settings.builder() - .put(globalSettings); - for (Map.Entry entry : config.settings().getAsMap().entrySet()) { - builder.put("xpack.security.authc.realms.ldap." + entry.getKey(), entry.getValue()); - } - Settings settings = builder.build(); - sslService = new SSLService(settings, new Environment(settings)); - LdapUserSearchSessionFactory sessionFactory = new LdapUserSearchSessionFactory(config, sslService); - - String[] users = new String[] { "cap", "hawkeye", "hulk", "ironman", "thor" }; - try { - for (String user : users) { - //auth - try (LdapSession ldap = session(sessionFactory, user, new SecureString(OpenLdapTests.PASSWORD))) { - assertThat(ldap.userDn(), is(equalTo(new MessageFormat("uid={0},ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com", - Locale.ROOT).format(new Object[]{user}, new StringBuffer(), null).toString()))); - assertThat(groups(ldap), hasItem(containsString("Avengers"))); - } - - //lookup - try (LdapSession ldap = unauthenticatedSession(sessionFactory, user)) { - assertThat(ldap.userDn(), is(equalTo(new MessageFormat("uid={0},ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com", - Locale.ROOT).format(new Object[]{user}, new StringBuffer(), null).toString()))); - assertThat(groups(ldap), hasItem(containsString("Avengers"))); - } - } - } finally { - sessionFactory.close(); - } - } - public void testConnectionPoolDefaultSettings() throws Exception { String groupSearchBase = "o=sevenSeas"; String userSearchBase = "o=sevenSeas"; diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/LdapMetaDataResolverTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/LdapMetaDataResolverTests.java index 81784a5ab86..fe9b79870fb 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/LdapMetaDataResolverTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/LdapMetaDataResolverTests.java @@ -5,13 +5,6 @@ */ package org.elasticsearch.xpack.security.authc.ldap.support; -import javax.security.auth.DestroyFailedException; -import java.io.IOException; -import java.nio.file.Path; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.UnrecoverableKeyException; -import java.security.cert.CertificateException; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -19,17 +12,10 @@ import java.util.List; import java.util.Map; import com.unboundid.ldap.sdk.Attribute; -import com.unboundid.ldap.sdk.LDAPConnection; -import com.unboundid.ldap.sdk.LDAPException; -import org.bouncycastle.operator.OperatorCreationException; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.junit.annotations.Network; -import org.elasticsearch.xpack.security.authc.ldap.LdapTestUtils; -import org.elasticsearch.xpack.security.authc.ldap.OpenLdapTests; -import org.junit.After; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.contains; @@ -42,7 +28,6 @@ public class LdapMetaDataResolverTests extends ESTestCase { private static final String HAWKEYE_DN = "uid=hawkeye,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; private LdapMetaDataResolver resolver; - private LDAPConnection connection; public void testParseSettings() throws Exception { resolver = new LdapMetaDataResolver(Settings.builder().putArray("metadata", "cn", "uid").build(), false); @@ -85,52 +70,9 @@ public class LdapMetaDataResolverTests extends ESTestCase { assertThat(map.get("uid"), equalTo("hawkeye")); } - @Network - @AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/1823") - public void testResolveSingleValuedAttributeFromConnection() throws Exception { - resolver = new LdapMetaDataResolver(Arrays.asList("givenName", "sn"), true); - setupOpenLdapConnection(); - final Map map = resolve(null); - assertThat(map.size(), equalTo(2)); - assertThat(map.get("givenName"), equalTo("Clint")); - assertThat(map.get("sn"), equalTo("Barton")); - } - - @Network - @AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/1823") - public void testResolveMultiValuedAttributeFromConnection() throws Exception { - resolver = new LdapMetaDataResolver(Arrays.asList("objectClass"), true); - setupOpenLdapConnection(); - final Map map = resolve(null); - assertThat(map.size(), equalTo(1)); - assertThat(map.get("objectClass"), instanceOf(List.class)); - assertThat((List) map.get("objectClass"), contains("top", "posixAccount", "inetOrgPerson")); - } - - @Network - @AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/1823") - public void testResolveMissingAttributeFromConnection() throws Exception { - resolver = new LdapMetaDataResolver(Arrays.asList("alias"), true); - setupOpenLdapConnection(); - final Map map = resolve(null); - assertThat(map.size(), equalTo(0)); - } - private Map resolve(Collection attributes) throws Exception { final PlainActionFuture> future = new PlainActionFuture<>(); - resolver.resolve(connection, HAWKEYE_DN, TimeValue.timeValueSeconds(1), logger, attributes, future); + resolver.resolve(null, HAWKEYE_DN, TimeValue.timeValueSeconds(1), logger, attributes, future); return future.get(); } - - private void setupOpenLdapConnection() throws Exception { - Path truststore = getDataPath("./ldaptrust.jks"); - this.connection = LdapTestUtils.openConnection(OpenLdapTests.OPEN_LDAP_URL, HAWKEYE_DN, OpenLdapTests.PASSWORD, truststore); - } - - @After - public void tearDownLdapConnection() throws Exception { - if (connection != null) { - connection.close(); - } - } } \ No newline at end of file diff --git a/plugin/src/test/java/org/elasticsearch/xpack/watcher/transport/action/get/GetWatchTests.java b/plugin/src/test/java/org/elasticsearch/xpack/watcher/transport/action/get/GetWatchTests.java index c843e01ba3b..81913534107 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/watcher/transport/action/get/GetWatchTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/watcher/transport/action/get/GetWatchTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.watcher.transport.action.get; +import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.xpack.watcher.condition.AlwaysCondition; import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource; @@ -56,9 +57,16 @@ public class GetWatchTests extends AbstractWatcherIntegrationTestCase { public void testGetNotFound() throws Exception { // does not matter if the watch does not exist or the index does not exist, we expect the same response + // if the watches index is an alias, remove the alias randomly, otherwise the index if (randomBoolean()) { try { - assertAcked(client().admin().indices().prepareDelete(Watch.INDEX)); + GetIndexResponse indexResponse = client().admin().indices().prepareGetIndex().setIndices(Watch.INDEX).get(); + boolean isWatchIndexAlias = Watch.INDEX.equals(indexResponse.indices()[0]) == false; + if (isWatchIndexAlias) { + assertAcked(client().admin().indices().prepareAliases().removeAlias(indexResponse.indices()[0], Watch.INDEX)); + } else { + assertAcked(client().admin().indices().prepareDelete(Watch.INDEX)); + } } catch (IndexNotFoundException e) {} } diff --git a/plugin/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/ldaptrust.jks b/plugin/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/ldaptrust.jks index 2b8287d88f0..b1dc2e31473 100644 Binary files a/plugin/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/ldaptrust.jks and b/plugin/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/ldaptrust.jks differ diff --git a/plugin/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks b/plugin/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks index f034f5b005a..457d62f51a4 100644 Binary files a/plugin/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks and b/plugin/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks differ diff --git a/qa/openldap-tests/build.gradle b/qa/openldap-tests/build.gradle new file mode 100644 index 00000000000..6bf54cd04a8 --- /dev/null +++ b/qa/openldap-tests/build.gradle @@ -0,0 +1,22 @@ +apply plugin: 'elasticsearch.standalone-test' +apply plugin: 'elasticsearch.vagrantsupport' + +dependencies { + testCompile project(path: ':x-pack-elasticsearch:plugin', configuration: 'runtime') + testCompile project(path: ':x-pack-elasticsearch:plugin', configuration: 'testArtifacts') +} + +task openLdapFixture { + dependsOn "vagrantCheckVersion", "virtualboxCheckVersion", ":x-pack-elasticsearch:test:openldap-fixture:up" +} + +if (project.rootProject.vagrantSupported) { + test.dependsOn openLdapFixture +} else { + test.enabled = false +} + +namingConventions { + // integ tests use Tests instead of IT + skipIntegTestInDisguise = true +} diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapTests.java b/qa/openldap-tests/src/test/java/org/elasticsearch/test/OpenLdapTests.java similarity index 64% rename from plugin/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapTests.java rename to qa/openldap-tests/src/test/java/org/elasticsearch/test/OpenLdapTests.java index 6d2699670b9..b418b8983b3 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapTests.java +++ b/qa/openldap-tests/src/test/java/org/elasticsearch/test/OpenLdapTests.java @@ -3,77 +3,88 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.security.authc.ldap; +package org.elasticsearch.test; +import com.unboundid.ldap.sdk.LDAPConnection; import com.unboundid.ldap.sdk.LDAPException; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException; import org.elasticsearch.env.Environment; import org.elasticsearch.xpack.security.authc.RealmConfig; +import org.elasticsearch.xpack.security.authc.ldap.LdapSessionFactory; +import org.elasticsearch.xpack.security.authc.ldap.LdapTestUtils; +import org.elasticsearch.xpack.security.authc.ldap.support.LdapMetaDataResolver; import org.elasticsearch.xpack.security.authc.ldap.support.LdapSearchScope; import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession; import org.elasticsearch.xpack.security.authc.ldap.support.LdapTestCase; import org.elasticsearch.xpack.security.authc.ldap.support.SessionFactory; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.junit.annotations.Network; import org.elasticsearch.xpack.ssl.SSLService; import org.elasticsearch.xpack.ssl.VerificationMode; import org.junit.Before; import java.nio.file.Path; import java.util.List; +import java.util.Map; import java.util.concurrent.ExecutionException; import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.instanceOf; -@Network -@AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/1823") public class OpenLdapTests extends ESTestCase { - public static final String OPEN_LDAP_URL = "ldaps://54.200.235.244:636"; + public static final String OPEN_LDAP_URL = "ldaps://localhost:60636"; public static final String PASSWORD = "NickFuryHeartsES"; - public static final SecureString PASSWORD_SECURE_STRING = new SecureString(PASSWORD.toCharArray()); + private static final String HAWKEYE_DN = "uid=hawkeye,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; + public static final String LDAPTRUST_PATH = "/org/elasticsearch/xpack/security/authc/ldap/support/ldaptrust.jks"; + private static final SecureString PASSWORD_SECURE_STRING = new SecureString(PASSWORD.toCharArray()); private boolean useGlobalSSL; private SSLService sslService; private Settings globalSettings; + @Override + public boolean enableWarningsCheck() { + return false; + } + @Before public void initializeSslSocketFactory() throws Exception { - Path truststore = getDataPath("../ldap/support/ldaptrust.jks"); + Path truststore = getDataPath(LDAPTRUST_PATH); /* * Prior to each test we reinitialize the socket factory with a new SSLService so that we get a new SSLContext. * If we re-use a SSLContext, previously connected sessions can get re-established which breaks hostname * verification tests since a re-established connection does not perform hostname verification. */ useGlobalSSL = randomBoolean(); + MockSecureSettings mockSecureSettings = new MockSecureSettings(); Settings.Builder builder = Settings.builder().put("path.home", createTempDir()); if (useGlobalSSL) { - builder.put("xpack.ssl.truststore.path", truststore) - .put("xpack.ssl.truststore.password", "changeit"); + builder.put("xpack.ssl.truststore.path", truststore); + mockSecureSettings.setString("xpack.ssl.truststore.secure_password", "changeit"); // fake realm to load config with certificate verification mode builder.put("xpack.security.authc.realms.bar.ssl.truststore.path", truststore); - builder.put("xpack.security.authc.realms.bar.ssl.truststore.password", "changeit"); + mockSecureSettings.setString("xpack.security.authc.realms.bar.ssl.truststore.secure_password", "changeit"); builder.put("xpack.security.authc.realms.bar.ssl.verification_mode", VerificationMode.CERTIFICATE); } else { // fake realms so ssl will get loaded builder.put("xpack.security.authc.realms.foo.ssl.truststore.path", truststore); - builder.put("xpack.security.authc.realms.foo.ssl.truststore.password", "changeit"); + mockSecureSettings.setString("xpack.security.authc.realms.foo.ssl.truststore.secure_password", "changeit"); builder.put("xpack.security.authc.realms.foo.ssl.verification_mode", VerificationMode.FULL); builder.put("xpack.security.authc.realms.bar.ssl.truststore.path", truststore); - builder.put("xpack.security.authc.realms.bar.ssl.truststore.password", "changeit"); + mockSecureSettings.setString("xpack.security.authc.realms.bar.ssl.truststore.secure_password", "changeit"); builder.put("xpack.security.authc.realms.bar.ssl.verification_mode", VerificationMode.CERTIFICATE); } - globalSettings = builder.build(); + globalSettings = builder.setSecureSettings(mockSecureSettings).build(); Environment environment = new Environment(globalSettings); sslService = new SSLService(globalSettings, environment); } @@ -88,6 +99,7 @@ public class OpenLdapTests extends ESTestCase { String[] users = new String[] { "blackwidow", "cap", "hawkeye", "hulk", "ironman", "thor" }; for (String user : users) { + logger.info("testing connect as user [{}]", user); try (LdapSession ldap = session(sessionFactory, user, PASSWORD_SECURE_STRING)) { assertThat(groups(ldap), hasItem(containsString("Avengers"))); } @@ -105,9 +117,9 @@ public class OpenLdapTests extends ESTestCase { String[] users = new String[] { "blackwidow", "cap", "hawkeye", "hulk", "ironman", "thor" }; for (String user : users) { - LdapSession ldap = session(sessionFactory, user, PASSWORD_SECURE_STRING); - assertThat(groups(ldap), hasItem(containsString("Avengers"))); - ldap.close(); + try (LdapSession ldap = session(sessionFactory, user, PASSWORD_SECURE_STRING)) { + assertThat(groups(ldap), hasItem(containsString("Avengers"))); + } } } @@ -116,7 +128,7 @@ public class OpenLdapTests extends ESTestCase { String userTemplate = "uid={0},ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; Settings settings = Settings.builder() .put(buildLdapSettings(OPEN_LDAP_URL, userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL)) - .put("group_search.filter", "(&(objectclass=posixGroup)(memberUID={0}))") + .put("group_search.filter", "(&(objectclass=posixGroup)(memberUid={0}))") .put("group_search.user_attribute", "uid") .build(); RealmConfig config = new RealmConfig("oldap-test", settings, globalSettings, new ThreadContext(Settings.EMPTY)); @@ -166,27 +178,67 @@ public class OpenLdapTests extends ESTestCase { anyOf(containsString("Hostname verification failed"), containsString("peer not authenticated"))); } - Settings buildLdapSettings(String ldapUrl, String userTemplate, String groupSearchBase, LdapSearchScope scope) { - Settings baseSettings = LdapTestCase.buildLdapSettings(ldapUrl, userTemplate, groupSearchBase, scope); - if (useGlobalSSL) { - return baseSettings; + public void testResolveSingleValuedAttributeFromConnection() throws Exception { + LdapMetaDataResolver resolver = new LdapMetaDataResolver(Settings.builder().putArray("metadata", "cn", "sn").build(), true); + try (LDAPConnection ldapConnection = setupOpenLdapConnection()) { + final Map map = resolve(ldapConnection, resolver); + assertThat(map.size(), equalTo(2)); + assertThat(map.get("cn"), equalTo("Clint Barton")); + assertThat(map.get("sn"), equalTo("Clint Barton")); } - return Settings.builder() - .put(baseSettings) - .put("ssl.truststore.path", getDataPath("../ldap/support/ldaptrust.jks")) + } + + public void testResolveMultiValuedAttributeFromConnection() throws Exception { + LdapMetaDataResolver resolver = new LdapMetaDataResolver(Settings.builder().putArray("metadata", "objectClass").build(), true); + try (LDAPConnection ldapConnection = setupOpenLdapConnection()) { + final Map map = resolve(ldapConnection, resolver); + assertThat(map.size(), equalTo(1)); + assertThat(map.get("objectClass"), instanceOf(List.class)); + assertThat((List) map.get("objectClass"), contains("top", "posixAccount", "inetOrgPerson")); + } + } + + public void testResolveMissingAttributeFromConnection() throws Exception { + LdapMetaDataResolver resolver = new LdapMetaDataResolver(Settings.builder().putArray("metadata", "alias").build(), true); + try (LDAPConnection ldapConnection = setupOpenLdapConnection()) { + final Map map = resolve(ldapConnection, resolver); + assertThat(map.size(), equalTo(0)); + } + } + + private Settings buildLdapSettings(String ldapUrl, String userTemplate, String groupSearchBase, LdapSearchScope scope) { + Settings.Builder builder = Settings.builder() + .put(LdapTestCase.buildLdapSettings(ldapUrl, userTemplate, groupSearchBase, scope)); + builder.put("group_search.user_attribute", "uid"); + if (useGlobalSSL) { + return builder.build(); + } + return builder + .put("ssl.truststore.path", getDataPath(LDAPTRUST_PATH)) .put("ssl.truststore.password", "changeit") .build(); } - protected LdapSession session(SessionFactory factory, String username, SecureString password) { + private LdapSession session(SessionFactory factory, String username, SecureString password) { PlainActionFuture future = new PlainActionFuture<>(); factory.session(username, password, future); return future.actionGet(); } - protected List groups(LdapSession ldapSession) { + private List groups(LdapSession ldapSession) { PlainActionFuture> future = new PlainActionFuture<>(); ldapSession.groups(future); return future.actionGet(); } + + private LDAPConnection setupOpenLdapConnection() throws Exception { + Path truststore = getDataPath(LDAPTRUST_PATH); + return LdapTestUtils.openConnection(OpenLdapTests.OPEN_LDAP_URL, HAWKEYE_DN, OpenLdapTests.PASSWORD, truststore); + } + + private Map resolve(LDAPConnection connection, LdapMetaDataResolver resolver) throws Exception { + final PlainActionFuture> future = new PlainActionFuture<>(); + resolver.resolve(connection, HAWKEYE_DN, TimeValue.timeValueSeconds(1), logger, null, future); + return future.get(); + } } diff --git a/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapUserSearchSessionFactoryTests.java b/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapUserSearchSessionFactoryTests.java new file mode 100644 index 00000000000..b5637db8553 --- /dev/null +++ b/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/OpenLdapUserSearchSessionFactoryTests.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.security.authc.ldap; + +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.MockSecureSettings; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.env.Environment; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.OpenLdapTests; +import org.elasticsearch.xpack.security.authc.RealmConfig; +import org.elasticsearch.xpack.security.authc.ldap.support.LdapSearchScope; +import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession; +import org.elasticsearch.xpack.security.authc.ldap.support.LdapTestCase; +import org.elasticsearch.xpack.security.authc.ldap.support.SessionFactory; +import org.elasticsearch.xpack.ssl.SSLService; +import org.junit.Before; + +import java.nio.file.Path; +import java.text.MessageFormat; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.test.OpenLdapTests.LDAPTRUST_PATH; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.is; + +public class OpenLdapUserSearchSessionFactoryTests extends ESTestCase { + + private Settings globalSettings; + + @Before + public void initializeSslSocketFactory() throws Exception { + Path keystore = getDataPath(LDAPTRUST_PATH); + + /* + * Prior to each test we reinitialize the socket factory with a new SSLService so that we get a new SSLContext. + * If we re-use a SSLContext, previously connected sessions can get re-established which breaks hostname + * verification tests since a re-established connection does not perform hostname verification. + */ + globalSettings = Settings.builder() + .put("path.home", createTempDir()) + .put("xpack.ssl.truststore.path", keystore) + .setSecureSettings(newSecureSettings("xpack.ssl.truststore.secure_password", "changeit")) + .build(); + } + + public void testUserSearchwithBindUserOpenLDAP() throws Exception { + String groupSearchBase = "ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; + String userSearchBase = "ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; + RealmConfig config = new RealmConfig("oldap-test", Settings.builder() + .put(LdapTestCase.buildLdapSettings(new String[] { OpenLdapTests.OPEN_LDAP_URL }, Strings.EMPTY_ARRAY, groupSearchBase, + LdapSearchScope.ONE_LEVEL)) + .put("user_search.base_dn", userSearchBase) + .put("group_search.user_attribute", "uid") + .put("bind_dn", "uid=blackwidow,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com") + .put("bind_password", OpenLdapTests.PASSWORD) + .put("user_search.pool.enabled", randomBoolean()) + .build(), globalSettings, new Environment(globalSettings), new ThreadContext(globalSettings)); + Settings.Builder builder = Settings.builder() + .put(globalSettings); + for (Map.Entry entry : config.settings().getAsMap().entrySet()) { + builder.put("xpack.security.authc.realms.ldap." + entry.getKey(), entry.getValue()); + } + Settings settings = builder.build(); + SSLService sslService = new SSLService(settings, new Environment(settings)); + + + String[] users = new String[] { "cap", "hawkeye", "hulk", "ironman", "thor" }; + try (LdapUserSearchSessionFactory sessionFactory = new LdapUserSearchSessionFactory(config, sslService)) { + for (String user : users) { + //auth + try (LdapSession ldap = session(sessionFactory, user, new SecureString(OpenLdapTests.PASSWORD))) { + assertThat(ldap.userDn(), is(equalTo(new MessageFormat("uid={0},ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com", + Locale.ROOT).format(new Object[]{user}, new StringBuffer(), null).toString()))); + assertThat(groups(ldap), hasItem(containsString("Avengers"))); + } + + //lookup + try (LdapSession ldap = unauthenticatedSession(sessionFactory, user)) { + assertThat(ldap.userDn(), is(equalTo(new MessageFormat("uid={0},ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com", + Locale.ROOT).format(new Object[]{user}, new StringBuffer(), null).toString()))); + assertThat(groups(ldap), hasItem(containsString("Avengers"))); + } + } + } + } + + private MockSecureSettings newSecureSettings(String key, String value) { + MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString(key, value); + return secureSettings; + } + + private LdapSession session(SessionFactory factory, String username, SecureString password) { + PlainActionFuture future = new PlainActionFuture<>(); + factory.session(username, password, future); + return future.actionGet(); + } + + private List groups(LdapSession ldapSession) { + Objects.requireNonNull(ldapSession); + PlainActionFuture> future = new PlainActionFuture<>(); + ldapSession.groups(future); + return future.actionGet(); + } + + private LdapSession unauthenticatedSession(SessionFactory factory, String username) { + PlainActionFuture future = new PlainActionFuture<>(); + factory.unauthenticatedSession(username, future); + return future.actionGet(); + } +} diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java b/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java similarity index 95% rename from plugin/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java rename to qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java index 40e015ec697..fd3ae074cd5 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java +++ b/qa/openldap-tests/src/test/java/org/elasticsearch/xpack/security/authc/ldap/SearchGroupsResolverTests.java @@ -5,11 +5,10 @@ */ package org.elasticsearch.xpack.security.authc.ldap; -import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.test.junit.annotations.Network; +import org.elasticsearch.test.OpenLdapTests; import org.elasticsearch.xpack.security.authc.ldap.support.LdapSearchScope; import org.elasticsearch.xpack.security.support.NoOpLogger; @@ -21,15 +20,15 @@ import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -@Network -@AwaitsFix(bugUrl = "https://github.com/elastic/x-pack-elasticsearch/issues/1823") +@SuppressWarnings("unchecked") public class SearchGroupsResolverTests extends GroupsResolverTestCase { - public static final String BRUCE_BANNER_DN = "uid=hulk,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; + private static final String BRUCE_BANNER_DN = "uid=hulk,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com"; public void testResolveSubTree() throws Exception { Settings settings = Settings.builder() .put("group_search.base_dn", "dc=oldap,dc=test,dc=elasticsearch,dc=com") + .put("group_search.user_attribute", "uid") .build(); SearchGroupsResolver resolver = new SearchGroupsResolver(settings); @@ -46,6 +45,7 @@ public class SearchGroupsResolverTests extends GroupsResolverTestCase { Settings settings = Settings.builder() .put("group_search.base_dn", "ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com") .put("group_search.scope", LdapSearchScope.ONE_LEVEL) + .put("group_search.user_attribute", "uid") .build(); SearchGroupsResolver resolver = new SearchGroupsResolver(settings); @@ -62,6 +62,7 @@ public class SearchGroupsResolverTests extends GroupsResolverTestCase { Settings settings = Settings.builder() .put("group_search.base_dn", "cn=Avengers,ou=People,dc=oldap,dc=test,dc=elasticsearch,dc=com") .put("group_search.scope", LdapSearchScope.BASE) + .put("group_search.user_attribute", "uid") .build(); SearchGroupsResolver resolver = new SearchGroupsResolver(settings); diff --git a/qa/openldap-tests/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/ldaptrust.jks b/qa/openldap-tests/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/ldaptrust.jks new file mode 100644 index 00000000000..b1dc2e31473 Binary files /dev/null and b/qa/openldap-tests/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/ldaptrust.jks differ diff --git a/qa/security-setup-password-tests/build.gradle b/qa/security-setup-password-tests/build.gradle new file mode 100644 index 00000000000..f019e70a2c5 --- /dev/null +++ b/qa/security-setup-password-tests/build.gradle @@ -0,0 +1,28 @@ +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' + +dependencies { + testCompile project(path: ':x-pack-elasticsearch:plugin', configuration: 'runtime') + testCompile project(path: ':x-pack-elasticsearch:plugin', configuration: 'testArtifacts') +} + +integTestRunner { + systemProperty 'tests.security.manager', 'false' +} + +integTestCluster { + plugin ':x-pack-elasticsearch:plugin' + keystoreSetting 'bootstrap.password', 'x-pack-test-password' + setupCommand 'setupTestAdmin', + 'bin/x-pack/users', 'useradd', "test_admin", '-p', 'x-pack-test-password', '-r', "superuser" + waitCondition = { node, ant -> + File tmpFile = new File(node.cwd, 'wait.success') + ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow", + dest: tmpFile.toString(), + username: 'test_admin', + password: 'x-pack-test-password', + ignoreerrors: true, + retries: 10) + return tmpFile.exists() + } +} diff --git a/qa/security-setup-password-tests/src/test/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordToolIT.java b/qa/security-setup-password-tests/src/test/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordToolIT.java new file mode 100644 index 00000000000..da15bcbc669 --- /dev/null +++ b/qa/security-setup-password-tests/src/test/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordToolIT.java @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.security.authc.esnative.tool; + +import org.elasticsearch.cli.MockTerminal; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.http.HttpHost; +import org.elasticsearch.client.http.message.BasicHeader; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.network.NetworkService; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xpack.security.SecurityClusterClientYamlTestCase; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.InetAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; + +public class SetupPasswordToolIT extends ESRestTestCase { + + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue("test_admin", new SecureString("x-pack-test-password".toCharArray())); + return Settings.builder() + .put(ThreadContext.PREFIX + ".Authorization", token) + .build(); + } + + @SuppressWarnings("unchecked") + public void testSetupPasswordToolAutoSetup() throws Exception { + SecurityClusterClientYamlTestCase.waitForSecurity(); + + final String testConfigDir = System.getProperty("tests.config.dir"); + logger.info("--> CONF: {}", testConfigDir); + final Path configPath = PathUtils.get(testConfigDir); + setSystemPropsForTool(configPath); + + Response nodesResponse = client().performRequest("GET", "/_nodes/http"); + Map nodesMap = entityAsMap(nodesResponse); + + Map nodes = (Map) nodesMap.get("nodes"); + Map firstNode = (Map) nodes.entrySet().iterator().next().getValue(); + Map firstNodeHttp = (Map) firstNode.get("http"); + String nodePublishAddress = (String) firstNodeHttp.get("publish_address"); + final int lastColonIndex = nodePublishAddress.lastIndexOf(':'); + InetAddress actualPublishAddress = InetAddresses.forString(nodePublishAddress.substring(0, lastColonIndex)); + InetAddress expectedPublishAddress = new NetworkService(Collections.emptyList()).resolvePublishHostAddresses(Strings.EMPTY_ARRAY); + final int port = Integer.valueOf(nodePublishAddress.substring(lastColonIndex + 1)); + + List lines = Files.readAllLines(configPath.resolve("elasticsearch.yml")); + lines = lines.stream().filter(s -> s.startsWith("http.port") == false && s.startsWith("http.publish_port") == false) + .collect(Collectors.toList()); + lines.add(randomFrom("http.port", "http.publish_port") + ": " + port); + if (expectedPublishAddress.equals(actualPublishAddress) == false) { + lines.add("http.publish_address: " + InetAddresses.toAddrString(actualPublishAddress)); + } + Files.write(configPath.resolve("elasticsearch.yml"), lines, StandardCharsets.UTF_8, StandardOpenOption.TRUNCATE_EXISTING); + + MockTerminal mockTerminal = new MockTerminal(); + SetupPasswordTool tool = new SetupPasswordTool(); + final int status; + if (randomBoolean()) { + mockTerminal.addTextInput("y"); // answer yes to continue prompt + status = tool.main(new String[] { "auto" }, mockTerminal); + } else { + status = tool.main(new String[] { "auto", "--batch" }, mockTerminal); + } + assertEquals(0, status); + String output = mockTerminal.getOutput(); + logger.info("CLI TOOL OUTPUT:\n{}", output); + String[] outputLines = output.split("\\n"); + Map userPasswordMap = new HashMap<>(); + Arrays.asList(outputLines).forEach(line -> { + if (line.startsWith("PASSWORD ")) { + String[] pieces = line.split(" "); + String user = pieces[1]; + String password = pieces[pieces.length - 1]; + logger.info("user [{}] password [{}]", user, password); + userPasswordMap.put(user, password); + } + }); + + assertEquals(3, userPasswordMap.size()); + userPasswordMap.entrySet().forEach(entry -> { + final String basicHeader = "Basic " + + Base64.getEncoder().encodeToString((entry.getKey() + ":" + entry.getValue()).getBytes(StandardCharsets.UTF_8)); + try { + Response authenticateResponse = client().performRequest("GET", "/_xpack/security/_authenticate", + new BasicHeader("Authorization", basicHeader)); + assertEquals(200, authenticateResponse.getStatusLine().getStatusCode()); + Map userInfoMap = entityAsMap(authenticateResponse); + assertEquals(entry.getKey(), userInfoMap.get("username")); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } + + @SuppressForbidden(reason = "need to set sys props for CLI tool") + private void setSystemPropsForTool(Path configPath) { + System.setProperty("es.path.conf", configPath.toString()); + System.setProperty("es.path.home", configPath.getParent().toString()); + } +} diff --git a/test/build.gradle b/test/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/openldap-fixture/Vagrantfile b/test/openldap-fixture/Vagrantfile new file mode 100644 index 00000000000..633512fe816 --- /dev/null +++ b/test/openldap-fixture/Vagrantfile @@ -0,0 +1,37 @@ + +# ELASTICSEARCH CONFIDENTIAL +# __________________ +# +# [2017] Elasticsearch Incorporated. All Rights Reserved. +# +# NOTICE: All information contained herein is, and remains +# the property of Elasticsearch Incorporated and its suppliers, +# if any. The intellectual and technical concepts contained +# herein are proprietary to Elasticsearch Incorporated +# and its suppliers and may be covered by U.S. and Foreign Patents, +# patents in process, and are protected by trade secret or copyright law. +# Dissemination of this information or reproduction of this material +# is strictly forbidden unless prior written permission is obtained +# from Elasticsearch Incorporated. + +# This Vagrantfile exists to define a virtual machine running OpenLDAP +# for usage as a testing fixture for the build process.. + +Vagrant.configure("2") do |config| + + config.vm.define "openldap" do |config| + config.vm.box = "elastic/ubuntu-16.04-x86_64" + end + + config.vm.hostname = "openldap.build.elastic.co" + + if Vagrant.has_plugin?("vagrant-cachier") + config.cache.scope = :box + end + + config.vm.network "forwarded_port", guest: 389, host: 60389, protocol: "tcp" + config.vm.network "forwarded_port", guest: 636, host: 60636, protocol: "tcp" + + config.vm.provision "shell", path: "src/main/resources/provision/installopenldap.sh" + +end diff --git a/test/openldap-fixture/build.gradle b/test/openldap-fixture/build.gradle new file mode 100644 index 00000000000..eadc81dd14c --- /dev/null +++ b/test/openldap-fixture/build.gradle @@ -0,0 +1,43 @@ +apply plugin: 'elasticsearch.build' + +Map vagrantEnvVars = [ + 'VAGRANT_CWD' : "${project.projectDir.absolutePath}", + 'VAGRANT_VAGRANTFILE' : 'Vagrantfile', + 'VAGRANT_PROJECT_DIR' : "${project.projectDir.absolutePath}" +] + +String box = "openldap" + +task update(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) { + command 'box' + subcommand 'update' + boxName box + environmentVars vagrantEnvVars +} + +task up(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) { + command 'up' + args '--provision', '--provider', 'virtualbox' + boxName box + environmentVars vagrantEnvVars + dependsOn update +} + +task halt(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) { + command 'halt' + boxName box + environmentVars vagrantEnvVars +} + +task destroy(type: org.elasticsearch.gradle.vagrant.VagrantCommandTask) { + command 'destroy' + args '-f' + boxName box + environmentVars vagrantEnvVars + dependsOn halt +} + +thirdPartyAudit.enabled = false +licenseHeaders.enabled = false +test.enabled = false +jarHell.enabled = false diff --git a/test/openldap-fixture/src/main/resources/provision/cert.pem b/test/openldap-fixture/src/main/resources/provision/cert.pem new file mode 100644 index 00000000000..6358b293d3f --- /dev/null +++ b/test/openldap-fixture/src/main/resources/provision/cert.pem @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIENjCCAx6gAwIBAgIJANOFCysZla1fMA0GCSqGSIb3DQEBBQUAMG8xCzAJBgNV +BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEQMA4G +A1UEChMHRWxhc3RpYzEWMBQGA1UECxMNRWxhc3RpY3NlYXJjaDERMA8GA1UEAxMI +T3BlbkxEQVAwHhcNMTcwNzI1MTY0MTE0WhcNMjcwNzIzMTY0MTE0WjBvMQswCQYD +VQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDU1vdW50YWluIFZpZXcxEDAO +BgNVBAoTB0VsYXN0aWMxFjAUBgNVBAsTDUVsYXN0aWNzZWFyY2gxETAPBgNVBAMT +CE9wZW5MREFQMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAynN3UJkl +90OTBdChW9/pvPjzT7zNnbREFc1AjlcbuAUhaXhlFEiqLlniXcYVx5Kq2VWZam0i +WJFHLYE4G4ywM5qhP+QP0njN+QKT+ajLr23RXmb9EWTTW90OLpx1wvQBey/TkJli +WY7T3+mQoDeVnaQV51jZPYKaud5fl9EtfcQCy5FI/wW2d72SJxCT74h7UbjRKlAA +1QKzZWBcQIqRLAQX9ec6RvymplKR4JBIUZtaJII0nDwzm0sEEGzDJWUQhlr0K4NG ++fovlGRGJ4mMirHZSkUI5pQJLaIg44KL2X0HtGITvrifcBC5uLoiwt/fdNPln03M +ciyanMWDIF636wIDAQABo4HUMIHRMB0GA1UdDgQWBBR0R4Hxw7ZmYsHSpW3eCBz1 +YWa+nTCBoQYDVR0jBIGZMIGWgBR0R4Hxw7ZmYsHSpW3eCBz1YWa+naFzpHEwbzEL +MAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3 +MRAwDgYDVQQKEwdFbGFzdGljMRYwFAYDVQQLEw1FbGFzdGljc2VhcmNoMREwDwYD +VQQDEwhPcGVuTERBUIIJANOFCysZla1fMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADggEBAA1WQu1Rj0hYARjhNImL5h+EuLA5935w1aDaXAp+RzBBYNUZzo5p +m6nIowth/uUwLkgzJspBmEEUPtb0WYTyRPBmDoThwFtpkRXHncyrOFYnX6HknnjS +xV41IAfo325/2oGKN9eUI1Dg2zl2fTWEZK6HTbefS97PFbGAAfKtJl8S6uGs4v8G +bBJhjwL2NhpCwtm8y+SkQXZgRjTO4c84fQASxipyeYWm1Q8IY4IXmBy5znS2OYX7 ++QrujJ58GPq98C/djeDrE1Zu687q8VRMvKj+p3j6/TBK1INTfdlHsyXtHbrJvi17 +TmaJehNgJQsJEjnVPUj7wDWPduVU8v2Lo3Y= +-----END CERTIFICATE----- diff --git a/test/openldap-fixture/src/main/resources/provision/installopenldap.sh b/test/openldap-fixture/src/main/resources/provision/installopenldap.sh new file mode 100644 index 00000000000..9a25ccceb02 --- /dev/null +++ b/test/openldap-fixture/src/main/resources/provision/installopenldap.sh @@ -0,0 +1,167 @@ +#!/bin/bash + +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. + +set -ex + +VDIR=/vagrant +RESOURCES=$VDIR/src/main/resources +PROV_DIR=$RESOURCES/provision +SSL_DIR=/etc/ssl/private +OLDAP_PASSWORD=NickFuryHeartsES +OLDAP_DOMAIN=oldap.test.elasticsearch.com +OLDAP_DOMAIN_DN=dc=oldap,dc=test,dc=elasticsearch,dc=com + +MARKER_FILE=/etc/marker + +if [ -f $MARKER_FILE ]; then + echo "Already provisioned..." + exit 0; +fi + +# Update package manager +apt-get update -qqy + +# /dev/random produces output very slowly on Ubuntu VM's. Install haveged to increase entropy. +apt-get install -qqy haveged +haveged + +# set the openldap configuration options +debconf-set-selections < /tmp/slapd +mv /tmp/slapd /etc/default/slapd + +# restart to listen on port 636 +service slapd restart + +# create the people container +ldapadd -D "cn=admin,dc=oldap,dc=test,dc=elasticsearch,dc=com" -w $OLDAP_PASSWORD <