Use ephemeral ports for idp-fixture (#40333)

This change removes the use of hardcoded port values for the
idp-fixture in favor of the mapped ephemeral ports. This should prevent
failures due to port conflicts in CI.

distribution/bwc/build.gradle

@@ -169,7 +169,7 @@ bwcVersions.forPreviousUnreleased { BwcVersions.UnreleasedVersionInfo unreleased
                         'JAVA_HOME',
                         getJavaHome(it, Integer.parseInt(
                                 lines
-                                        .findAll({ it.startsWith("ES_BUILD_JAVA=")})
+                                        .findAll({ it.startsWith("ES_BUILD_JAVA=") })
                                         .collect({ it.replace("ES_BUILD_JAVA=java", "").trim() })
                                         .collect({ it.replace("ES_BUILD_JAVA=openjdk", "").trim() })
                                         .join("!!")

x-pack/qa/openldap-tests/src/test/java/org/elasticsearch/test/OpenLdapTests.java

@@ -52,8 +52,8 @@ import static org.hamcrest.Matchers.startsWith;
 
 public class OpenLdapTests extends ESTestCase {
 
-    public static final String OPEN_LDAP_DNS_URL = "ldaps://localhost:60636";
+    public static final String OPEN_LDAP_DNS_URL = "ldaps://localhost:" + getFromProperty("636");
-    public static final String OPEN_LDAP_IP_URL = "ldaps://127.0.0.1:60636";
+    public static final String OPEN_LDAP_IP_URL = "ldaps://127.0.0.1:" + getFromProperty("636");
 
     public static final String PASSWORD = "NickFuryHeartsES";
     private static final String HAWKEYE_DN = "uid=hawkeye,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com";
@@ -307,4 +307,11 @@ public class OpenLdapTests extends ESTestCase {
         resolver.resolve(connection, HAWKEYE_DN, TimeValue.timeValueSeconds(1), logger, null, future);
         return future.get();
     }
+
+    private static String getFromProperty(String port) {
+        String key = "test.fixtures.openldap.tcp." + port;
+        final String value = System.getProperty(key);
+        assertNotNull("Expected the actual value for port " + port + " to be in system property " + key, value);
+        return value;
+    }
 }

x-pack/qa/saml-idp-tests/build.gradle

@@ -16,12 +16,29 @@ testFixtures.useFixture ":x-pack:test:idp-fixture"
 
 
 String outputDir = "${project.buildDir}/generated-resources/${project.name}"
-task copyIdpCertificate(type: Copy) {
+task copyIdpFiles(type: Copy) {
-    from idpFixtureProject.file('idp/shibboleth-idp/credentials/idp-browser.pem');
+    from idpFixtureProject.files('idp/shibboleth-idp/credentials/idp-browser.pem', 'idp/shibboleth-idp/metadata/idp-metadata.xml');
     into outputDir
 }
-project.sourceSets.test.output.dir(outputDir, builtBy: copyIdpCertificate)
+project.sourceSets.test.output.dir(outputDir, builtBy: copyIdpFiles)
-integTestCluster.dependsOn copyIdpCertificate
+
+task setupPorts {
+    dependsOn copyIdpFiles, idpFixtureProject.postProcessFixture
+    doLast {
+        String portString = idpFixtureProject.postProcessFixture.ext."test.fixtures.shibboleth-idp.tcp.4443"
+        int ephemeralPort = Integer.valueOf(portString)
+        File idpMetaFile = file(outputDir + '/idp-metadata.xml')
+        List<String> lines = idpMetaFile.readLines("UTF-8")
+        StringBuilder content = new StringBuilder()
+        for (String line : lines) {
+            content.append(line.replace("localhost:4443", "localhost:" + ephemeralPort))
+        }
+        idpMetaFile.delete()
+        idpMetaFile.createNewFile()
+        idpMetaFile.write(content.toString(), "UTF-8")
+    }
+}
+integTestCluster.dependsOn setupPorts
 
 integTestCluster {
   setting 'xpack.license.self_generated.type', 'trial'
@@ -51,8 +68,9 @@ integTestCluster {
   setting 'xpack.security.authc.realms.native.native.order', '3'
 
   setting 'xpack.ml.enabled', 'false'
+  setting 'logger.org.elasticsearch.xpack.security', 'TRACE'
 
-  extraConfigFile 'idp-metadata.xml', idpFixtureProject.file("idp/shibboleth-idp/metadata/idp-metadata.xml")
+  extraConfigFile 'idp-metadata.xml', file(outputDir + "/idp-metadata.xml")
 
   setupCommand 'setupTestAdmin',
             'bin/elasticsearch-users', 'useradd', "test_admin", '-p', 'x-pack-test-password', '-r', "superuser"

x-pack/qa/saml-idp-tests/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlAuthenticationIT.java

@@ -639,5 +639,4 @@ public class SamlAuthenticationIT extends ESRestTestCase {
             throw new ElasticsearchException("Cannot construct URI for httpServer @ {}:{}", e, host, port);
         }
     }
-
 }

x-pack/test/idp-fixture/docker-compose.yml

@@ -4,8 +4,8 @@ services:
     command: --copy-service --loglevel debug
     image: "osixia/openldap:1.2.3"
     ports:
-      - "30389:389"
+      - "389"
-      - "60636:636"
+      - "636"
     environment:
       LDAP_ADMIN_PASSWORD: "NickFuryHeartsES"
       LDAP_DOMAIN: "oldap.test.elasticsearch.com"
@@ -31,7 +31,7 @@ services:
       - JETTY_BROWSER_SSL_KEYSTORE_PASSWORD=secret
       - JETTY_BACKCHANNEL_SSL_KEYSTORE_PASSWORD=secret
     ports:
-      - "4443:4443"
+      - "4443"
     links:
       - openldap:openldap
     volumes: