Allow Integ Tests to run in a FIPS-140 JVM (#31989)

* Complete changes for running IT in a fips JVM

- Mute :x-pack:qa:sql:security:ssl:integTest as it
  cannot run in FIPS 140 JVM until the SQL CLI supports key/cert.
- Set default JVM keystore/truststore password in top level build
  script for all integTest tasks in a FIPS 140 JVM
- Changed top level x-pack build script to use keys and certificates
  for trust/key material when spinning up clusters for IT

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -131,6 +131,9 @@ class BuildPlugin implements Plugin<Project> {
                 runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, runtimeJavaHome))
             }
 
+            String inFipsJvmScript = 'print(java.security.Security.getProviders()[0].name.toLowerCase().contains("fips"));'
+            boolean inFipsJvm = Boolean.parseBoolean(runJavascript(project, runtimeJavaHome, inFipsJvmScript))
+
             // Build debugging info
             println '======================================='
             println 'Elasticsearch Build Hamster says Hello!'
@@ -202,6 +205,7 @@ class BuildPlugin implements Plugin<Project> {
             project.rootProject.ext.buildChecksDone = true
             project.rootProject.ext.minimumCompilerVersion = minimumCompilerVersion
             project.rootProject.ext.minimumRuntimeVersion = minimumRuntimeVersion
+            project.rootProject.ext.inFipsJvm = inFipsJvm
         }
 
         project.targetCompatibility = project.rootProject.ext.minimumRuntimeVersion
@@ -213,6 +217,7 @@ class BuildPlugin implements Plugin<Project> {
         project.ext.compilerJavaVersion = project.rootProject.ext.compilerJavaVersion
         project.ext.runtimeJavaVersion = project.rootProject.ext.runtimeJavaVersion
         project.ext.javaVersions = project.rootProject.ext.javaVersions
+        project.ext.inFipsJvm = project.rootProject.ext.inFipsJvm
     }
 
     private static String findCompilerJavaHome() {
@@ -770,7 +775,11 @@ class BuildPlugin implements Plugin<Project> {
                     systemProperty property.getKey(), property.getValue()
                 }
             }
-
+            // Set the system keystore/truststore password if we're running tests in a FIPS-140 JVM
+            if (project.inFipsJvm) {
+                systemProperty 'javax.net.ssl.trustStorePassword', 'password'
+                systemProperty 'javax.net.ssl.keyStorePassword', 'password'
+            }
             boolean assertionsEnabled = Boolean.parseBoolean(System.getProperty('tests.asserts', 'true'))
             enableSystemAssertions assertionsEnabled
             enableAssertions assertionsEnabled

plugins/discovery-gce/build.gradle

@@ -22,36 +22,6 @@ dependencies {
   compile "commons-codec:commons-codec:${versions.commonscodec}"
 }
 
-
-// needed to be consistent with ssl host checking
-String host = InetAddress.getLoopbackAddress().getHostAddress();
-
-// location of keystore and files to generate it
-File keystore = new File(project.buildDir, 'keystore/test-node.jks')
-
-// generate the keystore
-task createKey(type: LoggedExec) {
-  doFirst {
-    project.delete(keystore.parentFile)
-    keystore.parentFile.mkdirs()
-  }
-  executable = new File(project.runtimeJavaHome, 'bin/keytool')
-  standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
-  args '-genkey',
-          '-alias', 'test-node',
-          '-keystore', keystore,
-          '-keyalg', 'RSA',
-          '-keysize', '2048',
-          '-validity', '712',
-          '-dname', 'CN=' + host,
-          '-keypass', 'keypass',
-          '-storepass', 'keypass'
-}
-
-// add keystore to test classpath: it expects it there
-sourceSets.test.resources.srcDir(keystore.parentFile)
-processTestResources.dependsOn(createKey)
-
 dependencyLicenses {
   mapping from: /google-.*/, to: 'google'
 }

server/src/test/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java

@@ -205,7 +205,14 @@ public class ReloadSecureSettingsIT extends ESIntegTestCase {
                             assertThat(nodesMap.size(), equalTo(cluster().size()));
                             for (final NodesReloadSecureSettingsResponse.NodeResponse nodeResponse : nodesReloadResponse.getNodes()) {
                                 assertThat(nodeResponse.reloadException(), notNullValue());
-                                assertThat(nodeResponse.reloadException(), instanceOf(IOException.class));
+                                // Running in a JVM with a BouncyCastle FIPS Security Provider, decrypting the Keystore with the wrong
+                                // password returns a SecurityException if the DataInputStream can't be fully consumed
+                                if (inFipsJvm()) {
+                                    assertThat(nodeResponse.reloadException(), instanceOf(SecurityException.class));
+                                } else {
+                                    assertThat(nodeResponse.reloadException(), instanceOf(IOException.class));
+                                }
+
                             }
                         } catch (final AssertionError e) {
                             reloadSettingsError.set(e);

test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java

@@ -176,6 +176,7 @@ import java.net.InetSocketAddress;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.security.Security;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -2364,4 +2365,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
         });
     }
 
+    public static boolean inFipsJvm() {
+        return Security.getProviders()[0].getName().toLowerCase(Locale.ROOT).contains("fips");
+    }
 }

x-pack/plugin/build.gradle

@@ -104,39 +104,28 @@ integTestRunner {
   systemProperty 'tests.rest.blacklist', blacklist.join(',')
 }
 
-// location of generated keystores and certificates
+// location for keys and certificates
 File keystoreDir = new File(project.buildDir, 'keystore')
+File nodeKey = file("$keystoreDir/testnode.pem")
+File nodeCert = file("$keystoreDir/testnode.crt")
 
-// Generate the node's keystore
+// Add key and certs to test classpath: it expects them there
-File nodeKeystore = new File(keystoreDir, 'test-node.jks')
+// User cert and key PEM files instead of a JKS Keystore for the cluster's trust material so that
-task createNodeKeyStore(type: LoggedExec) {
+// it can run in a FIPS 140 JVM
-  doFirst {
+// TODO: Remove all existing uses of cross project file references when the new approach for referencing static files is available
-    if (nodeKeystore.parentFile.exists() == false) {
+// https://github.com/elastic/elasticsearch/pull/32201
-      nodeKeystore.parentFile.mkdirs()
+task copyKeyCerts(type: Copy) {
+  from(project(':x-pack:plugin:core').file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/')) {
+    include 'testnode.crt', 'testnode.pem'
     }
-    if (nodeKeystore.exists()) {
+  into keystoreDir
-      delete nodeKeystore
-    }
-  }
-  executable = new File(project.runtimeJavaHome, 'bin/keytool')
-  standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
-  args '-genkey',
-          '-alias', 'test-node',
-          '-keystore', nodeKeystore,
-          '-keyalg', 'RSA',
-          '-keysize', '2048',
-          '-validity', '712',
-          '-dname', 'CN=smoke-test-plugins-ssl',
-          '-keypass', 'keypass',
-          '-storepass', 'keypass'
 }
-
 // Add keystores to test classpath: it expects it there
 sourceSets.test.resources.srcDir(keystoreDir)
-processTestResources.dependsOn(createNodeKeyStore)
+processTestResources.dependsOn(copyKeyCerts)
 
 integTestCluster {
-  dependsOn createNodeKeyStore
+  dependsOn copyKeyCerts
   setting 'xpack.ml.enabled', 'true'
   setting 'xpack.security.enabled', 'true'
   setting 'logger.org.elasticsearch.xpack.ml.datafeed', 'TRACE'
@@ -145,17 +134,19 @@ integTestCluster {
   setting 'xpack.monitoring.exporters._local.enabled', 'false'
   setting 'xpack.security.authc.token.enabled', 'true'
   setting 'xpack.security.transport.ssl.enabled', 'true'
-  setting 'xpack.security.transport.ssl.keystore.path', nodeKeystore.name
+  setting 'xpack.security.transport.ssl.key', nodeKey.name
+  setting 'xpack.security.transport.ssl.certificate', nodeCert.name
   setting 'xpack.security.transport.ssl.verification_mode', 'certificate'
   setting 'xpack.security.audit.enabled', 'true'
   setting 'xpack.license.self_generated.type', 'trial'
   keystoreSetting 'bootstrap.password', 'x-pack-test-password'
-  keystoreSetting 'xpack.security.transport.ssl.keystore.secure_password', 'keypass'
+  keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
   distribution = 'zip' // this is important since we use the reindex module in ML
 
   setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser'
 
-  extraConfigFile nodeKeystore.name, nodeKeystore
+  extraConfigFile nodeKey.name, nodeKey
+  extraConfigFile nodeCert.name, nodeCert
 
   waitCondition = { NodeInfo node, AntBuilder ant ->
       File tmpFile = new File(node.cwd, 'wait.success')

x-pack/plugin/src/test/resources/rest-api-spec/test/ssl/10_basic.yml

@@ -4,7 +4,6 @@
       xpack.ssl.certificates: {}
 
   - length: { $body: 1 }
-  - match: { $body.0.path: "test-node.jks" }
+  - match: { $body.0.path: "testnode.crt" }
-  - match: { $body.0.format: "jks" }
+  - match: { $body.0.format: "PEM" }
-  - match: { $body.0.alias: "test-node" }
   - match: { $body.0.has_private_key: true }

x-pack/qa/full-cluster-restart/build.gradle

@@ -125,8 +125,8 @@ subprojects {
 
   String output = "${buildDir}/generated-resources/${project.name}"
   task copyTestNodeKeystore(type: Copy) {
-    from project(xpackModule('core'))
+    from project(':x-pack:plugin:core')
-            .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
+      .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
     into outputDir
   }
 

x-pack/qa/ml-native-multi-node-tests/build.gradle

@@ -18,59 +18,45 @@ integTestRunner {
     systemProperty 'es.set.netty.runtime.available.processors', 'false'
 }
 
-// location of generated keystores and certificates
+// location for keys and certificates
 File keystoreDir = new File(project.buildDir, 'keystore')
-
+File nodeKey = file("$keystoreDir/testnode.pem")
-// Generate the node's keystore
+File nodeCert = file("$keystoreDir/testnode.crt")
-File nodeKeystore = new File(keystoreDir, 'test-node.jks')
+// Add key and certs to test classpath: it expects it there
-task createNodeKeyStore(type: LoggedExec) {
+task copyKeyCerts(type: Copy) {
-    doFirst {
+    from(project(':x-pack:plugin:core').file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/')) {
-        if (nodeKeystore.parentFile.exists() == false) {
+        include 'testnode.crt', 'testnode.pem'
-            nodeKeystore.parentFile.mkdirs()
-        }
-        if (nodeKeystore.exists()) {
-            delete nodeKeystore
-        }
     }
-    executable = new File(project.runtimeJavaHome, 'bin/keytool')
+    into keystoreDir
-    standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
-    args '-genkey',
-            '-alias', 'test-node',
-            '-keystore', nodeKeystore,
-            '-keyalg', 'RSA',
-            '-keysize', '2048',
-            '-validity', '712',
-            '-dname', 'CN=smoke-test-plugins-ssl',
-            '-keypass', 'keypass',
-            '-storepass', 'keypass'
 }
-
+// Add keys and cets to test classpath: it expects it there
-// Add keystores to test classpath: it expects it there
 sourceSets.test.resources.srcDir(keystoreDir)
-processTestResources.dependsOn(createNodeKeyStore)
+processTestResources.dependsOn(copyKeyCerts)
 
 integTestCluster {
-    dependsOn createNodeKeyStore
+    dependsOn copyKeyCerts
     setting 'xpack.security.enabled', 'true'
     setting 'xpack.ml.enabled', 'true'
     setting 'logger.org.elasticsearch.xpack.ml.datafeed', 'TRACE'
     setting 'xpack.monitoring.enabled', 'false'
     setting 'xpack.security.authc.token.enabled', 'true'
     setting 'xpack.security.transport.ssl.enabled', 'true'
-    setting 'xpack.security.transport.ssl.keystore.path', nodeKeystore.name
+    setting 'xpack.security.transport.ssl.key', nodeKey.name
+    setting 'xpack.security.transport.ssl.certificate', nodeCert.name
     setting 'xpack.security.transport.ssl.verification_mode', 'certificate'
     setting 'xpack.security.audit.enabled', 'true'
     setting 'xpack.license.self_generated.type', 'trial'
 
     keystoreSetting 'bootstrap.password', 'x-pack-test-password'
-    keystoreSetting 'xpack.security.transport.ssl.keystore.secure_password', 'keypass'
+    keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
 
     numNodes = 3
 
     setupCommand 'setupDummyUser',
             'bin/elasticsearch-users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser'
 
-    extraConfigFile nodeKeystore.name, nodeKeystore
+    extraConfigFile nodeKey.name, nodeKey
+    extraConfigFile nodeCert.name, nodeCert
 
     waitCondition = { node, ant ->
         File tmpFile = new File(node.cwd, 'wait.success')

x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java

@@ -124,9 +124,11 @@ abstract class MlNativeAutodetectIntegTestCase extends ESIntegTestCase {
 
     @Override
     protected Settings externalClusterClientSettings() {
-        Path keyStore;
+        Path key;
+        Path certificate;
         try {
-            keyStore = PathUtils.get(getClass().getResource("/test-node.jks").toURI());
+            key = PathUtils.get(getClass().getResource("/testnode.pem").toURI());
+            certificate = PathUtils.get(getClass().getResource("/testnode.crt").toURI());
         } catch (URISyntaxException e) {
             throw new IllegalStateException("error trying to get keystore path", e);
         }
@@ -135,8 +137,9 @@ abstract class MlNativeAutodetectIntegTestCase extends ESIntegTestCase {
         builder.put(SecurityField.USER_SETTING.getKey(), "x_pack_rest_user:" + SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING);
         builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), true);
         builder.put("xpack.security.transport.ssl.enabled", true);
-        builder.put("xpack.security.transport.ssl.keystore.path", keyStore.toAbsolutePath().toString());
+        builder.put("xpack.security.transport.ssl.key", key.toAbsolutePath().toString());
-        builder.put("xpack.security.transport.ssl.keystore.password", "keypass");
+        builder.put("xpack.security.transport.ssl.certificate", certificate.toAbsolutePath().toString());
+        builder.put("xpack.security.transport.ssl.key_passphrase", "testnode");
         builder.put("xpack.security.transport.ssl.verification_mode", "certificate");
         return builder.build();
     }

x-pack/qa/rolling-upgrade/build.gradle

@@ -107,7 +107,7 @@ subprojects {
 
   String output = "${buildDir}/generated-resources/${project.name}"
   task copyTestNodeKeystore(type: Copy) {
-    from project(xpackModule('core'))
+    from project(':x-pack:plugin:core')
             .file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
     into outputDir
   }

x-pack/qa/smoke-test-plugins-ssl/build.gradle

@@ -4,7 +4,7 @@ import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.elasticsearch.gradle.test.NodeInfo
 
 import javax.net.ssl.HttpsURLConnection
-import javax.net.ssl.KeyManagerFactory
+import javax.net.ssl.KeyManager
 import javax.net.ssl.SSLContext
 import javax.net.ssl.TrustManagerFactory
 import java.nio.charset.StandardCharsets
@@ -26,135 +26,27 @@ task copyXPackPluginProps(type: Copy) {
 }
 project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackPluginProps)
 
-// needed to be consistent with ssl host checking
-Object san = new SanEvaluator()
-
 // location of generated keystores and certificates
 File keystoreDir = new File(project.buildDir, 'keystore')
-
+File nodeKeystore = file("$keystoreDir/testnode.jks")
-// Generate the node's keystore
+File nodeKey = file("$keystoreDir/testnode.pem")
-File nodeKeystore = new File(keystoreDir, 'test-node.jks')
+File nodeCert = file("$keystoreDir/testnode.crt")
-task createNodeKeyStore(type: LoggedExec) {
+File clientKeyStore = file("$keystoreDir/testclient.jks")
-  doFirst {
+File clientKey = file("$keystoreDir/testclient.pem")
-    if (nodeKeystore.parentFile.exists() == false) {
+File clientCert = file("$keystoreDir/testclient.crt")
-      nodeKeystore.parentFile.mkdirs()
-    }
-    if (nodeKeystore.exists()) {
-      delete nodeKeystore
-    }
-  }
-  executable = new File(project.runtimeJavaHome, 'bin/keytool')
-  standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
-  args '-genkey',
-          '-alias', 'test-node',
-          '-keystore', nodeKeystore,
-          '-keyalg', 'RSA',
-          '-keysize', '2048',
-          '-validity', '712',
-          '-dname', 'CN=smoke-test-plugins-ssl',
-          '-keypass', 'keypass',
-          '-storepass', 'keypass',
-          '-ext', san
-}
-
-// Generate the client's keystore
-File clientKeyStore = new File(keystoreDir, 'test-client.jks')
-task createClientKeyStore(type: LoggedExec) {
-  doFirst {
-    if (clientKeyStore.parentFile.exists() == false) {
-      clientKeyStore.parentFile.mkdirs()
-    }
-    if (clientKeyStore.exists()) {
-      delete clientKeyStore
-    }
-  }
-  executable = new File(project.runtimeJavaHome, 'bin/keytool')
-  standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
-  args '-genkey',
-          '-alias', 'test-client',
-          '-keystore', clientKeyStore,
-          '-keyalg', 'RSA',
-          '-keysize', '2048',
-          '-validity', '712',
-          '-dname', 'CN=smoke-test-plugins-ssl',
-          '-keypass', 'keypass',
-          '-storepass', 'keypass',
-          '-ext', san
-}
-
-// Export the node's certificate
-File nodeCertificate = new File(keystoreDir, 'test-node.cert')
-task exportNodeCertificate(type: LoggedExec) {
-  dependsOn createNodeKeyStore
-  doFirst {
-    if (nodeCertificate.parentFile.exists() == false) {
-      nodeCertificate.parentFile.mkdirs()
-    }
-    if (nodeCertificate.exists()) {
-      delete nodeCertificate
-    }
-  }
-  executable = new File(project.runtimeJavaHome, 'bin/keytool')
-  args '-export',
-          '-alias', 'test-node',
-          '-keystore', nodeKeystore,
-          '-storepass', 'keypass',
-          '-file', nodeCertificate
-}
-
-// Import the node certificate in the client's keystore
-task importNodeCertificateInClientKeyStore(type: LoggedExec) {
-  dependsOn createClientKeyStore, exportNodeCertificate
-  executable = new File(project.runtimeJavaHome, 'bin/keytool')
-  args '-import',
-          '-alias', 'test-node',
-          '-keystore', clientKeyStore,
-          '-storepass', 'keypass',
-          '-file', nodeCertificate,
-          '-noprompt'
-}
-
-// Export the client's certificate
-File clientCertificate = new File(keystoreDir, 'test-client.cert')
-task exportClientCertificate(type: LoggedExec) {
-  dependsOn createClientKeyStore
-  doFirst {
-    if (clientCertificate.parentFile.exists() == false) {
-      clientCertificate.parentFile.mkdirs()
-    }
-    if (clientCertificate.exists()) {
-      delete clientCertificate
-    }
-  }
-  executable = new File(project.runtimeJavaHome, 'bin/keytool')
-  args '-export',
-          '-alias', 'test-client',
-          '-keystore', clientKeyStore,
-          '-storepass', 'keypass',
-          '-file', clientCertificate
-}
-
-// Import the client certificate in the node's keystore
-task importClientCertificateInNodeKeyStore(type: LoggedExec) {
-  dependsOn createNodeKeyStore, exportClientCertificate
-  executable = new File(project.runtimeJavaHome, 'bin/keytool')
-  args '-import',
-          '-alias', 'test-client',
-          '-keystore', nodeKeystore,
-          '-storepass', 'keypass',
-          '-file', clientCertificate,
-          '-noprompt'
-}
-
-forbiddenPatterns {
-  exclude '**/*.cert'
-}
 
+// Add keystores to test classpath: it expects it there
+task copyKeyCerts(type: Copy) {
+  from('./') {
+      include '*.crt', '*.pem', '*.jks'
+  }
+  into keystoreDir
+}
 // Add keystores to test classpath: it expects it there
 sourceSets.test.resources.srcDir(keystoreDir)
-processTestResources.dependsOn(importNodeCertificateInClientKeyStore, importClientCertificateInNodeKeyStore)
+processTestResources.dependsOn(copyKeyCerts)
 
-integTestCluster.dependsOn(importClientCertificateInNodeKeyStore, importNodeCertificateInClientKeyStore)
+integTestCluster.dependsOn(copyKeyCerts)
 
 ext.pluginsCount = 0
 project(':plugins').getChildProjects().each { pluginName, pluginProject ->
@@ -167,8 +59,7 @@ integTestCluster {
   setting 'xpack.monitoring.collection.interval', '1s'
   setting 'xpack.monitoring.exporters._http.type', 'http'
   setting 'xpack.monitoring.exporters._http.enabled', 'false'
-  setting 'xpack.monitoring.exporters._http.ssl.truststore.path', clientKeyStore.name
+  setting 'xpack.ssl.certificate_authorities', 'testnode.crt'
-  setting 'xpack.monitoring.exporters._http.ssl.truststore.password', 'keypass'
   setting 'xpack.monitoring.exporters._http.auth.username', 'monitoring_agent'
   setting 'xpack.monitoring.exporters._http.auth.password', 'x-pack-test-password'
   setting 'xpack.monitoring.exporters._http.ssl.verification_mode', 'full'
@@ -176,14 +67,18 @@ integTestCluster {
   setting 'xpack.license.self_generated.type', 'trial'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.security.http.ssl.enabled', 'true'
-  setting 'xpack.security.http.ssl.keystore.path', nodeKeystore.name
+  setting 'xpack.security.http.ssl.key', 'testnode.pem'
-  keystoreSetting 'xpack.security.http.ssl.keystore.secure_password', 'keypass'
+  setting 'xpack.security.http.ssl.certificate', 'testnode.crt'
+  keystoreSetting 'xpack.security.http.ssl.secure_key_passphrase', 'testnode'
 
   setting 'xpack.ml.enabled', 'false'
-
+  // copy keystores, keys and certificates into config/
-  // copy keystores into config/
   extraConfigFile nodeKeystore.name, nodeKeystore
+  extraConfigFile nodeKey.name, nodeKey
+  extraConfigFile nodeCert.name, nodeCert
   extraConfigFile clientKeyStore.name, clientKeyStore
+  extraConfigFile clientKey.name, clientKey
+  extraConfigFile clientCert.name, clientCert
 
   setupCommand 'setupTestUser',
                'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
@@ -193,13 +88,12 @@ integTestCluster {
   waitCondition = { NodeInfo node, AntBuilder ant ->
     File tmpFile = new File(node.cwd, 'wait.success')
     KeyStore keyStore = KeyStore.getInstance("JKS");
-    keyStore.load(clientKeyStore.newInputStream(), 'keypass'.toCharArray());
+    keyStore.load(clientKeyStore.newInputStream(), 'testclient'.toCharArray());
-    KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
-    kmf.init(keyStore, 'keypass'.toCharArray());
     TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
     tmf.init(keyStore);
+    // We don't need a KeyManager as there won't be client auth required so pass an empty array
     SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
-    sslContext.init(kmf.getKeyManagers(), tmf.getTrustManagers(), new SecureRandom());
+    sslContext.init(new KeyManager[0], tmf.getTrustManagers(), new SecureRandom());
     for (int i = 0; i < 10; i++) {
       // we use custom wait logic here for HTTPS
       HttpsURLConnection httpURLConnection = null;
@@ -244,160 +138,4 @@ processTestResources {
     inputs.properties(expansions)
     MavenFilteringHack.filter(it, expansions)
   }
-}
+}
-
-/** A lazy evaluator to find the san to use for certificate generation. */
-class SanEvaluator {
-
-  private static String san = null
-
-  String toString() {
-    synchronized (SanEvaluator.class) {
-      if (san == null) {
-        san = getSubjectAlternativeNameString()
-      }
-    }
-    return san
-  }
-
-  // Code stolen from NetworkUtils/InetAddresses/NetworkAddress to support SAN
-  /** Return all interfaces (and subinterfaces) on the system */
-  private static List<NetworkInterface> getInterfaces() throws SocketException {
-    List<NetworkInterface> all = new ArrayList<>();
-    addAllInterfaces(all, Collections.list(NetworkInterface.getNetworkInterfaces()));
-    Collections.sort(all, new Comparator<NetworkInterface>() {
-      @Override
-      public int compare(NetworkInterface left, NetworkInterface right) {
-        return Integer.compare(left.getIndex(), right.getIndex());
-      }
-    });
-    return all;
-  }
-
-  /** Helper for getInterfaces, recursively adds subinterfaces to {@code target} */
-  private static void addAllInterfaces(List<NetworkInterface> target, List<NetworkInterface> level) {
-    if (!level.isEmpty()) {
-      target.addAll(level);
-      for (NetworkInterface intf : level) {
-        addAllInterfaces(target, Collections.list(intf.getSubInterfaces()));
-      }
-    }
-  }
-
-  private static String getSubjectAlternativeNameString() {
-    List<InetAddress> list = new ArrayList<>();
-    for (NetworkInterface intf : getInterfaces()) {
-      if (intf.isUp()) {
-        // NOTE: some operating systems (e.g. BSD stack) assign a link local address to the loopback interface
-        // while technically not a loopback address, some of these treat them as one (e.g. OS X "localhost") so we must too,
-        // otherwise things just won't work out of box. So we include all addresses from loopback interfaces.
-        for (InetAddress address : Collections.list(intf.getInetAddresses())) {
-          if (intf.isLoopback() || address.isLoopbackAddress()) {
-            list.add(address);
-          }
-        }
-      }
-    }
-    if (list.isEmpty()) {
-      throw new IllegalArgumentException("no up-and-running loopback addresses found, got " + getInterfaces());
-    }
-
-    StringBuilder builder = new StringBuilder("san=");
-    for (int i = 0; i < list.size(); i++) {
-      InetAddress address = list.get(i);
-      String hostAddress;
-      if (address instanceof Inet6Address) {
-        hostAddress = compressedIPV6Address((Inet6Address)address);
-      } else {
-        hostAddress = address.getHostAddress();
-      }
-      builder.append("ip:").append(hostAddress);
-      String hostname = address.getHostName();
-      if (hostname.equals(address.getHostAddress()) == false) {
-        builder.append(",dns:").append(hostname);
-      }
-
-      if (i != (list.size() - 1)) {
-        builder.append(",");
-      }
-    }
-
-    return builder.toString();
-  }
-
-  private static String compressedIPV6Address(Inet6Address inet6Address) {
-    byte[] bytes = inet6Address.getAddress();
-    int[] hextets = new int[8];
-    for (int i = 0; i < hextets.length; i++) {
-      hextets[i] =  (bytes[2 * i] & 255) << 8 | bytes[2 * i + 1] & 255;
-    }
-    compressLongestRunOfZeroes(hextets);
-    return hextetsToIPv6String(hextets);
-  }
-
-  /**
-   * Identify and mark the longest run of zeroes in an IPv6 address.
-   *
-   * <p>Only runs of two or more hextets are considered.  In case of a tie, the
-   * leftmost run wins.  If a qualifying run is found, its hextets are replaced
-   * by the sentinel value -1.
-   *
-   * @param hextets {@code int[]} mutable array of eight 16-bit hextets
-   */
-  private static void compressLongestRunOfZeroes(int[] hextets) {
-    int bestRunStart = -1;
-    int bestRunLength = -1;
-    int runStart = -1;
-    for (int i = 0; i < hextets.length + 1; i++) {
-      if (i < hextets.length && hextets[i] == 0) {
-        if (runStart < 0) {
-          runStart = i;
-        }
-      } else if (runStart >= 0) {
-        int runLength = i - runStart;
-        if (runLength > bestRunLength) {
-          bestRunStart = runStart;
-          bestRunLength = runLength;
-        }
-        runStart = -1;
-      }
-    }
-    if (bestRunLength >= 2) {
-      Arrays.fill(hextets, bestRunStart, bestRunStart + bestRunLength, -1);
-    }
-  }
-
-  /**
-   * Convert a list of hextets into a human-readable IPv6 address.
-   *
-   * <p>In order for "::" compression to work, the input should contain negative
-   * sentinel values in place of the elided zeroes.
-   *
-   * @param hextets {@code int[]} array of eight 16-bit hextets, or -1s
-   */
-  private static String hextetsToIPv6String(int[] hextets) {
-    /*
-     * While scanning the array, handle these state transitions:
-     *   start->num => "num"     start->gap => "::"
-     *   num->num   => ":num"    num->gap   => "::"
-     *   gap->num   => "num"     gap->gap   => ""
-     */
-    StringBuilder buf = new StringBuilder(39);
-    boolean lastWasNumber = false;
-    for (int i = 0; i < hextets.length; i++) {
-      boolean thisIsNumber = hextets[i] >= 0;
-      if (thisIsNumber) {
-        if (lastWasNumber) {
-          buf.append(':');
-        }
-        buf.append(Integer.toHexString(hextets[i]));
-      } else {
-        if (i == 0 || lastWasNumber) {
-          buf.append("::");
-        }
-      }
-      lastWasNumber = thisIsNumber;
-    }
-    return buf.toString();
-  }
-}

x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsSslClientYamlTestSuiteIT.java

@@ -29,7 +29,7 @@ public class SmokeTestPluginsSslClientYamlTestSuiteIT extends ESClientYamlSuiteT
 
     private static final String USER = "test_user";
     private static final String PASS = "x-pack-test-password";
-    private static final String KEYSTORE_PASS = "keypass";
+    private static final String KEYSTORE_PASS = "testnode";
 
     public SmokeTestPluginsSslClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
@@ -45,7 +45,7 @@ public class SmokeTestPluginsSslClientYamlTestSuiteIT extends ESClientYamlSuiteT
     @BeforeClass
     public static void getKeyStore() {
       try {
-          keyStore = PathUtils.get(SmokeTestPluginsSslClientYamlTestSuiteIT.class.getResource("/test-node.jks").toURI());
+          keyStore = PathUtils.get(SmokeTestPluginsSslClientYamlTestSuiteIT.class.getResource("/testnode.jks").toURI());
       } catch (URISyntaxException e) {
           throw new ElasticsearchException("exception while reading the store", e);
       }

x-pack/qa/smoke-test-plugins-ssl/testclient.pem

@@ -0,0 +1,30 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: DES-EDE3-CBC,C98A45E4AFC263C2
+
+wLuUEXldYc54r4ryWd6jw6UMGYwn6+ibGKHp4sD92l42lmI2UrCT/Mb/E0O+KMMy
+pHgc5/dBWkXgMiqDyLIhHk4kgT40rdw5W5lZkAA4Qt/Yzd+rbscTvzp09zrF6Fll
+czgoE7FrvhOKiEOakerTit4pIPYosdX606cpVQE2lq9oZs9HVMcLzdAZj8A/P/4g
+fo4X3+zqVYC/LH4n00bhNoeeej2o1lEJ+l9u9hptT2ATXle6pANa83Ldg4OxJyj8
+dkR9ahnAMCvYTSjEU7nwmGNPeFX0PIUjJKQivr410cYG104DC30Yy+XrIUfjTVUi
+agwlMpHoBq79/ZRUJR3xPLkIGgw4g+RPt45D9eKsEsV4vqy8SFlgaoJ2mKUKleZy
+i7D9ouzMKQ3sYE4eQVQ5o3K8ZPn5eozCwCVIp7jGSsuvDpLA9peZSwWPfc5y8JFD
+/64usCt1J8Mv/e9NVllC8ZA+ZmDitTiwLZysczpMOaFqqeUbk9EJst38n4nBzRV2
+quxvg9W/iveQIydFyftCtNfRkpbp0NCsLz293dBYwZacHsPcY27IBCwXHiICjiAW
+q7bnisXsgSaQMhMNRGW9YElZGb7ZWxoIzcyNBisGI8zxn48ObERVOmkOFxY/gs9T
+YmpVMliWtmRG6hb6iCh9b7z8THRquxgTGE9ZFBwtLUKg33aubtgAfnUh/Xq2Ue5K
+l+ZCqDGEi/FSIjVENUNNntAx/vXeNPbkoGLb/HSJwAh+sjpaLGQ54xixCtE9l3NY
+o2QAiZ804KLPaGtbbOv7wPumxQ+8mxG5FN0hTRrsMW9t8pBXw47iMy/T2H21TD5D
+E5XbM6kFeBrnsWnZJ2/ieXqDE4SX0tm3WEvZlDg7N7jV8QDM/D3Xdkb/sqJRabMG
+tQRgwkLiB+mZ5MAfGLogI2/lOEayrBVz4qYdXojewxY4LtaZ5HiUIlyA9CJelMvD
+nS52I6+FpaFhvuZC10qaM9Ph9TNyx+XKRUsPILuDiBRnYiHUKs1qASl5tjn2yyjM
+71WSo7A7btOckzhDZdMVf1T472f0LGsRYoQebMhotqCuR7yArZHzTeWB0CjL3tOz
+j3QlhKt2E1jx43bSK5tBasd9Bpmn2onvdwu1RRP8cyQBsXJSDy4/8t/g63+C3wod
+8VPrlKhK+TenK9EoEqJ2mNuNq+duOjTXfK/7GM5s0BFKv+i2ckpDi1NPckd2gXjF
+yUFZhmK6k0WC4jjWloMt+WQpi1rXMEXwCypgTrqWbvD0p6+X3uQmP57L4yHQcZoW
+Qcs5GnihJ0DIhw9vYDhBhNo0WY1oBO20nVCN3R/JIpp3uDtg64WvfvMSXzJIPBCY
+s+/GM5TtuD6mERDu3+qXxWwiy4PMQRcgjRTMEZ3A4Iv77YfQRkcd6S9qjUUuR/5D
+xs+J4ryb1biz9ofW7I+Dbz4SArWSgwcuh14AV9RBv6Rh9m83rjT2K0yvbe/+7hHW
+R8nzRMqJcGNGCHmRjA/cwoiv6+k2J/RbCJqnR3RmNex/85XaXBfZwRfHXVbzZQfa
+SrFaaNLf1hMwGLAJjIcQRxa3yZbjFXVx1Bp4hh8rKNWaOItjavNtNg==
+-----END RSA PRIVATE KEY-----

x-pack/qa/smoke-test-plugins-ssl/testnode.pem

@@ -0,0 +1,30 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: DES-EDE3-CBC,9D867F7E0C94D013
+
+dVoVCjPeg1wgS7rVtOvGfQcrZyLkx393aWRnFq45tbjKBVuITtJ9vI7o4QXOV/15
+Gnb6WhXGIdWrzsxEAd46K6hIuNSISd4Emsx6c2Q5hTqWXXfexbOZBNfTtXtdJPnJ
+1jAaikhtztLo3JSLTKNY5sNxd+XbaQyYVUWvueK6zOaIIMETvB+VPVFd9i1ROibk
+Sgdtyj01KjkoalifqK/tA0CIYNKL0S6/eoK3UhAlpIprlpV+cnXa940C6bjLeJPt
+PMAGGp5RrplxSgrSerw3I9DOWkHGtpqzIka3XneNUXJP8k4HUJ+aZkGH2ZILKS8d
+4KMIb+KZSpHEGn+6uGccWLtZZmAjWJrDw56JbQtSHdRYLBRSOjLbTvQoPu/2Hpli
+7HOxbotlvjptMunncq5aqK57SHA1dh0cwF7J3LUmGFJ67eoz+VV3b5qMn4MopSeI
+mS16Ydd3nGpjSrln/elM0CQxqWfcOAXRZpDpFUQoXcBrLVzvz2DBl/0CrTRLhgzi
+CO+5/IVcBWRlYpRNGgjjP7q0j6URID3jk5J06fYQXmBiwQT5j+GZqqzpMCJ9mIy2
+1O9SN1hebJnIcEU+E0njn/MGjlYdPywhaCy8pqElp6Q8TUEJpwLRFO/owCoBet/n
+ZmCXUjfCGhc1pWHufFcDEQ6xMgEWWY/tdwCZeSU7EhErTjCbfupg+55A5fpDml0m
+3wH4CFcuRjlqyx6Ywixm1ATeitDtJl5HQTw6b8OtEXwSgRmZ0eSqSRVk9QbVS7gu
+IpQe09/Zimb5HzjZqZ3fdqHlcW4xax8hyJeyIvF5ZJ57eY8CBvu/wP2GDn26QnvF
+xQqdfDbq1H4JmpwUHpbFwBoQK4Q6WFd1z4EA9bRQeo3H9PoqoOwMDjzajwLRF7b7
+q6tYH/n9PyHwdf1c4fFwgSmL1toXGfKlA9hjIaLsRSDD6srT5EdUk78bsnddwI51
+tu7C7P4JG+h1VdRNMNTlqtileWsIE7Nn2A1OkcUxZdF5mamENpDpJcHePLto6c8q
+FKiwyFMsxhgsj6HK2HqO+UA4sX5Ni4oHwiPmb//EZLn045M5i1AN26KosJmb8++D
+sgR5reWRy+UqJCTYblVg+7Dx++ggUnfxVyQEsWmw5r5f4KU5wXBkvoVMGtPNa9DE
+n/uLtObD1qkNL38pRsr2OGRchYCgEoKGqEISBP4knfGXLOlWiW/246j9QzI97r1u
+tvy7fKg28G7AUz9l6bpewsPHefBUeRQeieP9eJINaEpxkF/w2RpKDLpQjWxwDDOM
+s+D0mrBMJve17AmJ8rMw6dIQPZYNZ88/jz1uQuUwQ2YlbmtZbCG81k9YMFGEU9XS
+cyhJxj8hvYnt2PR5Z9/cJPyWOs0m/ufOeeQQ8SnU/lzmrQnpzUd2Z6p5i/B7LdRP
+n1kX+l1qynuPnjvBz4nJQE0p6nzW8RyCDSniC9mtYtZmhgC8icqxgbvS7uEOBIYJ
+NbK+0bEETTO34iY/JVTIqLOw3iQZYMeUpxpj6Phgx/oooxMTquMecPKNgeVtaBst
+qjTNPX0ti1/HYpZqzYi8SV8YjHSJWCVMsZjKPr3W/HIcCKqYoIfgzi83Ha2KMQx6
+-----END RSA PRIVATE KEY-----

x-pack/qa/sql/security/ssl/build.gradle

@@ -1,5 +1,5 @@
+import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.LoggedExec
-import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.test.NodeInfo
 
 import javax.net.ssl.HttpsURLConnection
@@ -22,7 +22,7 @@ Object san = new SanEvaluator()
 File keystoreDir = new File(project.buildDir, 'keystore')
 
 // Generate the node's keystore
-File nodeKeystore = new File(keystoreDir, 'test-node.jks')
+File nodeKeystore = file("$keystoreDir/test-node.jks")
 task createNodeKeyStore(type: LoggedExec) {
   doFirst {
     if (nodeKeystore.parentFile.exists() == false) {
@@ -47,7 +47,7 @@ task createNodeKeyStore(type: LoggedExec) {
 }
 
 // Generate the client's keystore
-File clientKeyStore = new File(keystoreDir, 'test-client.jks')
+File clientKeyStore = file("$keystoreDir/test-client.jks")
 task createClientKeyStore(type: LoggedExec) {
   doFirst {
     if (clientKeyStore.parentFile.exists() == false) {
@@ -72,7 +72,7 @@ task createClientKeyStore(type: LoggedExec) {
 }
 
 // Export the node's certificate
-File nodeCertificate = new File(keystoreDir, 'test-node.cert')
+File nodeCertificate = file("$keystoreDir/test-node.cert")
 task exportNodeCertificate(type: LoggedExec) {
   dependsOn createNodeKeyStore
   doFirst {
@@ -104,7 +104,7 @@ task importNodeCertificateInClientKeyStore(type: LoggedExec) {
 }
 
 // Export the client's certificate
-File clientCertificate = new File(keystoreDir, 'test-client.cert')
+File clientCertificate = file("$keystoreDir/test-client.cert")
 task exportClientCertificate(type: LoggedExec) {
   dependsOn createClientKeyStore
   doFirst {
@@ -145,7 +145,6 @@ processTestResources.dependsOn(importNodeCertificateInClientKeyStore, importClie
 
 integTestCluster.dependsOn(importClientCertificateInNodeKeyStore)
 
-
 integTestCluster {
   // The setup that we actually want
   setting 'xpack.security.http.ssl.enabled', 'true'
@@ -206,9 +205,18 @@ integTestCluster {
     return tmpFile.exists()
   }
 }
+Closure notRunningFips = {
+  Boolean.parseBoolean(BuildPlugin.runJavascript(project, project.runtimeJavaHome,
+          'print(java.security.Security.getProviders()[0].name.toLowerCase().contains("fips"));')) == false
+}
 
-
+// Do not attempt to form a cluster in a FIPS JVM, as doing so with a JKS keystore will fail.
-
+// TODO Revisit this when SQL CLI client can handle key/certificate instead of only Keystores.
+// https://github.com/elastic/elasticsearch/issues/32306
+tasks.matching({ it.name == "integTestCluster#init" }).all { onlyIf notRunningFips }
+tasks.matching({ it.name == "integTestCluster#start" }).all { onlyIf notRunningFips }
+tasks.matching({ it.name == "integTestCluster#wait" }).all { onlyIf notRunningFips }
+tasks.matching({ it.name == "integTestRunner" }).all { onlyIf notRunningFips }
 
 /** A lazy evaluator to find the san to use for certificate generation. */
 class SanEvaluator {
@@ -293,7 +301,7 @@ class SanEvaluator {
     byte[] bytes = inet6Address.getAddress();
     int[] hextets = new int[8];
     for (int i = 0; i < hextets.length; i++) {
-      hextets[i] =  (bytes[2 * i] & 255) << 8 | bytes[2 * i + 1] & 255;
+      hextets[i] = (bytes[2 * i] & 255) << 8 | bytes[2 * i + 1] & 255;
     }
     compressLongestRunOfZeroes(hextets);
     return hextetsToIPv6String(hextets);

x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/JdbcIntegrationTestCase.java

@@ -84,7 +84,7 @@ public abstract class JdbcIntegrationTestCase extends ESRestTestCase {
     public static void index(String index, CheckedConsumer<XContentBuilder, IOException> body) throws IOException {
         index(index, "1", body);
     }
-    
+
     public static void index(String index, String documentId, CheckedConsumer<XContentBuilder, IOException> body) throws IOException {
         Request request = new Request("PUT", "/" + index + "/doc/" + documentId);
         request.addParameter("refresh", "true");