Build: Fix third party audit task for xpack core (elastic/x-pack-elasticsearch#3656)

This commit re-enables thirdPartyAudit for x-pack core. Previously, when
xpack was a single plugin, it transitively picked up httpcore-nio
through the elasticsearch rest client. Now that xpack core does not
depend on the rest client, httpcore-nio must be added as a dependency.
Additionally, commons-logging was previously handled through the rest
client, but now xpack depends directly on this, thus excludes must be
added for the pesky missing classes there.

This commit also cleans up unnecessary parts of plugin/build.gradle no
longer necessary.

Original commit: elastic/x-pack-elasticsearch@70e936bdc3

plugin/build.gradle

@@ -19,10 +19,6 @@ es_meta_plugin {
              'ml', 'monitoring', 'security', 'upgrade', 'watcher']
 }
 
-ext.expansions = [
- 'project.version': version,
-]
-
 dependencies {
   testCompile project(path: ':x-pack-elasticsearch:plugin:core', configuration: 'testArtifacts')
 }
@@ -40,6 +36,13 @@ artifacts {
 }
 
 integTestRunner {
+  /*
+   * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
+   * other if we allow them to set the number of available processors as it's set-once in Netty.
+   */
+  systemProperty 'es.set.netty.runtime.available.processors', 'false'
+
+
   // TODO: fix this rest test to not depend on a hardcoded port!
   def blacklist = ['getting_started/10_monitor_cluster_health/*']
   boolean snapshot = "true".equals(System.getProperty("build.snapshot", "true"))
@@ -140,14 +143,6 @@ integTestCluster {
   }
 }
 
-integTestRunner {
-  /*
-   * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
-   * other if we allow them to set the number of available processors as it's set-once in Netty.
-   */
-  systemProperty 'es.set.netty.runtime.available.processors', 'false'
-}
-
 run {
   setting 'xpack.ml.enabled', 'true'
   setting 'xpack.graph.enabled', 'true'

plugin/core/build.gradle

@@ -5,6 +5,9 @@ import java.nio.file.Path
 import java.nio.file.StandardCopyOption
 
 apply plugin: 'elasticsearch.esplugin'
+
+archivesBaseName = 'x-pack-core'
+
 esplugin {
   name 'x-pack-core'
   description 'Elasticsearch Expanded Pack Plugin - Core'
@@ -18,27 +21,16 @@ esplugin {
 integTest.enabled = false
 
 dependencyLicenses {
-  mapping from: /netty-.*/, to: 'netty'
   mapping from: /bc.*/, to: 'bouncycastle'
-  mapping from: /owasp-java-html-sanitizer.*/, to: 'owasp-java-html-sanitizer'
-  mapping from: /transport-netty.*/, to: 'elasticsearch'
-  mapping from: /transport-nio.*/, to: 'elasticsearch'
-  mapping from: /elasticsearch-nio.*/, to: 'elasticsearch'
-  mapping from: /elasticsearch-rest-client.*/, to: 'elasticsearch'
   mapping from: /http.*/, to: 'httpclient' // pulled in by rest client
   mapping from: /commons-.*/, to: 'commons' // pulled in by rest client
-  ignoreSha 'elasticsearch-rest-client'
-  ignoreSha 'transport-netty4'
-  ignoreSha 'transport-nio'
-  ignoreSha 'elasticsearch-nio'
-  ignoreSha 'elasticsearch-rest-client-sniffer'
-  ignoreSha 'x-pack-core'
 }
 
 dependencies {
     provided "org.elasticsearch:elasticsearch:${version}"
     compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
     compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
+    compile "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}"
     compile "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
 
     compile "commons-logging:commons-logging:${versions.commonslogging}"
@@ -50,7 +42,6 @@ dependencies {
     compile 'org.bouncycastle:bcpkix-jdk15on:1.58'
     compile project(path: ':modules:transport-netty4', configuration: 'runtime')
 
-    //testCompile project(path: ':core:cli', configuration: 'runtime')
     testCompile 'org.elasticsearch:securemock:1.2'
     testCompile "org.elasticsearch:mocksocket:${versions.mocksocket}"
     testCompile "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
@@ -60,6 +51,10 @@ dependencies {
     testCompile project(path: ':modules:analysis-common', configuration: 'runtime')
 }
 
+ext.expansions = [
+  'project.version': version
+]
+
 processResources {
   from(sourceSets.main.resources.srcDirs) {
     exclude '**/public.key'
@@ -81,14 +76,9 @@ forbiddenPatterns {
     exclude '**/*.zip'
 }
 
-archivesBaseName = 'x-pack-core'
-
 compileJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked"
 compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked"
 
-// TODO: fix these!
-thirdPartyAudit.enabled = false
-
 licenseHeaders {
   approvedLicenses << 'BCrypt (BSD-like)'
   additionalLicense 'BCRYP', 'BCrypt (BSD-like)', 'Copyright (c) 2006 Damien Miller <djm@mindrot.org>'
@@ -100,6 +90,7 @@ sourceSets.test.java {
     srcDir '../../license-tools/src/main/java'
 }
 
+// TODO: remove this jar once xpack extensions have been removed
 // assemble the API JAR for the transport-client and extension authors; this JAR is the core JAR by another name
 project.afterEvaluate {
     task apiJar {
@@ -131,108 +122,6 @@ project.afterEvaluate {
     }
 }
 
-//
-// integTestRunner {
-//     // TODO: fix this rest test to not depend on a hardcoded port!
-//     def blacklist = ['getting_started/10_monitor_cluster_health/*']
-//     boolean snapshot = "true".equals(System.getProperty("build.snapshot", "true"))
-//     if (!snapshot) {
-//         // these tests attempt to install basic/internal licenses signed against the dev/public.key
-//         // Since there is no infrastructure in place (anytime soon) to generate licenses using the production
-//         // private key, these tests are whitelisted in non-snapshot test runs
-//         blacklist.addAll(['xpack/15_basic/*', 'license/20_put_license/*'])
-//     }
-//     systemProperty 'tests.rest.blacklist', blacklist.join(',')
-// }
-
-// // location of generated keystores and certificates
-// File keystoreDir = new File(project.buildDir, 'keystore')
-
-// // Generate the node's keystore
-// File nodeKeystore = new File(keystoreDir, 'test-node.jks')
-// task createNodeKeyStore(type: LoggedExec) {
-//     doFirst {
-//         if (nodeKeystore.parentFile.exists() == false) {
-//             nodeKeystore.parentFile.mkdirs()
-//         }
-//         if (nodeKeystore.exists()) {
-//             delete nodeKeystore
-//         }
-//     }
-//     executable = new File(project.javaHome, 'bin/keytool')
-//     standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
-//     args '-genkey',
-//             '-alias', 'test-node',
-//             '-keystore', nodeKeystore,
-//             '-keyalg', 'RSA',
-//             '-keysize', '2048',
-//             '-validity', '712',
-//             '-dname', 'CN=smoke-test-plugins-ssl',
-//             '-keypass', 'keypass',
-//             '-storepass', 'keypass'
-// }
-
-// Add keystores to test classpath: it expects it there
-//sourceSets.test.resources.srcDir(keystoreDir)
-//processTestResources.dependsOn(createNodeKeyStore)
-
-// integTestCluster {
-//     dependsOn createNodeKeyStore
-//     setting 'xpack.ml.enabled', 'true'
-//     setting 'logger.org.elasticsearch.xpack.ml.datafeed', 'TRACE'
-//     // Integration tests are supposed to enable/disable exporters before/after each test
-//     setting 'xpack.monitoring.exporters._local.type', 'local'
-//     setting 'xpack.monitoring.exporters._local.enabled', 'false'
-//     setting 'xpack.monitoring.collection.interval', '-1'
-//     setting 'xpack.security.authc.token.enabled', 'true'
-//     setting 'xpack.security.transport.ssl.enabled', 'true'
-//     setting 'xpack.security.transport.ssl.keystore.path', nodeKeystore.name
-//     setting 'xpack.security.transport.ssl.verification_mode', 'certificate'
-//     setting 'xpack.security.audit.enabled', 'true'
-//     keystoreSetting 'bootstrap.password', 'x-pack-test-password'
-//     keystoreSetting 'xpack.security.transport.ssl.keystore.secure_password', 'keypass'
-//     distribution = 'zip' // this is important since we use the reindex module in ML
-
-//     setupCommand 'setupTestUser', 'bin/x-pack/users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser'
-
-//     extraConfigFile nodeKeystore.name, nodeKeystore
-
-//     waitCondition = { NodeInfo node, AntBuilder ant ->
-//         File tmpFile = new File(node.cwd, 'wait.success')
-
-//         for (int i = 0; i < 10; i++) {
-//             // we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned
-//             HttpURLConnection httpURLConnection = null;
-//             try {
-//                 httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health?wait_for_nodes=${numNodes}&wait_for_status=yellow").openConnection();
-//                 httpURLConnection.setRequestProperty("Authorization", "Basic " +
-//                         Base64.getEncoder().encodeToString("x_pack_rest_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)));
-//                 httpURLConnection.setRequestMethod("GET");
-//                 httpURLConnection.connect();
-//                 if (httpURLConnection.getResponseCode() == 200) {
-//                     tmpFile.withWriter StandardCharsets.UTF_8.name(), {
-//                         it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name()))
-//                     }
-//                 }
-//             } catch (Exception e) {
-//                 if (i == 9) {
-//                     logger.error("final attempt of calling cluster health failed", e)
-//                 } else {
-//                     logger.debug("failed to call cluster health", e)
-//                 }
-//             } finally {
-//                 if (httpURLConnection != null) {
-//                     httpURLConnection.disconnect();
-//                 }
-//             }
-
-//             // did not start, so wait a bit before trying again
-//             Thread.sleep(500L);
-//         }
-//         return tmpFile.exists()
-//     }
-//}
-
 test {
     /*
      * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
@@ -249,6 +138,7 @@ integTestRunner {
     systemProperty 'es.set.netty.runtime.available.processors', 'false'
 }
 
+
 // TODO: don't publish test artifacts just to run messy tests, fix the tests!
 // https://github.com/elastic/x-plugins/issues/724
 configurations {
@@ -264,38 +154,12 @@ artifacts {
     testArtifacts testJar
 }
 
-// pulled in as external dependency to work on java 9
+thirdPartyAudit.excludes = [
-if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
+  //commons-logging optional dependencies
-    thirdPartyAudit.excludes += [
+  'org.apache.avalon.framework.logger.Logger',
-            'com.sun.activation.registries.MailcapParseException',
+  'org.apache.log.Hierarchy',
-            'javax.activation.ActivationDataFlavor',
+  'org.apache.log.Logger',
-            'javax.activation.CommandInfo',
+  //commons-logging provided dependencies
-            'javax.activation.CommandMap',
+  'javax.servlet.ServletContextEvent',
-            'javax.activation.CommandObject',
+  'javax.servlet.ServletContextListener'
-            'javax.activation.DataContentHandler',
-            'javax.activation.DataContentHandlerFactory',
-            'javax.activation.DataHandler$1',
-            'javax.activation.DataHandler',
-            'javax.activation.DataHandlerDataSource',
-            'javax.activation.DataSource',
-            'javax.activation.DataSourceDataContentHandler',
-            'javax.activation.FileDataSource',
-            'javax.activation.FileTypeMap',
-            'javax.activation.MimeType',
-            'javax.activation.MimeTypeParameterList',
-            'javax.activation.MimeTypeParseException',
-            'javax.activation.ObjectDataContentHandler',
-            'javax.activation.SecuritySupport$1',
-            'javax.activation.SecuritySupport$2',
-            'javax.activation.SecuritySupport$3',
-            'javax.activation.SecuritySupport$4',
-            'javax.activation.SecuritySupport$5',
-            'javax.activation.SecuritySupport',
-            'javax.activation.URLDataSource',
-            'javax.activation.UnsupportedDataTypeException'
 ]
-}
-
-run {
-    distribution = 'zip'
-}