From 9d87b63ca4dc8eaebbdb355fe0672d3c6f2b6e25 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 22 Jan 2018 22:58:34 -0800 Subject: [PATCH] Build: Fix third party audit task for xpack core (elastic/x-pack-elasticsearch#3656) This commit re-enables thirdPartyAudit for x-pack core. Previously, when xpack was a single plugin, it transitively picked up httpcore-nio through the elasticsearch rest client. Now that xpack core does not depend on the rest client, httpcore-nio must be added as a dependency. Additionally, commons-logging was previously handled through the rest client, but now xpack depends directly on this, thus excludes must be added for the pesky missing classes there. This commit also cleans up unnecessary parts of plugin/build.gradle no longer necessary. Original commit: elastic/x-pack-elasticsearch@70e936bdc3b14026729c725ac1b9478a0b81017e --- plugin/build.gradle | 19 +- plugin/core/build.gradle | 174 ++---------------- .../licenses/httpcore-nio-4.4.5.jar.sha1 | 0 3 files changed, 26 insertions(+), 167 deletions(-) rename plugin/{security => core}/licenses/httpcore-nio-4.4.5.jar.sha1 (100%) diff --git a/plugin/build.gradle b/plugin/build.gradle index 6f5ebc75ae4..843601cc4c9 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -19,10 +19,6 @@ es_meta_plugin { 'ml', 'monitoring', 'security', 'upgrade', 'watcher'] } -ext.expansions = [ - 'project.version': version, -] - dependencies { testCompile project(path: ':x-pack-elasticsearch:plugin:core', configuration: 'testArtifacts') } @@ -40,6 +36,13 @@ artifacts { } integTestRunner { + /* + * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each + * other if we allow them to set the number of available processors as it's set-once in Netty. + */ + systemProperty 'es.set.netty.runtime.available.processors', 'false' + + // TODO: fix this rest test to not depend on a hardcoded port! def blacklist = ['getting_started/10_monitor_cluster_health/*'] boolean snapshot = "true".equals(System.getProperty("build.snapshot", "true")) @@ -140,14 +143,6 @@ integTestCluster { } } -integTestRunner { - /* - * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each - * other if we allow them to set the number of available processors as it's set-once in Netty. - */ - systemProperty 'es.set.netty.runtime.available.processors', 'false' -} - run { setting 'xpack.ml.enabled', 'true' setting 'xpack.graph.enabled', 'true' diff --git a/plugin/core/build.gradle b/plugin/core/build.gradle index 5e14e72534f..6419bc3c965 100644 --- a/plugin/core/build.gradle +++ b/plugin/core/build.gradle @@ -5,6 +5,9 @@ import java.nio.file.Path import java.nio.file.StandardCopyOption apply plugin: 'elasticsearch.esplugin' + +archivesBaseName = 'x-pack-core' + esplugin { name 'x-pack-core' description 'Elasticsearch Expanded Pack Plugin - Core' @@ -18,27 +21,16 @@ esplugin { integTest.enabled = false dependencyLicenses { - mapping from: /netty-.*/, to: 'netty' mapping from: /bc.*/, to: 'bouncycastle' - mapping from: /owasp-java-html-sanitizer.*/, to: 'owasp-java-html-sanitizer' - mapping from: /transport-netty.*/, to: 'elasticsearch' - mapping from: /transport-nio.*/, to: 'elasticsearch' - mapping from: /elasticsearch-nio.*/, to: 'elasticsearch' - mapping from: /elasticsearch-rest-client.*/, to: 'elasticsearch' mapping from: /http.*/, to: 'httpclient' // pulled in by rest client mapping from: /commons-.*/, to: 'commons' // pulled in by rest client - ignoreSha 'elasticsearch-rest-client' - ignoreSha 'transport-netty4' - ignoreSha 'transport-nio' - ignoreSha 'elasticsearch-nio' - ignoreSha 'elasticsearch-rest-client-sniffer' - ignoreSha 'x-pack-core' } dependencies { provided "org.elasticsearch:elasticsearch:${version}" compile "org.apache.httpcomponents:httpclient:${versions.httpclient}" compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" + compile "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}" compile "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}" compile "commons-logging:commons-logging:${versions.commonslogging}" @@ -50,7 +42,6 @@ dependencies { compile 'org.bouncycastle:bcpkix-jdk15on:1.58' compile project(path: ':modules:transport-netty4', configuration: 'runtime') - //testCompile project(path: ':core:cli', configuration: 'runtime') testCompile 'org.elasticsearch:securemock:1.2' testCompile "org.elasticsearch:mocksocket:${versions.mocksocket}" testCompile "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}" @@ -60,6 +51,10 @@ dependencies { testCompile project(path: ':modules:analysis-common', configuration: 'runtime') } +ext.expansions = [ + 'project.version': version +] + processResources { from(sourceSets.main.resources.srcDirs) { exclude '**/public.key' @@ -81,14 +76,9 @@ forbiddenPatterns { exclude '**/*.zip' } -archivesBaseName = 'x-pack-core' - compileJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked" compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked" -// TODO: fix these! -thirdPartyAudit.enabled = false - licenseHeaders { approvedLicenses << 'BCrypt (BSD-like)' additionalLicense 'BCRYP', 'BCrypt (BSD-like)', 'Copyright (c) 2006 Damien Miller ' @@ -100,6 +90,7 @@ sourceSets.test.java { srcDir '../../license-tools/src/main/java' } +// TODO: remove this jar once xpack extensions have been removed // assemble the API JAR for the transport-client and extension authors; this JAR is the core JAR by another name project.afterEvaluate { task apiJar { @@ -131,108 +122,6 @@ project.afterEvaluate { } } -// -// integTestRunner { -// // TODO: fix this rest test to not depend on a hardcoded port! -// def blacklist = ['getting_started/10_monitor_cluster_health/*'] -// boolean snapshot = "true".equals(System.getProperty("build.snapshot", "true")) -// if (!snapshot) { -// // these tests attempt to install basic/internal licenses signed against the dev/public.key -// // Since there is no infrastructure in place (anytime soon) to generate licenses using the production -// // private key, these tests are whitelisted in non-snapshot test runs -// blacklist.addAll(['xpack/15_basic/*', 'license/20_put_license/*']) -// } -// systemProperty 'tests.rest.blacklist', blacklist.join(',') -// } - -// // location of generated keystores and certificates -// File keystoreDir = new File(project.buildDir, 'keystore') - -// // Generate the node's keystore -// File nodeKeystore = new File(keystoreDir, 'test-node.jks') -// task createNodeKeyStore(type: LoggedExec) { -// doFirst { -// if (nodeKeystore.parentFile.exists() == false) { -// nodeKeystore.parentFile.mkdirs() -// } -// if (nodeKeystore.exists()) { -// delete nodeKeystore -// } -// } -// executable = new File(project.javaHome, 'bin/keytool') -// standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8')) -// args '-genkey', -// '-alias', 'test-node', -// '-keystore', nodeKeystore, -// '-keyalg', 'RSA', -// '-keysize', '2048', -// '-validity', '712', -// '-dname', 'CN=smoke-test-plugins-ssl', -// '-keypass', 'keypass', -// '-storepass', 'keypass' -// } - -// Add keystores to test classpath: it expects it there -//sourceSets.test.resources.srcDir(keystoreDir) -//processTestResources.dependsOn(createNodeKeyStore) - -// integTestCluster { -// dependsOn createNodeKeyStore -// setting 'xpack.ml.enabled', 'true' -// setting 'logger.org.elasticsearch.xpack.ml.datafeed', 'TRACE' -// // Integration tests are supposed to enable/disable exporters before/after each test -// setting 'xpack.monitoring.exporters._local.type', 'local' -// setting 'xpack.monitoring.exporters._local.enabled', 'false' -// setting 'xpack.monitoring.collection.interval', '-1' -// setting 'xpack.security.authc.token.enabled', 'true' -// setting 'xpack.security.transport.ssl.enabled', 'true' -// setting 'xpack.security.transport.ssl.keystore.path', nodeKeystore.name -// setting 'xpack.security.transport.ssl.verification_mode', 'certificate' -// setting 'xpack.security.audit.enabled', 'true' -// keystoreSetting 'bootstrap.password', 'x-pack-test-password' -// keystoreSetting 'xpack.security.transport.ssl.keystore.secure_password', 'keypass' -// distribution = 'zip' // this is important since we use the reindex module in ML - -// setupCommand 'setupTestUser', 'bin/x-pack/users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser' - -// extraConfigFile nodeKeystore.name, nodeKeystore - -// waitCondition = { NodeInfo node, AntBuilder ant -> -// File tmpFile = new File(node.cwd, 'wait.success') - -// for (int i = 0; i < 10; i++) { -// // we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned -// HttpURLConnection httpURLConnection = null; -// try { -// httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health?wait_for_nodes=${numNodes}&wait_for_status=yellow").openConnection(); -// httpURLConnection.setRequestProperty("Authorization", "Basic " + -// Base64.getEncoder().encodeToString("x_pack_rest_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8))); -// httpURLConnection.setRequestMethod("GET"); -// httpURLConnection.connect(); -// if (httpURLConnection.getResponseCode() == 200) { -// tmpFile.withWriter StandardCharsets.UTF_8.name(), { -// it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name())) -// } -// } -// } catch (Exception e) { -// if (i == 9) { -// logger.error("final attempt of calling cluster health failed", e) -// } else { -// logger.debug("failed to call cluster health", e) -// } -// } finally { -// if (httpURLConnection != null) { -// httpURLConnection.disconnect(); -// } -// } - -// // did not start, so wait a bit before trying again -// Thread.sleep(500L); -// } -// return tmpFile.exists() -// } -//} - test { /* * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each @@ -249,6 +138,7 @@ integTestRunner { systemProperty 'es.set.netty.runtime.available.processors', 'false' } + // TODO: don't publish test artifacts just to run messy tests, fix the tests! // https://github.com/elastic/x-plugins/issues/724 configurations { @@ -264,38 +154,12 @@ artifacts { testArtifacts testJar } -// pulled in as external dependency to work on java 9 -if (JavaVersion.current() <= JavaVersion.VERSION_1_8) { - thirdPartyAudit.excludes += [ - 'com.sun.activation.registries.MailcapParseException', - 'javax.activation.ActivationDataFlavor', - 'javax.activation.CommandInfo', - 'javax.activation.CommandMap', - 'javax.activation.CommandObject', - 'javax.activation.DataContentHandler', - 'javax.activation.DataContentHandlerFactory', - 'javax.activation.DataHandler$1', - 'javax.activation.DataHandler', - 'javax.activation.DataHandlerDataSource', - 'javax.activation.DataSource', - 'javax.activation.DataSourceDataContentHandler', - 'javax.activation.FileDataSource', - 'javax.activation.FileTypeMap', - 'javax.activation.MimeType', - 'javax.activation.MimeTypeParameterList', - 'javax.activation.MimeTypeParseException', - 'javax.activation.ObjectDataContentHandler', - 'javax.activation.SecuritySupport$1', - 'javax.activation.SecuritySupport$2', - 'javax.activation.SecuritySupport$3', - 'javax.activation.SecuritySupport$4', - 'javax.activation.SecuritySupport$5', - 'javax.activation.SecuritySupport', - 'javax.activation.URLDataSource', - 'javax.activation.UnsupportedDataTypeException' - ] -} - -run { - distribution = 'zip' -} +thirdPartyAudit.excludes = [ + //commons-logging optional dependencies + 'org.apache.avalon.framework.logger.Logger', + 'org.apache.log.Hierarchy', + 'org.apache.log.Logger', + //commons-logging provided dependencies + 'javax.servlet.ServletContextEvent', + 'javax.servlet.ServletContextListener' +] diff --git a/plugin/security/licenses/httpcore-nio-4.4.5.jar.sha1 b/plugin/core/licenses/httpcore-nio-4.4.5.jar.sha1 similarity index 100% rename from plugin/security/licenses/httpcore-nio-4.4.5.jar.sha1 rename to plugin/core/licenses/httpcore-nio-4.4.5.jar.sha1