import org.elasticsearch.gradle.LoggedExec import org.elasticsearch.gradle.MavenFilteringHack import org.elasticsearch.gradle.test.NodeInfo import java.nio.charset.StandardCharsets import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardCopyOption group 'org.elasticsearch.plugin' apply plugin: 'elasticsearch.esplugin' esplugin { name 'x-pack' description 'Elasticsearch Expanded Pack Plugin' classname 'org.elasticsearch.xpack.XPackPlugin' hasNativeController true requiresKeystore true licenseFile project(':x-pack-elasticsearch').file('LICENSE.txt') noticeFile project(':x-pack-elasticsearch').file('NOTICE.txt') } archivesBaseName = 'x-pack' // for api jar // TODO: fix this! https://github.com/elastic/x-plugins/issues/1066 ext.compactProfile = 'full' dependencyLicenses { mapping from: /netty-.*/, to: 'netty' mapping from: /bc.*/, to: 'bouncycastle' mapping from: /owasp-java-html-sanitizer.*/, to: 'owasp-java-html-sanitizer' mapping from: /transport-netty.*/, to: 'elasticsearch' mapping from: /tribe.*/, to: 'elasticsearch' mapping from: /elasticsearch-rest-client.*/, to: 'elasticsearch' mapping from: /server.*/, to: 'elasticsearch' mapping from: /jdbc-proto.*/, to: 'elasticsearch' mapping from: /cli-proto.*/, to: 'elasticsearch' mapping from: /shared-proto.*/, to: 'elasticsearch' mapping from: /http.*/, to: 'httpclient' // pulled in by rest client mapping from: /commons-.*/, to: 'commons' // pulled in by rest client ignoreSha 'elasticsearch-rest-client' ignoreSha 'transport-netty4' ignoreSha 'tribe' ignoreSha 'server' ignoreSha 'jdbc-proto' ignoreSha 'cli-proto' ignoreSha 'shared-proto' ignoreSha 'elasticsearch-rest-client-sniffer' } licenseHeaders { approvedLicenses << 'BCrypt (BSD-like)' additionalLicense 'BCRYP', 'BCrypt (BSD-like)', 'Copyright (c) 2006 Damien Miller ' } configurations { nativeBundle { resolutionStrategy.dependencySubstitution { if (findProject(':machine-learning-cpp') != null) { substitute module("org.elasticsearch.ml:ml-cpp") with project(":machine-learning-cpp") } else { substitute module("org.elasticsearch.ml:ml-cpp") with project("${project.path}:ml-cpp-snapshot") } } } } dependencies { // security deps compile project(path: ':modules:tribe', configuration: 'runtime') compile project(path: ':modules:transport-netty4', configuration: 'runtime') compile 'com.unboundid:unboundid-ldapsdk:3.2.0' compile 'org.bouncycastle:bcprov-jdk15on:1.58' compile 'org.bouncycastle:bcpkix-jdk15on:1.58' testCompile 'com.google.jimfs:jimfs:1.1' // watcher deps compile 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:r239' compile 'com.google.guava:guava:16.0.1' // needed by watcher for the html sanitizer and security tests for jimfs compile 'com.sun.mail:javax.mail:1.5.6' // HACK: java 9 removed javax.activation from the default modules, so instead of trying to add modules, which would have // to be conditionalized for java 8/9, we pull in the classes directly compile 'javax.activation:activation:1.1.1' testCompile 'org.subethamail:subethasmtp:3.1.7' // needed for subethasmtp, has @GuardedBy annotation testCompile 'com.google.code.findbugs:jsr305:3.0.1' // monitoring deps compile "org.elasticsearch.client:elasticsearch-rest-client:${version}" compile "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}" // ml deps compile 'net.sf.supercsv:super-csv:2.4.0' nativeBundle "org.elasticsearch.ml:ml-cpp:${project.version}@zip" testCompile 'org.ini4j:ini4j:0.5.2' // sql's server components and its transitive dependencies compile project(':x-pack-elasticsearch:sql:server') // common test deps testCompile 'org.elasticsearch:securemock:1.2' testCompile "org.elasticsearch:mocksocket:${versions.mocksocket}" testCompile 'org.slf4j:slf4j-log4j12:1.6.2' testCompile 'org.slf4j:slf4j-api:1.6.2' testCompile project(path: ':modules:reindex', configuration: 'runtime') testCompile project(path: ':modules:parent-join', configuration: 'runtime') testCompile project(path: ':modules:analysis-common', configuration: 'runtime') } // make LicenseSigner available for testing signed licenses sourceSets.test.java { srcDir '../license-tools/src/main/java' } compileJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked" compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked" ext.expansions = [ 'project.version': version, ] processResources { from(sourceSets.main.resources.srcDirs) { exclude '**/public.key' inputs.properties(expansions) MavenFilteringHack.filter(it, expansions) } boolean snapshot = "true".equals(System.getProperty("build.snapshot", "true")) if (snapshot) { from 'keys/dev/public.key' } else { from 'keys/prod/public.key' } } forbiddenPatterns { exclude '**/*.key' exclude '**/*.p12' exclude '**/*.der' exclude '**/*.zip' } forbiddenApisMain { signaturesURLs += file('forbidden/ldap-signatures.txt').toURI().toURL() } task extractNativeLicenses(type: Copy) { dependsOn configurations.nativeBundle into "${buildDir}" from { project.zipTree(configurations.nativeBundle.singleFile) } include 'platform/licenses/**' // This is to reduce the risk of credentials used to access the native bundle not // having propagated throughout AWS by the time it's downloaded; the time needed // to compile the Java is extra time during which the propagation can take place shouldRunAfter compileJava } // TODO: standardize packaging config for plugins bundlePlugin { dependsOn configurations.nativeBundle from('bin/x-pack') { into 'bin' } from('config/x-pack') { into 'config' } from { project.zipTree(configurations.nativeBundle.singleFile) } // We don't ship the individual nativeBundle licenses - instead // they get combined into the top level NOTICES file we ship exclude 'platform/licenses/**' } // add api jar for extension authors to compile against // note this is just the normal x-pack jar for now, with a different name project.afterEvaluate { task apiJar { dependsOn('generatePomFileForApijarPublication', project.jar) doFirst { Path jarFile = project.jar.outputs.files.singleFile.toPath() String apiFileName = jarFile.fileName.toString().replace(project.version, "api-${project.version}") Files.copy(jarFile, jarFile.resolveSibling(apiFileName), StandardCopyOption.REPLACE_EXISTING) String pomFileName = jarFile.fileName.toString().replace('.jar', '.pom') String apiPomFileName = apiFileName.replace('.jar', '.pom') Files.copy(jarFile.resolveSibling(pomFileName), jarFile.resolveSibling(apiPomFileName), StandardCopyOption.REPLACE_EXISTING) } } assemble.dependsOn(apiJar) project.publishing { publications { apijar(MavenPublication) { from project.components.java artifactId = 'x-pack-api' pom.withXml { XmlProvider xml -> Node root = xml.asNode() root.appendNode('name', project.pluginProperties.extension.name) root.appendNode('description', project.pluginProperties.extension.description) } } } } // Add an extra licenses directory to the combined notices project.tasks.findByName('generateNotice').dependsOn extractNativeLicenses project.tasks.findByName('generateNotice').licensesDir new File("${project.buildDir}/platform/licenses") } integTestRunner { // TODO: fix this rest test to not depend on a hardcoded port! def blacklist = ['getting_started/10_monitor_cluster_health/*'] boolean snapshot = "true".equals(System.getProperty("build.snapshot", "true")) if (!snapshot) { // these tests attempt to install basic/internal licenses signed against the dev/public.key // Since there is no infrastructure in place (anytime soon) to generate licenses using the production // private key, these tests are whitelisted in non-snapshot test runs blacklist.addAll(['xpack/15_basic/*', 'license/20_put_license/*']) } systemProperty 'tests.rest.blacklist', blacklist.join(',') } // location of generated keystores and certificates File keystoreDir = new File(project.buildDir, 'keystore') // Generate the node's keystore File nodeKeystore = new File(keystoreDir, 'test-node.jks') task createNodeKeyStore(type: LoggedExec) { doFirst { if (nodeKeystore.parentFile.exists() == false) { nodeKeystore.parentFile.mkdirs() } if (nodeKeystore.exists()) { delete nodeKeystore } } executable = new File(project.javaHome, 'bin/keytool') standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8')) args '-genkey', '-alias', 'test-node', '-keystore', nodeKeystore, '-keyalg', 'RSA', '-keysize', '2048', '-validity', '712', '-dname', 'CN=smoke-test-plugins-ssl', '-keypass', 'keypass', '-storepass', 'keypass' } // Add keystores to test classpath: it expects it there sourceSets.test.resources.srcDir(keystoreDir) processTestResources.dependsOn(createNodeKeyStore) integTestCluster { dependsOn createNodeKeyStore setting 'xpack.ml.enabled', 'true' setting 'logger.org.elasticsearch.xpack.ml.datafeed', 'TRACE' // Integration tests are supposed to enable/disable exporters before/after each test setting 'xpack.monitoring.exporters._local.type', 'local' setting 'xpack.monitoring.exporters._local.enabled', 'false' setting 'xpack.monitoring.collection.interval', '-1' setting 'xpack.security.authc.token.enabled', 'true' setting 'xpack.security.transport.ssl.enabled', 'true' setting 'xpack.security.transport.ssl.keystore.path', nodeKeystore.name setting 'xpack.security.transport.ssl.verification_mode', 'certificate' keystoreSetting 'bootstrap.password', 'x-pack-test-password' keystoreSetting 'xpack.security.transport.ssl.keystore.secure_password', 'keypass' distribution = 'zip' // this is important since we use the reindex module in ML setupCommand 'setupTestUser', 'bin/x-pack/users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser' extraConfigFile nodeKeystore.name, nodeKeystore waitCondition = { NodeInfo node, AntBuilder ant -> File tmpFile = new File(node.cwd, 'wait.success') for (int i = 0; i < 10; i++) { // we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned HttpURLConnection httpURLConnection = null; try { httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health?wait_for_nodes=${numNodes}&wait_for_status=yellow").openConnection(); httpURLConnection.setRequestProperty("Authorization", "Basic " + Base64.getEncoder().encodeToString("x_pack_rest_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8))); httpURLConnection.setRequestMethod("GET"); httpURLConnection.connect(); if (httpURLConnection.getResponseCode() == 200) { tmpFile.withWriter StandardCharsets.UTF_8.name(), { it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name())) } } } catch (Exception e) { if (i == 9) { logger.error("final attempt of calling cluster health failed", e) } else { logger.debug("failed to call cluster health", e) } } finally { if (httpURLConnection != null) { httpURLConnection.disconnect(); } } // did not start, so wait a bit before trying again Thread.sleep(500L); } return tmpFile.exists() } } test { /* * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each * other if we allow them to set the number of available processors as it's set-once in Netty. */ systemProperty 'es.set.netty.runtime.available.processors', 'false' } integTestRunner { /* * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each * other if we allow them to set the number of available processors as it's set-once in Netty. */ systemProperty 'es.set.netty.runtime.available.processors', 'false' } // TODO: don't publish test artifacts just to run messy tests, fix the tests! // https://github.com/elastic/x-plugins/issues/724 configurations { testArtifacts.extendsFrom testRuntime } task testJar(type: Jar) { classifier "test" from sourceSets.test.output } artifacts { // normal es plugins do not publish the jar but we need to since users need it for Transport Clients and extensions archives jar testArtifacts testJar } // classes are missing, e.g. com.ibm.icu.lang.UCharacter thirdPartyAudit.excludes = [ // uses internal java api: sun.misc.Unsafe 'com.google.common.cache.Striped64', 'com.google.common.cache.Striped64$1', 'com.google.common.cache.Striped64$Cell', 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator', 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1', // pulled in as external dependency to work on java 9 'com.sun.activation.registries.LineTokenizer', 'com.sun.activation.registries.LogSupport', 'com.sun.activation.registries.MailcapFile', 'com.sun.activation.registries.MailcapTokenizer', 'com.sun.activation.registries.MimeTypeEntry', 'com.sun.activation.registries.MimeTypeFile', 'javax.activation.MailcapCommandMap', 'javax.activation.MimetypesFileTypeMap', ] // pulled in as external dependency to work on java 9 if (JavaVersion.current() <= JavaVersion.VERSION_1_8) { thirdPartyAudit.excludes += [ 'com.sun.activation.registries.MailcapParseException', 'javax.activation.ActivationDataFlavor', 'javax.activation.CommandInfo', 'javax.activation.CommandMap', 'javax.activation.CommandObject', 'javax.activation.DataContentHandler', 'javax.activation.DataContentHandlerFactory', 'javax.activation.DataHandler$1', 'javax.activation.DataHandler', 'javax.activation.DataHandlerDataSource', 'javax.activation.DataSource', 'javax.activation.DataSourceDataContentHandler', 'javax.activation.FileDataSource', 'javax.activation.FileTypeMap', 'javax.activation.MimeType', 'javax.activation.MimeTypeParameterList', 'javax.activation.MimeTypeParseException', 'javax.activation.ObjectDataContentHandler', 'javax.activation.SecuritySupport$1', 'javax.activation.SecuritySupport$2', 'javax.activation.SecuritySupport$3', 'javax.activation.SecuritySupport$4', 'javax.activation.SecuritySupport$5', 'javax.activation.SecuritySupport', 'javax.activation.URLDataSource', 'javax.activation.UnsupportedDataTypeException' ] } run { setting 'xpack.ml.enabled', 'true' setting 'xpack.graph.enabled', 'true' setting 'xpack.security.enabled', 'true' setting 'xpack.monitoring.enabled', 'true' setting 'xpack.watcher.enabled', 'true' keystoreSetting 'bootstrap.password', 'password' }