mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-02-06 04:58:50 +00:00
718518fe85
Within the same JVM, setting the number of processors available to Netty can only be done once. However, tests randomize the number of processors and so without intervention would attempt to set this value multiple times. Therefore, we need to use a flag that prevents setting this value in tests. Relates elastic/x-pack-elasticsearch#1266 Original commit: elastic/x-pack-elasticsearch@d127149725
462 lines
17 KiB
Groovy
462 lines
17 KiB
Groovy
import com.amazonaws.AmazonServiceException
|
|
import com.amazonaws.ClientConfiguration
|
|
import com.amazonaws.auth.AWSCredentials
|
|
import com.amazonaws.auth.BasicAWSCredentials
|
|
import com.amazonaws.services.s3.AmazonS3Client
|
|
import com.amazonaws.services.s3.model.HeadBucketRequest
|
|
import com.bettercloud.vault.Vault
|
|
import com.bettercloud.vault.VaultConfig
|
|
import com.bettercloud.vault.response.LogicalResponse
|
|
import org.elasticsearch.gradle.MavenFilteringHack
|
|
import org.elasticsearch.gradle.test.NodeInfo
|
|
|
|
import java.nio.charset.StandardCharsets
|
|
import java.nio.file.Files
|
|
import java.nio.file.Path
|
|
import java.nio.file.StandardCopyOption
|
|
import java.nio.file.attribute.PosixFilePermission
|
|
import java.nio.file.attribute.PosixFilePermissions
|
|
|
|
group 'org.elasticsearch.plugin'
|
|
|
|
apply plugin: 'elasticsearch.esplugin'
|
|
esplugin {
|
|
name 'x-pack'
|
|
description 'Elasticsearch Expanded Pack Plugin'
|
|
classname 'org.elasticsearch.xpack.XPackPlugin'
|
|
hasNativeController true
|
|
licenseFile project(':x-pack-elasticsearch').file('LICENSE.txt')
|
|
noticeFile project(':x-pack-elasticsearch').file('NOTICE.txt')
|
|
}
|
|
archivesBaseName = 'x-pack' // for api jar
|
|
|
|
buildscript {
|
|
repositories {
|
|
mavenCentral()
|
|
}
|
|
dependencies {
|
|
classpath group: 'com.bettercloud', name: 'vault-java-driver', version:"1.1.0"
|
|
classpath 'com.amazonaws:aws-java-sdk-s3:1.10.33'
|
|
}
|
|
}
|
|
|
|
Closure setAwsCreds = {
|
|
|
|
/**
|
|
* The Elastic Secrets vault is served via HTTPS with a Let's Encrypt certificate. The root certificates that cross-signed the Let's
|
|
* Encrypt certificates were not trusted by the JDK until 8u101. Therefore, we enforce that the JDK is at least 8u101 here.
|
|
*/
|
|
final String javaVersion = System.getProperty('java.version')
|
|
final String javaVendor = System.getProperty('java.vendor')
|
|
def matcher = javaVersion =~ /1\.8\.0(?:_(\d+))?/
|
|
boolean matches = matcher.matches()
|
|
assert matches
|
|
|
|
final int update
|
|
if (matcher.group(1) == null) {
|
|
update = 0
|
|
} else {
|
|
update = matcher.group(1).toInteger()
|
|
}
|
|
if (update < 101) {
|
|
throw new GradleException("JDK ${javaVendor} ${javaVersion} does not have necessary root certificates " +
|
|
"(https://bugs.openjdk.java.net/browse/JDK-8154757), update your JDK to at least JDK 8u101+")
|
|
}
|
|
// get an authentication token with vault
|
|
String homePath = System.properties['user.home']
|
|
File githubToken = file("${homePath}/.elastic/github.token")
|
|
final String VAULT_URL = 'https://secrets.elastic.co:8200'
|
|
final String VAULT_ROLE_ID = "8e90dd88-5a8e-9c12-0da9-5439f293ff97"
|
|
final String VAULT_SECRET_ID = System.env.VAULT_SECRET_ID
|
|
String authBody = null
|
|
URL vaultUrl = null
|
|
if (githubToken.exists()) {
|
|
try {
|
|
Set<PosixFilePermission> perms = Files.getPosixFilePermissions(githubToken.toPath())
|
|
if (perms.equals(PosixFilePermissions.fromString("rw-------")) == false) {
|
|
throw new GradleException('github.token must have 600 permissions')
|
|
}
|
|
} catch (UnsupportedOperationException e) {
|
|
// Assume this isn't a POSIX file system
|
|
}
|
|
vaultUrl = new URL(VAULT_URL + '/v1/auth/github/login')
|
|
authBody = "{\"token\": \"${githubToken.getText('UTF-8').trim()}\"}"
|
|
} else if (VAULT_SECRET_ID != null) {
|
|
vaultUrl = new URL(VAULT_URL + '/v1/auth/approle/login')
|
|
authBody = "{\"role_id\": \"${VAULT_ROLE_ID}\", \"secret_id\": \"${VAULT_SECRET_ID}\"}"
|
|
} else {
|
|
throw new GradleException('Missing ~/.elastic/github.token file or VAULT_SECRET_ID environment variable, needed to authenticate with vault for secrets')
|
|
}
|
|
HttpURLConnection vaultConn = (HttpURLConnection) vaultUrl.openConnection()
|
|
vaultConn.setRequestProperty('Content-Type', 'application/json')
|
|
vaultConn.setRequestMethod('PUT')
|
|
vaultConn.setDoOutput(true)
|
|
vaultConn.outputStream.withWriter('UTF-8') { writer ->
|
|
writer.write(authBody)
|
|
}
|
|
vaultConn.connect()
|
|
Object authResponse = new groovy.json.JsonSlurper().parseText(vaultConn.content.text)
|
|
VaultConfig config = new VaultConfig(VAULT_URL, authResponse.auth.client_token)
|
|
Vault vault = new Vault(config)
|
|
LogicalResponse secret = vault.logical().read("aws-dev/creds/prelertartifacts")
|
|
project.ext.mlAwsAccessKey = secret.data.get('access_key')
|
|
project.ext.mlAwsSecretKey = secret.data.get('secret_key')
|
|
// Retry for up to 1 minute to give AWS a chance to propagate the credentials
|
|
int retries = 120
|
|
while (retries > 0) {
|
|
AWSCredentials creds = new BasicAWSCredentials(project.mlAwsAccessKey, project.mlAwsSecretKey)
|
|
|
|
ClientConfiguration clientConfiguration = new ClientConfiguration()
|
|
// the response metadata cache is only there for diagnostics purposes,
|
|
// but can force objects from every response to the old generation.
|
|
clientConfiguration.setResponseMetadataCacheSize(0)
|
|
|
|
AmazonS3Client client = new AmazonS3Client(creds, clientConfiguration)
|
|
try {
|
|
client.headBucket(new HeadBucketRequest('prelert-artifacts'))
|
|
break;
|
|
} catch (AmazonServiceException e) {
|
|
if (e.getStatusCode() != 403 || retries == 0) {
|
|
throw new GradleException('Could not access ml-cpp artifacts. Timed out after 60 attempts', e)
|
|
}
|
|
}
|
|
sleep(500)
|
|
retries--
|
|
}
|
|
}
|
|
|
|
gradle.taskGraph.whenReady { taskGraph ->
|
|
// Vault auth to get keys for access to cpp artifacts
|
|
|
|
if (taskGraph.hasTask(bundlePlugin)) {
|
|
if (project.hasProperty("mlAwsAccessKey") == false && project.hasProperty("mlAwsSecretKey") == false) {
|
|
if (project.gradle.startParameter.isOffline()) {
|
|
// if the project is offline, then we shouldn't try to contact AWS and instead can just
|
|
// use a cached, but possibly expired artifact so that the build works...
|
|
project.ext.mlAwsAccessKey = ""
|
|
project.ext.mlAwsSecretKey = ""
|
|
} else {
|
|
setAwsCreds()
|
|
}
|
|
}
|
|
|
|
repositories {
|
|
maven {
|
|
url "s3://prelert-artifacts/maven"
|
|
credentials(AwsCredentials) {
|
|
accessKey "${project.mlAwsAccessKey}"
|
|
secretKey "${project.mlAwsSecretKey}"
|
|
}
|
|
}
|
|
}
|
|
// For some unknown reason, this from statement triggers
|
|
// a resolve of the nativeBundle configuration. It should
|
|
// be delayed until the bundlePlugin task is executed, but
|
|
// it is not. So for now, we add the extra files to bundlePlugin
|
|
// here right after configuring the prelert maven, which
|
|
// will only happen when bundlePlugin will be run anyways.
|
|
project.bundlePlugin.from {
|
|
zipTree(configurations.nativeBundle.singleFile)
|
|
}
|
|
project.extractNativeLicenses.from {
|
|
zipTree(configurations.nativeBundle.singleFile)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TODO: fix this! https://github.com/elastic/x-plugins/issues/1066
|
|
ext.compactProfile = 'full'
|
|
|
|
dependencyLicenses {
|
|
mapping from: /netty-.*/, to: 'netty'
|
|
mapping from: /bc.*/, to: 'bouncycastle'
|
|
mapping from: /owasp-java-html-sanitizer.*/, to: 'owasp-java-html-sanitizer'
|
|
mapping from: /transport-netty.*/, to: 'elasticsearch'
|
|
mapping from: /rest.*/, to: 'elasticsearch'
|
|
mapping from: /http.*/, to: 'httpclient' // pulled in by rest client
|
|
mapping from: /commons-.*/, to: 'commons' // pulled in by rest client
|
|
mapping from: /sniffer.*/, to: 'elasticsearch'
|
|
ignoreSha 'rest'
|
|
ignoreSha 'transport-netty4'
|
|
ignoreSha 'sniffer'
|
|
}
|
|
|
|
licenseHeaders {
|
|
approvedLicenses << 'BCrypt (BSD-like)'
|
|
additionalLicense 'BCRYP', 'BCrypt (BSD-like)', 'Copyright (c) 2006 Damien Miller <djm@mindrot.org>'
|
|
}
|
|
|
|
configurations {
|
|
nativeBundle {
|
|
// Check for new native code on every build, otherwise builds can
|
|
// fail for 24 hours after a change to the C++ output format
|
|
resolutionStrategy.cacheChangingModulesFor 0, 'seconds'
|
|
}
|
|
}
|
|
|
|
if (findProject(':machine-learning-cpp') != null) {
|
|
configurations.nativeBundle {
|
|
resolutionStrategy.dependencySubstitution {
|
|
substitute module("org.elasticsearch.ml:ml-cpp") with project(":machine-learning-cpp")
|
|
}
|
|
}
|
|
bundlePlugin.dependsOn ':machine-learning-cpp:buildUberZip'
|
|
}
|
|
|
|
dependencies {
|
|
// security deps
|
|
compile project(path: ':modules:transport-netty4', configuration: 'runtime')
|
|
compile 'com.unboundid:unboundid-ldapsdk:3.2.0'
|
|
compile 'org.bouncycastle:bcprov-jdk15on:1.55'
|
|
compile 'org.bouncycastle:bcpkix-jdk15on:1.55'
|
|
testCompile 'com.google.jimfs:jimfs:1.1'
|
|
|
|
// watcher deps
|
|
compile 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:r239'
|
|
compile 'com.google.guava:guava:16.0.1' // needed by watcher for the html sanitizer and security tests for jimfs
|
|
compile 'com.sun.mail:javax.mail:1.5.6'
|
|
// HACK: java 9 removed javax.activation from the default modules, so instead of trying to add modules, which would have
|
|
// to be conditionalized for java 8/9, we pull in the classes directly
|
|
compile 'javax.activation:activation:1.1.1'
|
|
|
|
testCompile 'org.subethamail:subethasmtp:3.1.7'
|
|
// needed for subethasmtp, has @GuardedBy annotation
|
|
testCompile 'com.google.code.findbugs:jsr305:3.0.1'
|
|
|
|
// monitoring deps
|
|
compile "org.elasticsearch.client:rest:${version}"
|
|
compile "org.elasticsearch.client:sniffer:${version}"
|
|
|
|
// ml deps
|
|
compile 'net.sf.supercsv:super-csv:2.4.0'
|
|
nativeBundle ("org.elasticsearch.ml:ml-cpp:${project.version}@zip") {
|
|
changing = true
|
|
}
|
|
testCompile 'org.ini4j:ini4j:0.5.2'
|
|
|
|
// common test deps
|
|
testCompile 'org.elasticsearch:securemock:1.2'
|
|
testCompile "org.elasticsearch:mocksocket:${versions.mocksocket}"
|
|
testCompile 'org.slf4j:slf4j-log4j12:1.6.2'
|
|
testCompile 'org.slf4j:slf4j-api:1.6.2'
|
|
}
|
|
|
|
// make LicenseSigner available for testing signed licenses
|
|
sourceSets.test.java {
|
|
srcDir '../license-tools/src/main/java'
|
|
}
|
|
|
|
compileJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked"
|
|
compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked"
|
|
|
|
ext.expansions = [
|
|
'project.version': version,
|
|
]
|
|
|
|
processResources {
|
|
from(sourceSets.main.resources.srcDirs) {
|
|
exclude '**/public.key'
|
|
inputs.properties(expansions)
|
|
MavenFilteringHack.filter(it, expansions)
|
|
}
|
|
boolean snapshot = "true".equals(System.getProperty("build.snapshot", "true"))
|
|
if (snapshot) {
|
|
from 'keys/dev/public.key'
|
|
} else {
|
|
from 'keys/prod/public.key'
|
|
}
|
|
}
|
|
|
|
forbiddenPatterns {
|
|
exclude '**/*.key'
|
|
exclude '**/*.p12'
|
|
exclude '**/*.der'
|
|
exclude '**/*.zip'
|
|
}
|
|
|
|
task extractNativeLicenses(type: Copy) {
|
|
into "${buildDir}"
|
|
include 'platform/licenses/**'
|
|
}
|
|
|
|
// TODO: standardize packaging config for plugins
|
|
bundlePlugin {
|
|
from('bin/x-pack') {
|
|
into 'bin'
|
|
}
|
|
from('config/x-pack') {
|
|
into 'config'
|
|
}
|
|
// We don't ship the individual nativeBundle licenses - instead
|
|
// they get combined into the top level NOTICES file we ship
|
|
exclude 'platform/licenses/**'
|
|
}
|
|
|
|
// add api jar for extension authors to compile against
|
|
// note this is just the normal x-pack jar for now, with a different name
|
|
project.afterEvaluate {
|
|
task apiJar {
|
|
dependsOn('generatePomFileForApijarPublication', project.jar)
|
|
doFirst {
|
|
Path jarFile = project.jar.outputs.files.singleFile.toPath()
|
|
String apiFileName = jarFile.fileName.toString().replace(project.version, "api-${project.version}")
|
|
Files.copy(jarFile, jarFile.resolveSibling(apiFileName), StandardCopyOption.REPLACE_EXISTING)
|
|
|
|
String pomFileName = jarFile.fileName.toString().replace('.jar', '.pom')
|
|
String apiPomFileName = apiFileName.replace('.jar', '.pom')
|
|
Files.copy(jarFile.resolveSibling(pomFileName), jarFile.resolveSibling(apiPomFileName),
|
|
StandardCopyOption.REPLACE_EXISTING)
|
|
}
|
|
}
|
|
assemble.dependsOn(apiJar)
|
|
project.publishing {
|
|
publications {
|
|
apijar(MavenPublication) {
|
|
from project.components.java
|
|
artifactId = 'x-pack-api'
|
|
pom.withXml { XmlProvider xml ->
|
|
Node root = xml.asNode()
|
|
root.appendNode('name', project.pluginProperties.extension.name)
|
|
root.appendNode('description', project.pluginProperties.extension.description)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Add an extra licenses directory to the combined notices
|
|
project.tasks.findByName('generateNotice').dependsOn extractNativeLicenses
|
|
project.tasks.findByName('generateNotice').licensesDir new File("${project.buildDir}/platform/licenses")
|
|
}
|
|
|
|
integTestRunner {
|
|
// TODO: fix this rest test to not depend on a hardcoded port!
|
|
systemProperty 'tests.rest.blacklist', 'getting_started/10_monitor_cluster_health/*'
|
|
}
|
|
|
|
integTestCluster {
|
|
setting 'xpack.ml.enabled', 'true'
|
|
setting 'logger.org.elasticsearch.xpack.ml.datafeed', 'TRACE'
|
|
// Integration tests are supposed to enable/disable exporters before/after each test
|
|
setting 'xpack.monitoring.exporters._local.type', 'local'
|
|
setting 'xpack.monitoring.exporters._local.enabled', 'false'
|
|
setting 'xpack.monitoring.collection.interval', '-1'
|
|
|
|
waitCondition = { NodeInfo node, AntBuilder ant ->
|
|
File tmpFile = new File(node.cwd, 'wait.success')
|
|
for (int i = 0; i < 10; i++) {
|
|
// we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned
|
|
HttpURLConnection httpURLConnection = null;
|
|
try {
|
|
httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health?wait_for_nodes=${numNodes}&wait_for_status=yellow").openConnection();
|
|
httpURLConnection.setRequestProperty("Authorization", "Basic " +
|
|
Base64.getEncoder().encodeToString("elastic:changeme".getBytes(StandardCharsets.UTF_8)));
|
|
httpURLConnection.setRequestMethod("GET");
|
|
httpURLConnection.connect();
|
|
if (httpURLConnection.getResponseCode() == 200) {
|
|
tmpFile.withWriter StandardCharsets.UTF_8.name(), {
|
|
it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name()))
|
|
}
|
|
}
|
|
} catch (Exception e) {
|
|
if (i == 9) {
|
|
logger.error("final attempt of calling cluster health failed", e)
|
|
} else {
|
|
logger.debug("failed to call cluster health", e)
|
|
}
|
|
} finally {
|
|
if (httpURLConnection != null) {
|
|
httpURLConnection.disconnect();
|
|
}
|
|
}
|
|
|
|
// did not start, so wait a bit before trying again
|
|
Thread.sleep(500L);
|
|
}
|
|
return tmpFile.exists()
|
|
}
|
|
}
|
|
|
|
test {
|
|
/*
|
|
* We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
|
|
* other if we allow them to set the number of available processors as it's set-once in Netty.
|
|
*/
|
|
systemProperty 'es.set.netty.runtime.available.processors', 'false'
|
|
}
|
|
|
|
integTestRunner {
|
|
/*
|
|
* We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
|
|
* other if we allow them to set the number of available processors as it's set-once in Netty.
|
|
*/
|
|
systemProperty 'es.set.netty.runtime.available.processors', 'false'
|
|
}
|
|
|
|
// TODO: don't publish test artifacts just to run messy tests, fix the tests!
|
|
// https://github.com/elastic/x-plugins/issues/724
|
|
configurations {
|
|
testArtifacts.extendsFrom testRuntime
|
|
}
|
|
task testJar(type: Jar) {
|
|
classifier "test"
|
|
from sourceSets.test.output
|
|
}
|
|
artifacts {
|
|
// normal es plugins do not publish the jar but we need to since users need it for Transport Clients and extensions
|
|
archives jar
|
|
testArtifacts testJar
|
|
}
|
|
|
|
// classes are missing, e.g. com.ibm.icu.lang.UCharacter
|
|
thirdPartyAudit.excludes = [
|
|
// uses internal java api: sun.misc.Unsafe
|
|
'com.google.common.cache.Striped64',
|
|
'com.google.common.cache.Striped64$1',
|
|
'com.google.common.cache.Striped64$Cell',
|
|
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
|
|
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
|
|
|
|
// pulled in as external dependency to work on java 9
|
|
'com.sun.activation.registries.LineTokenizer',
|
|
'com.sun.activation.registries.LogSupport',
|
|
'com.sun.activation.registries.MailcapFile',
|
|
'com.sun.activation.registries.MailcapParseException',
|
|
'com.sun.activation.registries.MailcapTokenizer',
|
|
'com.sun.activation.registries.MimeTypeEntry',
|
|
'com.sun.activation.registries.MimeTypeFile',
|
|
'javax.activation.ActivationDataFlavor',
|
|
'javax.activation.CommandInfo',
|
|
'javax.activation.CommandMap',
|
|
'javax.activation.CommandObject',
|
|
'javax.activation.DataContentHandler',
|
|
'javax.activation.DataContentHandlerFactory',
|
|
'javax.activation.DataHandler$1',
|
|
'javax.activation.DataHandler',
|
|
'javax.activation.DataHandlerDataSource',
|
|
'javax.activation.DataSource',
|
|
'javax.activation.DataSourceDataContentHandler',
|
|
'javax.activation.FileDataSource',
|
|
'javax.activation.FileTypeMap',
|
|
'javax.activation.MailcapCommandMap',
|
|
'javax.activation.MimeType',
|
|
'javax.activation.MimeTypeParameterList',
|
|
'javax.activation.MimeTypeParseException',
|
|
'javax.activation.MimetypesFileTypeMap',
|
|
'javax.activation.ObjectDataContentHandler',
|
|
'javax.activation.SecuritySupport$1',
|
|
'javax.activation.SecuritySupport$2',
|
|
'javax.activation.SecuritySupport$3',
|
|
'javax.activation.SecuritySupport$4',
|
|
'javax.activation.SecuritySupport$5',
|
|
'javax.activation.SecuritySupport',
|
|
'javax.activation.URLDataSource',
|
|
'javax.activation.UnsupportedDataTypeException'
|
|
]
|
|
|
|
run {
|
|
setting 'xpack.ml.enabled', 'true'
|
|
setting 'xpack.graph.enabled', 'true'
|
|
setting 'xpack.security.enabled', 'true'
|
|
setting 'xpack.monitoring.enabled', 'true'
|
|
setting 'xpack.watcher.enabled', 'true'
|
|
}
|