mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-02-06 04:58:50 +00:00
0a683a0e18
This change removes the InternalClient and the InternalSecurityClient. These are replaced with usage of the ThreadContext and a transient value, `action.origin`, to indicate which component the request came from. The security code has been updated to look for this value and ensure the request is executed as the proper user. This work comes from elastic/x-pack-elasticsearch#2808 where @s1monw suggested that we do this. While working on this, I came across index template registries and rather than updating them to use the new method, I replaced the ML one with the template upgrade framework so that we could remove this template registry. The watcher template registry is still needed as the template must be updated for rolling upgrades to work (see elastic/x-pack-elasticsearch#2950). Original commit: elastic/x-pack-elasticsearch@7dbf2f263e
397 lines
15 KiB
Groovy
397 lines
15 KiB
Groovy
import org.elasticsearch.gradle.LoggedExec
|
|
import org.elasticsearch.gradle.MavenFilteringHack
|
|
import org.elasticsearch.gradle.test.NodeInfo
|
|
|
|
import java.nio.charset.StandardCharsets
|
|
import java.nio.file.Files
|
|
import java.nio.file.Path
|
|
import java.nio.file.StandardCopyOption
|
|
|
|
group 'org.elasticsearch.plugin'
|
|
|
|
apply plugin: 'elasticsearch.esplugin'
|
|
esplugin {
|
|
name 'x-pack'
|
|
description 'Elasticsearch Expanded Pack Plugin'
|
|
classname 'org.elasticsearch.xpack.XPackPlugin'
|
|
hasNativeController true
|
|
requiresKeystore true
|
|
licenseFile project(':x-pack-elasticsearch').file('LICENSE.txt')
|
|
noticeFile project(':x-pack-elasticsearch').file('NOTICE.txt')
|
|
}
|
|
archivesBaseName = 'x-pack' // for api jar
|
|
|
|
// TODO: fix this! https://github.com/elastic/x-plugins/issues/1066
|
|
ext.compactProfile = 'full'
|
|
|
|
dependencyLicenses {
|
|
mapping from: /netty-.*/, to: 'netty'
|
|
mapping from: /bc.*/, to: 'bouncycastle'
|
|
mapping from: /owasp-java-html-sanitizer.*/, to: 'owasp-java-html-sanitizer'
|
|
mapping from: /transport-netty.*/, to: 'elasticsearch'
|
|
mapping from: /elasticsearch-rest-client.*/, to: 'elasticsearch'
|
|
mapping from: /http.*/, to: 'httpclient' // pulled in by rest client
|
|
mapping from: /commons-.*/, to: 'commons' // pulled in by rest client
|
|
ignoreSha 'elasticsearch-rest-client'
|
|
ignoreSha 'x-pack-client-api-objects'
|
|
ignoreSha 'transport-netty4'
|
|
ignoreSha 'elasticsearch-rest-client-sniffer'
|
|
}
|
|
|
|
licenseHeaders {
|
|
approvedLicenses << 'BCrypt (BSD-like)'
|
|
additionalLicense 'BCRYP', 'BCrypt (BSD-like)', 'Copyright (c) 2006 Damien Miller <djm@mindrot.org>'
|
|
}
|
|
|
|
configurations {
|
|
nativeBundle {
|
|
resolutionStrategy.dependencySubstitution {
|
|
if (findProject(':machine-learning-cpp') != null) {
|
|
substitute module("org.elasticsearch.ml:ml-cpp") with project(":machine-learning-cpp")
|
|
} else {
|
|
substitute module("org.elasticsearch.ml:ml-cpp") with project("${project.path}:ml-cpp-snapshot")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
dependencies {
|
|
// CLI deps
|
|
compile project(path: ':core:cli', configuration: 'runtime')
|
|
|
|
// Request and Response objects
|
|
compile "org.elasticsearch:x-pack-client-api-objects:${version}"
|
|
|
|
// security deps
|
|
compile project(path: ':modules:transport-netty4', configuration: 'runtime')
|
|
compile 'com.unboundid:unboundid-ldapsdk:3.2.0'
|
|
compile 'org.bouncycastle:bcprov-jdk15on:1.58'
|
|
compile 'org.bouncycastle:bcpkix-jdk15on:1.58'
|
|
testCompile 'com.google.jimfs:jimfs:1.1'
|
|
|
|
// watcher deps
|
|
compile 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:r239'
|
|
compile 'com.google.guava:guava:16.0.1' // needed by watcher for the html sanitizer and security tests for jimfs
|
|
compile 'com.sun.mail:javax.mail:1.5.6'
|
|
// HACK: java 9 removed javax.activation from the default modules, so instead of trying to add modules, which would have
|
|
// to be conditionalized for java 8/9, we pull in the classes directly
|
|
compile 'javax.activation:activation:1.1.1'
|
|
|
|
testCompile 'org.subethamail:subethasmtp:3.1.7'
|
|
// needed for subethasmtp, has @GuardedBy annotation
|
|
testCompile 'com.google.code.findbugs:jsr305:3.0.1'
|
|
|
|
// monitoring deps
|
|
compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
|
|
compile "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}"
|
|
|
|
// ml deps
|
|
compile 'net.sf.supercsv:super-csv:2.4.0'
|
|
nativeBundle "org.elasticsearch.ml:ml-cpp:${project.version}@zip"
|
|
testCompile 'org.ini4j:ini4j:0.5.2'
|
|
|
|
// common test deps
|
|
testCompile 'org.elasticsearch:securemock:1.2'
|
|
testCompile "org.elasticsearch:mocksocket:${versions.mocksocket}"
|
|
testCompile 'org.slf4j:slf4j-log4j12:1.6.2'
|
|
testCompile 'org.slf4j:slf4j-api:1.6.2'
|
|
testCompile project(path: ':modules:reindex', configuration: 'runtime')
|
|
testCompile project(path: ':modules:parent-join', configuration: 'runtime')
|
|
testCompile project(path: ':modules:analysis-common', configuration: 'runtime')
|
|
}
|
|
|
|
// make LicenseSigner available for testing signed licenses
|
|
sourceSets.test.java {
|
|
srcDir '../license-tools/src/main/java'
|
|
}
|
|
|
|
compileJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked"
|
|
compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked"
|
|
|
|
ext.expansions = [
|
|
'project.version': version,
|
|
]
|
|
|
|
processResources {
|
|
from(sourceSets.main.resources.srcDirs) {
|
|
exclude '**/public.key'
|
|
inputs.properties(expansions)
|
|
MavenFilteringHack.filter(it, expansions)
|
|
}
|
|
boolean snapshot = "true".equals(System.getProperty("build.snapshot", "true"))
|
|
if (snapshot) {
|
|
from 'keys/dev/public.key'
|
|
} else {
|
|
from 'keys/prod/public.key'
|
|
}
|
|
}
|
|
|
|
forbiddenPatterns {
|
|
exclude '**/*.key'
|
|
exclude '**/*.p12'
|
|
exclude '**/*.der'
|
|
exclude '**/*.zip'
|
|
}
|
|
|
|
forbiddenApisMain {
|
|
signaturesURLs += file('forbidden/ldap-signatures.txt').toURI().toURL()
|
|
}
|
|
|
|
task extractNativeLicenses(type: Copy) {
|
|
dependsOn configurations.nativeBundle
|
|
into "${buildDir}"
|
|
from {
|
|
project.zipTree(configurations.nativeBundle.singleFile)
|
|
}
|
|
include 'platform/licenses/**'
|
|
// This is to reduce the risk of credentials used to access the native bundle not
|
|
// having propagated throughout AWS by the time it's downloaded; the time needed
|
|
// to compile the Java is extra time during which the propagation can take place
|
|
shouldRunAfter compileJava
|
|
}
|
|
|
|
// TODO: standardize packaging config for plugins
|
|
bundlePlugin {
|
|
dependsOn configurations.nativeBundle
|
|
from('bin/x-pack') {
|
|
into 'bin'
|
|
}
|
|
from('config/x-pack') {
|
|
into 'config'
|
|
}
|
|
from {
|
|
project.zipTree(configurations.nativeBundle.singleFile)
|
|
}
|
|
// We don't ship the individual nativeBundle licenses - instead
|
|
// they get combined into the top level NOTICES file we ship
|
|
exclude 'platform/licenses/**'
|
|
}
|
|
|
|
// add api jar for extension authors to compile against
|
|
// note this is just the normal x-pack jar for now, with a different name
|
|
project.afterEvaluate {
|
|
task apiJar {
|
|
dependsOn('generatePomFileForApijarPublication', project.jar)
|
|
doFirst {
|
|
Path jarFile = project.jar.outputs.files.singleFile.toPath()
|
|
String apiFileName = jarFile.fileName.toString().replace(project.version, "api-${project.version}")
|
|
Files.copy(jarFile, jarFile.resolveSibling(apiFileName), StandardCopyOption.REPLACE_EXISTING)
|
|
|
|
String pomFileName = jarFile.fileName.toString().replace('.jar', '.pom')
|
|
String apiPomFileName = apiFileName.replace('.jar', '.pom')
|
|
Files.copy(jarFile.resolveSibling(pomFileName), jarFile.resolveSibling(apiPomFileName),
|
|
StandardCopyOption.REPLACE_EXISTING)
|
|
}
|
|
}
|
|
assemble.dependsOn(apiJar)
|
|
project.publishing {
|
|
publications {
|
|
apijar(MavenPublication) {
|
|
from project.components.java
|
|
artifactId = 'x-pack-api'
|
|
pom.withXml { XmlProvider xml ->
|
|
Node root = xml.asNode()
|
|
root.appendNode('name', project.pluginProperties.extension.name)
|
|
root.appendNode('description', project.pluginProperties.extension.description)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Add an extra licenses directory to the combined notices
|
|
project.tasks.findByName('generateNotice').dependsOn extractNativeLicenses
|
|
project.tasks.findByName('generateNotice').licensesDir new File("${project.buildDir}/platform/licenses")
|
|
}
|
|
|
|
integTestRunner {
|
|
// TODO: fix this rest test to not depend on a hardcoded port!
|
|
def blacklist = ['getting_started/10_monitor_cluster_health/*']
|
|
boolean snapshot = "true".equals(System.getProperty("build.snapshot", "true"))
|
|
if (!snapshot) {
|
|
// these tests attempt to install basic/internal licenses signed against the dev/public.key
|
|
// Since there is no infrastructure in place (anytime soon) to generate licenses using the production
|
|
// private key, these tests are whitelisted in non-snapshot test runs
|
|
blacklist.addAll(['xpack/15_basic/*', 'license/20_put_license/*'])
|
|
}
|
|
systemProperty 'tests.rest.blacklist', blacklist.join(',')
|
|
}
|
|
|
|
// location of generated keystores and certificates
|
|
File keystoreDir = new File(project.buildDir, 'keystore')
|
|
|
|
// Generate the node's keystore
|
|
File nodeKeystore = new File(keystoreDir, 'test-node.jks')
|
|
task createNodeKeyStore(type: LoggedExec) {
|
|
doFirst {
|
|
if (nodeKeystore.parentFile.exists() == false) {
|
|
nodeKeystore.parentFile.mkdirs()
|
|
}
|
|
if (nodeKeystore.exists()) {
|
|
delete nodeKeystore
|
|
}
|
|
}
|
|
executable = new File(project.javaHome, 'bin/keytool')
|
|
standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
|
|
args '-genkey',
|
|
'-alias', 'test-node',
|
|
'-keystore', nodeKeystore,
|
|
'-keyalg', 'RSA',
|
|
'-keysize', '2048',
|
|
'-validity', '712',
|
|
'-dname', 'CN=smoke-test-plugins-ssl',
|
|
'-keypass', 'keypass',
|
|
'-storepass', 'keypass'
|
|
}
|
|
|
|
// Add keystores to test classpath: it expects it there
|
|
sourceSets.test.resources.srcDir(keystoreDir)
|
|
processTestResources.dependsOn(createNodeKeyStore)
|
|
|
|
integTestCluster {
|
|
dependsOn createNodeKeyStore
|
|
setting 'xpack.ml.enabled', 'true'
|
|
setting 'logger.org.elasticsearch.xpack.ml.datafeed', 'TRACE'
|
|
// Integration tests are supposed to enable/disable exporters before/after each test
|
|
setting 'xpack.monitoring.exporters._local.type', 'local'
|
|
setting 'xpack.monitoring.exporters._local.enabled', 'false'
|
|
setting 'xpack.monitoring.collection.interval', '-1'
|
|
setting 'xpack.security.authc.token.enabled', 'true'
|
|
setting 'xpack.security.transport.ssl.enabled', 'true'
|
|
setting 'xpack.security.transport.ssl.keystore.path', nodeKeystore.name
|
|
setting 'xpack.security.transport.ssl.verification_mode', 'certificate'
|
|
setting 'xpack.security.audit.enabled', 'true'
|
|
keystoreSetting 'bootstrap.password', 'x-pack-test-password'
|
|
keystoreSetting 'xpack.security.transport.ssl.keystore.secure_password', 'keypass'
|
|
distribution = 'zip' // this is important since we use the reindex module in ML
|
|
|
|
setupCommand 'setupTestUser', 'bin/x-pack/users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser'
|
|
|
|
extraConfigFile nodeKeystore.name, nodeKeystore
|
|
|
|
waitCondition = { NodeInfo node, AntBuilder ant ->
|
|
File tmpFile = new File(node.cwd, 'wait.success')
|
|
|
|
for (int i = 0; i < 10; i++) {
|
|
// we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned
|
|
HttpURLConnection httpURLConnection = null;
|
|
try {
|
|
httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health?wait_for_nodes=${numNodes}&wait_for_status=yellow").openConnection();
|
|
httpURLConnection.setRequestProperty("Authorization", "Basic " +
|
|
Base64.getEncoder().encodeToString("x_pack_rest_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)));
|
|
httpURLConnection.setRequestMethod("GET");
|
|
httpURLConnection.connect();
|
|
if (httpURLConnection.getResponseCode() == 200) {
|
|
tmpFile.withWriter StandardCharsets.UTF_8.name(), {
|
|
it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name()))
|
|
}
|
|
}
|
|
} catch (Exception e) {
|
|
if (i == 9) {
|
|
logger.error("final attempt of calling cluster health failed", e)
|
|
} else {
|
|
logger.debug("failed to call cluster health", e)
|
|
}
|
|
} finally {
|
|
if (httpURLConnection != null) {
|
|
httpURLConnection.disconnect();
|
|
}
|
|
}
|
|
|
|
// did not start, so wait a bit before trying again
|
|
Thread.sleep(500L);
|
|
}
|
|
return tmpFile.exists()
|
|
}
|
|
}
|
|
|
|
test {
|
|
/*
|
|
* We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
|
|
* other if we allow them to set the number of available processors as it's set-once in Netty.
|
|
*/
|
|
systemProperty 'es.set.netty.runtime.available.processors', 'false'
|
|
}
|
|
|
|
integTestRunner {
|
|
/*
|
|
* We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
|
|
* other if we allow them to set the number of available processors as it's set-once in Netty.
|
|
*/
|
|
systemProperty 'es.set.netty.runtime.available.processors', 'false'
|
|
}
|
|
|
|
// TODO: don't publish test artifacts just to run messy tests, fix the tests!
|
|
// https://github.com/elastic/x-plugins/issues/724
|
|
configurations {
|
|
testArtifacts.extendsFrom testRuntime
|
|
}
|
|
task testJar(type: Jar) {
|
|
appendix 'test'
|
|
from sourceSets.test.output
|
|
}
|
|
artifacts {
|
|
// normal es plugins do not publish the jar but we need to since users need it for Transport Clients and extensions
|
|
archives jar
|
|
testArtifacts testJar
|
|
}
|
|
|
|
// classes are missing, e.g. com.ibm.icu.lang.UCharacter
|
|
thirdPartyAudit.excludes = [
|
|
// uses internal java api: sun.misc.Unsafe
|
|
'com.google.common.cache.Striped64',
|
|
'com.google.common.cache.Striped64$1',
|
|
'com.google.common.cache.Striped64$Cell',
|
|
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
|
|
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
|
|
|
|
// pulled in as external dependency to work on java 9
|
|
'com.sun.activation.registries.LineTokenizer',
|
|
'com.sun.activation.registries.LogSupport',
|
|
'com.sun.activation.registries.MailcapFile',
|
|
'com.sun.activation.registries.MailcapTokenizer',
|
|
'com.sun.activation.registries.MimeTypeEntry',
|
|
'com.sun.activation.registries.MimeTypeFile',
|
|
'javax.activation.MailcapCommandMap',
|
|
'javax.activation.MimetypesFileTypeMap',
|
|
]
|
|
|
|
// pulled in as external dependency to work on java 9
|
|
if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
|
|
thirdPartyAudit.excludes += [
|
|
'com.sun.activation.registries.MailcapParseException',
|
|
'javax.activation.ActivationDataFlavor',
|
|
'javax.activation.CommandInfo',
|
|
'javax.activation.CommandMap',
|
|
'javax.activation.CommandObject',
|
|
'javax.activation.DataContentHandler',
|
|
'javax.activation.DataContentHandlerFactory',
|
|
'javax.activation.DataHandler$1',
|
|
'javax.activation.DataHandler',
|
|
'javax.activation.DataHandlerDataSource',
|
|
'javax.activation.DataSource',
|
|
'javax.activation.DataSourceDataContentHandler',
|
|
'javax.activation.FileDataSource',
|
|
'javax.activation.FileTypeMap',
|
|
'javax.activation.MimeType',
|
|
'javax.activation.MimeTypeParameterList',
|
|
'javax.activation.MimeTypeParseException',
|
|
'javax.activation.ObjectDataContentHandler',
|
|
'javax.activation.SecuritySupport$1',
|
|
'javax.activation.SecuritySupport$2',
|
|
'javax.activation.SecuritySupport$3',
|
|
'javax.activation.SecuritySupport$4',
|
|
'javax.activation.SecuritySupport$5',
|
|
'javax.activation.SecuritySupport',
|
|
'javax.activation.URLDataSource',
|
|
'javax.activation.UnsupportedDataTypeException'
|
|
]
|
|
}
|
|
|
|
run {
|
|
setting 'xpack.ml.enabled', 'true'
|
|
setting 'xpack.graph.enabled', 'true'
|
|
setting 'xpack.security.enabled', 'true'
|
|
setting 'xpack.monitoring.enabled', 'true'
|
|
setting 'xpack.watcher.enabled', 'true'
|
|
keystoreSetting 'bootstrap.password', 'password'
|
|
}
|