honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-09 22:45:04 +00:00
Code Issues Packages Projects Releases Wiki Activity
OpenSearch/plugins/repository-s3/build.gradle
Mark Vieira 70cfedf542
Refactor global build info plugin to leverage JavaInstallationRegistry (#54026) 
This commit removes the configuration time vs execution time distinction
with regards to certain BuildParms properties. Because of the cost of
determining Java versions for configuration JDK locations we deferred
this until execution time. This had two main downsides. First, we had
to implement all this build logic in tasks, which required a bunch of
additional plumbing and complexity. Second, because some information
wasn't known during configuration time, we had to nest any build logic
that depended on this in awkward callbacks.

We now defer to the JavaInstallationRegistry recently added in Gradle.
This utility uses a much more efficient method for probing Java
installations vs our jrunscript implementation. This, combined with some
optimizations to avoid probing the current JVM as well as deferring
some evaluation via Providers when probing installations for BWC builds
we can maintain effectively the same configuration time performance
while removing a bunch of complexity and runtime cost (snapshotting
inputs for the GenerateGlobalBuildInfoTask was very expensive). The end
result should be a much more responsive build execution in almost all
scenarios.

(cherry picked from commit ecdbd37f2e0f0447ed574b306adb64c19adc3ce1)
2020-03-23 15:30:10 -07:00

421 lines
17 KiB
Groovy
Raw Blame History

import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.info.BuildParams
import org.elasticsearch.gradle.test.RestIntegTestTask
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
apply plugin: 'elasticsearch.test.fixtures'
esplugin {
description 'The S3 repository plugin adds S3 repositories'
classname 'org.elasticsearch.repositories.s3.S3RepositoryPlugin'
}
versions << [
'aws': '1.11.749'
]
dependencies {
compile "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
compile "com.amazonaws:aws-java-sdk-core:${versions.aws}"
compile "com.amazonaws:jmespath-java:${versions.aws}"
compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
compile "commons-logging:commons-logging:${versions.commonslogging}"
compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
compile "commons-codec:commons-codec:${versions.commonscodec}"
compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
compile "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
compile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
compile "joda-time:joda-time:${versions.joda}"
// HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
// and whitelist this hack in JarHell
compile 'javax.xml.bind:jaxb-api:2.2.2'
testCompile project(':test:fixtures:s3-fixture')
}
dependencyLicenses {
mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
mapping from: /jmespath-java.*/, to: 'aws-java-sdk'
mapping from: /jackson-.*/, to: 'jackson'
mapping from: /jaxb-.*/, to: 'jaxb'
}
bundlePlugin {
from('config/repository-s3') {
into 'config'
}
}
task testRepositoryCreds(type: Test) {
include '**/RepositoryCredentialsTests.class'
systemProperty 'es.allow_insecure_settings', 'true'
}
check.dependsOn(testRepositoryCreds)
test {
// these are tested explicitly in separate test tasks
exclude '**/RepositoryCredentialsTests.class'
exclude '**/S3RepositoryThirdPartyTests.class'
}
boolean useFixture = false
// We test against two repositories, one which uses the usual two-part "permanent" credentials and
// the other which uses three-part "temporary" or "session" credentials.
String s3PermanentAccessKey = System.getenv("amazon_s3_access_key")
String s3PermanentSecretKey = System.getenv("amazon_s3_secret_key")
String s3PermanentBucket = System.getenv("amazon_s3_bucket")
String s3PermanentBasePath = System.getenv("amazon_s3_base_path")
String s3TemporaryAccessKey = System.getenv("amazon_s3_access_key_temporary")
String s3TemporarySecretKey = System.getenv("amazon_s3_secret_key_temporary")
String s3TemporarySessionToken = System.getenv("amazon_s3_session_token_temporary")
String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary")
String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary")
String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2")
String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2")
String s3ECSBucket = System.getenv("amazon_s3_bucket_ecs")
String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs")
boolean s3DisableChunkedEncoding = (new Random(Long.parseUnsignedLong(BuildParams.testSeed.tokenize(':').get(0), 16))).nextBoolean()
// If all these variables are missing then we are testing against the internal fixture instead, which has the following
// credentials hard-coded in.
if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath) {
s3PermanentAccessKey = 'access_key'
s3PermanentSecretKey = 'secret_key'
s3PermanentBucket = 'bucket'
s3PermanentBasePath = 'base_path'
useFixture = true
} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) {
throw new IllegalArgumentException("not all options specified to run against external S3 service as permanent credentials are present")
}
if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) {
s3TemporaryAccessKey = 'session_token_access_key'
s3TemporarySecretKey = 'session_token_secret_key'
s3TemporaryBucket = 'session_token_bucket'
s3TemporaryBasePath = 'session_token_base_path'
s3TemporarySessionToken = 'session_token'
} else if (!s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken) {
throw new IllegalArgumentException("not all options specified to run against external S3 service as temporary credentials are present")
}
if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) {
s3EC2Bucket = 'ec2_bucket'
s3EC2BasePath = 'ec2_base_path'
s3ECSBucket = 'ecs_bucket'
s3ECSBasePath = 'ecs_base_path'
} else if (!s3EC2Bucket || !s3EC2BasePath || !s3ECSBucket || !s3ECSBasePath) {
throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present")
}
task thirdPartyTest(type: Test) {
include '**/S3RepositoryThirdPartyTests.class'
systemProperty 'test.s3.account', s3PermanentAccessKey
systemProperty 'test.s3.key', s3PermanentSecretKey
systemProperty 'test.s3.bucket', s3PermanentBucket
nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + BuildParams.testSeed
}
if (useFixture) {
testFixtures.useFixture(':test:fixtures:minio-fixture', 'minio-fixture')
def minioAddress = {
int minioPort = project(':test:fixtures:minio-fixture').postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000"
assert minioPort > 0
'http://127.0.0.1:' + minioPort
}
normalization {
runtimeClasspath {
// ignore generated address file for the purposes of build avoidance
ignore 's3Fixture.address'
}
}
thirdPartyTest {
dependsOn tasks.bundlePlugin
nonInputProperties.systemProperty 'test.s3.endpoint', "${-> minioAddress.call()}"
}
task integTestMinio(type: RestIntegTestTask) {
description = "Runs REST tests using the Minio repository."
dependsOn tasks.bundlePlugin
runner {
// Minio only supports a single access key, see https://github.com/minio/minio/pull/5968
systemProperty 'tests.rest.blacklist', [
'repository_s3/30_repository_temporary_credentials/*',
'repository_s3/40_repository_ec2_credentials/*',
'repository_s3/50_repository_ecs_credentials/*'
].join(",")
}
}
check.dependsOn(integTestMinio)
testClusters.integTestMinio {
keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
setting 's3.client.integration_test_permanent.endpoint', minioAddress, IGNORE_VALUE
plugin file(tasks.bundlePlugin.archiveFile)
}
integTest.runner {
systemProperty 'tests.rest.blacklist', 'repository_s3/50_repository_ecs_credentials/*'
}
} else {
integTest.runner {
systemProperty 'tests.rest.blacklist',
[
'repository_s3/30_repository_temporary_credentials/*',
'repository_s3/40_repository_ec2_credentials/*',
'repository_s3/50_repository_ecs_credentials/*'
].join(",")
}
}
check.dependsOn(thirdPartyTest)
processTestResources {
Map<String, Object> expansions = [
'permanent_bucket': s3PermanentBucket,
'permanent_base_path': s3PermanentBasePath + "_integration_tests",
'temporary_bucket': s3TemporaryBucket,
'temporary_base_path': s3TemporaryBasePath + "_integration_tests",
'ec2_bucket': s3EC2Bucket,
'ec2_base_path': s3EC2BasePath,
'ecs_bucket': s3ECSBucket,
'ecs_base_path': s3ECSBasePath,
'disable_chunked_encoding': s3DisableChunkedEncoding,
]
inputs.properties(expansions)
MavenFilteringHack.filter(it, expansions)
}
testFixtures.useFixture(':test:fixtures:s3-fixture')
def fixtureAddress = { fixture ->
assert useFixture: 'closure should not be used without a fixture'
int ephemeralPort = project(':test:fixtures:s3-fixture').postProcessFixture.ext."test.fixtures.${fixture}.tcp.80"
assert ephemeralPort > 0
'http://127.0.0.1:' + ephemeralPort
}
testClusters.integTest {
keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
keystore 's3.client.integration_test_temporary.access_key', s3TemporaryAccessKey
keystore 's3.client.integration_test_temporary.secret_key', s3TemporarySecretKey
keystore 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken
if (useFixture) {
setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('s3-fixture')}" }, IGNORE_VALUE
setting 's3.client.integration_test_temporary.endpoint', { "${-> fixtureAddress('s3-fixture-with-session-token')}" }, IGNORE_VALUE
setting 's3.client.integration_test_ec2.endpoint', { "${-> fixtureAddress('s3-fixture-with-ec2')}" }, IGNORE_VALUE
// to redirect InstanceProfileCredentialsProvider to custom auth point
systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "${-> fixtureAddress('s3-fixture-with-ec2')}" }, IGNORE_VALUE
} else {
println "Using an external service to test the repository-s3 plugin"
}
}
task s3ThirdPartyTests {
dependsOn check
}
if (useFixture) {
task integTestECS(type: RestIntegTestTask.class) {
description = "Runs tests using the ECS repository."
dependsOn('bundlePlugin')
runner {
systemProperty 'tests.rest.blacklist', [
'repository_s3/10_basic/*',
'repository_s3/20_repository_permanent_credentials/*',
'repository_s3/30_repository_temporary_credentials/*',
'repository_s3/40_repository_ec2_credentials/*'
].join(",")
}
}
check.dependsOn(integTestECS)
testClusters.integTestECS {
setting 's3.client.integration_test_ecs.endpoint', { "${-> fixtureAddress('s3-fixture-with-ecs')}" }, IGNORE_VALUE
plugin file(tasks.bundlePlugin.archiveFile)
environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "${-> fixtureAddress('s3-fixture-with-ecs')}/ecs_credentials_endpoint" }, IGNORE_VALUE
}
gradle.taskGraph.whenReady {
if (it.hasTask(s3ThirdPartyTests)) {
throw new IllegalStateException("Tried to run third party tests but not all of the necessary environment variables 'amazon_s3_access_key', " +
"'amazon_s3_secret_key', 'amazon_s3_bucket', and 'amazon_s3_base_path' are set.");
}
}
}
thirdPartyAudit.ignoreMissingClasses(
// classes are missing
'javax.servlet.ServletContextEvent',
'javax.servlet.ServletContextListener',
'org.apache.avalon.framework.logger.Logger',
'org.apache.log.Hierarchy',
'org.apache.log.Logger',
'software.amazon.ion.IonReader',
'software.amazon.ion.IonSystem',
'software.amazon.ion.IonType',
'software.amazon.ion.IonWriter',
'software.amazon.ion.Timestamp',
'software.amazon.ion.system.IonBinaryWriterBuilder',
'software.amazon.ion.system.IonSystemBuilder',
'software.amazon.ion.system.IonTextWriterBuilder',
'software.amazon.ion.system.IonWriterBuilder',
// We don't use the kms dependency
'com.amazonaws.services.kms.AWSKMS',
'com.amazonaws.services.kms.AWSKMSClient',
'com.amazonaws.services.kms.model.DecryptRequest',
'com.amazonaws.services.kms.model.DecryptResult',
'com.amazonaws.services.kms.model.EncryptRequest',
'com.amazonaws.services.kms.model.EncryptResult',
'com.amazonaws.services.kms.model.GenerateDataKeyRequest',
'com.amazonaws.services.kms.model.GenerateDataKeyResult'
)
// jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
if (BuildParams.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
thirdPartyAudit.ignoreJarHellWithJDK(
'javax.xml.bind.Binder',
'javax.xml.bind.ContextFinder$1',
'javax.xml.bind.ContextFinder',
'javax.xml.bind.DataBindingException',
'javax.xml.bind.DatatypeConverter',
'javax.xml.bind.DatatypeConverterImpl$CalendarFormatter',
'javax.xml.bind.DatatypeConverterImpl',
'javax.xml.bind.DatatypeConverterInterface',
'javax.xml.bind.Element',
'javax.xml.bind.GetPropertyAction',
'javax.xml.bind.JAXB$Cache',
'javax.xml.bind.JAXB',
'javax.xml.bind.JAXBContext',
'javax.xml.bind.JAXBElement$GlobalScope',
'javax.xml.bind.JAXBElement',
'javax.xml.bind.JAXBException',
'javax.xml.bind.JAXBIntrospector',
'javax.xml.bind.JAXBPermission',
'javax.xml.bind.MarshalException',
'javax.xml.bind.Marshaller$Listener',
'javax.xml.bind.Marshaller',
'javax.xml.bind.Messages',
'javax.xml.bind.NotIdentifiableEvent',
'javax.xml.bind.ParseConversionEvent',
'javax.xml.bind.PrintConversionEvent',
'javax.xml.bind.PropertyException',
'javax.xml.bind.SchemaOutputResolver',
'javax.xml.bind.TypeConstraintException',
'javax.xml.bind.UnmarshalException',
'javax.xml.bind.Unmarshaller$Listener',
'javax.xml.bind.Unmarshaller',
'javax.xml.bind.UnmarshallerHandler',
'javax.xml.bind.ValidationEvent',
'javax.xml.bind.ValidationEventHandler',
'javax.xml.bind.ValidationEventLocator',
'javax.xml.bind.ValidationException',
'javax.xml.bind.Validator',
'javax.xml.bind.WhiteSpaceProcessor',
'javax.xml.bind.annotation.DomHandler',
'javax.xml.bind.annotation.W3CDomHandler',
'javax.xml.bind.annotation.XmlAccessOrder',
'javax.xml.bind.annotation.XmlAccessType',
'javax.xml.bind.annotation.XmlAccessorOrder',
'javax.xml.bind.annotation.XmlAccessorType',
'javax.xml.bind.annotation.XmlAnyAttribute',
'javax.xml.bind.annotation.XmlAnyElement',
'javax.xml.bind.annotation.XmlAttachmentRef',
'javax.xml.bind.annotation.XmlAttribute',
'javax.xml.bind.annotation.XmlElement$DEFAULT',
'javax.xml.bind.annotation.XmlElement',
'javax.xml.bind.annotation.XmlElementDecl$GLOBAL',
'javax.xml.bind.annotation.XmlElementDecl',
'javax.xml.bind.annotation.XmlElementRef$DEFAULT',
'javax.xml.bind.annotation.XmlElementRef',
'javax.xml.bind.annotation.XmlElementRefs',
'javax.xml.bind.annotation.XmlElementWrapper',
'javax.xml.bind.annotation.XmlElements',
'javax.xml.bind.annotation.XmlEnum',
'javax.xml.bind.annotation.XmlEnumValue',
'javax.xml.bind.annotation.XmlID',
'javax.xml.bind.annotation.XmlIDREF',
'javax.xml.bind.annotation.XmlInlineBinaryData',
'javax.xml.bind.annotation.XmlList',
'javax.xml.bind.annotation.XmlMimeType',
'javax.xml.bind.annotation.XmlMixed',
'javax.xml.bind.annotation.XmlNs',
'javax.xml.bind.annotation.XmlNsForm',
'javax.xml.bind.annotation.XmlRegistry',
'javax.xml.bind.annotation.XmlRootElement',
'javax.xml.bind.annotation.XmlSchema',
'javax.xml.bind.annotation.XmlSchemaType$DEFAULT',
'javax.xml.bind.annotation.XmlSchemaType',
'javax.xml.bind.annotation.XmlSchemaTypes',
'javax.xml.bind.annotation.XmlSeeAlso',
'javax.xml.bind.annotation.XmlTransient',
'javax.xml.bind.annotation.XmlType$DEFAULT',
'javax.xml.bind.annotation.XmlType',
'javax.xml.bind.annotation.XmlValue',
'javax.xml.bind.annotation.adapters.CollapsedStringAdapter',
'javax.xml.bind.annotation.adapters.HexBinaryAdapter',
'javax.xml.bind.annotation.adapters.NormalizedStringAdapter',
'javax.xml.bind.annotation.adapters.XmlAdapter',
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter$DEFAULT',
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter',
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapters',
'javax.xml.bind.attachment.AttachmentMarshaller',
'javax.xml.bind.attachment.AttachmentUnmarshaller',
'javax.xml.bind.helpers.AbstractMarshallerImpl',
'javax.xml.bind.helpers.AbstractUnmarshallerImpl',
'javax.xml.bind.helpers.DefaultValidationEventHandler',
'javax.xml.bind.helpers.Messages',
'javax.xml.bind.helpers.NotIdentifiableEventImpl',
'javax.xml.bind.helpers.ParseConversionEventImpl',
'javax.xml.bind.helpers.PrintConversionEventImpl',
'javax.xml.bind.helpers.ValidationEventImpl',
'javax.xml.bind.helpers.ValidationEventLocatorImpl',
'javax.xml.bind.util.JAXBResult',
'javax.xml.bind.util.JAXBSource$1',
'javax.xml.bind.util.JAXBSource',
'javax.xml.bind.util.Messages',
'javax.xml.bind.util.ValidationEventCollector'
)
} else {
thirdPartyAudit.ignoreMissingClasses 'javax.activation.DataHandler'
}
Reference in New Issue View Git Blame Copy Permalink