honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-08 05:58:44 +00:00
Code Issues Packages Projects Releases Wiki Activity
OpenSearch/plugins/repository-s3/build.gradle
Jake Landis 604c6dd528
7.x - Create plugin for yamlTest task (#56841) (#59090) 
This commit creates a new Gradle plugin to provide a separate task name
and source set for running YAML based REST tests. The only project
converted to use the new plugin in this PR is distribution/archives/integ-test-zip.
For which the testing has been moved to :rest-api-spec since it makes the most
sense and it avoids a small but awkward change to the distribution plugin.

The remaining cases in modules, plugins, and x-pack will be handled in followups.

This plugin is distinctly different from the plugin introduced in #55896 since
the YAML REST tests are intended to be black box tests over HTTP. As such they
should not (by default) have access to the classpath for that which they are testing.

The YAML based REST tests will be moved to separate source sets (yamlRestTest).
The which source is the target for the test resources is dependent on if this
new plugin is applied. If it is not applied, it will default to the test source
set.

Further, this introduces a breaking change for plugin developers that
use the YAML testing framework. They will now need to either use the new source set
and matching task, or configure the rest resources to use the old "test" source set that
matches the old integTest task. (The former should be preferred).

As part of this change (which is also breaking for plugin developers) the
rest resources plugin has been removed from the build plugin and now requires
either explicit application or application via the new YAML REST test plugin.

Plugin developers should be able to fix the breaking changes to the YAML tests
by adding apply plugin: 'elasticsearch.yaml-rest-test' and moving the YAML tests
under a yamlRestTest folder (instead of test)
2020-07-06 14:16:26 -05:00

420 lines
17 KiB
Groovy
Raw Blame History

import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.info.BuildParams
import org.elasticsearch.gradle.test.RestIntegTestTask
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
apply plugin: 'elasticsearch.rest-resources'
esplugin {
description 'The S3 repository plugin adds S3 repositories'
classname 'org.elasticsearch.repositories.s3.S3RepositoryPlugin'
}
versions << [
'aws': '1.11.749'
]
dependencies {
api "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
api "com.amazonaws:aws-java-sdk-core:${versions.aws}"
api "com.amazonaws:jmespath-java:${versions.aws}"
api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
api "commons-logging:commons-logging:${versions.commonslogging}"
api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
api "commons-codec:commons-codec:${versions.commonscodec}"
api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
api "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
api "joda-time:joda-time:${versions.joda}"
// HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
// and whitelist this hack in JarHell
api 'javax.xml.bind:jaxb-api:2.2.2'
testImplementation project(':test:fixtures:s3-fixture')
}
tasks.named("dependencyLicenses").configure {
mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
mapping from: /jmespath-java.*/, to: 'aws-java-sdk'
mapping from: /jackson-.*/, to: 'jackson'
mapping from: /jaxb-.*/, to: 'jaxb'
}
bundlePlugin {
from('config/repository-s3') {
into 'config'
}
}
task testRepositoryCreds(type: Test) {
include '**/RepositoryCredentialsTests.class'
systemProperty 'es.allow_insecure_settings', 'true'
}
check.dependsOn(testRepositoryCreds)
test {
// this is tested explicitly in separate test tasks
exclude '**/RepositoryCredentialsTests.class'
exclude '**/S3RepositoryThirdPartyTests.class'
}
boolean useFixture = false
def fixtureAddress = { fixture, name, port ->
assert useFixture: 'closure should not be used without a fixture'
int ephemeralPort = project(":test:fixtures:${fixture}").postProcessFixture.ext."test.fixtures.${name}.tcp.${port}"
assert ephemeralPort > 0
'http://127.0.0.1:' + ephemeralPort
}
// We test against two repositories, one which uses the usual two-part "permanent" credentials and
// the other which uses three-part "temporary" or "session" credentials.
String s3PermanentAccessKey = System.getenv("amazon_s3_access_key")
String s3PermanentSecretKey = System.getenv("amazon_s3_secret_key")
String s3PermanentBucket = System.getenv("amazon_s3_bucket")
String s3PermanentBasePath = System.getenv("amazon_s3_base_path")
String s3TemporaryAccessKey = System.getenv("amazon_s3_access_key_temporary")
String s3TemporarySecretKey = System.getenv("amazon_s3_secret_key_temporary")
String s3TemporarySessionToken = System.getenv("amazon_s3_session_token_temporary")
String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary")
String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary")
String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2")
String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2")
String s3ECSBucket = System.getenv("amazon_s3_bucket_ecs")
String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs")
boolean s3DisableChunkedEncoding = (new Random(Long.parseUnsignedLong(BuildParams.testSeed.tokenize(':').get(0), 16))).nextBoolean()
// If all these variables are missing then we are testing against the internal fixture instead, which has the following
// credentials hard-coded in.
if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath) {
s3PermanentAccessKey = 'access_key'
s3PermanentSecretKey = 'secret_key'
s3PermanentBucket = 'bucket'
s3PermanentBasePath = 'base_path'
apply plugin: 'elasticsearch.test.fixtures'
useFixture = true
} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) {
throw new IllegalArgumentException("not all options specified to run against external S3 service as permanent credentials are present")
}
if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) {
s3TemporaryAccessKey = 'session_token_access_key'
s3TemporarySecretKey = 'session_token_secret_key'
s3TemporaryBucket = 'session_token_bucket'
s3TemporaryBasePath = 'session_token_base_path'
s3TemporarySessionToken = 'session_token'
} else if (!s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken) {
throw new IllegalArgumentException("not all options specified to run against external S3 service as temporary credentials are present")
}
if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) {
s3EC2Bucket = 'ec2_bucket'
s3EC2BasePath = 'ec2_base_path'
s3ECSBucket = 'ecs_bucket'
s3ECSBasePath = 'ecs_base_path'
} else if (!s3EC2Bucket || !s3EC2BasePath || !s3ECSBucket || !s3ECSBasePath) {
throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present")
}
processTestResources {
Map<String, Object> expansions = [
'permanent_bucket': s3PermanentBucket,
'permanent_base_path': s3PermanentBasePath + "_integration_tests",
'temporary_bucket': s3TemporaryBucket,
'temporary_base_path': s3TemporaryBasePath + "_integration_tests",
'ec2_bucket': s3EC2Bucket,
'ec2_base_path': s3EC2BasePath,
'ecs_bucket': s3ECSBucket,
'ecs_base_path': s3ECSBasePath,
'disable_chunked_encoding': s3DisableChunkedEncoding,
]
inputs.properties(expansions)
MavenFilteringHack.filter(it, expansions)
}
test {
// this is tested explicitly in a separate test task
exclude '**/S3RepositoryThirdPartyTests.class'
}
// IntegTest
integTest {
runner {
systemProperty 'tests.rest.blacklist', (
useFixture ?
['repository_s3/50_repository_ecs_credentials/*']
:
[
'repository_s3/30_repository_temporary_credentials/*',
'repository_s3/40_repository_ec2_credentials/*',
'repository_s3/50_repository_ecs_credentials/*'
]
).join(",")
}
}
testClusters.integTest {
keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
keystore 's3.client.integration_test_temporary.access_key', s3TemporaryAccessKey
keystore 's3.client.integration_test_temporary.secret_key', s3TemporarySecretKey
keystore 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken
if (useFixture) {
testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture')
testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-session-token')
testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-ec2')
normalization {
runtimeClasspath {
// ignore generated address file for the purposes of build avoidance
ignore 's3Fixture.address'
}
}
setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture', '80')}" }, IGNORE_VALUE
setting 's3.client.integration_test_temporary.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-session-token', '80')}" }, IGNORE_VALUE
setting 's3.client.integration_test_ec2.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ec2', '80')}" }, IGNORE_VALUE
// to redirect InstanceProfileCredentialsProvider to custom auth point
systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ec2', '80')}" }, IGNORE_VALUE
} else {
println "Using an external service to test the repository-s3 plugin"
}
}
// MinIO
if (useFixture) {
testFixtures.useFixture(':test:fixtures:minio-fixture', 'minio-fixture')
task integTestMinio(type: RestIntegTestTask) {
description = "Runs REST tests using the Minio repository."
dependsOn tasks.bundlePlugin
runner {
// Minio only supports a single access key, see https://github.com/minio/minio/pull/5968
systemProperty 'tests.rest.blacklist', [
'repository_s3/30_repository_temporary_credentials/*',
'repository_s3/40_repository_ec2_credentials/*',
'repository_s3/50_repository_ecs_credentials/*'
].join(",")
}
}
check.dependsOn(integTestMinio)
testClusters.integTestMinio {
keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('minio-fixture', 'minio-fixture', '9000')}" }, IGNORE_VALUE
plugin tasks.bundlePlugin.archiveFile
}
}
// ECS
if (useFixture) {
testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-ecs')
task integTestECS(type: RestIntegTestTask.class) {
description = "Runs tests using the ECS repository."
dependsOn('bundlePlugin')
runner {
systemProperty 'tests.rest.blacklist', [
'repository_s3/10_basic/*',
'repository_s3/20_repository_permanent_credentials/*',
'repository_s3/30_repository_temporary_credentials/*',
'repository_s3/40_repository_ec2_credentials/*'
].join(",")
}
}
check.dependsOn(integTestECS)
testClusters.integTestECS {
setting 's3.client.integration_test_ecs.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ecs', '80')}" }, IGNORE_VALUE
plugin tasks.bundlePlugin.archiveFile
environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ecs', '80')}/ecs_credentials_endpoint" }, IGNORE_VALUE
}
}
// 3rd Party Tests
task s3ThirdPartyTest(type: Test) {
include '**/S3RepositoryThirdPartyTests.class'
systemProperty 'test.s3.account', s3PermanentAccessKey
systemProperty 'test.s3.key', s3PermanentSecretKey
systemProperty 'test.s3.bucket', s3PermanentBucket
nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + BuildParams.testSeed
if (useFixture) {
dependsOn tasks.integTestMinio
nonInputProperties.systemProperty 'test.s3.endpoint', "${-> fixtureAddress('minio-fixture', 'minio-fixture', '9000') }"
} else {
dependsOn tasks.integTest
}
}
check.dependsOn(s3ThirdPartyTest)
thirdPartyAudit.ignoreMissingClasses(
// classes are missing
'javax.servlet.ServletContextEvent',
'javax.servlet.ServletContextListener',
'org.apache.avalon.framework.logger.Logger',
'org.apache.log.Hierarchy',
'org.apache.log.Logger',
'software.amazon.ion.IonReader',
'software.amazon.ion.IonSystem',
'software.amazon.ion.IonType',
'software.amazon.ion.IonWriter',
'software.amazon.ion.Timestamp',
'software.amazon.ion.system.IonBinaryWriterBuilder',
'software.amazon.ion.system.IonSystemBuilder',
'software.amazon.ion.system.IonTextWriterBuilder',
'software.amazon.ion.system.IonWriterBuilder',
// We don't use the kms dependency
'com.amazonaws.services.kms.AWSKMS',
'com.amazonaws.services.kms.AWSKMSClient',
'com.amazonaws.services.kms.model.DecryptRequest',
'com.amazonaws.services.kms.model.DecryptResult',
'com.amazonaws.services.kms.model.EncryptRequest',
'com.amazonaws.services.kms.model.EncryptResult',
'com.amazonaws.services.kms.model.GenerateDataKeyRequest',
'com.amazonaws.services.kms.model.GenerateDataKeyResult'
)
// jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
if (BuildParams.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
thirdPartyAudit.ignoreJarHellWithJDK(
'javax.xml.bind.Binder',
'javax.xml.bind.ContextFinder$1',
'javax.xml.bind.ContextFinder',
'javax.xml.bind.DataBindingException',
'javax.xml.bind.DatatypeConverter',
'javax.xml.bind.DatatypeConverterImpl$CalendarFormatter',
'javax.xml.bind.DatatypeConverterImpl',
'javax.xml.bind.DatatypeConverterInterface',
'javax.xml.bind.Element',
'javax.xml.bind.GetPropertyAction',
'javax.xml.bind.JAXB$Cache',
'javax.xml.bind.JAXB',
'javax.xml.bind.JAXBContext',
'javax.xml.bind.JAXBElement$GlobalScope',
'javax.xml.bind.JAXBElement',
'javax.xml.bind.JAXBException',
'javax.xml.bind.JAXBIntrospector',
'javax.xml.bind.JAXBPermission',
'javax.xml.bind.MarshalException',
'javax.xml.bind.Marshaller$Listener',
'javax.xml.bind.Marshaller',
'javax.xml.bind.Messages',
'javax.xml.bind.NotIdentifiableEvent',
'javax.xml.bind.ParseConversionEvent',
'javax.xml.bind.PrintConversionEvent',
'javax.xml.bind.PropertyException',
'javax.xml.bind.SchemaOutputResolver',
'javax.xml.bind.TypeConstraintException',
'javax.xml.bind.UnmarshalException',
'javax.xml.bind.Unmarshaller$Listener',
'javax.xml.bind.Unmarshaller',
'javax.xml.bind.UnmarshallerHandler',
'javax.xml.bind.ValidationEvent',
'javax.xml.bind.ValidationEventHandler',
'javax.xml.bind.ValidationEventLocator',
'javax.xml.bind.ValidationException',
'javax.xml.bind.Validator',
'javax.xml.bind.WhiteSpaceProcessor',
'javax.xml.bind.annotation.DomHandler',
'javax.xml.bind.annotation.W3CDomHandler',
'javax.xml.bind.annotation.XmlAccessOrder',
'javax.xml.bind.annotation.XmlAccessType',
'javax.xml.bind.annotation.XmlAccessorOrder',
'javax.xml.bind.annotation.XmlAccessorType',
'javax.xml.bind.annotation.XmlAnyAttribute',
'javax.xml.bind.annotation.XmlAnyElement',
'javax.xml.bind.annotation.XmlAttachmentRef',
'javax.xml.bind.annotation.XmlAttribute',
'javax.xml.bind.annotation.XmlElement$DEFAULT',
'javax.xml.bind.annotation.XmlElement',
'javax.xml.bind.annotation.XmlElementDecl$GLOBAL',
'javax.xml.bind.annotation.XmlElementDecl',
'javax.xml.bind.annotation.XmlElementRef$DEFAULT',
'javax.xml.bind.annotation.XmlElementRef',
'javax.xml.bind.annotation.XmlElementRefs',
'javax.xml.bind.annotation.XmlElementWrapper',
'javax.xml.bind.annotation.XmlElements',
'javax.xml.bind.annotation.XmlEnum',
'javax.xml.bind.annotation.XmlEnumValue',
'javax.xml.bind.annotation.XmlID',
'javax.xml.bind.annotation.XmlIDREF',
'javax.xml.bind.annotation.XmlInlineBinaryData',
'javax.xml.bind.annotation.XmlList',
'javax.xml.bind.annotation.XmlMimeType',
'javax.xml.bind.annotation.XmlMixed',
'javax.xml.bind.annotation.XmlNs',
'javax.xml.bind.annotation.XmlNsForm',
'javax.xml.bind.annotation.XmlRegistry',
'javax.xml.bind.annotation.XmlRootElement',
'javax.xml.bind.annotation.XmlSchema',
'javax.xml.bind.annotation.XmlSchemaType$DEFAULT',
'javax.xml.bind.annotation.XmlSchemaType',
'javax.xml.bind.annotation.XmlSchemaTypes',
'javax.xml.bind.annotation.XmlSeeAlso',
'javax.xml.bind.annotation.XmlTransient',
'javax.xml.bind.annotation.XmlType$DEFAULT',
'javax.xml.bind.annotation.XmlType',
'javax.xml.bind.annotation.XmlValue',
'javax.xml.bind.annotation.adapters.CollapsedStringAdapter',
'javax.xml.bind.annotation.adapters.HexBinaryAdapter',
'javax.xml.bind.annotation.adapters.NormalizedStringAdapter',
'javax.xml.bind.annotation.adapters.XmlAdapter',
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter$DEFAULT',
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter',
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapters',
'javax.xml.bind.attachment.AttachmentMarshaller',
'javax.xml.bind.attachment.AttachmentUnmarshaller',
'javax.xml.bind.helpers.AbstractMarshallerImpl',
'javax.xml.bind.helpers.AbstractUnmarshallerImpl',
'javax.xml.bind.helpers.DefaultValidationEventHandler',
'javax.xml.bind.helpers.Messages',
'javax.xml.bind.helpers.NotIdentifiableEventImpl',
'javax.xml.bind.helpers.ParseConversionEventImpl',
'javax.xml.bind.helpers.PrintConversionEventImpl',
'javax.xml.bind.helpers.ValidationEventImpl',
'javax.xml.bind.helpers.ValidationEventLocatorImpl',
'javax.xml.bind.util.JAXBResult',
'javax.xml.bind.util.JAXBSource$1',
'javax.xml.bind.util.JAXBSource',
'javax.xml.bind.util.Messages',
'javax.xml.bind.util.ValidationEventCollector'
)
} else {
thirdPartyAudit.ignoreMissingClasses 'javax.activation.DataHandler'
}
Reference in New Issue View Git Blame Copy Permalink