Jason Tedor 371cb9a8ce
Remove Log4j 1.2 API as a dependency (#42702)
We had this as a dependency for legacy dependencies that still needed
the Log4j 1.2 API. This appears to no longer be necessary, so this
commit removes this artifact as a dependency.

To remove this dependency, we had to fix a few places where we were
accidentally relying on Log4j 1.2 instead of Log4j 2 (easy to do, since
both APIs were on the compile-time classpath).

Finally, we can remove our custom Netty logger factory. This was needed
when we were on Log4j 1.2 and handled logging in our own unique
way. When we migrated to Log4j 2 we could have dropped this
dependency. However, even then Netty would still pick up Log4j 1.2 since
it was on the classpath, thus the advantage to removing this as a
dependency now.
2019-05-30 16:08:07 -04:00

449 lines
19 KiB
Groovy

import org.elasticsearch.gradle.BuildPlugin
import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.test.AntFixture
import org.elasticsearch.gradle.test.RestIntegTestTask
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
esplugin {
description 'The S3 repository plugin adds S3 repositories'
classname 'org.elasticsearch.repositories.s3.S3RepositoryPlugin'
}
versions << [
'aws': '1.11.505'
]
dependencies {
compile "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
compile "com.amazonaws:aws-java-sdk-kms:${versions.aws}"
compile "com.amazonaws:aws-java-sdk-core:${versions.aws}"
compile "com.amazonaws:jmespath-java:${versions.aws}"
compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
compile "commons-logging:commons-logging:${versions.commonslogging}"
compile "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
compile "commons-codec:commons-codec:${versions.commonscodec}"
compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
compile 'com.fasterxml.jackson.core:jackson-databind:2.8.11.3'
compile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
compile "joda-time:joda-time:${versions.joda}"
// HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
// and whitelist this hack in JarHell
compile 'javax.xml.bind:jaxb-api:2.2.2'
}
dependencyLicenses {
mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
mapping from: /jmespath-java.*/, to: 'aws-java-sdk'
mapping from: /jackson-.*/, to: 'jackson'
mapping from: /jaxb-.*/, to: 'jaxb'
}
bundlePlugin {
from('config/repository-s3') {
into 'config'
}
}
task testRepositoryCreds(type: Test) {
include '**/RepositoryCredentialsTests.class'
include '**/S3BlobStoreRepositoryTests.class'
systemProperty 'es.allow_insecure_settings', 'true'
}
check.dependsOn(testRepositoryCreds)
test {
// these are tested explicitly in separate test tasks
exclude '**/*CredentialsTests.class'
exclude '**/S3BlobStoreRepositoryTests.class'
exclude '**/S3RepositoryThirdPartyTests.class'
}
boolean useFixture = false
// We test against two repositories, one which uses the usual two-part "permanent" credentials and
// the other which uses three-part "temporary" or "session" credentials.
String s3PermanentAccessKey = System.getenv("amazon_s3_access_key")
String s3PermanentSecretKey = System.getenv("amazon_s3_secret_key")
String s3PermanentBucket = System.getenv("amazon_s3_bucket")
String s3PermanentBasePath = System.getenv("amazon_s3_base_path")
String s3TemporaryAccessKey = System.getenv("amazon_s3_access_key_temporary")
String s3TemporarySecretKey = System.getenv("amazon_s3_secret_key_temporary")
String s3TemporarySessionToken = System.getenv("amazon_s3_session_token_temporary")
String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary")
String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary")
String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2")
String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2")
String s3ECSBucket = System.getenv("amazon_s3_bucket_ecs")
String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs")
// If all these variables are missing then we are testing against the internal fixture instead, which has the following
// credentials hard-coded in.
if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath) {
s3PermanentAccessKey = 's3_integration_test_permanent_access_key'
s3PermanentSecretKey = 's3_integration_test_permanent_secret_key'
s3PermanentBucket = 'permanent-bucket-test'
s3PermanentBasePath = 'integration_test'
useFixture = true
} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) {
throw new IllegalArgumentException("not all options specified to run against external S3 service as permanent credentials are present")
}
if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) {
s3TemporaryAccessKey = 's3_integration_test_temporary_access_key'
s3TemporarySecretKey = 's3_integration_test_temporary_secret_key'
s3TemporaryBucket = 'temporary-bucket-test'
s3TemporaryBasePath = 'integration_test'
s3TemporarySessionToken = 's3_integration_test_temporary_session_token'
} else if (!s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken) {
throw new IllegalArgumentException("not all options specified to run against external S3 service as temporary credentials are present")
}
if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) {
s3EC2Bucket = 'ec2-bucket-test'
s3EC2BasePath = 'integration_test'
s3ECSBucket = 'ecs-bucket-test'
s3ECSBasePath = 'integration_test'
} else if (!s3EC2Bucket || !s3EC2BasePath || !s3ECSBucket || !s3ECSBasePath) {
throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present")
}
task thirdPartyTest(type: Test) {
include '**/S3RepositoryThirdPartyTests.class'
systemProperty 'test.s3.account', s3PermanentAccessKey
systemProperty 'test.s3.key', s3PermanentSecretKey
systemProperty 'test.s3.bucket', s3PermanentBucket
systemProperty 'test.s3.base', s3PermanentBasePath
}
if (useFixture) {
apply plugin: 'elasticsearch.test.fixtures'
task writeDockerFile {
File minioDockerfile = new File("${project.buildDir}/minio-docker/Dockerfile")
outputs.file(minioDockerfile)
doLast {
minioDockerfile.parentFile.mkdirs()
minioDockerfile.text = "FROM minio/minio:RELEASE.2019-01-23T23-18-58Z\n" +
"RUN mkdir -p /minio/data/${s3PermanentBucket}\n" +
"ENV MINIO_ACCESS_KEY ${s3PermanentAccessKey}\n" +
"ENV MINIO_SECRET_KEY ${s3PermanentSecretKey}"
}
}
preProcessFixture {
dependsOn(writeDockerFile)
}
def minioAddress = {
int minioPort = postProcessFixture.ext."test.fixtures.minio-fixture.tcp.9000"
assert minioPort > 0
return 'http://127.0.0.1:' + minioPort
}
File minioAddressFile = new File(project.buildDir, 'generated-resources/s3Fixture.address')
// We can't lazy evaluate a system property for the Minio address passed to JUnit so we write it to a resource file
// and pass its name instead.
task writeMinioAddress {
dependsOn tasks.bundlePlugin, tasks.postProcessFixture
outputs.file(minioAddressFile)
doLast {
file(minioAddressFile).text = "${ -> minioAddress.call() }"
}
}
thirdPartyTest {
dependsOn writeMinioAddress
inputs.file(minioAddressFile)
systemProperty 'test.s3.endpoint', minioAddressFile.name
}
BuildPlugin.requireDocker(tasks.thirdPartyTest)
task integTestMinio(type: RestIntegTestTask) {
description = "Runs REST tests using the Minio repository."
dependsOn tasks.bundlePlugin, tasks.postProcessFixture
runner {
// Minio only supports a single access key, see https://github.com/minio/minio/pull/5968
systemProperty 'tests.rest.blacklist', [
'repository_s3/30_repository_temporary_credentials/*',
'repository_s3/40_repository_ec2_credentials/*',
'repository_s3/50_repository_ecs_credentials/*'
].join(",")
}
}
check.dependsOn(integTestMinio)
BuildPlugin.requireDocker(tasks.integTestMinio)
testClusters.integTestMinio {
keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
setting 's3.client.integration_test_permanent.endpoint', minioAddress
plugin file(tasks.bundlePlugin.archiveFile)
}
integTest.runner {
systemProperty 'tests.rest.blacklist', 'repository_s3/50_repository_ecs_credentials/*'
}
} else {
integTest.runner {
systemProperty 'tests.rest.blacklist',
[
'repository_s3/30_repository_temporary_credentials/*',
'repository_s3/40_repository_ec2_credentials/*',
'repository_s3/50_repository_ecs_credentials/*'
].join(",")
}
}
check.dependsOn(thirdPartyTest)
File parentFixtures = new File(project.buildDir, "fixtures")
File s3FixtureFile = new File(parentFixtures, 's3Fixture.properties')
task s3FixtureProperties {
outputs.file(s3FixtureFile)
def s3FixtureOptions = [
"tests.seed" : project.testSeed,
"s3Fixture.permanent_bucket_name" : s3PermanentBucket,
"s3Fixture.permanent_key" : s3PermanentAccessKey,
"s3Fixture.temporary_bucket_name" : s3TemporaryBucket,
"s3Fixture.temporary_key" : s3TemporaryAccessKey,
"s3Fixture.temporary_session_token": s3TemporarySessionToken,
"s3Fixture.ec2_bucket_name" : s3EC2Bucket,
"s3Fixture.ecs_bucket_name" : s3ECSBucket
]
doLast {
file(s3FixtureFile).text = s3FixtureOptions.collect { k, v -> "$k = $v" }.join("\n")
}
}
/** A task to start the AmazonS3Fixture which emulates an S3 service **/
task s3Fixture(type: AntFixture) {
dependsOn testClasses
dependsOn s3FixtureProperties
inputs.file(s3FixtureFile)
env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
executable = new File(project.runtimeJavaHome, 'bin/java')
args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3FixtureFile.getAbsolutePath()
}
processTestResources {
Map<String, Object> expansions = [
'permanent_bucket': s3PermanentBucket,
'permanent_base_path': s3PermanentBasePath,
'temporary_bucket': s3TemporaryBucket,
'temporary_base_path': s3TemporaryBasePath,
'ec2_bucket': s3EC2Bucket,
'ec2_base_path': s3EC2BasePath,
'ecs_bucket': s3ECSBucket,
'ecs_base_path': s3ECSBasePath
]
inputs.properties(expansions)
MavenFilteringHack.filter(it, expansions)
}
integTest {
dependsOn s3Fixture
}
testClusters.integTest {
keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
keystore 's3.client.integration_test_temporary.access_key', s3TemporaryAccessKey
keystore 's3.client.integration_test_temporary.secret_key', s3TemporarySecretKey
keystore 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken
if (useFixture) {
setting 's3.client.integration_test_permanent.endpoint', { "http://${s3Fixture.addressAndPort}" }
setting 's3.client.integration_test_temporary.endpoint', { "http://${s3Fixture.addressAndPort}" }
setting 's3.client.integration_test_ec2.endpoint', { "http://${s3Fixture.addressAndPort}" }
// to redirect InstanceProfileCredentialsProvider to custom auth point
systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "http://${s3Fixture.addressAndPort}" }
} else {
println "Using an external service to test the repository-s3 plugin"
}
}
if (useFixture) {
task integTestECS(type: RestIntegTestTask.class) {
description = "Runs tests using the ECS repository."
dependsOn(project.s3Fixture)
runner {
systemProperty 'tests.rest.blacklist', [
'repository_s3/10_basic/*',
'repository_s3/20_repository_permanent_credentials/*',
'repository_s3/30_repository_temporary_credentials/*',
'repository_s3/40_repository_ec2_credentials/*'
].join(",")
}
}
check.dependsOn(integTestECS)
testClusters.integTestECS {
setting 's3.client.integration_test_ecs.endpoint', { "http://${s3Fixture.addressAndPort}" }
plugin file(tasks.bundlePlugin.archiveFile)
environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "http://${s3Fixture.addressAndPort}/ecs_credentials_endpoint" }
}
}
thirdPartyAudit.ignoreMissingClasses (
// classes are missing
'javax.servlet.ServletContextEvent',
'javax.servlet.ServletContextListener',
'org.apache.avalon.framework.logger.Logger',
'org.apache.log.Hierarchy',
'org.apache.log.Logger',
'software.amazon.ion.IonReader',
'software.amazon.ion.IonSystem',
'software.amazon.ion.IonType',
'software.amazon.ion.IonWriter',
'software.amazon.ion.Timestamp',
'software.amazon.ion.system.IonBinaryWriterBuilder',
'software.amazon.ion.system.IonSystemBuilder',
'software.amazon.ion.system.IonTextWriterBuilder',
'software.amazon.ion.system.IonWriterBuilder'
)
// jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
rootProject.globalInfo.ready {
if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
thirdPartyAudit.ignoreJarHellWithJDK(
'javax.xml.bind.Binder',
'javax.xml.bind.ContextFinder$1',
'javax.xml.bind.ContextFinder',
'javax.xml.bind.DataBindingException',
'javax.xml.bind.DatatypeConverter',
'javax.xml.bind.DatatypeConverterImpl$CalendarFormatter',
'javax.xml.bind.DatatypeConverterImpl',
'javax.xml.bind.DatatypeConverterInterface',
'javax.xml.bind.Element',
'javax.xml.bind.GetPropertyAction',
'javax.xml.bind.JAXB$Cache',
'javax.xml.bind.JAXB',
'javax.xml.bind.JAXBContext',
'javax.xml.bind.JAXBElement$GlobalScope',
'javax.xml.bind.JAXBElement',
'javax.xml.bind.JAXBException',
'javax.xml.bind.JAXBIntrospector',
'javax.xml.bind.JAXBPermission',
'javax.xml.bind.MarshalException',
'javax.xml.bind.Marshaller$Listener',
'javax.xml.bind.Marshaller',
'javax.xml.bind.Messages',
'javax.xml.bind.NotIdentifiableEvent',
'javax.xml.bind.ParseConversionEvent',
'javax.xml.bind.PrintConversionEvent',
'javax.xml.bind.PropertyException',
'javax.xml.bind.SchemaOutputResolver',
'javax.xml.bind.TypeConstraintException',
'javax.xml.bind.UnmarshalException',
'javax.xml.bind.Unmarshaller$Listener',
'javax.xml.bind.Unmarshaller',
'javax.xml.bind.UnmarshallerHandler',
'javax.xml.bind.ValidationEvent',
'javax.xml.bind.ValidationEventHandler',
'javax.xml.bind.ValidationEventLocator',
'javax.xml.bind.ValidationException',
'javax.xml.bind.Validator',
'javax.xml.bind.WhiteSpaceProcessor',
'javax.xml.bind.annotation.DomHandler',
'javax.xml.bind.annotation.W3CDomHandler',
'javax.xml.bind.annotation.XmlAccessOrder',
'javax.xml.bind.annotation.XmlAccessType',
'javax.xml.bind.annotation.XmlAccessorOrder',
'javax.xml.bind.annotation.XmlAccessorType',
'javax.xml.bind.annotation.XmlAnyAttribute',
'javax.xml.bind.annotation.XmlAnyElement',
'javax.xml.bind.annotation.XmlAttachmentRef',
'javax.xml.bind.annotation.XmlAttribute',
'javax.xml.bind.annotation.XmlElement$DEFAULT',
'javax.xml.bind.annotation.XmlElement',
'javax.xml.bind.annotation.XmlElementDecl$GLOBAL',
'javax.xml.bind.annotation.XmlElementDecl',
'javax.xml.bind.annotation.XmlElementRef$DEFAULT',
'javax.xml.bind.annotation.XmlElementRef',
'javax.xml.bind.annotation.XmlElementRefs',
'javax.xml.bind.annotation.XmlElementWrapper',
'javax.xml.bind.annotation.XmlElements',
'javax.xml.bind.annotation.XmlEnum',
'javax.xml.bind.annotation.XmlEnumValue',
'javax.xml.bind.annotation.XmlID',
'javax.xml.bind.annotation.XmlIDREF',
'javax.xml.bind.annotation.XmlInlineBinaryData',
'javax.xml.bind.annotation.XmlList',
'javax.xml.bind.annotation.XmlMimeType',
'javax.xml.bind.annotation.XmlMixed',
'javax.xml.bind.annotation.XmlNs',
'javax.xml.bind.annotation.XmlNsForm',
'javax.xml.bind.annotation.XmlRegistry',
'javax.xml.bind.annotation.XmlRootElement',
'javax.xml.bind.annotation.XmlSchema',
'javax.xml.bind.annotation.XmlSchemaType$DEFAULT',
'javax.xml.bind.annotation.XmlSchemaType',
'javax.xml.bind.annotation.XmlSchemaTypes',
'javax.xml.bind.annotation.XmlSeeAlso',
'javax.xml.bind.annotation.XmlTransient',
'javax.xml.bind.annotation.XmlType$DEFAULT',
'javax.xml.bind.annotation.XmlType',
'javax.xml.bind.annotation.XmlValue',
'javax.xml.bind.annotation.adapters.CollapsedStringAdapter',
'javax.xml.bind.annotation.adapters.HexBinaryAdapter',
'javax.xml.bind.annotation.adapters.NormalizedStringAdapter',
'javax.xml.bind.annotation.adapters.XmlAdapter',
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter$DEFAULT',
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter',
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapters',
'javax.xml.bind.attachment.AttachmentMarshaller',
'javax.xml.bind.attachment.AttachmentUnmarshaller',
'javax.xml.bind.helpers.AbstractMarshallerImpl',
'javax.xml.bind.helpers.AbstractUnmarshallerImpl',
'javax.xml.bind.helpers.DefaultValidationEventHandler',
'javax.xml.bind.helpers.Messages',
'javax.xml.bind.helpers.NotIdentifiableEventImpl',
'javax.xml.bind.helpers.ParseConversionEventImpl',
'javax.xml.bind.helpers.PrintConversionEventImpl',
'javax.xml.bind.helpers.ValidationEventImpl',
'javax.xml.bind.helpers.ValidationEventLocatorImpl',
'javax.xml.bind.util.JAXBResult',
'javax.xml.bind.util.JAXBSource$1',
'javax.xml.bind.util.JAXBSource',
'javax.xml.bind.util.Messages',
'javax.xml.bind.util.ValidationEventCollector'
)
} else {
thirdPartyAudit.ignoreMissingClasses 'javax.activation.DataHandler'
}
}