OpenSearch/plugins/repository-s3/build.gradle

/*
 * SPDX-License-Identifier: Apache-2.0
 *
 * The OpenSearch Contributors require contributions made to
 * this file be licensed under the Apache-2.0 license or a
 * compatible open source license.
 *
 * Modifications Copyright OpenSearch Contributors. See
 * GitHub history for details.
 */

/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import org.opensearch.gradle.MavenFilteringHack
import org.opensearch.gradle.info.BuildParams
import org.opensearch.gradle.test.RestIntegTestTask
import org.opensearch.gradle.test.rest.YamlRestTestPlugin
import org.opensearch.gradle.test.InternalClusterTestPlugin

import static org.opensearch.gradle.PropertyNormalization.IGNORE_VALUE

apply plugin: 'opensearch.yaml-rest-test'
apply plugin: 'opensearch.internal-cluster-test'

opensearchplugin {
  description 'The S3 repository plugin adds S3 repositories'
  classname 'org.opensearch.repositories.s3.S3RepositoryPlugin'
}

versions << [
  'aws': '1.11.749'
]

dependencies {
  api "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
  api "com.amazonaws:aws-java-sdk-core:${versions.aws}"
  api "com.amazonaws:jmespath-java:${versions.aws}"
  api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
  api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
  api "commons-logging:commons-logging:${versions.commonslogging}"
  api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
  api "commons-codec:commons-codec:${versions.commonscodec}"
  api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
  api "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
  api "joda-time:joda-time:${versions.joda}"

  // HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
  // and whitelist this hack in JarHell
  api 'javax.xml.bind:jaxb-api:2.2.2'

  testImplementation project(':test:fixtures:s3-fixture')
}

restResources {
  restApi {
    includeCore '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count'
  }
}

tasks.named("dependencyLicenses").configure {
  mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
  mapping from: /jmespath-java.*/, to: 'aws-java-sdk'
  mapping from: /jackson-.*/, to: 'jackson'
  mapping from: /jaxb-.*/, to: 'jaxb'
}

bundlePlugin {
  from('config/repository-s3') {
    into 'config'
  }
}

task testRepositoryCreds(type: Test) {
  include '**/RepositoryCredentialsTests.class'
  systemProperty 'opensearch.allow_insecure_settings', 'true'
}
check.dependsOn(testRepositoryCreds)

test {
  // this is tested explicitly in separate test tasks
  exclude '**/RepositoryCredentialsTests.class'
  exclude '**/S3RepositoryThirdPartyTests.class'
}

boolean useFixture = false

def fixtureAddress = { fixture, name, port ->
  assert useFixture: 'closure should not be used without a fixture'
  int ephemeralPort = project(":test:fixtures:${fixture}").postProcessFixture.ext."test.fixtures.${name}.tcp.${port}"
  assert ephemeralPort > 0
  'http://127.0.0.1:' + ephemeralPort
}

// We test against two repositories, one which uses the usual two-part "permanent" credentials and
// the other which uses three-part "temporary" or "session" credentials.

String s3PermanentAccessKey = System.getenv("amazon_s3_access_key")
String s3PermanentSecretKey = System.getenv("amazon_s3_secret_key")
String s3PermanentBucket = System.getenv("amazon_s3_bucket")
String s3PermanentBasePath = System.getenv("amazon_s3_base_path")

String s3TemporaryAccessKey = System.getenv("amazon_s3_access_key_temporary")
String s3TemporarySecretKey = System.getenv("amazon_s3_secret_key_temporary")
String s3TemporarySessionToken = System.getenv("amazon_s3_session_token_temporary")
String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary")
String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary")

String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2")
String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2")

String s3ECSBucket = System.getenv("amazon_s3_bucket_ecs")
String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs")

boolean s3DisableChunkedEncoding = (new Random(Long.parseUnsignedLong(BuildParams.testSeed.tokenize(':').get(0), 16))).nextBoolean()

// If all these variables are missing then we are testing against the internal fixture instead, which has the following
// credentials hard-coded in.

if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath) {
  s3PermanentAccessKey = 'access_key'
  s3PermanentSecretKey = 'secret_key'
  s3PermanentBucket = 'bucket'
  s3PermanentBasePath = 'base_path'

  apply plugin: 'opensearch.test.fixtures'
  useFixture = true

} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) {
  throw new IllegalArgumentException("not all options specified to run against external S3 service as permanent credentials are present")
}

if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) {
  s3TemporaryAccessKey = 'session_token_access_key'
  s3TemporarySecretKey = 'session_token_secret_key'
  s3TemporaryBucket = 'session_token_bucket'
  s3TemporaryBasePath = 'session_token_base_path'
  s3TemporarySessionToken = 'session_token'

} else if (!s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken) {
  throw new IllegalArgumentException("not all options specified to run against external S3 service as temporary credentials are present")
}

if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) {
  s3EC2Bucket = 'ec2_bucket'
  s3EC2BasePath = 'ec2_base_path'
  s3ECSBucket = 'ecs_bucket'
  s3ECSBasePath = 'ecs_base_path'
} else if (!s3EC2Bucket || !s3EC2BasePath || !s3ECSBucket || !s3ECSBasePath) {
  throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present")
}

processYamlRestTestResources {
  Map<String, Object> expansions = [
    'permanent_bucket': s3PermanentBucket,
    'permanent_base_path': s3PermanentBasePath + "_integration_tests",
    'temporary_bucket': s3TemporaryBucket,
    'temporary_base_path': s3TemporaryBasePath + "_integration_tests",
    'ec2_bucket': s3EC2Bucket,
    'ec2_base_path': s3EC2BasePath,
    'ecs_bucket': s3ECSBucket,
    'ecs_base_path': s3ECSBasePath,
    'disable_chunked_encoding': s3DisableChunkedEncoding,
  ]
  inputs.properties(expansions)
  MavenFilteringHack.filter(it, expansions)
}

internalClusterTest {
  // this is tested explicitly in a separate test task
  exclude '**/S3RepositoryThirdPartyTests.class'
}

yamlRestTest {
    systemProperty 'tests.rest.blacklist', (
      useFixture ?
        ['repository_s3/50_repository_ecs_credentials/*']
        :
        [
          'repository_s3/30_repository_temporary_credentials/*',
          'repository_s3/40_repository_ec2_credentials/*',
          'repository_s3/50_repository_ecs_credentials/*'
        ]
    ).join(",")
}

testClusters.yamlRestTest {
  keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
  keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey

  keystore 's3.client.integration_test_temporary.access_key', s3TemporaryAccessKey
  keystore 's3.client.integration_test_temporary.secret_key', s3TemporarySecretKey
  keystore 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken

  if (useFixture) {
    testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture')
    testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-session-token')
    testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-ec2')

    normalization {
      runtimeClasspath {
        // ignore generated address file for the purposes of build avoidance
        ignore 's3Fixture.address'
      }
    }

    setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture', '80')}" }, IGNORE_VALUE
    setting 's3.client.integration_test_temporary.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-session-token', '80')}" }, IGNORE_VALUE
    setting 's3.client.integration_test_ec2.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ec2', '80')}" }, IGNORE_VALUE

    // to redirect InstanceProfileCredentialsProvider to custom auth point
    systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ec2', '80')}" }, IGNORE_VALUE
  } else {
    println "Using an external service to test the repository-s3 plugin"
  }
}

// MinIO
if (useFixture) {
  testFixtures.useFixture(':test:fixtures:minio-fixture', 'minio-fixture')

  task yamlRestTestMinio(type: RestIntegTestTask) {
    description = "Runs REST tests using the Minio repository."
    dependsOn tasks.bundlePlugin
    SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
    SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME)
    setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
    setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())

    // Minio only supports a single access key, see https://github.com/minio/minio/pull/5968
    systemProperty 'tests.rest.blacklist', [
      'repository_s3/30_repository_temporary_credentials/*',
      'repository_s3/40_repository_ec2_credentials/*',
      'repository_s3/50_repository_ecs_credentials/*'
    ].join(",")
  }
  check.dependsOn(yamlRestTestMinio)

  testClusters.yamlRestTestMinio {
    keystore 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey
    keystore 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey
    setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('minio-fixture', 'minio-fixture', '9000')}" }, IGNORE_VALUE
    plugin tasks.bundlePlugin.archiveFile
  }
}

// ECS
if (useFixture) {
  testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-ecs')
  task yamlRestTestECS(type: RestIntegTestTask.class) {
    description = "Runs tests using the ECS repository."
    dependsOn('bundlePlugin')
    SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
    SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME)
    setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
    setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
    systemProperty 'tests.rest.blacklist', [
      'repository_s3/10_basic/*',
      'repository_s3/20_repository_permanent_credentials/*',
      'repository_s3/30_repository_temporary_credentials/*',
      'repository_s3/40_repository_ec2_credentials/*'
    ].join(",")
  }
  check.dependsOn(yamlRestTestECS)

  testClusters.yamlRestTestECS {
    setting 's3.client.integration_test_ecs.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ecs', '80')}" }, IGNORE_VALUE
    plugin tasks.bundlePlugin.archiveFile
    environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ecs', '80')}/ecs_credentials_endpoint" }, IGNORE_VALUE
  }
}

// 3rd Party Tests
TaskProvider s3ThirdPartyTest = tasks.register("s3ThirdPartyTest", Test) {
  SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
  SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME)
  setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs())
  setClasspath(internalTestSourceSet.getRuntimeClasspath())
  include '**/S3RepositoryThirdPartyTests.class'
  systemProperty 'test.s3.account', s3PermanentAccessKey
  systemProperty 'test.s3.key', s3PermanentSecretKey
  systemProperty 'test.s3.bucket', s3PermanentBucket
  nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + BuildParams.testSeed
  if (useFixture) {
    nonInputProperties.systemProperty 'test.s3.endpoint', "${-> fixtureAddress('minio-fixture', 'minio-fixture', '9000') }"
  }
}
tasks.named("check").configure { dependsOn(s3ThirdPartyTest) }

thirdPartyAudit.ignoreMissingClasses(
  // classes are missing
  'javax.jms.Message',
  'javax.servlet.ServletContextEvent',
  'javax.servlet.ServletContextListener',
  'org.apache.avalon.framework.logger.Logger',
  'org.apache.log.Hierarchy',
  'org.apache.log.Logger',
  'software.amazon.ion.IonReader',
  'software.amazon.ion.IonSystem',
  'software.amazon.ion.IonType',
  'software.amazon.ion.IonWriter',
  'software.amazon.ion.Timestamp',
  'software.amazon.ion.system.IonBinaryWriterBuilder',
  'software.amazon.ion.system.IonSystemBuilder',
  'software.amazon.ion.system.IonTextWriterBuilder',
  'software.amazon.ion.system.IonWriterBuilder',
  // We don't use the kms dependency
  'com.amazonaws.services.kms.AWSKMS',
  'com.amazonaws.services.kms.AWSKMSClient',
  'com.amazonaws.services.kms.model.DecryptRequest',
  'com.amazonaws.services.kms.model.DecryptResult',
  'com.amazonaws.services.kms.model.EncryptRequest',
  'com.amazonaws.services.kms.model.EncryptResult',
  'com.amazonaws.services.kms.model.GenerateDataKeyRequest',
  'com.amazonaws.services.kms.model.GenerateDataKeyResult'
)

// jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
if (BuildParams.runtimeJavaVersion <= JavaVersion.VERSION_1_8) {
  thirdPartyAudit.ignoreJarHellWithJDK(
    'javax.xml.bind.Binder',
    'javax.xml.bind.ContextFinder$1',
    'javax.xml.bind.ContextFinder',
    'javax.xml.bind.DataBindingException',
    'javax.xml.bind.DatatypeConverter',
    'javax.xml.bind.DatatypeConverterImpl$CalendarFormatter',
    'javax.xml.bind.DatatypeConverterImpl',
    'javax.xml.bind.DatatypeConverterInterface',
    'javax.xml.bind.Element',
    'javax.xml.bind.GetPropertyAction',
    'javax.xml.bind.JAXB$Cache',
    'javax.xml.bind.JAXB',
    'javax.xml.bind.JAXBContext',
    'javax.xml.bind.JAXBElement$GlobalScope',
    'javax.xml.bind.JAXBElement',
    'javax.xml.bind.JAXBException',
    'javax.xml.bind.JAXBIntrospector',
    'javax.xml.bind.JAXBPermission',
    'javax.xml.bind.MarshalException',
    'javax.xml.bind.Marshaller$Listener',
    'javax.xml.bind.Marshaller',
    'javax.xml.bind.Messages',
    'javax.xml.bind.NotIdentifiableEvent',
    'javax.xml.bind.ParseConversionEvent',
    'javax.xml.bind.PrintConversionEvent',
    'javax.xml.bind.PropertyException',
    'javax.xml.bind.SchemaOutputResolver',
    'javax.xml.bind.TypeConstraintException',
    'javax.xml.bind.UnmarshalException',
    'javax.xml.bind.Unmarshaller$Listener',
    'javax.xml.bind.Unmarshaller',
    'javax.xml.bind.UnmarshallerHandler',
    'javax.xml.bind.ValidationEvent',
    'javax.xml.bind.ValidationEventHandler',
    'javax.xml.bind.ValidationEventLocator',
    'javax.xml.bind.ValidationException',
    'javax.xml.bind.Validator',
    'javax.xml.bind.WhiteSpaceProcessor',
    'javax.xml.bind.annotation.DomHandler',
    'javax.xml.bind.annotation.W3CDomHandler',
    'javax.xml.bind.annotation.XmlAccessOrder',
    'javax.xml.bind.annotation.XmlAccessType',
    'javax.xml.bind.annotation.XmlAccessorOrder',
    'javax.xml.bind.annotation.XmlAccessorType',
    'javax.xml.bind.annotation.XmlAnyAttribute',
    'javax.xml.bind.annotation.XmlAnyElement',
    'javax.xml.bind.annotation.XmlAttachmentRef',
    'javax.xml.bind.annotation.XmlAttribute',
    'javax.xml.bind.annotation.XmlElement$DEFAULT',
    'javax.xml.bind.annotation.XmlElement',
    'javax.xml.bind.annotation.XmlElementDecl$GLOBAL',
    'javax.xml.bind.annotation.XmlElementDecl',
    'javax.xml.bind.annotation.XmlElementRef$DEFAULT',
    'javax.xml.bind.annotation.XmlElementRef',
    'javax.xml.bind.annotation.XmlElementRefs',
    'javax.xml.bind.annotation.XmlElementWrapper',
    'javax.xml.bind.annotation.XmlElements',
    'javax.xml.bind.annotation.XmlEnum',
    'javax.xml.bind.annotation.XmlEnumValue',
    'javax.xml.bind.annotation.XmlID',
    'javax.xml.bind.annotation.XmlIDREF',
    'javax.xml.bind.annotation.XmlInlineBinaryData',
    'javax.xml.bind.annotation.XmlList',
    'javax.xml.bind.annotation.XmlMimeType',
    'javax.xml.bind.annotation.XmlMixed',
    'javax.xml.bind.annotation.XmlNs',
    'javax.xml.bind.annotation.XmlNsForm',
    'javax.xml.bind.annotation.XmlRegistry',
    'javax.xml.bind.annotation.XmlRootElement',
    'javax.xml.bind.annotation.XmlSchema',
    'javax.xml.bind.annotation.XmlSchemaType$DEFAULT',
    'javax.xml.bind.annotation.XmlSchemaType',
    'javax.xml.bind.annotation.XmlSchemaTypes',
    'javax.xml.bind.annotation.XmlSeeAlso',
    'javax.xml.bind.annotation.XmlTransient',
    'javax.xml.bind.annotation.XmlType$DEFAULT',
    'javax.xml.bind.annotation.XmlType',
    'javax.xml.bind.annotation.XmlValue',
    'javax.xml.bind.annotation.adapters.CollapsedStringAdapter',
    'javax.xml.bind.annotation.adapters.HexBinaryAdapter',
    'javax.xml.bind.annotation.adapters.NormalizedStringAdapter',
    'javax.xml.bind.annotation.adapters.XmlAdapter',
    'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter$DEFAULT',
    'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter',
    'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapters',
    'javax.xml.bind.attachment.AttachmentMarshaller',
    'javax.xml.bind.attachment.AttachmentUnmarshaller',
    'javax.xml.bind.helpers.AbstractMarshallerImpl',
    'javax.xml.bind.helpers.AbstractUnmarshallerImpl',
    'javax.xml.bind.helpers.DefaultValidationEventHandler',
    'javax.xml.bind.helpers.Messages',
    'javax.xml.bind.helpers.NotIdentifiableEventImpl',
    'javax.xml.bind.helpers.ParseConversionEventImpl',
    'javax.xml.bind.helpers.PrintConversionEventImpl',
    'javax.xml.bind.helpers.ValidationEventImpl',
    'javax.xml.bind.helpers.ValidationEventLocatorImpl',
    'javax.xml.bind.util.JAXBResult',
    'javax.xml.bind.util.JAXBSource$1',
    'javax.xml.bind.util.JAXBSource',
    'javax.xml.bind.util.Messages',
    'javax.xml.bind.util.ValidationEventCollector'
  )
} else {
  thirdPartyAudit.ignoreMissingClasses 'javax.activation.DataHandler'
}