OpenSearch/plugins/repository-gcs/build.gradle

/*
 * SPDX-License-Identifier: Apache-2.0
 *
 * The OpenSearch Contributors require contributions made to
 * this file be licensed under the Apache-2.0 license or a
 * compatible open source license.
 *
 * Modifications Copyright OpenSearch Contributors. See
 * GitHub history for details.
 */

import java.nio.file.Files
import java.security.KeyPair
import java.security.KeyPairGenerator
import org.opensearch.gradle.MavenFilteringHack
import org.opensearch.gradle.info.BuildParams
import org.opensearch.gradle.test.RestIntegTestTask
import org.opensearch.gradle.test.rest.YamlRestTestPlugin
import org.opensearch.gradle.test.InternalClusterTestPlugin

import java.nio.file.Files
import java.security.KeyPair
import java.security.KeyPairGenerator

import static org.opensearch.gradle.PropertyNormalization.IGNORE_VALUE
/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
apply plugin: 'opensearch.yaml-rest-test'
apply plugin: 'opensearch.internal-cluster-test'

opensearchplugin {
  description 'The GCS repository plugin adds Google Cloud Storage support for repositories.'
  classname 'org.opensearch.repositories.gcs.GoogleCloudStoragePlugin'
}

dependencies {
  api 'com.google.cloud:google-cloud-storage:1.113.1'
  api 'com.google.cloud:google-cloud-core:1.93.3'
  runtimeOnly 'com.google.guava:guava:26.0-jre'
  api 'com.google.http-client:google-http-client:1.35.0'
  api "commons-logging:commons-logging:${versions.commonslogging}"
  api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
  api "commons-codec:commons-codec:${versions.commonscodec}"
  api 'com.google.api:api-common:1.8.1'
  api 'com.google.api:gax:1.54.0'
  api 'org.threeten:threetenbp:1.4.4'
  api 'com.google.protobuf:protobuf-java-util:3.11.3'
  api 'com.google.protobuf:protobuf-java:3.11.3'
  api 'com.google.code.gson:gson:2.7'
  api 'com.google.api.grpc:proto-google-common-protos:1.16.0'
  api 'com.google.api.grpc:proto-google-iam-v1:0.12.0'
  api 'com.google.cloud:google-cloud-core-http:1.93.3'
  api 'com.google.auth:google-auth-library-credentials:0.20.0'
  api 'com.google.auth:google-auth-library-oauth2-http:0.20.0'
  api 'com.google.oauth-client:google-oauth-client:1.31.0'
  api 'com.google.api-client:google-api-client:1.30.10'
  api 'com.google.http-client:google-http-client-appengine:1.35.0'
  api 'com.google.http-client:google-http-client-jackson2:1.35.0'
  api 'com.google.api:gax-httpjson:0.62.0'
  api 'io.grpc:grpc-context:1.29.0'
  api 'io.opencensus:opencensus-api:0.18.0'
  api 'io.opencensus:opencensus-contrib-http-util:0.18.0'
  api 'com.google.apis:google-api-services-storage:v1-rev20200814-1.30.10'

  testImplementation project(':test:fixtures:gcs-fixture')
}

restResources {
  restApi {
    includeCore '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count'
  }
}

tasks.named("dependencyLicenses").configure {
  mapping from: /google-cloud-.*/, to: 'google-cloud'
  mapping from: /google-auth-.*/, to: 'google-auth'
  mapping from: /google-http-.*/, to: 'google-http'
  mapping from: /opencensus.*/, to: 'opencensus'
  mapping from: /protobuf.*/, to: 'protobuf'
  mapping from: /proto-google.*/, to: 'proto-google'
}

thirdPartyAudit {
  ignoreViolations(
    // uses internal java api: sun.misc.Unsafe
    'com.google.protobuf.UnsafeUtil',
    'com.google.protobuf.UnsafeUtil$1',
    'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
    'com.google.protobuf.UnsafeUtil$MemoryAccessor',
    'com.google.protobuf.MessageSchema',
    'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
    'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
    'com.google.common.cache.Striped64',
    'com.google.common.cache.Striped64$1',
    'com.google.common.cache.Striped64$Cell',
    'com.google.common.hash.Striped64',
    'com.google.common.hash.Striped64$1',
    'com.google.common.hash.Striped64$Cell',
    'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
    'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
    'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
    'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3',
    'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
    'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
    'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
    'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
    'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
  )

  ignoreMissingClasses(
    'com.google.appengine.api.datastore.Blob',
    'com.google.appengine.api.datastore.DatastoreService',
    'com.google.appengine.api.datastore.DatastoreServiceFactory',
    'com.google.appengine.api.datastore.Entity',
    'com.google.appengine.api.datastore.Key',
    'com.google.appengine.api.datastore.KeyFactory',
    'com.google.appengine.api.datastore.PreparedQuery',
    'com.google.appengine.api.datastore.Query',
    'com.google.appengine.api.memcache.Expiration',
    'com.google.appengine.api.memcache.MemcacheService',
    'com.google.appengine.api.memcache.MemcacheServiceFactory',
    'com.google.appengine.api.urlfetch.FetchOptions$Builder',
    'com.google.appengine.api.urlfetch.FetchOptions',
    'com.google.appengine.api.urlfetch.HTTPHeader',
    'com.google.appengine.api.urlfetch.HTTPMethod',
    'com.google.appengine.api.urlfetch.HTTPRequest',
    'com.google.appengine.api.urlfetch.HTTPResponse',
    'com.google.appengine.api.urlfetch.URLFetchService',
    'com.google.appengine.api.urlfetch.URLFetchServiceFactory',
    // commons-logging optional dependencies
    'org.apache.avalon.framework.logger.Logger',
    'org.apache.log.Hierarchy',
    'org.apache.log.Logger',
    // optional apache http client dependencies
    'org.apache.http.ConnectionReuseStrategy',
    'org.apache.http.Header',
    'org.apache.http.HttpEntity',
    'org.apache.http.HttpEntityEnclosingRequest',
    'org.apache.http.HttpHost',
    'org.apache.http.HttpRequest',
    'org.apache.http.HttpResponse',
    'org.apache.http.HttpVersion',
    'org.apache.http.RequestLine',
    'org.apache.http.StatusLine',
    'org.apache.http.client.AuthenticationHandler',
    'org.apache.http.client.HttpClient',
    'org.apache.http.client.HttpRequestRetryHandler',
    'org.apache.http.client.RedirectHandler',
    'org.apache.http.client.RequestDirector',
    'org.apache.http.client.UserTokenHandler',
    'org.apache.http.client.methods.HttpDelete',
    'org.apache.http.client.methods.HttpEntityEnclosingRequestBase',
    'org.apache.http.client.methods.HttpGet',
    'org.apache.http.client.methods.HttpHead',
    'org.apache.http.client.methods.HttpOptions',
    'org.apache.http.client.methods.HttpPost',
    'org.apache.http.client.methods.HttpPut',
    'org.apache.http.client.methods.HttpRequestBase',
    'org.apache.http.client.methods.HttpTrace',
    'org.apache.http.config.SocketConfig',
    'org.apache.http.config.SocketConfig$Builder',
    'org.apache.http.conn.ClientConnectionManager',
    'org.apache.http.conn.ConnectionKeepAliveStrategy',
    'org.apache.http.conn.params.ConnManagerParams',
    'org.apache.http.conn.params.ConnPerRouteBean',
    'org.apache.http.conn.params.ConnRouteParams',
    'org.apache.http.conn.routing.HttpRoutePlanner',
    'org.apache.http.conn.scheme.PlainSocketFactory',
    'org.apache.http.conn.scheme.Scheme',
    'org.apache.http.conn.scheme.SchemeRegistry',
    'org.apache.http.conn.ssl.SSLConnectionSocketFactory',
    'org.apache.http.conn.ssl.SSLSocketFactory',
    'org.apache.http.conn.ssl.X509HostnameVerifier',
    'org.apache.http.entity.AbstractHttpEntity',
    'org.apache.http.impl.client.DefaultHttpClient',
    'org.apache.http.impl.client.DefaultHttpRequestRetryHandler',
    'org.apache.http.impl.client.HttpClientBuilder',
    'org.apache.http.impl.conn.PoolingHttpClientConnectionManager',
    'org.apache.http.impl.conn.ProxySelectorRoutePlanner',
    'org.apache.http.impl.conn.SystemDefaultRoutePlanner',
    'org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager',
    'org.apache.http.message.BasicHttpResponse',
    'org.apache.http.params.BasicHttpParams',
    'org.apache.http.params.HttpConnectionParams',
    'org.apache.http.params.HttpParams',
    'org.apache.http.params.HttpProtocolParams',
    'org.apache.http.protocol.HttpContext',
    'org.apache.http.protocol.HttpProcessor',
    'org.apache.http.protocol.HttpRequestExecutor',
    // commons-logging provided dependencies
    'javax.servlet.ServletContextEvent',
    'javax.servlet.ServletContextListener',
  )
}

boolean useFixture = false

def fixtureAddress = { fixture ->
  assert useFixture: 'closure should not be used without a fixture'
  int ephemeralPort = project(':test:fixtures:gcs-fixture').postProcessFixture.ext."test.fixtures.${fixture}.tcp.80"
  assert ephemeralPort > 0
  'http://127.0.0.1:' + ephemeralPort
}

String gcsServiceAccount = System.getenv("google_storage_service_account")
String gcsBucket = System.getenv("google_storage_bucket")
String gcsBasePath = System.getenv("google_storage_base_path")
File serviceAccountFile = null

if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
  serviceAccountFile = new File(project.buildDir, 'generated-resources/service_account_test.json')
  gcsBucket = 'bucket'
  gcsBasePath = 'integration_test'
  useFixture = true

  apply plugin: 'opensearch.test.fixtures'
  testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture')
  testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture-third-party')

} else if (!gcsServiceAccount || !gcsBucket || !gcsBasePath) {
  throw new IllegalArgumentException("not all options specified to run tests against external GCS service are present")
} else {
  serviceAccountFile = new File(gcsServiceAccount)
}

def encodedCredentials = {
  Base64.encoder.encodeToString(Files.readAllBytes(serviceAccountFile.toPath()))
}

/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/
task createServiceAccountFile() {
  doLast {
    KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA")
    keyPairGenerator.initialize(1024)
    KeyPair keyPair = keyPairGenerator.generateKeyPair()
    String encodedKey = Base64.getEncoder().encodeToString(keyPair.private.getEncoded())

    serviceAccountFile.parentFile.mkdirs()
    serviceAccountFile.setText("{\n" +
      '  "type": "service_account",\n' +
      '  "project_id": "integration_test",\n' +
      '  "private_key_id": "' + UUID.randomUUID().toString() + '",\n' +
      '  "private_key": "-----BEGIN PRIVATE KEY-----\\n' + encodedKey + '\\n-----END PRIVATE KEY-----\\n",\n' +
      '  "client_email": "integration_test@appspot.gserviceaccount.com",\n' +
      '  "client_id": "123456789101112130594"\n' +
      '}', 'UTF-8')
  }
}

Map<String, Object> expansions = [
  'bucket'   : gcsBucket,
  'base_path': gcsBasePath + "_integration_tests"
]

processYamlRestTestResources {
  inputs.properties(expansions)
  MavenFilteringHack.filter(it, expansions)
}

internalClusterTest {
  // this is tested explicitly in a separate test task
  exclude '**/GoogleCloudStorageThirdPartyTests.class'
}

final Closure testClustersConfiguration = {
  keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile, IGNORE_VALUE

  if (useFixture) {
    /* Use a closure on the string to delay evaluation until tests are executed */
    setting 'gcs.client.integration_test.endpoint', { "${-> fixtureAddress('gcs-fixture')}" }, IGNORE_VALUE
    setting 'gcs.client.integration_test.token_uri', { "${-> fixtureAddress('gcs-fixture')}/o/oauth2/token" }, IGNORE_VALUE
  } else {
    println "Using an external service to test the repository-gcs plugin"
  }
}

yamlRestTest {
  if (useFixture) {
    dependsOn createServiceAccountFile
  }
}

testClusters {
  all testClustersConfiguration
}

/*
 * We only use a small amount of data in these tests, which means that the resumable upload path is not tested. We add
 * an additional test that forces the large blob threshold to be small to exercise the resumable upload path.
 */
task largeBlobYamlRestTest(type: RestIntegTestTask) {
  dependsOn bundlePlugin
  if (useFixture) {
    dependsOn createServiceAccountFile
  }
    SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
    SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME)
    setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
    setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
}

check.dependsOn largeBlobYamlRestTest

testClusters {
  largeBlobYamlRestTest {
    plugin bundlePlugin.archiveFile

    // force large blob uploads by setting the threshold small, forcing this code path to be tested
    systemProperty 'opensearch.repository_gcs.large_blob_threshold_byte_size', '256'
  }
}

task gcsThirdPartyTest(type: Test) {
  SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
  SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME)
  setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs())
  setClasspath(internalTestSourceSet.getRuntimeClasspath())
  include '**/GoogleCloudStorageThirdPartyTests.class'
  systemProperty 'tests.security.manager', false
  systemProperty 'test.google.bucket', gcsBucket
  nonInputProperties.systemProperty 'test.google.base', gcsBasePath + "_third_party_tests_" + BuildParams.testSeed
  nonInputProperties.systemProperty 'test.google.account', "${-> encodedCredentials.call()}"
  if (useFixture) {
    dependsOn createServiceAccountFile
    nonInputProperties.systemProperty 'test.google.endpoint', "${-> fixtureAddress('gcs-fixture-third-party')}"
    nonInputProperties.systemProperty 'test.google.tokenURI', "${-> fixtureAddress('gcs-fixture-third-party')}/o/oauth2/token"
  }
}
check.dependsOn(gcsThirdPartyTest)