346 lines
14 KiB
Groovy
346 lines
14 KiB
Groovy
/*
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*
|
|
* The OpenSearch Contributors require contributions made to
|
|
* this file be licensed under the Apache-2.0 license or a
|
|
* compatible open source license.
|
|
*
|
|
* Modifications Copyright OpenSearch Contributors. See
|
|
* GitHub history for details.
|
|
*/
|
|
|
|
import java.nio.file.Files
|
|
import java.security.KeyPair
|
|
import java.security.KeyPairGenerator
|
|
import org.opensearch.gradle.MavenFilteringHack
|
|
import org.opensearch.gradle.info.BuildParams
|
|
import org.opensearch.gradle.test.RestIntegTestTask
|
|
import org.opensearch.gradle.test.rest.YamlRestTestPlugin
|
|
import org.opensearch.gradle.test.InternalClusterTestPlugin
|
|
|
|
import java.nio.file.Files
|
|
import java.security.KeyPair
|
|
import java.security.KeyPairGenerator
|
|
|
|
import static org.opensearch.gradle.PropertyNormalization.IGNORE_VALUE
|
|
/*
|
|
* Licensed to Elasticsearch under one or more contributor
|
|
* license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright
|
|
* ownership. Elasticsearch licenses this file to you under
|
|
* the Apache License, Version 2.0 (the "License"); you may
|
|
* not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing,
|
|
* software distributed under the License is distributed on an
|
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
* KIND, either express or implied. See the License for the
|
|
* specific language governing permissions and limitations
|
|
* under the License.
|
|
*/
|
|
apply plugin: 'opensearch.yaml-rest-test'
|
|
apply plugin: 'opensearch.internal-cluster-test'
|
|
|
|
opensearchplugin {
|
|
description 'The GCS repository plugin adds Google Cloud Storage support for repositories.'
|
|
classname 'org.opensearch.repositories.gcs.GoogleCloudStoragePlugin'
|
|
}
|
|
|
|
dependencies {
|
|
api 'com.google.cloud:google-cloud-storage:1.113.1'
|
|
api 'com.google.cloud:google-cloud-core:1.93.3'
|
|
runtimeOnly 'com.google.guava:guava:26.0-jre'
|
|
api 'com.google.http-client:google-http-client:1.35.0'
|
|
api "commons-logging:commons-logging:${versions.commonslogging}"
|
|
api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
|
|
api "commons-codec:commons-codec:${versions.commonscodec}"
|
|
api 'com.google.api:api-common:1.8.1'
|
|
api 'com.google.api:gax:1.54.0'
|
|
api 'org.threeten:threetenbp:1.4.4'
|
|
api 'com.google.protobuf:protobuf-java-util:3.11.3'
|
|
api 'com.google.protobuf:protobuf-java:3.11.3'
|
|
api 'com.google.code.gson:gson:2.7'
|
|
api 'com.google.api.grpc:proto-google-common-protos:1.16.0'
|
|
api 'com.google.api.grpc:proto-google-iam-v1:0.12.0'
|
|
api 'com.google.cloud:google-cloud-core-http:1.93.3'
|
|
api 'com.google.auth:google-auth-library-credentials:0.20.0'
|
|
api 'com.google.auth:google-auth-library-oauth2-http:0.20.0'
|
|
api 'com.google.oauth-client:google-oauth-client:1.31.0'
|
|
api 'com.google.api-client:google-api-client:1.30.10'
|
|
api 'com.google.http-client:google-http-client-appengine:1.35.0'
|
|
api 'com.google.http-client:google-http-client-jackson2:1.35.0'
|
|
api 'com.google.api:gax-httpjson:0.62.0'
|
|
api 'io.grpc:grpc-context:1.29.0'
|
|
api 'io.opencensus:opencensus-api:0.18.0'
|
|
api 'io.opencensus:opencensus-contrib-http-util:0.18.0'
|
|
api 'com.google.apis:google-api-services-storage:v1-rev20200814-1.30.10'
|
|
|
|
testImplementation project(':test:fixtures:gcs-fixture')
|
|
}
|
|
|
|
restResources {
|
|
restApi {
|
|
includeCore '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count'
|
|
}
|
|
}
|
|
|
|
tasks.named("dependencyLicenses").configure {
|
|
mapping from: /google-cloud-.*/, to: 'google-cloud'
|
|
mapping from: /google-auth-.*/, to: 'google-auth'
|
|
mapping from: /google-http-.*/, to: 'google-http'
|
|
mapping from: /opencensus.*/, to: 'opencensus'
|
|
mapping from: /protobuf.*/, to: 'protobuf'
|
|
mapping from: /proto-google.*/, to: 'proto-google'
|
|
}
|
|
|
|
thirdPartyAudit {
|
|
ignoreViolations(
|
|
// uses internal java api: sun.misc.Unsafe
|
|
'com.google.protobuf.UnsafeUtil',
|
|
'com.google.protobuf.UnsafeUtil$1',
|
|
'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
|
|
'com.google.protobuf.UnsafeUtil$MemoryAccessor',
|
|
'com.google.protobuf.MessageSchema',
|
|
'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
|
|
'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
|
|
'com.google.common.cache.Striped64',
|
|
'com.google.common.cache.Striped64$1',
|
|
'com.google.common.cache.Striped64$Cell',
|
|
'com.google.common.hash.Striped64',
|
|
'com.google.common.hash.Striped64$1',
|
|
'com.google.common.hash.Striped64$Cell',
|
|
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
|
|
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
|
|
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
|
|
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3',
|
|
'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
|
|
'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
|
|
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
|
|
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
|
|
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
|
|
)
|
|
|
|
ignoreMissingClasses(
|
|
'com.google.appengine.api.datastore.Blob',
|
|
'com.google.appengine.api.datastore.DatastoreService',
|
|
'com.google.appengine.api.datastore.DatastoreServiceFactory',
|
|
'com.google.appengine.api.datastore.Entity',
|
|
'com.google.appengine.api.datastore.Key',
|
|
'com.google.appengine.api.datastore.KeyFactory',
|
|
'com.google.appengine.api.datastore.PreparedQuery',
|
|
'com.google.appengine.api.datastore.Query',
|
|
'com.google.appengine.api.memcache.Expiration',
|
|
'com.google.appengine.api.memcache.MemcacheService',
|
|
'com.google.appengine.api.memcache.MemcacheServiceFactory',
|
|
'com.google.appengine.api.urlfetch.FetchOptions$Builder',
|
|
'com.google.appengine.api.urlfetch.FetchOptions',
|
|
'com.google.appengine.api.urlfetch.HTTPHeader',
|
|
'com.google.appengine.api.urlfetch.HTTPMethod',
|
|
'com.google.appengine.api.urlfetch.HTTPRequest',
|
|
'com.google.appengine.api.urlfetch.HTTPResponse',
|
|
'com.google.appengine.api.urlfetch.URLFetchService',
|
|
'com.google.appengine.api.urlfetch.URLFetchServiceFactory',
|
|
// commons-logging optional dependencies
|
|
'org.apache.avalon.framework.logger.Logger',
|
|
'org.apache.log.Hierarchy',
|
|
'org.apache.log.Logger',
|
|
// optional apache http client dependencies
|
|
'org.apache.http.ConnectionReuseStrategy',
|
|
'org.apache.http.Header',
|
|
'org.apache.http.HttpEntity',
|
|
'org.apache.http.HttpEntityEnclosingRequest',
|
|
'org.apache.http.HttpHost',
|
|
'org.apache.http.HttpRequest',
|
|
'org.apache.http.HttpResponse',
|
|
'org.apache.http.HttpVersion',
|
|
'org.apache.http.RequestLine',
|
|
'org.apache.http.StatusLine',
|
|
'org.apache.http.client.AuthenticationHandler',
|
|
'org.apache.http.client.HttpClient',
|
|
'org.apache.http.client.HttpRequestRetryHandler',
|
|
'org.apache.http.client.RedirectHandler',
|
|
'org.apache.http.client.RequestDirector',
|
|
'org.apache.http.client.UserTokenHandler',
|
|
'org.apache.http.client.methods.HttpDelete',
|
|
'org.apache.http.client.methods.HttpEntityEnclosingRequestBase',
|
|
'org.apache.http.client.methods.HttpGet',
|
|
'org.apache.http.client.methods.HttpHead',
|
|
'org.apache.http.client.methods.HttpOptions',
|
|
'org.apache.http.client.methods.HttpPost',
|
|
'org.apache.http.client.methods.HttpPut',
|
|
'org.apache.http.client.methods.HttpRequestBase',
|
|
'org.apache.http.client.methods.HttpTrace',
|
|
'org.apache.http.config.SocketConfig',
|
|
'org.apache.http.config.SocketConfig$Builder',
|
|
'org.apache.http.conn.ClientConnectionManager',
|
|
'org.apache.http.conn.ConnectionKeepAliveStrategy',
|
|
'org.apache.http.conn.params.ConnManagerParams',
|
|
'org.apache.http.conn.params.ConnPerRouteBean',
|
|
'org.apache.http.conn.params.ConnRouteParams',
|
|
'org.apache.http.conn.routing.HttpRoutePlanner',
|
|
'org.apache.http.conn.scheme.PlainSocketFactory',
|
|
'org.apache.http.conn.scheme.Scheme',
|
|
'org.apache.http.conn.scheme.SchemeRegistry',
|
|
'org.apache.http.conn.ssl.SSLConnectionSocketFactory',
|
|
'org.apache.http.conn.ssl.SSLSocketFactory',
|
|
'org.apache.http.conn.ssl.X509HostnameVerifier',
|
|
'org.apache.http.entity.AbstractHttpEntity',
|
|
'org.apache.http.impl.client.DefaultHttpClient',
|
|
'org.apache.http.impl.client.DefaultHttpRequestRetryHandler',
|
|
'org.apache.http.impl.client.HttpClientBuilder',
|
|
'org.apache.http.impl.conn.PoolingHttpClientConnectionManager',
|
|
'org.apache.http.impl.conn.ProxySelectorRoutePlanner',
|
|
'org.apache.http.impl.conn.SystemDefaultRoutePlanner',
|
|
'org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager',
|
|
'org.apache.http.message.BasicHttpResponse',
|
|
'org.apache.http.params.BasicHttpParams',
|
|
'org.apache.http.params.HttpConnectionParams',
|
|
'org.apache.http.params.HttpParams',
|
|
'org.apache.http.params.HttpProtocolParams',
|
|
'org.apache.http.protocol.HttpContext',
|
|
'org.apache.http.protocol.HttpProcessor',
|
|
'org.apache.http.protocol.HttpRequestExecutor',
|
|
// commons-logging provided dependencies
|
|
'javax.servlet.ServletContextEvent',
|
|
'javax.servlet.ServletContextListener',
|
|
)
|
|
}
|
|
|
|
boolean useFixture = false
|
|
|
|
def fixtureAddress = { fixture ->
|
|
assert useFixture: 'closure should not be used without a fixture'
|
|
int ephemeralPort = project(':test:fixtures:gcs-fixture').postProcessFixture.ext."test.fixtures.${fixture}.tcp.80"
|
|
assert ephemeralPort > 0
|
|
'http://127.0.0.1:' + ephemeralPort
|
|
}
|
|
|
|
String gcsServiceAccount = System.getenv("google_storage_service_account")
|
|
String gcsBucket = System.getenv("google_storage_bucket")
|
|
String gcsBasePath = System.getenv("google_storage_base_path")
|
|
File serviceAccountFile = null
|
|
|
|
if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
|
|
serviceAccountFile = new File(project.buildDir, 'generated-resources/service_account_test.json')
|
|
gcsBucket = 'bucket'
|
|
gcsBasePath = 'integration_test'
|
|
useFixture = true
|
|
|
|
apply plugin: 'opensearch.test.fixtures'
|
|
testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture')
|
|
testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture-third-party')
|
|
|
|
} else if (!gcsServiceAccount || !gcsBucket || !gcsBasePath) {
|
|
throw new IllegalArgumentException("not all options specified to run tests against external GCS service are present")
|
|
} else {
|
|
serviceAccountFile = new File(gcsServiceAccount)
|
|
}
|
|
|
|
def encodedCredentials = {
|
|
Base64.encoder.encodeToString(Files.readAllBytes(serviceAccountFile.toPath()))
|
|
}
|
|
|
|
/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/
|
|
task createServiceAccountFile() {
|
|
doLast {
|
|
KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA")
|
|
keyPairGenerator.initialize(1024)
|
|
KeyPair keyPair = keyPairGenerator.generateKeyPair()
|
|
String encodedKey = Base64.getEncoder().encodeToString(keyPair.private.getEncoded())
|
|
|
|
serviceAccountFile.parentFile.mkdirs()
|
|
serviceAccountFile.setText("{\n" +
|
|
' "type": "service_account",\n' +
|
|
' "project_id": "integration_test",\n' +
|
|
' "private_key_id": "' + UUID.randomUUID().toString() + '",\n' +
|
|
' "private_key": "-----BEGIN PRIVATE KEY-----\\n' + encodedKey + '\\n-----END PRIVATE KEY-----\\n",\n' +
|
|
' "client_email": "integration_test@appspot.gserviceaccount.com",\n' +
|
|
' "client_id": "123456789101112130594"\n' +
|
|
'}', 'UTF-8')
|
|
}
|
|
}
|
|
|
|
Map<String, Object> expansions = [
|
|
'bucket' : gcsBucket,
|
|
'base_path': gcsBasePath + "_integration_tests"
|
|
]
|
|
|
|
processYamlRestTestResources {
|
|
inputs.properties(expansions)
|
|
MavenFilteringHack.filter(it, expansions)
|
|
}
|
|
|
|
internalClusterTest {
|
|
// this is tested explicitly in a separate test task
|
|
exclude '**/GoogleCloudStorageThirdPartyTests.class'
|
|
}
|
|
|
|
final Closure testClustersConfiguration = {
|
|
keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile, IGNORE_VALUE
|
|
|
|
if (useFixture) {
|
|
/* Use a closure on the string to delay evaluation until tests are executed */
|
|
setting 'gcs.client.integration_test.endpoint', { "${-> fixtureAddress('gcs-fixture')}" }, IGNORE_VALUE
|
|
setting 'gcs.client.integration_test.token_uri', { "${-> fixtureAddress('gcs-fixture')}/o/oauth2/token" }, IGNORE_VALUE
|
|
} else {
|
|
println "Using an external service to test the repository-gcs plugin"
|
|
}
|
|
}
|
|
|
|
yamlRestTest {
|
|
if (useFixture) {
|
|
dependsOn createServiceAccountFile
|
|
}
|
|
}
|
|
|
|
testClusters {
|
|
all testClustersConfiguration
|
|
}
|
|
|
|
/*
|
|
* We only use a small amount of data in these tests, which means that the resumable upload path is not tested. We add
|
|
* an additional test that forces the large blob threshold to be small to exercise the resumable upload path.
|
|
*/
|
|
task largeBlobYamlRestTest(type: RestIntegTestTask) {
|
|
dependsOn bundlePlugin
|
|
if (useFixture) {
|
|
dependsOn createServiceAccountFile
|
|
}
|
|
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
|
|
SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME)
|
|
setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
|
|
setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
|
|
}
|
|
|
|
check.dependsOn largeBlobYamlRestTest
|
|
|
|
testClusters {
|
|
largeBlobYamlRestTest {
|
|
plugin bundlePlugin.archiveFile
|
|
|
|
// force large blob uploads by setting the threshold small, forcing this code path to be tested
|
|
systemProperty 'opensearch.repository_gcs.large_blob_threshold_byte_size', '256'
|
|
}
|
|
}
|
|
|
|
task gcsThirdPartyTest(type: Test) {
|
|
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
|
|
SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME)
|
|
setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs())
|
|
setClasspath(internalTestSourceSet.getRuntimeClasspath())
|
|
include '**/GoogleCloudStorageThirdPartyTests.class'
|
|
systemProperty 'tests.security.manager', false
|
|
systemProperty 'test.google.bucket', gcsBucket
|
|
nonInputProperties.systemProperty 'test.google.base', gcsBasePath + "_third_party_tests_" + BuildParams.testSeed
|
|
nonInputProperties.systemProperty 'test.google.account', "${-> encodedCredentials.call()}"
|
|
if (useFixture) {
|
|
dependsOn createServiceAccountFile
|
|
nonInputProperties.systemProperty 'test.google.endpoint', "${-> fixtureAddress('gcs-fixture-third-party')}"
|
|
nonInputProperties.systemProperty 'test.google.tokenURI', "${-> fixtureAddress('gcs-fixture-third-party')}/o/oauth2/token"
|
|
}
|
|
}
|
|
check.dependsOn(gcsThirdPartyTest)
|