mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-03-01 16:39:11 +00:00
d6e17170c3
There are currently half a dozen ways to add plugins and modules for test clusters to use. All of them require the calling project to peek into the plugin or module they want to use to grab its bundlePlugin task, and then both depend on that task, as well as extract the archive path the task will produce. This creates cross project dependencies that are difficult to detect, and if the dependent plugin/module has not yet been configured, the build will fail because the task does not yet exist. This commit makes the plugin and module methods for testclusters symmetetric, and simply adding a file provider directly, or a project path that will produce the plugin/module zip. Internally this new variant uses normal configuration/dependencies across projects to get the zip artifact. It also has the added benefit of no longer needing the caller to add to the test task a dependsOn for bundlePlugin task.
335 lines
14 KiB
Groovy
335 lines
14 KiB
Groovy
import java.nio.file.Files
|
|
import java.security.KeyPair
|
|
import java.security.KeyPairGenerator
|
|
import org.elasticsearch.gradle.MavenFilteringHack
|
|
import org.elasticsearch.gradle.info.BuildParams
|
|
import org.elasticsearch.gradle.test.RestIntegTestTask
|
|
import org.elasticsearch.gradle.test.rest.YamlRestTestPlugin
|
|
import org.elasticsearch.gradle.test.InternalClusterTestPlugin
|
|
|
|
import java.nio.file.Files
|
|
import java.security.KeyPair
|
|
import java.security.KeyPairGenerator
|
|
|
|
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
|
|
/*
|
|
* Licensed to Elasticsearch under one or more contributor
|
|
* license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright
|
|
* ownership. Elasticsearch licenses this file to you under
|
|
* the Apache License, Version 2.0 (the "License"); you may
|
|
* not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing,
|
|
* software distributed under the License is distributed on an
|
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
* KIND, either express or implied. See the License for the
|
|
* specific language governing permissions and limitations
|
|
* under the License.
|
|
*/
|
|
apply plugin: 'elasticsearch.yaml-rest-test'
|
|
apply plugin: 'elasticsearch.internal-cluster-test'
|
|
|
|
esplugin {
|
|
description 'The GCS repository plugin adds Google Cloud Storage support for repositories.'
|
|
classname 'org.elasticsearch.repositories.gcs.GoogleCloudStoragePlugin'
|
|
}
|
|
|
|
dependencies {
|
|
api 'com.google.cloud:google-cloud-storage:1.106.0'
|
|
api 'com.google.cloud:google-cloud-core:1.93.3'
|
|
runtimeOnly 'com.google.guava:guava:26.0-jre'
|
|
api 'com.google.http-client:google-http-client:1.34.2'
|
|
api "commons-logging:commons-logging:${versions.commonslogging}"
|
|
api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
|
|
api "commons-codec:commons-codec:${versions.commonscodec}"
|
|
api 'com.google.api:api-common:1.8.1'
|
|
api 'com.google.api:gax:1.54.0'
|
|
api 'org.threeten:threetenbp:1.4.1'
|
|
api 'com.google.protobuf:protobuf-java-util:3.11.3'
|
|
api 'com.google.protobuf:protobuf-java:3.11.3'
|
|
api 'com.google.code.gson:gson:2.7'
|
|
api 'com.google.api.grpc:proto-google-common-protos:1.16.0'
|
|
api 'com.google.api.grpc:proto-google-iam-v1:0.12.0'
|
|
api 'com.google.cloud:google-cloud-core-http:1.93.3'
|
|
api 'com.google.auth:google-auth-library-credentials:0.20.0'
|
|
api 'com.google.auth:google-auth-library-oauth2-http:0.20.0'
|
|
api 'com.google.oauth-client:google-oauth-client:1.28.0'
|
|
api 'com.google.api-client:google-api-client:1.30.9'
|
|
api 'com.google.http-client:google-http-client-appengine:1.34.2'
|
|
api 'com.google.http-client:google-http-client-jackson2:1.34.2'
|
|
api 'com.google.api:gax-httpjson:0.62.0'
|
|
api 'io.grpc:grpc-context:1.12.0'
|
|
api 'io.opencensus:opencensus-api:0.18.0'
|
|
api 'io.opencensus:opencensus-contrib-http-util:0.18.0'
|
|
api 'com.google.apis:google-api-services-storage:v1-rev20200226-1.30.9'
|
|
|
|
testImplementation project(':test:fixtures:gcs-fixture')
|
|
}
|
|
|
|
restResources {
|
|
restApi {
|
|
includeCore '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count'
|
|
}
|
|
}
|
|
|
|
tasks.named("dependencyLicenses").configure {
|
|
mapping from: /google-cloud-.*/, to: 'google-cloud'
|
|
mapping from: /google-auth-.*/, to: 'google-auth'
|
|
mapping from: /google-http-.*/, to: 'google-http'
|
|
mapping from: /opencensus.*/, to: 'opencensus'
|
|
mapping from: /protobuf.*/, to: 'protobuf'
|
|
mapping from: /proto-google.*/, to: 'proto-google'
|
|
}
|
|
|
|
thirdPartyAudit {
|
|
ignoreViolations(
|
|
// uses internal java api: sun.misc.Unsafe
|
|
'com.google.protobuf.UnsafeUtil',
|
|
'com.google.protobuf.UnsafeUtil$1',
|
|
'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
|
|
'com.google.protobuf.UnsafeUtil$MemoryAccessor',
|
|
'com.google.protobuf.MessageSchema',
|
|
'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
|
|
'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
|
|
'com.google.common.cache.Striped64',
|
|
'com.google.common.cache.Striped64$1',
|
|
'com.google.common.cache.Striped64$Cell',
|
|
'com.google.common.hash.Striped64',
|
|
'com.google.common.hash.Striped64$1',
|
|
'com.google.common.hash.Striped64$Cell',
|
|
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
|
|
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
|
|
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
|
|
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3',
|
|
'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
|
|
'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
|
|
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
|
|
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
|
|
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
|
|
)
|
|
|
|
ignoreMissingClasses(
|
|
'com.google.appengine.api.datastore.Blob',
|
|
'com.google.appengine.api.datastore.DatastoreService',
|
|
'com.google.appengine.api.datastore.DatastoreServiceFactory',
|
|
'com.google.appengine.api.datastore.Entity',
|
|
'com.google.appengine.api.datastore.Key',
|
|
'com.google.appengine.api.datastore.KeyFactory',
|
|
'com.google.appengine.api.datastore.PreparedQuery',
|
|
'com.google.appengine.api.datastore.Query',
|
|
'com.google.appengine.api.memcache.Expiration',
|
|
'com.google.appengine.api.memcache.MemcacheService',
|
|
'com.google.appengine.api.memcache.MemcacheServiceFactory',
|
|
'com.google.appengine.api.urlfetch.FetchOptions$Builder',
|
|
'com.google.appengine.api.urlfetch.FetchOptions',
|
|
'com.google.appengine.api.urlfetch.HTTPHeader',
|
|
'com.google.appengine.api.urlfetch.HTTPMethod',
|
|
'com.google.appengine.api.urlfetch.HTTPRequest',
|
|
'com.google.appengine.api.urlfetch.HTTPResponse',
|
|
'com.google.appengine.api.urlfetch.URLFetchService',
|
|
'com.google.appengine.api.urlfetch.URLFetchServiceFactory',
|
|
// commons-logging optional dependencies
|
|
'org.apache.avalon.framework.logger.Logger',
|
|
'org.apache.log.Hierarchy',
|
|
'org.apache.log.Logger',
|
|
// optional apache http client dependencies
|
|
'org.apache.http.ConnectionReuseStrategy',
|
|
'org.apache.http.Header',
|
|
'org.apache.http.HttpEntity',
|
|
'org.apache.http.HttpEntityEnclosingRequest',
|
|
'org.apache.http.HttpHost',
|
|
'org.apache.http.HttpRequest',
|
|
'org.apache.http.HttpResponse',
|
|
'org.apache.http.HttpVersion',
|
|
'org.apache.http.RequestLine',
|
|
'org.apache.http.StatusLine',
|
|
'org.apache.http.client.AuthenticationHandler',
|
|
'org.apache.http.client.HttpClient',
|
|
'org.apache.http.client.HttpRequestRetryHandler',
|
|
'org.apache.http.client.RedirectHandler',
|
|
'org.apache.http.client.RequestDirector',
|
|
'org.apache.http.client.UserTokenHandler',
|
|
'org.apache.http.client.methods.HttpDelete',
|
|
'org.apache.http.client.methods.HttpEntityEnclosingRequestBase',
|
|
'org.apache.http.client.methods.HttpGet',
|
|
'org.apache.http.client.methods.HttpHead',
|
|
'org.apache.http.client.methods.HttpOptions',
|
|
'org.apache.http.client.methods.HttpPost',
|
|
'org.apache.http.client.methods.HttpPut',
|
|
'org.apache.http.client.methods.HttpRequestBase',
|
|
'org.apache.http.client.methods.HttpTrace',
|
|
'org.apache.http.config.SocketConfig',
|
|
'org.apache.http.config.SocketConfig$Builder',
|
|
'org.apache.http.conn.ClientConnectionManager',
|
|
'org.apache.http.conn.ConnectionKeepAliveStrategy',
|
|
'org.apache.http.conn.params.ConnManagerParams',
|
|
'org.apache.http.conn.params.ConnPerRouteBean',
|
|
'org.apache.http.conn.params.ConnRouteParams',
|
|
'org.apache.http.conn.routing.HttpRoutePlanner',
|
|
'org.apache.http.conn.scheme.PlainSocketFactory',
|
|
'org.apache.http.conn.scheme.Scheme',
|
|
'org.apache.http.conn.scheme.SchemeRegistry',
|
|
'org.apache.http.conn.ssl.SSLConnectionSocketFactory',
|
|
'org.apache.http.conn.ssl.SSLSocketFactory',
|
|
'org.apache.http.conn.ssl.X509HostnameVerifier',
|
|
'org.apache.http.entity.AbstractHttpEntity',
|
|
'org.apache.http.impl.client.DefaultHttpClient',
|
|
'org.apache.http.impl.client.DefaultHttpRequestRetryHandler',
|
|
'org.apache.http.impl.client.HttpClientBuilder',
|
|
'org.apache.http.impl.conn.PoolingHttpClientConnectionManager',
|
|
'org.apache.http.impl.conn.ProxySelectorRoutePlanner',
|
|
'org.apache.http.impl.conn.SystemDefaultRoutePlanner',
|
|
'org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager',
|
|
'org.apache.http.message.BasicHttpResponse',
|
|
'org.apache.http.params.BasicHttpParams',
|
|
'org.apache.http.params.HttpConnectionParams',
|
|
'org.apache.http.params.HttpParams',
|
|
'org.apache.http.params.HttpProtocolParams',
|
|
'org.apache.http.protocol.HttpContext',
|
|
'org.apache.http.protocol.HttpProcessor',
|
|
'org.apache.http.protocol.HttpRequestExecutor',
|
|
// commons-logging provided dependencies
|
|
'javax.servlet.ServletContextEvent',
|
|
'javax.servlet.ServletContextListener'
|
|
)
|
|
}
|
|
|
|
boolean useFixture = false
|
|
|
|
def fixtureAddress = { fixture ->
|
|
assert useFixture: 'closure should not be used without a fixture'
|
|
int ephemeralPort = project(':test:fixtures:gcs-fixture').postProcessFixture.ext."test.fixtures.${fixture}.tcp.80"
|
|
assert ephemeralPort > 0
|
|
'http://127.0.0.1:' + ephemeralPort
|
|
}
|
|
|
|
String gcsServiceAccount = System.getenv("google_storage_service_account")
|
|
String gcsBucket = System.getenv("google_storage_bucket")
|
|
String gcsBasePath = System.getenv("google_storage_base_path")
|
|
File serviceAccountFile = null
|
|
|
|
if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
|
|
serviceAccountFile = new File(project.buildDir, 'generated-resources/service_account_test.json')
|
|
gcsBucket = 'bucket'
|
|
gcsBasePath = 'integration_test'
|
|
useFixture = true
|
|
|
|
apply plugin: 'elasticsearch.test.fixtures'
|
|
testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture')
|
|
testFixtures.useFixture(':test:fixtures:gcs-fixture', 'gcs-fixture-third-party')
|
|
|
|
} else if (!gcsServiceAccount || !gcsBucket || !gcsBasePath) {
|
|
throw new IllegalArgumentException("not all options specified to run tests against external GCS service are present")
|
|
} else {
|
|
serviceAccountFile = new File(gcsServiceAccount)
|
|
}
|
|
|
|
def encodedCredentials = {
|
|
Base64.encoder.encodeToString(Files.readAllBytes(serviceAccountFile.toPath()))
|
|
}
|
|
|
|
/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/
|
|
task createServiceAccountFile() {
|
|
doLast {
|
|
KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA")
|
|
keyPairGenerator.initialize(1024)
|
|
KeyPair keyPair = keyPairGenerator.generateKeyPair()
|
|
String encodedKey = Base64.getEncoder().encodeToString(keyPair.private.getEncoded())
|
|
|
|
serviceAccountFile.parentFile.mkdirs()
|
|
serviceAccountFile.setText("{\n" +
|
|
' "type": "service_account",\n' +
|
|
' "project_id": "integration_test",\n' +
|
|
' "private_key_id": "' + UUID.randomUUID().toString() + '",\n' +
|
|
' "private_key": "-----BEGIN PRIVATE KEY-----\\n' + encodedKey + '\\n-----END PRIVATE KEY-----\\n",\n' +
|
|
' "client_email": "integration_test@appspot.gserviceaccount.com",\n' +
|
|
' "client_id": "123456789101112130594"\n' +
|
|
'}', 'UTF-8')
|
|
}
|
|
}
|
|
|
|
Map<String, Object> expansions = [
|
|
'bucket' : gcsBucket,
|
|
'base_path': gcsBasePath + "_integration_tests"
|
|
]
|
|
|
|
processYamlRestTestResources {
|
|
inputs.properties(expansions)
|
|
MavenFilteringHack.filter(it, expansions)
|
|
}
|
|
|
|
internalClusterTest {
|
|
// this is tested explicitly in a separate test task
|
|
exclude '**/GoogleCloudStorageThirdPartyTests.class'
|
|
}
|
|
|
|
final Closure testClustersConfiguration = {
|
|
keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile, IGNORE_VALUE
|
|
|
|
if (useFixture) {
|
|
/* Use a closure on the string to delay evaluation until tests are executed */
|
|
setting 'gcs.client.integration_test.endpoint', { "${-> fixtureAddress('gcs-fixture')}" }, IGNORE_VALUE
|
|
setting 'gcs.client.integration_test.token_uri', { "${-> fixtureAddress('gcs-fixture')}/o/oauth2/token" }, IGNORE_VALUE
|
|
} else {
|
|
println "Using an external service to test the repository-gcs plugin"
|
|
}
|
|
}
|
|
|
|
yamlRestTest {
|
|
if (useFixture) {
|
|
dependsOn createServiceAccountFile
|
|
}
|
|
}
|
|
|
|
testClusters {
|
|
all testClustersConfiguration
|
|
}
|
|
|
|
/*
|
|
* We only use a small amount of data in these tests, which means that the resumable upload path is not tested. We add
|
|
* an additional test that forces the large blob threshold to be small to exercise the resumable upload path.
|
|
*/
|
|
task largeBlobYamlRestTest(type: RestIntegTestTask) {
|
|
dependsOn bundlePlugin
|
|
if (useFixture) {
|
|
dependsOn createServiceAccountFile
|
|
}
|
|
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
|
|
SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME)
|
|
setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
|
|
setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
|
|
}
|
|
|
|
check.dependsOn largeBlobYamlRestTest
|
|
|
|
testClusters {
|
|
largeBlobYamlRestTest {
|
|
plugin bundlePlugin.archiveFile
|
|
|
|
// force large blob uploads by setting the threshold small, forcing this code path to be tested
|
|
systemProperty 'es.repository_gcs.large_blob_threshold_byte_size', '256'
|
|
}
|
|
}
|
|
|
|
task gcsThirdPartyTest(type: Test) {
|
|
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
|
|
SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME)
|
|
setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs())
|
|
setClasspath(internalTestSourceSet.getRuntimeClasspath())
|
|
include '**/GoogleCloudStorageThirdPartyTests.class'
|
|
systemProperty 'tests.security.manager', false
|
|
systemProperty 'test.google.bucket', gcsBucket
|
|
nonInputProperties.systemProperty 'test.google.base', gcsBasePath + "_third_party_tests_" + BuildParams.testSeed
|
|
nonInputProperties.systemProperty 'test.google.account', "${-> encodedCredentials.call()}"
|
|
if (useFixture) {
|
|
dependsOn createServiceAccountFile
|
|
nonInputProperties.systemProperty 'test.google.endpoint', "${-> fixtureAddress('gcs-fixture-third-party')}"
|
|
nonInputProperties.systemProperty 'test.google.tokenURI', "${-> fixtureAddress('gcs-fixture-third-party')}/o/oauth2/token"
|
|
}
|
|
}
|
|
check.dependsOn(gcsThirdPartyTest)
|