306 lines
14 KiB
Groovy
306 lines
14 KiB
Groovy
/*
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*
|
|
* The OpenSearch Contributors require contributions made to
|
|
* this file be licensed under the Apache-2.0 license or a
|
|
* compatible open source license.
|
|
*
|
|
* Modifications Copyright OpenSearch Contributors. See
|
|
* GitHub history for details.
|
|
*/
|
|
|
|
/*
|
|
* Licensed to Elasticsearch under one or more contributor
|
|
* license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright
|
|
* ownership. Elasticsearch licenses this file to you under
|
|
* the Apache License, Version 2.0 (the "License"); you may
|
|
* not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing,
|
|
* software distributed under the License is distributed on an
|
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
* KIND, either express or implied. See the License for the
|
|
* specific language governing permissions and limitations
|
|
* under the License.
|
|
*/
|
|
|
|
import org.opensearch.gradle.MavenFilteringHack
|
|
import org.opensearch.gradle.info.BuildParams
|
|
import org.opensearch.gradle.test.InternalClusterTestPlugin
|
|
|
|
import static org.opensearch.gradle.PropertyNormalization.DEFAULT
|
|
import static org.opensearch.gradle.PropertyNormalization.IGNORE_VALUE
|
|
|
|
apply plugin: 'opensearch.yaml-rest-test'
|
|
apply plugin: 'opensearch.internal-cluster-test'
|
|
|
|
opensearchplugin {
|
|
description 'The Azure Repository plugin adds support for Azure storage repositories.'
|
|
classname 'org.opensearch.repositories.azure.AzureRepositoryPlugin'
|
|
}
|
|
|
|
dependencies {
|
|
api 'com.azure:azure-core:1.22.0'
|
|
api 'com.azure:azure-storage-common:12.14.0'
|
|
api 'com.azure:azure-core-http-netty:1.11.7'
|
|
api "io.netty:netty-codec-dns:${versions.netty}"
|
|
api "io.netty:netty-codec-socks:${versions.netty}"
|
|
api "io.netty:netty-codec-http2:${versions.netty}"
|
|
api "io.netty:netty-handler-proxy:${versions.netty}"
|
|
api "io.netty:netty-resolver-dns:${versions.netty}"
|
|
api "io.netty:netty-transport-native-unix-common:${versions.netty}"
|
|
implementation project(':modules:transport-netty4')
|
|
api 'com.azure:azure-storage-blob:12.14.1'
|
|
api 'org.reactivestreams:reactive-streams:1.0.3'
|
|
api 'io.projectreactor:reactor-core:3.4.15'
|
|
api 'io.projectreactor.netty:reactor-netty:1.0.13'
|
|
api 'io.projectreactor.netty:reactor-netty-core:1.0.13'
|
|
api 'io.projectreactor.netty:reactor-netty-http:1.0.13'
|
|
api "org.slf4j:slf4j-api:${versions.slf4j}"
|
|
api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
|
|
api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
|
|
api "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.jackson}"
|
|
api "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${versions.jackson}"
|
|
api "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:${versions.jackson}"
|
|
api 'org.codehaus.woodstox:stax2-api:4.2.1'
|
|
implementation 'com.fasterxml.woodstox:woodstox-core:6.1.1'
|
|
runtimeOnly 'com.google.guava:guava:30.1.1-jre'
|
|
api 'org.apache.commons:commons-lang3:3.4'
|
|
testImplementation project(':test:fixtures:azure-fixture')
|
|
}
|
|
|
|
restResources {
|
|
restApi {
|
|
includeCore '_common', 'cluster', 'nodes', 'snapshot', 'bulk', 'count', 'indices'
|
|
}
|
|
}
|
|
|
|
tasks.named("dependencyLicenses").configure {
|
|
mapping from: /azure-.*/, to: 'azure'
|
|
mapping from: /jackson-.*/, to: 'jackson'
|
|
mapping from: /jersey-.*/, to: 'jersey'
|
|
mapping from: /jaxb-.*/, to: 'jaxb'
|
|
mapping from: /stax-.*/, to: 'stax'
|
|
mapping from: /netty-.*/, to: 'netty'
|
|
mapping from: /reactor-.*/, to: 'reactor'
|
|
}
|
|
|
|
thirdPartyAudit {
|
|
ignoreMissingClasses(
|
|
// Optional and not enabled by Elasticsearch
|
|
'com.google.common.util.concurrent.internal.InternalFutureFailureAccess',
|
|
'com.google.common.util.concurrent.internal.InternalFutures',
|
|
'com.azure.storage.internal.avro.implementation.AvroObject',
|
|
'com.azure.storage.internal.avro.implementation.AvroReader',
|
|
'com.azure.storage.internal.avro.implementation.AvroReaderFactory',
|
|
'com.azure.storage.internal.avro.implementation.schema.AvroSchema',
|
|
'com.ctc.wstx.shaded.msv_core.driver.textui.Driver',
|
|
'io.micrometer.core.instrument.Clock',
|
|
'io.micrometer.core.instrument.Counter',
|
|
'io.micrometer.core.instrument.Counter$Builder',
|
|
'io.micrometer.core.instrument.DistributionSummary',
|
|
'io.micrometer.core.instrument.DistributionSummary$Builder',
|
|
'io.micrometer.core.instrument.Gauge',
|
|
'io.micrometer.core.instrument.Gauge$Builder',
|
|
'io.micrometer.core.instrument.Meter',
|
|
'io.micrometer.core.instrument.MeterRegistry',
|
|
'io.micrometer.core.instrument.Metrics',
|
|
'io.micrometer.core.instrument.Tag',
|
|
'io.micrometer.core.instrument.Tags',
|
|
'io.micrometer.core.instrument.Timer',
|
|
'io.micrometer.core.instrument.Timer$Builder',
|
|
'io.micrometer.core.instrument.Timer$Sample',
|
|
'io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics',
|
|
'io.micrometer.core.instrument.composite.CompositeMeterRegistry',
|
|
'io.micrometer.core.instrument.search.Search',
|
|
'io.netty.channel.epoll.Epoll',
|
|
'io.netty.channel.epoll.EpollDatagramChannel',
|
|
'io.netty.channel.epoll.EpollDomainDatagramChannel',
|
|
'io.netty.channel.epoll.EpollDomainSocketChannel',
|
|
'io.netty.channel.epoll.EpollEventLoopGroup',
|
|
'io.netty.channel.epoll.EpollServerDomainSocketChannel',
|
|
'io.netty.channel.epoll.EpollServerSocketChannel',
|
|
'io.netty.channel.epoll.EpollSocketChannel',
|
|
'io.netty.channel.kqueue.KQueue',
|
|
'io.netty.channel.kqueue.KQueueDatagramChannel',
|
|
'io.netty.channel.kqueue.KQueueDomainDatagramChannel',
|
|
'io.netty.channel.kqueue.KQueueDomainSocketChannel',
|
|
'io.netty.channel.kqueue.KQueueEventLoopGroup',
|
|
'io.netty.channel.kqueue.KQueueServerDomainSocketChannel',
|
|
'io.netty.channel.kqueue.KQueueServerSocketChannel',
|
|
'io.netty.channel.kqueue.KQueueSocketChannel',
|
|
'io.netty.handler.codec.haproxy.HAProxyMessage',
|
|
'io.netty.handler.codec.haproxy.HAProxyMessageDecoder',
|
|
'io.netty.incubator.channel.uring.IOUring',
|
|
'io.netty.incubator.channel.uring.IOUringDatagramChannel',
|
|
'io.netty.incubator.channel.uring.IOUringEventLoopGroup',
|
|
'io.netty.incubator.channel.uring.IOUringServerSocketChannel',
|
|
'io.netty.incubator.channel.uring.IOUringSocketChannel',
|
|
'javax.activation.DataHandler',
|
|
'javax.activation.DataSource',
|
|
'javax.xml.bind.JAXBElement',
|
|
'javax.xml.bind.annotation.XmlAccessOrder',
|
|
'javax.xml.bind.annotation.XmlAccessType',
|
|
'javax.xml.bind.annotation.XmlAccessorOrder',
|
|
'javax.xml.bind.annotation.XmlAccessorType',
|
|
'javax.xml.bind.annotation.XmlAttribute',
|
|
'javax.xml.bind.annotation.XmlElement',
|
|
'javax.xml.bind.annotation.XmlElement$DEFAULT',
|
|
'javax.xml.bind.annotation.XmlElementRef',
|
|
'javax.xml.bind.annotation.XmlElementRefs',
|
|
'javax.xml.bind.annotation.XmlElementWrapper',
|
|
'javax.xml.bind.annotation.XmlElements',
|
|
'javax.xml.bind.annotation.XmlEnum',
|
|
'javax.xml.bind.annotation.XmlEnumValue',
|
|
'javax.xml.bind.annotation.XmlID',
|
|
'javax.xml.bind.annotation.XmlIDREF',
|
|
'javax.xml.bind.annotation.XmlRootElement',
|
|
'javax.xml.bind.annotation.XmlSeeAlso',
|
|
'javax.xml.bind.annotation.XmlTransient',
|
|
'javax.xml.bind.annotation.XmlType',
|
|
'javax.xml.bind.annotation.XmlValue',
|
|
'javax.xml.bind.annotation.adapters.XmlAdapter',
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter',
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter$DEFAULT',
|
|
'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapters',
|
|
'kotlin.TypeCastException',
|
|
'kotlin.collections.ArraysKt',
|
|
'kotlin.jvm.JvmClassMappingKt',
|
|
'kotlin.jvm.functions.Function0',
|
|
'kotlin.jvm.functions.Function1',
|
|
'kotlin.jvm.internal.FunctionReference',
|
|
'kotlin.jvm.internal.Intrinsics',
|
|
'kotlin.jvm.internal.Reflection',
|
|
'kotlin.jvm.internal.markers.KMappedMarker',
|
|
'kotlin.reflect.KClass',
|
|
'kotlin.reflect.KDeclarationContainer',
|
|
'kotlin.sequences.Sequence',
|
|
'org.osgi.framework.BundleActivator',
|
|
'org.osgi.framework.BundleContext',
|
|
'org.slf4j.impl.StaticLoggerBinder',
|
|
'org.slf4j.impl.StaticMDCBinder',
|
|
'org.slf4j.impl.StaticMarkerBinder',
|
|
'reactor.blockhound.BlockHound$Builder',
|
|
'reactor.blockhound.integration.BlockHoundIntegration'
|
|
)
|
|
|
|
ignoreViolations(
|
|
// uses internal java api: sun.misc.Unsafe
|
|
'com.google.common.cache.Striped64',
|
|
'com.google.common.cache.Striped64$1',
|
|
'com.google.common.cache.Striped64$Cell',
|
|
'com.google.common.hash.Striped64',
|
|
'com.google.common.hash.Striped64$1',
|
|
'com.google.common.hash.Striped64$Cell',
|
|
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
|
|
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
|
|
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3',
|
|
'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
|
|
'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
|
|
'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
|
|
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
|
|
'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
|
|
'reactor.core.publisher.Traces$SharedSecretsCallSiteSupplierFactory$TracingException'
|
|
)
|
|
}
|
|
|
|
boolean useFixture = false
|
|
|
|
def azureAddress = {
|
|
assert useFixture: 'closure should not be used without a fixture'
|
|
int ephemeralPort = project(':test:fixtures:azure-fixture').postProcessFixture.ext."test.fixtures.azure-fixture.tcp.8091"
|
|
assert ephemeralPort > 0
|
|
return 'ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://127.0.0.1:' + ephemeralPort + '/'
|
|
}
|
|
|
|
String azureAccount = System.getenv("azure_storage_account")
|
|
String azureKey = System.getenv("azure_storage_key")
|
|
String azureContainer = System.getenv("azure_storage_container")
|
|
String azureBasePath = System.getenv("azure_storage_base_path")
|
|
String azureSasToken = System.getenv("azure_storage_sas_token")
|
|
|
|
if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) {
|
|
azureAccount = 'azure_integration_test_account'
|
|
azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64
|
|
azureContainer = 'container'
|
|
azureBasePath = ''
|
|
azureSasToken = ''
|
|
useFixture = true
|
|
|
|
apply plugin: 'opensearch.test.fixtures'
|
|
testFixtures.useFixture ':test:fixtures:azure-fixture', 'azure-fixture'
|
|
}
|
|
|
|
Map<String, Object> expansions = [
|
|
'container': azureContainer,
|
|
'base_path': azureBasePath + "_integration_tests"
|
|
]
|
|
|
|
processYamlRestTestResources {
|
|
inputs.properties(expansions)
|
|
MavenFilteringHack.filter(it, expansions)
|
|
}
|
|
|
|
internalClusterTest {
|
|
// this is tested explicitly in a separate test task
|
|
exclude '**/AzureStorageCleanupThirdPartyTests.class'
|
|
}
|
|
|
|
testClusters {
|
|
yamlRestTest {
|
|
keystore 'azure.client.integration_test.account', azureAccount
|
|
if (azureKey != null && azureKey.isEmpty() == false) {
|
|
keystore 'azure.client.integration_test.key', azureKey
|
|
}
|
|
if (azureSasToken != null && azureSasToken.isEmpty() == false) {
|
|
keystore 'azure.client.integration_test.sas_token', azureSasToken
|
|
}
|
|
if (useFixture) {
|
|
setting 'azure.client.integration_test.endpoint_suffix', azureAddress
|
|
String firstPartOfSeed = BuildParams.testSeed.tokenize(':').get(0)
|
|
setting 'thread_pool.repository_azure.max', (Math.abs(Long.parseUnsignedLong(firstPartOfSeed, 16) % 10) + 1).toString(), System.getProperty('ignore.tests.seed') == null ? DEFAULT : IGNORE_VALUE
|
|
}
|
|
}
|
|
}
|
|
|
|
task azureThirdPartyTest(type: Test) {
|
|
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
|
|
SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME)
|
|
setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs())
|
|
setClasspath(internalTestSourceSet.getRuntimeClasspath())
|
|
dependsOn tasks.internalClusterTest
|
|
include '**/AzureStorageCleanupThirdPartyTests.class'
|
|
systemProperty 'test.azure.account', azureAccount ? azureAccount : ""
|
|
systemProperty 'test.azure.key', azureKey ? azureKey : ""
|
|
systemProperty 'test.azure.sas_token', azureSasToken ? azureSasToken : ""
|
|
systemProperty 'test.azure.container', azureContainer ? azureContainer : ""
|
|
systemProperty 'test.azure.base', (azureBasePath ? azureBasePath : "") + "_third_party_tests_" + BuildParams.testSeed
|
|
if (useFixture) {
|
|
nonInputProperties.systemProperty 'test.azure.endpoint_suffix', "${-> azureAddress.call() }"
|
|
}
|
|
}
|
|
|
|
task azureThirdPartyDefaultXmlTest(type: Test) {
|
|
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
|
|
SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME)
|
|
setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs())
|
|
setClasspath(internalTestSourceSet.getRuntimeClasspath())
|
|
dependsOn tasks.internalClusterTest
|
|
include '**/AzureStorageCleanupThirdPartyTests.class'
|
|
systemProperty 'javax.xml.stream.XMLInputFactory', "com.sun.xml.internal.stream.XMLInputFactoryImpl"
|
|
systemProperty 'test.azure.account', azureAccount ? azureAccount : ""
|
|
systemProperty 'test.azure.key', azureKey ? azureKey : ""
|
|
systemProperty 'test.azure.sas_token', azureSasToken ? azureSasToken : ""
|
|
systemProperty 'test.azure.container', azureContainer ? azureContainer : ""
|
|
systemProperty 'test.azure.base', (azureBasePath ? azureBasePath : "") + "_third_party_tests_" + BuildParams.testSeed
|
|
if (useFixture) {
|
|
nonInputProperties.systemProperty 'test.azure.endpoint_suffix', "${-> azureAddress.call() }"
|
|
}
|
|
}
|
|
|
|
check.dependsOn(azureThirdPartyTest, azureThirdPartyDefaultXmlTest)
|