/* * SPDX-License-Identifier: Apache-2.0 * * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. * * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. */ /* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright * ownership. Elasticsearch licenses this file to you under * the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ import org.opensearch.gradle.MavenFilteringHack import org.opensearch.gradle.info.BuildParams import org.opensearch.gradle.test.InternalClusterTestPlugin import static org.opensearch.gradle.PropertyNormalization.DEFAULT import static org.opensearch.gradle.PropertyNormalization.IGNORE_VALUE apply plugin: 'opensearch.yaml-rest-test' apply plugin: 'opensearch.internal-cluster-test' opensearchplugin { description 'The Azure Repository plugin adds support for Azure storage repositories.' classname 'org.opensearch.repositories.azure.AzureRepositoryPlugin' } dependencies { api 'com.azure:azure-core:1.27.0' api 'com.azure:azure-storage-common:12.15.0' api 'com.azure:azure-core-http-netty:1.11.9' api "io.netty:netty-codec-dns:${versions.netty}" api "io.netty:netty-codec-socks:${versions.netty}" api "io.netty:netty-codec-http2:${versions.netty}" api "io.netty:netty-handler-proxy:${versions.netty}" api "io.netty:netty-resolver-dns:${versions.netty}" api "io.netty:netty-transport-native-unix-common:${versions.netty}" implementation project(':modules:transport-netty4') api 'com.azure:azure-storage-blob:12.16.1' api 'org.reactivestreams:reactive-streams:1.0.3' api 'io.projectreactor:reactor-core:3.4.17' api 'io.projectreactor.netty:reactor-netty:1.0.18' api 'io.projectreactor.netty:reactor-netty-core:1.0.16' api 'io.projectreactor.netty:reactor-netty-http:1.0.18' api "org.slf4j:slf4j-api:${versions.slf4j}" api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" api "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.jackson}" api "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${versions.jackson}" api "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:${versions.jackson}" api 'org.codehaus.woodstox:stax2-api:4.2.1' implementation 'com.fasterxml.woodstox:woodstox-core:6.2.8' runtimeOnly 'com.google.guava:guava:31.1-jre' api 'org.apache.commons:commons-lang3:3.12.0' testImplementation project(':test:fixtures:azure-fixture') } restResources { restApi { includeCore '_common', 'cluster', 'nodes', 'snapshot', 'bulk', 'count', 'indices' } } tasks.named("dependencyLicenses").configure { mapping from: /azure-.*/, to: 'azure' mapping from: /jackson-.*/, to: 'jackson' mapping from: /jersey-.*/, to: 'jersey' mapping from: /jaxb-.*/, to: 'jaxb' mapping from: /stax-.*/, to: 'stax' mapping from: /netty-.*/, to: 'netty' mapping from: /reactor-.*/, to: 'reactor' } thirdPartyAudit { ignoreMissingClasses( // Optional and not enabled by Elasticsearch 'com.google.common.util.concurrent.internal.InternalFutureFailureAccess', 'com.google.common.util.concurrent.internal.InternalFutures', 'com.azure.storage.internal.avro.implementation.AvroObject', 'com.azure.storage.internal.avro.implementation.AvroReader', 'com.azure.storage.internal.avro.implementation.AvroReaderFactory', 'com.azure.storage.internal.avro.implementation.schema.AvroSchema', 'com.ctc.wstx.shaded.msv_core.driver.textui.Driver', 'io.micrometer.core.instrument.Clock', 'io.micrometer.core.instrument.Counter', 'io.micrometer.core.instrument.Counter$Builder', 'io.micrometer.core.instrument.DistributionSummary', 'io.micrometer.core.instrument.DistributionSummary$Builder', 'io.micrometer.core.instrument.Gauge', 'io.micrometer.core.instrument.Gauge$Builder', 'io.micrometer.core.instrument.Meter', 'io.micrometer.core.instrument.MeterRegistry', 'io.micrometer.core.instrument.Metrics', 'io.micrometer.core.instrument.Tag', 'io.micrometer.core.instrument.Tags', 'io.micrometer.core.instrument.Timer', 'io.micrometer.core.instrument.Timer$Builder', 'io.micrometer.core.instrument.Timer$Sample', 'io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics', 'io.micrometer.core.instrument.composite.CompositeMeterRegistry', 'io.micrometer.core.instrument.search.Search', 'io.netty.channel.epoll.Epoll', 'io.netty.channel.epoll.EpollDatagramChannel', 'io.netty.channel.epoll.EpollServerSocketChannel', 'io.netty.channel.epoll.EpollSocketChannel', 'io.netty.channel.kqueue.KQueue', 'io.netty.channel.kqueue.KQueueDatagramChannel', 'io.netty.channel.kqueue.KQueueServerSocketChannel', 'io.netty.channel.kqueue.KQueueSocketChannel', 'io.netty.handler.codec.haproxy.HAProxyMessage', 'io.netty.handler.codec.haproxy.HAProxyMessageDecoder', 'io.netty.incubator.channel.uring.IOUring', 'io.netty.incubator.channel.uring.IOUringDatagramChannel', 'io.netty.incubator.channel.uring.IOUringServerSocketChannel', 'io.netty.incubator.channel.uring.IOUringSocketChannel', 'javax.activation.DataHandler', 'javax.activation.DataSource', 'javax.xml.bind.JAXBElement', 'javax.xml.bind.annotation.XmlAccessOrder', 'javax.xml.bind.annotation.XmlAccessType', 'javax.xml.bind.annotation.XmlAccessorOrder', 'javax.xml.bind.annotation.XmlAccessorType', 'javax.xml.bind.annotation.XmlAttribute', 'javax.xml.bind.annotation.XmlElement', 'javax.xml.bind.annotation.XmlElement$DEFAULT', 'javax.xml.bind.annotation.XmlElementRef', 'javax.xml.bind.annotation.XmlElementRefs', 'javax.xml.bind.annotation.XmlElementWrapper', 'javax.xml.bind.annotation.XmlElements', 'javax.xml.bind.annotation.XmlEnum', 'javax.xml.bind.annotation.XmlEnumValue', 'javax.xml.bind.annotation.XmlID', 'javax.xml.bind.annotation.XmlIDREF', 'javax.xml.bind.annotation.XmlRootElement', 'javax.xml.bind.annotation.XmlSeeAlso', 'javax.xml.bind.annotation.XmlTransient', 'javax.xml.bind.annotation.XmlType', 'javax.xml.bind.annotation.XmlValue', 'javax.xml.bind.annotation.adapters.XmlAdapter', 'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter', 'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter$DEFAULT', 'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapters', 'kotlin.collections.ArraysKt', 'kotlin.jvm.JvmClassMappingKt', 'kotlin.jvm.functions.Function0', 'kotlin.jvm.functions.Function1', 'kotlin.jvm.internal.FunctionReference', 'kotlin.jvm.internal.Intrinsics', 'kotlin.jvm.internal.Reflection', 'kotlin.jvm.internal.markers.KMappedMarker', 'kotlin.reflect.KClass', 'kotlin.reflect.KDeclarationContainer', 'kotlin.sequences.Sequence', 'org.osgi.framework.BundleActivator', 'org.osgi.framework.BundleContext', 'org.slf4j.impl.StaticLoggerBinder', 'org.slf4j.impl.StaticMDCBinder', 'org.slf4j.impl.StaticMarkerBinder', 'reactor.blockhound.BlockHound$Builder', 'reactor.blockhound.integration.BlockHoundIntegration' ) ignoreViolations( // uses internal java api: sun.misc.Unsafe 'com.google.common.cache.Striped64', 'com.google.common.cache.Striped64$1', 'com.google.common.cache.Striped64$Cell', 'com.google.common.hash.Striped64', 'com.google.common.hash.Striped64$1', 'com.google.common.hash.Striped64$Cell', 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1', 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2', 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$3', 'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper', 'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1', 'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray', 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator', 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1', 'reactor.core.publisher.Traces$SharedSecretsCallSiteSupplierFactory$TracingException' ) } boolean useFixture = false def azureAddress = { assert useFixture: 'closure should not be used without a fixture' int ephemeralPort = project(':test:fixtures:azure-fixture').postProcessFixture.ext."test.fixtures.azure-fixture.tcp.8091" assert ephemeralPort > 0 return 'ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://127.0.0.1:' + ephemeralPort + '/' } String azureAccount = System.getenv("azure_storage_account") String azureKey = System.getenv("azure_storage_key") String azureContainer = System.getenv("azure_storage_container") String azureBasePath = System.getenv("azure_storage_base_path") String azureSasToken = System.getenv("azure_storage_sas_token") if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) { azureAccount = 'azure_integration_test_account' azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64 azureContainer = 'container' azureBasePath = '' azureSasToken = '' useFixture = true apply plugin: 'opensearch.test.fixtures' testFixtures.useFixture ':test:fixtures:azure-fixture', 'azure-fixture' } Map expansions = [ 'container': azureContainer, 'base_path': azureBasePath + "_integration_tests" ] processYamlRestTestResources { inputs.properties(expansions) MavenFilteringHack.filter(it, expansions) } internalClusterTest { // this is tested explicitly in a separate test task exclude '**/AzureStorageCleanupThirdPartyTests.class' } testClusters { yamlRestTest { keystore 'azure.client.integration_test.account', azureAccount if (azureKey != null && azureKey.isEmpty() == false) { keystore 'azure.client.integration_test.key', azureKey } if (azureSasToken != null && azureSasToken.isEmpty() == false) { keystore 'azure.client.integration_test.sas_token', azureSasToken } if (useFixture) { setting 'azure.client.integration_test.endpoint_suffix', azureAddress String firstPartOfSeed = BuildParams.testSeed.tokenize(':').get(0) setting 'thread_pool.repository_azure.max', (Math.abs(Long.parseUnsignedLong(firstPartOfSeed, 16) % 10) + 1).toString(), System.getProperty('ignore.tests.seed') == null ? DEFAULT : IGNORE_VALUE } } } task azureThirdPartyTest(type: Test) { SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME) setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs()) setClasspath(internalTestSourceSet.getRuntimeClasspath()) dependsOn tasks.internalClusterTest include '**/AzureStorageCleanupThirdPartyTests.class' systemProperty 'test.azure.account', azureAccount ? azureAccount : "" systemProperty 'test.azure.key', azureKey ? azureKey : "" systemProperty 'test.azure.sas_token', azureSasToken ? azureSasToken : "" systemProperty 'test.azure.container', azureContainer ? azureContainer : "" systemProperty 'test.azure.base', (azureBasePath ? azureBasePath : "") + "_third_party_tests_" + BuildParams.testSeed if (useFixture) { nonInputProperties.systemProperty 'test.azure.endpoint_suffix', "${-> azureAddress.call() }" } } task azureThirdPartyDefaultXmlTest(type: Test) { SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); SourceSet internalTestSourceSet = sourceSets.getByName(InternalClusterTestPlugin.SOURCE_SET_NAME) setTestClassesDirs(internalTestSourceSet.getOutput().getClassesDirs()) setClasspath(internalTestSourceSet.getRuntimeClasspath()) dependsOn tasks.internalClusterTest include '**/AzureStorageCleanupThirdPartyTests.class' systemProperty 'javax.xml.stream.XMLInputFactory', "com.sun.xml.internal.stream.XMLInputFactoryImpl" systemProperty 'test.azure.account', azureAccount ? azureAccount : "" systemProperty 'test.azure.key', azureKey ? azureKey : "" systemProperty 'test.azure.sas_token', azureSasToken ? azureSasToken : "" systemProperty 'test.azure.container', azureContainer ? azureContainer : "" systemProperty 'test.azure.base', (azureBasePath ? azureBasePath : "") + "_third_party_tests_" + BuildParams.testSeed if (useFixture) { nonInputProperties.systemProperty 'test.azure.endpoint_suffix', "${-> azureAddress.call() }" } } check.dependsOn(azureThirdPartyTest, azureThirdPartyDefaultXmlTest)