From a2864191d60998053c38581c159b6744f641ec52 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 26 May 2020 14:10:07 -0700 Subject: [PATCH] Move common repository configuration to java plugin (#57057) This commit moves the common maven repository configuration to the ES java plugin. All java projects need this common set of repos. Note that the Elastic download and maven repos are removed, as they are not necessary anymore since distribution download was split into the DistributionDownloadPlugin. --- build.gradle | 2 - .../elasticsearch/gradle/BuildPlugin.groovy | 120 ------------------ .../test/StandaloneRestTestPlugin.groovy | 2 +- .../gradle/ElasticsearchJavaPlugin.java | 79 ++++++++++++ .../gradle/BuildPluginTests.java | 51 -------- .../test/GradleIntegrationTestCase.java | 4 +- 6 files changed, 83 insertions(+), 175 deletions(-) delete mode 100644 buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginTests.java diff --git a/build.gradle b/build.gradle index 35a12111d72..4618d0f7cf4 100644 --- a/build.gradle +++ b/build.gradle @@ -54,8 +54,6 @@ allprojects { description = "Elasticsearch subproject ${project.path}" } -BuildPlugin.configureRepositories(project) - String licenseCommit if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) { licenseCommit = BuildParams.gitRevision ?: "master" // leniency for non git builds diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index e3266f00061..daaa15828a9 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -18,7 +18,6 @@ */ package org.elasticsearch.gradle - import groovy.transform.CompileStatic import org.apache.commons.io.IOUtils import org.elasticsearch.gradle.info.BuildParams @@ -47,7 +46,6 @@ import org.gradle.api.plugins.ExtraPropertiesExtension import org.gradle.api.plugins.JavaPlugin import org.gradle.api.tasks.bundling.Jar import org.gradle.api.tasks.testing.Test -import org.gradle.authentication.http.HttpHeaderAuthentication import org.gradle.util.GradleVersion import java.nio.charset.StandardCharsets @@ -90,7 +88,6 @@ class BuildPlugin implements Plugin { project.getTasks().register("buildResources", ExportElasticsearchBuildResourcesTask) - configureRepositories(project) project.extensions.getByType(ExtraPropertiesExtension).set('versions', VersionProperties.versions) PrecommitTasks.create(project, true) configureFips140(project) @@ -154,123 +151,6 @@ class BuildPlugin implements Plugin { } } - /** - * Makes dependencies non-transitive. - * - * Gradle allows setting all dependencies as non-transitive very easily. - * Sadly this mechanism does not translate into maven pom generation. In order - * to effectively make the pom act as if it has no transitive dependencies, - * we must exclude each transitive dependency of each direct dependency. - * - * Determining the transitive deps of a dependency which has been resolved as - * non-transitive is difficult because the process of resolving removes the - * transitive deps. To sidestep this issue, we create a configuration per - * direct dependency version. This specially named and unique configuration - * will contain all of the transitive dependencies of this particular - * dependency. We can then use this configuration during pom generation - * to iterate the transitive dependencies and add excludes. - */ - static void configureConfigurations(Project project) { - // we want to test compileOnly deps! - project.configurations.getByName(JavaPlugin.TEST_COMPILE_CONFIGURATION_NAME).extendsFrom(project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME)) - - // we are not shipping these jars, we act like dumb consumers of these things - if (project.path.startsWith(':test:fixtures') || project.path == ':build-tools') { - return - } - // fail on any conflicting dependency versions - project.configurations.all({ Configuration configuration -> - if (configuration.name.endsWith('Fixture')) { - // just a self contained test-fixture configuration, likely transitive and hellacious - return - } - configuration.resolutionStrategy { - failOnVersionConflict() - } - }) - - // force all dependencies added directly to compile/testCompile to be non-transitive, except for ES itself - Closure disableTransitiveDeps = { Dependency dep -> - if (dep instanceof ModuleDependency && !(dep instanceof ProjectDependency) - && dep.group.startsWith('org.elasticsearch') == false) { - dep.transitive = false - } - } - - project.configurations.getByName(JavaPlugin.COMPILE_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps) - project.configurations.getByName(JavaPlugin.TEST_COMPILE_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps) - project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps) - project.configurations.getByName(JavaPlugin.RUNTIME_ONLY_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps) - } - - /** Adds repositories used by ES dependencies */ - static void configureRepositories(Project project) { - project.getRepositories().all { repository -> - if (repository instanceof MavenArtifactRepository) { - final MavenArtifactRepository maven = (MavenArtifactRepository) repository - assertRepositoryURIIsSecure(maven.name, project.path, maven.getUrl()) - repository.getArtifactUrls().each { uri -> assertRepositoryURIIsSecure(maven.name, project.path, uri) } - } else if (repository instanceof IvyArtifactRepository) { - final IvyArtifactRepository ivy = (IvyArtifactRepository) repository - assertRepositoryURIIsSecure(ivy.name, project.path, ivy.getUrl()) - } - } - RepositoryHandler repos = project.repositories - if (System.getProperty('repos.mavenLocal') != null) { - // with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is - // useful for development ie. bwc tests where we install stuff in the local repository - // such that we don't have to pass hardcoded files to gradle - repos.mavenLocal() - } - repos.jcenter() - repos.ivy { IvyArtifactRepository repo -> - repo.name = 'elasticsearch' - repo.url = 'https://artifacts.elastic.co/downloads' - repo.patternLayout { IvyPatternRepositoryLayout layout -> - layout.artifact 'elasticsearch/[module]-[revision](-[classifier]).[ext]' - } - // this header is not a credential but we hack the capability to send this header to avoid polluting our download stats - repo.credentials(HttpHeaderCredentials, { HttpHeaderCredentials creds -> - creds.name = 'X-Elastic-No-KPI' - creds.value = '1' - } as Action) - repo.authentication.create('header', HttpHeaderAuthentication) - } - repos.maven { MavenArtifactRepository repo -> - repo.name = 'elastic' - repo.url = 'https://artifacts.elastic.co/maven' - } - String luceneVersion = VersionProperties.lucene - if (luceneVersion.contains('-snapshot')) { - // extract the revision number from the version with a regex matcher - List matches = (luceneVersion =~ /\w+-snapshot-([a-z0-9]+)/).getAt(0) as List - String revision = matches.get(1) - MavenArtifactRepository luceneRepo = repos.maven { MavenArtifactRepository repo -> - repo.name = 'lucene-snapshots' - repo.url = "https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}" - } - repos.exclusiveContent { ExclusiveContentRepository exclusiveRepo -> - exclusiveRepo.filter { - it.includeVersionByRegex(/org\.apache\.lucene/, '.*', ".*-snapshot-${revision}") - } - exclusiveRepo.forRepositories(luceneRepo) - } - } - } - - static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) { - if (uri != null && ["file", "https", "s3"].contains(uri.getScheme()) == false) { - final String message = String.format( - Locale.ROOT, - "repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]", - repositoryName, - projectPath, - uri.toURL()) - throw new GradleException(message) - } - } - - private static class TestFailureReportingPlugin implements Plugin { @Override void apply(Project project) { diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy index 23298344936..c4bf0e4db1d 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy @@ -62,7 +62,7 @@ class StandaloneRestTestPlugin implements Plugin { project.pluginManager.apply(TestClustersPlugin) project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask) - BuildPlugin.configureRepositories(project) + ElasticsearchJavaPlugin.configureRepositories(project) ElasticsearchJavaPlugin.configureTestTasks(project) ElasticsearchJavaPlugin.configureInputNormalization(project) BuildPlugin.configureFips140(project) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java index e3d6d12e356..279596fe530 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java @@ -36,6 +36,9 @@ import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.ModuleDependency; import org.gradle.api.artifacts.ProjectDependency; import org.gradle.api.artifacts.ResolutionStrategy; +import org.gradle.api.artifacts.dsl.RepositoryHandler; +import org.gradle.api.artifacts.repositories.IvyArtifactRepository; +import org.gradle.api.artifacts.repositories.MavenArtifactRepository; import org.gradle.api.file.FileCollection; import org.gradle.api.plugins.BasePlugin; import org.gradle.api.plugins.JavaPlugin; @@ -55,12 +58,18 @@ import org.gradle.language.base.plugins.LifecycleBasePlugin; import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; +import java.net.MalformedURLException; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.function.Consumer; import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure; import static org.elasticsearch.gradle.util.Util.toStringable; @@ -76,6 +85,7 @@ public class ElasticsearchJavaPlugin implements Plugin { project.getPluginManager().apply(JavaPlugin.class); configureConfigurations(project); + configureRepositories(project); configureCompile(project); configureInputNormalization(project); configureTestTasks(project); @@ -136,6 +146,75 @@ public class ElasticsearchJavaPlugin implements Plugin { disableTransitiveDeps.accept(JavaPlugin.RUNTIME_ONLY_CONFIGURATION_NAME); } + private static final Pattern LUCENE_SNAPSHOT_REGEX = Pattern.compile("\\w+-snapshot-([a-z0-9]+)"); + + /** Adds repositories used by ES dependencies */ + public static void configureRepositories(Project project) { + // ensure all repositories use secure urls + // TODO: remove this with gradle 7.0, which no longer allows insecure urls + project.getRepositories().all(repository -> { + if (repository instanceof MavenArtifactRepository) { + final MavenArtifactRepository maven = (MavenArtifactRepository) repository; + assertRepositoryURIIsSecure(maven.getName(), project.getPath(), maven.getUrl()); + for (URI uri : maven.getArtifactUrls()) { + assertRepositoryURIIsSecure(maven.getName(), project.getPath(), uri); + } + } else if (repository instanceof IvyArtifactRepository) { + final IvyArtifactRepository ivy = (IvyArtifactRepository) repository; + assertRepositoryURIIsSecure(ivy.getName(), project.getPath(), ivy.getUrl()); + } + }); + RepositoryHandler repos = project.getRepositories(); + if (System.getProperty("repos.mavenLocal") != null) { + // with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is + // useful for development ie. bwc tests where we install stuff in the local repository + // such that we don't have to pass hardcoded files to gradle + repos.mavenLocal(); + } + repos.jcenter(); + + String luceneVersion = VersionProperties.getLucene(); + if (luceneVersion.contains("-snapshot")) { + // extract the revision number from the version with a regex matcher + Matcher matcher = LUCENE_SNAPSHOT_REGEX.matcher(luceneVersion); + if (matcher.find() == false) { + throw new GradleException("Malformed lucene snapshot version: " + luceneVersion); + } + String revision = matcher.group(1); + MavenArtifactRepository luceneRepo = repos.maven(repo -> { + repo.setName("lucene-snapshots"); + repo.setUrl("https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + revision); + }); + repos.exclusiveContent(exclusiveRepo -> { + exclusiveRepo.filter( + descriptor -> descriptor.includeVersionByRegex("org\\.apache\\.lucene", ".*", ".*-snapshot-" + revision) + ); + exclusiveRepo.forRepositories(luceneRepo); + }); + } + } + + private static final List SECURE_URL_SCHEMES = Arrays.asList("file", "https", "s3"); + + private static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) { + if (uri != null && SECURE_URL_SCHEMES.contains(uri.getScheme()) == false) { + String url; + try { + url = uri.toURL().toString(); + } catch (MalformedURLException e) { + throw new IllegalStateException(e); + } + final String message = String.format( + Locale.ROOT, + "repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]", + repositoryName, + projectPath, + url + ); + throw new GradleException(message); + } + } + /** Adds compiler settings to the project */ public static void configureCompile(Project project) { project.getExtensions().getExtraProperties().set("compactProfile", "full"); diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginTests.java deleted file mode 100644 index 4389af00c03..00000000000 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginTests.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.gradle; - -import org.elasticsearch.gradle.test.GradleUnitTestCase; -import org.gradle.api.GradleException; -import org.junit.Test; - -import java.net.URI; -import java.net.URISyntaxException; - -public class BuildPluginTests extends GradleUnitTestCase { - - @Test(expected = GradleException.class) - public void testRepositoryURIThatUsesHttpScheme() throws URISyntaxException { - final URI uri = new URI("http://s3.amazonaws.com/artifacts.elastic.co/maven"); - BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri); - } - - public void testRepositoryThatUsesFileScheme() throws URISyntaxException { - final URI uri = new URI("file:/tmp/maven"); - BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri); - } - - public void testRepositoryURIThatUsesHttpsScheme() throws URISyntaxException { - final URI uri = new URI("https://s3.amazonaws.com/artifacts.elastic.co/maven"); - BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri); - } - - public void testRepositoryURIThatUsesS3Scheme() throws URISyntaxException { - final URI uri = new URI("s3://artifacts.elastic.co/maven"); - BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri); - } - -} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java index 8f0f2e69551..da291b24721 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java @@ -18,6 +18,8 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.hamcrest.CoreMatchers.containsString; + public abstract class GradleIntegrationTestCase extends GradleUnitTestCase { @Rule @@ -71,7 +73,7 @@ public abstract class GradleIntegrationTestCase extends GradleUnitTestCase { } protected void assertOutputContains(String output, String line) { - assertTrue("Expected the following line in output:\n\n" + line + "\n\nOutput is:\n" + output, output.contains(line)); + assertThat("Expected the following line in output:\n\n" + line + "\n\nOutput is:\n" + output, output, containsString(line)); } protected void assertOutputDoesNotContain(String output, String line) {