diff --git a/build.gradle b/build.gradle index 35a12111d72..4618d0f7cf4 100644 --- a/build.gradle +++ b/build.gradle @@ -54,8 +54,6 @@ allprojects { description = "Elasticsearch subproject ${project.path}" } -BuildPlugin.configureRepositories(project) - String licenseCommit if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) { licenseCommit = BuildParams.gitRevision ?: "master" // leniency for non git builds diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index e3266f00061..daaa15828a9 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -18,7 +18,6 @@ */ package org.elasticsearch.gradle - import groovy.transform.CompileStatic import org.apache.commons.io.IOUtils import org.elasticsearch.gradle.info.BuildParams @@ -47,7 +46,6 @@ import org.gradle.api.plugins.ExtraPropertiesExtension import org.gradle.api.plugins.JavaPlugin import org.gradle.api.tasks.bundling.Jar import org.gradle.api.tasks.testing.Test -import org.gradle.authentication.http.HttpHeaderAuthentication import org.gradle.util.GradleVersion import java.nio.charset.StandardCharsets @@ -90,7 +88,6 @@ class BuildPlugin implements Plugin { project.getTasks().register("buildResources", ExportElasticsearchBuildResourcesTask) - configureRepositories(project) project.extensions.getByType(ExtraPropertiesExtension).set('versions', VersionProperties.versions) PrecommitTasks.create(project, true) configureFips140(project) @@ -154,123 +151,6 @@ class BuildPlugin implements Plugin { } } - /** - * Makes dependencies non-transitive. - * - * Gradle allows setting all dependencies as non-transitive very easily. - * Sadly this mechanism does not translate into maven pom generation. In order - * to effectively make the pom act as if it has no transitive dependencies, - * we must exclude each transitive dependency of each direct dependency. - * - * Determining the transitive deps of a dependency which has been resolved as - * non-transitive is difficult because the process of resolving removes the - * transitive deps. To sidestep this issue, we create a configuration per - * direct dependency version. This specially named and unique configuration - * will contain all of the transitive dependencies of this particular - * dependency. We can then use this configuration during pom generation - * to iterate the transitive dependencies and add excludes. - */ - static void configureConfigurations(Project project) { - // we want to test compileOnly deps! - project.configurations.getByName(JavaPlugin.TEST_COMPILE_CONFIGURATION_NAME).extendsFrom(project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME)) - - // we are not shipping these jars, we act like dumb consumers of these things - if (project.path.startsWith(':test:fixtures') || project.path == ':build-tools') { - return - } - // fail on any conflicting dependency versions - project.configurations.all({ Configuration configuration -> - if (configuration.name.endsWith('Fixture')) { - // just a self contained test-fixture configuration, likely transitive and hellacious - return - } - configuration.resolutionStrategy { - failOnVersionConflict() - } - }) - - // force all dependencies added directly to compile/testCompile to be non-transitive, except for ES itself - Closure disableTransitiveDeps = { Dependency dep -> - if (dep instanceof ModuleDependency && !(dep instanceof ProjectDependency) - && dep.group.startsWith('org.elasticsearch') == false) { - dep.transitive = false - } - } - - project.configurations.getByName(JavaPlugin.COMPILE_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps) - project.configurations.getByName(JavaPlugin.TEST_COMPILE_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps) - project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps) - project.configurations.getByName(JavaPlugin.RUNTIME_ONLY_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps) - } - - /** Adds repositories used by ES dependencies */ - static void configureRepositories(Project project) { - project.getRepositories().all { repository -> - if (repository instanceof MavenArtifactRepository) { - final MavenArtifactRepository maven = (MavenArtifactRepository) repository - assertRepositoryURIIsSecure(maven.name, project.path, maven.getUrl()) - repository.getArtifactUrls().each { uri -> assertRepositoryURIIsSecure(maven.name, project.path, uri) } - } else if (repository instanceof IvyArtifactRepository) { - final IvyArtifactRepository ivy = (IvyArtifactRepository) repository - assertRepositoryURIIsSecure(ivy.name, project.path, ivy.getUrl()) - } - } - RepositoryHandler repos = project.repositories - if (System.getProperty('repos.mavenLocal') != null) { - // with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is - // useful for development ie. bwc tests where we install stuff in the local repository - // such that we don't have to pass hardcoded files to gradle - repos.mavenLocal() - } - repos.jcenter() - repos.ivy { IvyArtifactRepository repo -> - repo.name = 'elasticsearch' - repo.url = 'https://artifacts.elastic.co/downloads' - repo.patternLayout { IvyPatternRepositoryLayout layout -> - layout.artifact 'elasticsearch/[module]-[revision](-[classifier]).[ext]' - } - // this header is not a credential but we hack the capability to send this header to avoid polluting our download stats - repo.credentials(HttpHeaderCredentials, { HttpHeaderCredentials creds -> - creds.name = 'X-Elastic-No-KPI' - creds.value = '1' - } as Action) - repo.authentication.create('header', HttpHeaderAuthentication) - } - repos.maven { MavenArtifactRepository repo -> - repo.name = 'elastic' - repo.url = 'https://artifacts.elastic.co/maven' - } - String luceneVersion = VersionProperties.lucene - if (luceneVersion.contains('-snapshot')) { - // extract the revision number from the version with a regex matcher - List matches = (luceneVersion =~ /\w+-snapshot-([a-z0-9]+)/).getAt(0) as List - String revision = matches.get(1) - MavenArtifactRepository luceneRepo = repos.maven { MavenArtifactRepository repo -> - repo.name = 'lucene-snapshots' - repo.url = "https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}" - } - repos.exclusiveContent { ExclusiveContentRepository exclusiveRepo -> - exclusiveRepo.filter { - it.includeVersionByRegex(/org\.apache\.lucene/, '.*', ".*-snapshot-${revision}") - } - exclusiveRepo.forRepositories(luceneRepo) - } - } - } - - static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) { - if (uri != null && ["file", "https", "s3"].contains(uri.getScheme()) == false) { - final String message = String.format( - Locale.ROOT, - "repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]", - repositoryName, - projectPath, - uri.toURL()) - throw new GradleException(message) - } - } - - private static class TestFailureReportingPlugin implements Plugin { @Override void apply(Project project) { diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy index 23298344936..c4bf0e4db1d 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy @@ -62,7 +62,7 @@ class StandaloneRestTestPlugin implements Plugin { project.pluginManager.apply(TestClustersPlugin) project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask) - BuildPlugin.configureRepositories(project) + ElasticsearchJavaPlugin.configureRepositories(project) ElasticsearchJavaPlugin.configureTestTasks(project) ElasticsearchJavaPlugin.configureInputNormalization(project) BuildPlugin.configureFips140(project) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java index e3d6d12e356..279596fe530 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java @@ -36,6 +36,9 @@ import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.ModuleDependency; import org.gradle.api.artifacts.ProjectDependency; import org.gradle.api.artifacts.ResolutionStrategy; +import org.gradle.api.artifacts.dsl.RepositoryHandler; +import org.gradle.api.artifacts.repositories.IvyArtifactRepository; +import org.gradle.api.artifacts.repositories.MavenArtifactRepository; import org.gradle.api.file.FileCollection; import org.gradle.api.plugins.BasePlugin; import org.gradle.api.plugins.JavaPlugin; @@ -55,12 +58,18 @@ import org.gradle.language.base.plugins.LifecycleBasePlugin; import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; +import java.net.MalformedURLException; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.function.Consumer; import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure; import static org.elasticsearch.gradle.util.Util.toStringable; @@ -76,6 +85,7 @@ public class ElasticsearchJavaPlugin implements Plugin { project.getPluginManager().apply(JavaPlugin.class); configureConfigurations(project); + configureRepositories(project); configureCompile(project); configureInputNormalization(project); configureTestTasks(project); @@ -136,6 +146,75 @@ public class ElasticsearchJavaPlugin implements Plugin { disableTransitiveDeps.accept(JavaPlugin.RUNTIME_ONLY_CONFIGURATION_NAME); } + private static final Pattern LUCENE_SNAPSHOT_REGEX = Pattern.compile("\\w+-snapshot-([a-z0-9]+)"); + + /** Adds repositories used by ES dependencies */ + public static void configureRepositories(Project project) { + // ensure all repositories use secure urls + // TODO: remove this with gradle 7.0, which no longer allows insecure urls + project.getRepositories().all(repository -> { + if (repository instanceof MavenArtifactRepository) { + final MavenArtifactRepository maven = (MavenArtifactRepository) repository; + assertRepositoryURIIsSecure(maven.getName(), project.getPath(), maven.getUrl()); + for (URI uri : maven.getArtifactUrls()) { + assertRepositoryURIIsSecure(maven.getName(), project.getPath(), uri); + } + } else if (repository instanceof IvyArtifactRepository) { + final IvyArtifactRepository ivy = (IvyArtifactRepository) repository; + assertRepositoryURIIsSecure(ivy.getName(), project.getPath(), ivy.getUrl()); + } + }); + RepositoryHandler repos = project.getRepositories(); + if (System.getProperty("repos.mavenLocal") != null) { + // with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is + // useful for development ie. bwc tests where we install stuff in the local repository + // such that we don't have to pass hardcoded files to gradle + repos.mavenLocal(); + } + repos.jcenter(); + + String luceneVersion = VersionProperties.getLucene(); + if (luceneVersion.contains("-snapshot")) { + // extract the revision number from the version with a regex matcher + Matcher matcher = LUCENE_SNAPSHOT_REGEX.matcher(luceneVersion); + if (matcher.find() == false) { + throw new GradleException("Malformed lucene snapshot version: " + luceneVersion); + } + String revision = matcher.group(1); + MavenArtifactRepository luceneRepo = repos.maven(repo -> { + repo.setName("lucene-snapshots"); + repo.setUrl("https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + revision); + }); + repos.exclusiveContent(exclusiveRepo -> { + exclusiveRepo.filter( + descriptor -> descriptor.includeVersionByRegex("org\\.apache\\.lucene", ".*", ".*-snapshot-" + revision) + ); + exclusiveRepo.forRepositories(luceneRepo); + }); + } + } + + private static final List SECURE_URL_SCHEMES = Arrays.asList("file", "https", "s3"); + + private static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) { + if (uri != null && SECURE_URL_SCHEMES.contains(uri.getScheme()) == false) { + String url; + try { + url = uri.toURL().toString(); + } catch (MalformedURLException e) { + throw new IllegalStateException(e); + } + final String message = String.format( + Locale.ROOT, + "repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]", + repositoryName, + projectPath, + url + ); + throw new GradleException(message); + } + } + /** Adds compiler settings to the project */ public static void configureCompile(Project project) { project.getExtensions().getExtraProperties().set("compactProfile", "full"); diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginTests.java deleted file mode 100644 index 4389af00c03..00000000000 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginTests.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.gradle; - -import org.elasticsearch.gradle.test.GradleUnitTestCase; -import org.gradle.api.GradleException; -import org.junit.Test; - -import java.net.URI; -import java.net.URISyntaxException; - -public class BuildPluginTests extends GradleUnitTestCase { - - @Test(expected = GradleException.class) - public void testRepositoryURIThatUsesHttpScheme() throws URISyntaxException { - final URI uri = new URI("http://s3.amazonaws.com/artifacts.elastic.co/maven"); - BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri); - } - - public void testRepositoryThatUsesFileScheme() throws URISyntaxException { - final URI uri = new URI("file:/tmp/maven"); - BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri); - } - - public void testRepositoryURIThatUsesHttpsScheme() throws URISyntaxException { - final URI uri = new URI("https://s3.amazonaws.com/artifacts.elastic.co/maven"); - BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri); - } - - public void testRepositoryURIThatUsesS3Scheme() throws URISyntaxException { - final URI uri = new URI("s3://artifacts.elastic.co/maven"); - BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri); - } - -} diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java index 8f0f2e69551..da291b24721 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java @@ -18,6 +18,8 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.hamcrest.CoreMatchers.containsString; + public abstract class GradleIntegrationTestCase extends GradleUnitTestCase { @Rule @@ -71,7 +73,7 @@ public abstract class GradleIntegrationTestCase extends GradleUnitTestCase { } protected void assertOutputContains(String output, String line) { - assertTrue("Expected the following line in output:\n\n" + line + "\n\nOutput is:\n" + output, output.contains(line)); + assertThat("Expected the following line in output:\n\n" + line + "\n\nOutput is:\n" + output, output, containsString(line)); } protected void assertOutputDoesNotContain(String output, String line) {