honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Move common repository configuration to java plugin (#57057) 

This commit moves the common maven repository configuration to the ES
java plugin. All java projects need this common set of repos. Note that
the Elastic download and maven repos are removed, as they are not
necessary anymore since distribution download was split into the
DistributionDownloadPlugin.
This commit is contained in:
Ryan Ernst 2020-05-26 14:10:07 -07:00 committed by Ryan Ernst
parent 81060820e9
commit a2864191d6
No known key found for this signature in database
GPG Key ID: 5F7EA39E15F54DCE
6 changed files with 83 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.gradle
View File

@ -54,8 +54,6 @@ allprojects {
description = "Elasticsearch subproject ${project.path}"
}
BuildPlugin.configureRepositories(project)
String licenseCommit
if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) {
licenseCommit = BuildParams.gitRevision ?: "master" // leniency for non git builds

120
buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
View File

@ -18,7 +18,6 @@
*/
package org.elasticsearch.gradle
import groovy.transform.CompileStatic
import org.apache.commons.io.IOUtils
import org.elasticsearch.gradle.info.BuildParams
@ -47,7 +46,6 @@ import org.gradle.api.plugins.ExtraPropertiesExtension
import org.gradle.api.plugins.JavaPlugin
import org.gradle.api.tasks.bundling.Jar
import org.gradle.api.tasks.testing.Test
import org.gradle.authentication.http.HttpHeaderAuthentication
import org.gradle.util.GradleVersion
import java.nio.charset.StandardCharsets
@ -90,7 +88,6 @@ class BuildPlugin implements Plugin<Project> {
project.getTasks().register("buildResources", ExportElasticsearchBuildResourcesTask)
configureRepositories(project)
project.extensions.getByType(ExtraPropertiesExtension).set('versions', VersionProperties.versions)
PrecommitTasks.create(project, true)
configureFips140(project)
@ -154,123 +151,6 @@ class BuildPlugin implements Plugin<Project> {
}
}
/**
* Makes dependencies non-transitive.
*
* Gradle allows setting all dependencies as non-transitive very easily.
* Sadly this mechanism does not translate into maven pom generation. In order
* to effectively make the pom act as if it has no transitive dependencies,
* we must exclude each transitive dependency of each direct dependency.
*
* Determining the transitive deps of a dependency which has been resolved as
* non-transitive is difficult because the process of resolving removes the
* transitive deps. To sidestep this issue, we create a configuration per
* direct dependency version. This specially named and unique configuration
* will contain all of the transitive dependencies of this particular
* dependency. We can then use this configuration during pom generation
* to iterate the transitive dependencies and add excludes.
*/
static void configureConfigurations(Project project) {
// we want to test compileOnly deps!
project.configurations.getByName(JavaPlugin.TEST_COMPILE_CONFIGURATION_NAME).extendsFrom(project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME))
// we are not shipping these jars, we act like dumb consumers of these things
if (project.path.startsWith(':test:fixtures') || project.path == ':build-tools') {
return
}
// fail on any conflicting dependency versions
project.configurations.all({ Configuration configuration ->
if (configuration.name.endsWith('Fixture')) {
// just a self contained test-fixture configuration, likely transitive and hellacious
return
}
configuration.resolutionStrategy {
failOnVersionConflict()
}
})
// force all dependencies added directly to compile/testCompile to be non-transitive, except for ES itself
Closure disableTransitiveDeps = { Dependency dep ->
if (dep instanceof ModuleDependency && !(dep instanceof ProjectDependency)
&& dep.group.startsWith('org.elasticsearch') == false) {
dep.transitive = false
}
}
project.configurations.getByName(JavaPlugin.COMPILE_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
project.configurations.getByName(JavaPlugin.TEST_COMPILE_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
project.configurations.getByName(JavaPlugin.RUNTIME_ONLY_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
}
/** Adds repositories used by ES dependencies */
static void configureRepositories(Project project) {
project.getRepositories().all { repository ->
if (repository instanceof MavenArtifactRepository) {
final MavenArtifactRepository maven = (MavenArtifactRepository) repository
assertRepositoryURIIsSecure(maven.name, project.path, maven.getUrl())
repository.getArtifactUrls().each { uri -> assertRepositoryURIIsSecure(maven.name, project.path, uri) }
} else if (repository instanceof IvyArtifactRepository) {
final IvyArtifactRepository ivy = (IvyArtifactRepository) repository
assertRepositoryURIIsSecure(ivy.name, project.path, ivy.getUrl())
}
}
RepositoryHandler repos = project.repositories
if (System.getProperty('repos.mavenLocal') != null) {
// with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is
// useful for development ie. bwc tests where we install stuff in the local repository
// such that we don't have to pass hardcoded files to gradle
repos.mavenLocal()
}
repos.jcenter()
repos.ivy { IvyArtifactRepository repo ->
repo.name = 'elasticsearch'
repo.url = 'https://artifacts.elastic.co/downloads'
repo.patternLayout { IvyPatternRepositoryLayout layout ->
layout.artifact 'elasticsearch/[module]-[revision](-[classifier]).[ext]'
}
// this header is not a credential but we hack the capability to send this header to avoid polluting our download stats
repo.credentials(HttpHeaderCredentials, { HttpHeaderCredentials creds ->
creds.name = 'X-Elastic-No-KPI'
creds.value = '1'
} as Action<HttpHeaderCredentials>)
repo.authentication.create('header', HttpHeaderAuthentication)
}
repos.maven { MavenArtifactRepository repo ->
repo.name = 'elastic'
repo.url = 'https://artifacts.elastic.co/maven'
}
String luceneVersion = VersionProperties.lucene
if (luceneVersion.contains('-snapshot')) {
// extract the revision number from the version with a regex matcher
List<String> matches = (luceneVersion =~ /\w+-snapshot-([a-z0-9]+)/).getAt(0) as List<String>
String revision = matches.get(1)
MavenArtifactRepository luceneRepo = repos.maven { MavenArtifactRepository repo ->
repo.name = 'lucene-snapshots'
repo.url = "https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/${revision}"
}
repos.exclusiveContent { ExclusiveContentRepository exclusiveRepo ->
exclusiveRepo.filter {
it.includeVersionByRegex(/org\.apache\.lucene/, '.*', ".*-snapshot-${revision}")
}
exclusiveRepo.forRepositories(luceneRepo)
}
}
}
static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) {
if (uri != null && ["file", "https", "s3"].contains(uri.getScheme()) == false) {
final String message = String.format(
Locale.ROOT,
"repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]",
repositoryName,
projectPath,
uri.toURL())
throw new GradleException(message)
}
}
private static class TestFailureReportingPlugin implements Plugin<Project> {
@Override
void apply(Project project) {

2
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy
View File

@ -62,7 +62,7 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
project.pluginManager.apply(TestClustersPlugin)
project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask)
BuildPlugin.configureRepositories(project)
ElasticsearchJavaPlugin.configureRepositories(project)
ElasticsearchJavaPlugin.configureTestTasks(project)
ElasticsearchJavaPlugin.configureInputNormalization(project)
BuildPlugin.configureFips140(project)

79
buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java
View File

@ -36,6 +36,9 @@ import org.gradle.api.artifacts.Configuration;
import org.gradle.api.artifacts.ModuleDependency;
import org.gradle.api.artifacts.ProjectDependency;
import org.gradle.api.artifacts.ResolutionStrategy;
import org.gradle.api.artifacts.dsl.RepositoryHandler;
import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
import org.gradle.api.artifacts.repositories.MavenArtifactRepository;
import org.gradle.api.file.FileCollection;
import org.gradle.api.plugins.BasePlugin;
import org.gradle.api.plugins.JavaPlugin;
@ -55,12 +58,18 @@ import org.gradle.language.base.plugins.LifecycleBasePlugin;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure;
import static org.elasticsearch.gradle.util.Util.toStringable;
@ -76,6 +85,7 @@ public class ElasticsearchJavaPlugin implements Plugin<Project> {
project.getPluginManager().apply(JavaPlugin.class);
configureConfigurations(project);
configureRepositories(project);
configureCompile(project);
configureInputNormalization(project);
configureTestTasks(project);
@ -136,6 +146,75 @@ public class ElasticsearchJavaPlugin implements Plugin<Project> {
disableTransitiveDeps.accept(JavaPlugin.RUNTIME_ONLY_CONFIGURATION_NAME);
}
private static final Pattern LUCENE_SNAPSHOT_REGEX = Pattern.compile("\\w+-snapshot-([a-z0-9]+)");
/** Adds repositories used by ES dependencies */
public static void configureRepositories(Project project) {
// ensure all repositories use secure urls
// TODO: remove this with gradle 7.0, which no longer allows insecure urls
project.getRepositories().all(repository -> {
if (repository instanceof MavenArtifactRepository) {
final MavenArtifactRepository maven = (MavenArtifactRepository) repository;
assertRepositoryURIIsSecure(maven.getName(), project.getPath(), maven.getUrl());
for (URI uri : maven.getArtifactUrls()) {
assertRepositoryURIIsSecure(maven.getName(), project.getPath(), uri);
}
} else if (repository instanceof IvyArtifactRepository) {
final IvyArtifactRepository ivy = (IvyArtifactRepository) repository;
assertRepositoryURIIsSecure(ivy.getName(), project.getPath(), ivy.getUrl());
}
});
RepositoryHandler repos = project.getRepositories();
if (System.getProperty("repos.mavenLocal") != null) {
// with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is
// useful for development ie. bwc tests where we install stuff in the local repository
// such that we don't have to pass hardcoded files to gradle
repos.mavenLocal();
}
repos.jcenter();
String luceneVersion = VersionProperties.getLucene();
if (luceneVersion.contains("-snapshot")) {
// extract the revision number from the version with a regex matcher
Matcher matcher = LUCENE_SNAPSHOT_REGEX.matcher(luceneVersion);
if (matcher.find() == false) {
throw new GradleException("Malformed lucene snapshot version: " + luceneVersion);
}
String revision = matcher.group(1);
MavenArtifactRepository luceneRepo = repos.maven(repo -> {
repo.setName("lucene-snapshots");
repo.setUrl("https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + revision);
});
repos.exclusiveContent(exclusiveRepo -> {
exclusiveRepo.filter(
descriptor -> descriptor.includeVersionByRegex("org\\.apache\\.lucene", ".*", ".*-snapshot-" + revision)
);
exclusiveRepo.forRepositories(luceneRepo);
});
}
}
private static final List<String> SECURE_URL_SCHEMES = Arrays.asList("file", "https", "s3");
private static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) {
if (uri != null && SECURE_URL_SCHEMES.contains(uri.getScheme()) == false) {
String url;
try {
url = uri.toURL().toString();
} catch (MalformedURLException e) {
throw new IllegalStateException(e);
}
final String message = String.format(
Locale.ROOT,
"repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]",
repositoryName,
projectPath,
url
);
throw new GradleException(message);
}
}
/** Adds compiler settings to the project */
public static void configureCompile(Project project) {
project.getExtensions().getExtraProperties().set("compactProfile", "full");

51
buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginTests.java
View File

@ -1,51 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle;
import org.elasticsearch.gradle.test.GradleUnitTestCase;
import org.gradle.api.GradleException;
import org.junit.Test;
import java.net.URI;
import java.net.URISyntaxException;
public class BuildPluginTests extends GradleUnitTestCase {
@Test(expected = GradleException.class)
public void testRepositoryURIThatUsesHttpScheme() throws URISyntaxException {
final URI uri = new URI("http://s3.amazonaws.com/artifacts.elastic.co/maven");
BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri);
}
public void testRepositoryThatUsesFileScheme() throws URISyntaxException {
final URI uri = new URI("file:/tmp/maven");
BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri);
}
public void testRepositoryURIThatUsesHttpsScheme() throws URISyntaxException {
final URI uri = new URI("https://s3.amazonaws.com/artifacts.elastic.co/maven");
BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri);
}
public void testRepositoryURIThatUsesS3Scheme() throws URISyntaxException {
final URI uri = new URI("s3://artifacts.elastic.co/maven");
BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri);
}
}

4
buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java
View File

@ -18,6 +18,8 @@ import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.hamcrest.CoreMatchers.containsString;
public abstract class GradleIntegrationTestCase extends GradleUnitTestCase {
@Rule
@ -71,7 +73,7 @@ public abstract class GradleIntegrationTestCase extends GradleUnitTestCase {
}
protected void assertOutputContains(String output, String line) {
assertTrue("Expected the following line in output:\n\n" + line + "\n\nOutput is:\n" + output, output.contains(line));
assertThat("Expected the following line in output:\n\n" + line + "\n\nOutput is:\n" + output, output, containsString(line));
}
protected void assertOutputDoesNotContain(String output, String line) {