mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-02-22 04:45:37 +00:00
- Fixes how libs in distribution are resolved - Required minor rework on common repository setup to allow distribution projects to resolve thirdparty artifacts - Use Default configurations when resolving tools for distribution packaging - Related to #57920
This commit is contained in:
Rene Groeschke
GitHub
parent
4c04fd1e05
commit
68dd431bc9
@ -24,6 +24,7 @@ import groovy.transform.CompileStatic
|
||||
import org.elasticsearch.gradle.BuildPlugin
|
||||
import org.elasticsearch.gradle.ElasticsearchJavaPlugin
|
||||
import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
|
||||
import org.elasticsearch.gradle.RepositoriesSetupPlugin
|
||||
import org.elasticsearch.gradle.info.BuildParams
|
||||
import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
|
||||
import org.elasticsearch.gradle.precommit.PrecommitTasks
|
||||
@ -59,9 +60,9 @@ class StandaloneRestTestPlugin implements Plugin<Project> {
|
||||
project.rootProject.pluginManager.apply(GlobalBuildInfoPlugin)
|
||||
project.pluginManager.apply(JavaBasePlugin)
|
||||
project.pluginManager.apply(TestClustersPlugin)
|
||||
project.pluginManager.apply(RepositoriesSetupPlugin)
|
||||
|
||||
project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask)
|
||||
ElasticsearchJavaPlugin.configureRepositories(project)
|
||||
ElasticsearchJavaPlugin.configureTestTasks(project)
|
||||
ElasticsearchJavaPlugin.configureInputNormalization(project)
|
||||
ElasticsearchJavaPlugin.configureCompile(project)
|
||||
|
||||
@ -36,9 +36,6 @@ import org.gradle.api.artifacts.Configuration;
|
||||
import org.gradle.api.artifacts.ModuleDependency;
|
||||
import org.gradle.api.artifacts.ProjectDependency;
|
||||
import org.gradle.api.artifacts.ResolutionStrategy;
|
||||
import org.gradle.api.artifacts.dsl.RepositoryHandler;
|
||||
import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
|
||||
import org.gradle.api.artifacts.repositories.MavenArtifactRepository;
|
||||
import org.gradle.api.execution.TaskActionListener;
|
||||
import org.gradle.api.file.FileCollection;
|
||||
import org.gradle.api.plugins.BasePlugin;
|
||||
@ -59,16 +56,10 @@ import org.gradle.language.base.plugins.LifecycleBasePlugin;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URI;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Function;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure;
|
||||
import static org.elasticsearch.gradle.util.Util.toStringable;
|
||||
@ -83,11 +74,11 @@ public class ElasticsearchJavaPlugin implements Plugin<Project> {
|
||||
project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
|
||||
// apply global test task failure listener
|
||||
project.getRootProject().getPluginManager().apply(TestFailureReportingPlugin.class);
|
||||
|
||||
// common repositories setup
|
||||
project.getPluginManager().apply(RepositoriesSetupPlugin.class);
|
||||
project.getPluginManager().apply(JavaLibraryPlugin.class);
|
||||
|
||||
configureConfigurations(project);
|
||||
configureRepositories(project);
|
||||
configureCompile(project);
|
||||
configureInputNormalization(project);
|
||||
configureTestTasks(project);
|
||||
@ -149,77 +140,6 @@ public class ElasticsearchJavaPlugin implements Plugin<Project> {
|
||||
disableTransitiveDeps.accept(JavaPlugin.TEST_IMPLEMENTATION_CONFIGURATION_NAME);
|
||||
}
|
||||
|
||||
private static final Pattern LUCENE_SNAPSHOT_REGEX = Pattern.compile("\\w+-snapshot-([a-z0-9]+)");
|
||||
|
||||
/**
|
||||
* Adds repositories used by ES dependencies
|
||||
*/
|
||||
public static void configureRepositories(Project project) {
|
||||
// ensure all repositories use secure urls
|
||||
// TODO: remove this with gradle 7.0, which no longer allows insecure urls
|
||||
project.getRepositories().all(repository -> {
|
||||
if (repository instanceof MavenArtifactRepository) {
|
||||
final MavenArtifactRepository maven = (MavenArtifactRepository) repository;
|
||||
assertRepositoryURIIsSecure(maven.getName(), project.getPath(), maven.getUrl());
|
||||
for (URI uri : maven.getArtifactUrls()) {
|
||||
assertRepositoryURIIsSecure(maven.getName(), project.getPath(), uri);
|
||||
}
|
||||
} else if (repository instanceof IvyArtifactRepository) {
|
||||
final IvyArtifactRepository ivy = (IvyArtifactRepository) repository;
|
||||
assertRepositoryURIIsSecure(ivy.getName(), project.getPath(), ivy.getUrl());
|
||||
}
|
||||
});
|
||||
RepositoryHandler repos = project.getRepositories();
|
||||
if (System.getProperty("repos.mavenLocal") != null) {
|
||||
// with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is
|
||||
// useful for development ie. bwc tests where we install stuff in the local repository
|
||||
// such that we don't have to pass hardcoded files to gradle
|
||||
repos.mavenLocal();
|
||||
}
|
||||
repos.jcenter();
|
||||
|
||||
String luceneVersion = VersionProperties.getLucene();
|
||||
if (luceneVersion.contains("-snapshot")) {
|
||||
// extract the revision number from the version with a regex matcher
|
||||
Matcher matcher = LUCENE_SNAPSHOT_REGEX.matcher(luceneVersion);
|
||||
if (matcher.find() == false) {
|
||||
throw new GradleException("Malformed lucene snapshot version: " + luceneVersion);
|
||||
}
|
||||
String revision = matcher.group(1);
|
||||
MavenArtifactRepository luceneRepo = repos.maven(repo -> {
|
||||
repo.setName("lucene-snapshots");
|
||||
repo.setUrl("https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + revision);
|
||||
});
|
||||
repos.exclusiveContent(exclusiveRepo -> {
|
||||
exclusiveRepo.filter(
|
||||
descriptor -> descriptor.includeVersionByRegex("org\\.apache\\.lucene", ".*", ".*-snapshot-" + revision)
|
||||
);
|
||||
exclusiveRepo.forRepositories(luceneRepo);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private static final List<String> SECURE_URL_SCHEMES = Arrays.asList("file", "https", "s3");
|
||||
|
||||
private static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) {
|
||||
if (uri != null && SECURE_URL_SCHEMES.contains(uri.getScheme()) == false) {
|
||||
String url;
|
||||
try {
|
||||
url = uri.toURL().toString();
|
||||
} catch (MalformedURLException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
final String message = String.format(
|
||||
Locale.ROOT,
|
||||
"repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]",
|
||||
repositoryName,
|
||||
projectPath,
|
||||
url
|
||||
);
|
||||
throw new GradleException(message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds compiler settings to the project
|
||||
*/
|
||||
|
||||
@ -0,0 +1,114 @@
|
||||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.gradle;
|
||||
|
||||
import org.gradle.api.GradleException;
|
||||
import org.gradle.api.Plugin;
|
||||
import org.gradle.api.Project;
|
||||
import org.gradle.api.artifacts.dsl.RepositoryHandler;
|
||||
import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
|
||||
import org.gradle.api.artifacts.repositories.MavenArtifactRepository;
|
||||
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URI;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class RepositoriesSetupPlugin implements Plugin<Project> {
|
||||
|
||||
private static final List<String> SECURE_URL_SCHEMES = Arrays.asList("file", "https", "s3");
|
||||
private static final Pattern LUCENE_SNAPSHOT_REGEX = Pattern.compile("\\w+-snapshot-([a-z0-9]+)");
|
||||
|
||||
@Override
|
||||
public void apply(Project project) {
|
||||
configureRepositories(project);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds repositories used by ES projects and dependencies
|
||||
*/
|
||||
public static void configureRepositories(Project project) {
|
||||
// ensure all repositories use secure urls
|
||||
// TODO: remove this with gradle 7.0, which no longer allows insecure urls
|
||||
project.getRepositories().all(repository -> {
|
||||
if (repository instanceof MavenArtifactRepository) {
|
||||
final MavenArtifactRepository maven = (MavenArtifactRepository) repository;
|
||||
assertRepositoryURIIsSecure(maven.getName(), project.getPath(), maven.getUrl());
|
||||
for (URI uri : maven.getArtifactUrls()) {
|
||||
assertRepositoryURIIsSecure(maven.getName(), project.getPath(), uri);
|
||||
}
|
||||
} else if (repository instanceof IvyArtifactRepository) {
|
||||
final IvyArtifactRepository ivy = (IvyArtifactRepository) repository;
|
||||
assertRepositoryURIIsSecure(ivy.getName(), project.getPath(), ivy.getUrl());
|
||||
}
|
||||
});
|
||||
RepositoryHandler repos = project.getRepositories();
|
||||
if (System.getProperty("repos.mavenLocal") != null) {
|
||||
// with -Drepos.mavenLocal=true we can force checking the local .m2 repo which is
|
||||
// useful for development ie. bwc tests where we install stuff in the local repository
|
||||
// such that we don't have to pass hardcoded files to gradle
|
||||
repos.mavenLocal();
|
||||
}
|
||||
repos.jcenter();
|
||||
|
||||
String luceneVersion = VersionProperties.getLucene();
|
||||
if (luceneVersion.contains("-snapshot")) {
|
||||
// extract the revision number from the version with a regex matcher
|
||||
Matcher matcher = LUCENE_SNAPSHOT_REGEX.matcher(luceneVersion);
|
||||
if (matcher.find() == false) {
|
||||
throw new GradleException("Malformed lucene snapshot version: " + luceneVersion);
|
||||
}
|
||||
String revision = matcher.group(1);
|
||||
MavenArtifactRepository luceneRepo = repos.maven(repo -> {
|
||||
repo.setName("lucene-snapshots");
|
||||
repo.setUrl("https://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + revision);
|
||||
});
|
||||
repos.exclusiveContent(exclusiveRepo -> {
|
||||
exclusiveRepo.filter(
|
||||
descriptor -> descriptor.includeVersionByRegex("org\\.apache\\.lucene", ".*", ".*-snapshot-" + revision)
|
||||
);
|
||||
exclusiveRepo.forRepositories(luceneRepo);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) {
|
||||
if (uri != null && SECURE_URL_SCHEMES.contains(uri.getScheme()) == false) {
|
||||
String url;
|
||||
try {
|
||||
url = uri.toURL().toString();
|
||||
} catch (MalformedURLException e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
final String message = String.format(
|
||||
Locale.ROOT,
|
||||
"repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]",
|
||||
repositoryName,
|
||||
projectPath,
|
||||
url
|
||||
);
|
||||
throw new GradleException(message);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,20 @@
|
||||
#
|
||||
# Licensed to Elasticsearch under one or more contributor
|
||||
# license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright
|
||||
# ownership. Elasticsearch licenses this file to you under
|
||||
# the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
|
||||
implementation-class=org.elasticsearch.gradle.RepositoriesSetupPlugin
|
||||
@ -259,6 +259,7 @@ copyModule(processSystemdOutputs, project(':modules:systemd'))
|
||||
configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
|
||||
|
||||
apply plugin: 'elasticsearch.jdk-download'
|
||||
apply plugin: 'elasticsearch.repositories'
|
||||
|
||||
// Setup all required JDKs
|
||||
project.jdks {
|
||||
@ -278,6 +279,31 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
|
||||
/*****************************************************************************
|
||||
* Properties to expand when copying packaging files *
|
||||
*****************************************************************************/
|
||||
configurations {
|
||||
['libs', 'libsPluginCli', 'libsKeystoreCli', 'libsSecurityCli'].each {
|
||||
create(it) {
|
||||
canBeConsumed = false
|
||||
canBeResolved = true
|
||||
attributes {
|
||||
attribute(Category.CATEGORY_ATTRIBUTE, objects.named(Category, Category.LIBRARY))
|
||||
attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage, Usage.JAVA_RUNTIME))
|
||||
attribute(Bundling.BUNDLING_ATTRIBUTE, objects.named(Bundling, Bundling.EXTERNAL))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dependencies {
|
||||
libs project(':server')
|
||||
libs project(':libs:elasticsearch-plugin-classloader')
|
||||
libs project(':distribution:tools:java-version-checker')
|
||||
libs project(':distribution:tools:launchers')
|
||||
|
||||
libsPluginCli project(':distribution:tools:plugin-cli')
|
||||
libsKeystoreCli project(path: ':distribution:tools:keystore-cli')
|
||||
libsSecurityCli project(':x-pack:plugin:security:cli')
|
||||
}
|
||||
|
||||
project.ext {
|
||||
|
||||
/*****************************************************************************
|
||||
@ -286,22 +312,16 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
|
||||
libFiles = { oss ->
|
||||
copySpec {
|
||||
// delay by using closures, since they have not yet been configured, so no jar task exists yet
|
||||
from { project(':server').jar }
|
||||
from { project(':server').configurations.runtimeClasspath }
|
||||
from { project(':libs:elasticsearch-plugin-classloader').jar }
|
||||
from { project(':distribution:tools:java-version-checker').jar }
|
||||
from { project(':distribution:tools:launchers').jar }
|
||||
from(configurations.libs)
|
||||
into('tools/plugin-cli') {
|
||||
from { project(':distribution:tools:plugin-cli').jar }
|
||||
from { project(':distribution:tools:plugin-cli').configurations.runtimeClasspath }
|
||||
from(configurations.libsPluginCli)
|
||||
}
|
||||
into('tools/keystore-cli') {
|
||||
from { project(':distribution:tools:keystore-cli').jar }
|
||||
from(configurations.libsKeystoreCli)
|
||||
}
|
||||
if (oss == false) {
|
||||
into('tools/security-cli') {
|
||||
from { project(':x-pack:plugin:security:cli').jar }
|
||||
from { project(':x-pack:plugin:security:cli').configurations.runtimeClasspath }
|
||||
from(configurations.libsSecurityCli)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -613,14 +633,14 @@ subprojects {
|
||||
}
|
||||
|
||||
['archives:windows-zip', 'archives:oss-windows-zip',
|
||||
'archives:darwin-tar', 'archives:oss-darwin-tar',
|
||||
'archives:linux-aarch64-tar', 'archives:oss-linux-aarch64-tar',
|
||||
'archives:linux-tar', 'archives:oss-linux-tar',
|
||||
'archives:integ-test-zip',
|
||||
'packages:rpm', 'packages:deb',
|
||||
'packages:aarch64-rpm', 'packages:aarch64-deb',
|
||||
'packages:oss-rpm', 'packages:oss-deb',
|
||||
'packages:aarch64-oss-rpm', 'packages:aarch64-oss-deb'
|
||||
'archives:darwin-tar', 'archives:oss-darwin-tar',
|
||||
'archives:linux-aarch64-tar', 'archives:oss-linux-aarch64-tar',
|
||||
'archives:linux-tar', 'archives:oss-linux-tar',
|
||||
'archives:integ-test-zip',
|
||||
'packages:rpm', 'packages:deb',
|
||||
'packages:aarch64-rpm', 'packages:aarch64-deb',
|
||||
'packages:oss-rpm', 'packages:oss-deb',
|
||||
'packages:aarch64-oss-rpm', 'packages:aarch64-oss-deb'
|
||||
].forEach { subName ->
|
||||
Project subproject = project("${project.path}:${subName}")
|
||||
Configuration configuration = configurations.create(subproject.name)
|
||||
|
||||
@ -21,7 +21,7 @@ import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
|
||||
apply plugin: 'elasticsearch.build'
|
||||
|
||||
dependencies {
|
||||
api parent.project('java-version-checker')
|
||||
compileOnly project(':distribution:tools:java-version-checker')
|
||||
testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
|
||||
testImplementation "junit:junit:${versions.junit}"
|
||||
testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user