honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support downloading JDKs with legacy version format (#51587) 

(cherry picked from commit b70f925ccb735dc84d59598de06df6bf35bd4bdc)
This commit is contained in:
Mark Vieira 2020-01-30 09:34:55 -08:00
parent 36b2663e98
commit bb963b996c
No known key found for this signature in database
GPG Key ID: CA947EF7E6D4B105
5 changed files with 208 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
View File

@ -45,11 +45,12 @@ import org.gradle.authentication.http.HttpHeaderAuthentication;
import java.io.File;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.function.Supplier;
import static org.elasticsearch.gradle.Util.capitalize;
/**
* A plugin to manage getting and extracting distributions of Elasticsearch.
*
@ -328,10 +329,6 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
);
}
private static String capitalize(String s) {
return s.substring(0, 1).toUpperCase(Locale.ROOT) + s.substring(1);
}
private static String extractTaskName(ElasticsearchDistribution distribution) {
String taskName = "extractElasticsearch";
if (distribution.getType() != Type.INTEG_TEST_ZIP) {

63
buildSrc/src/main/java/org/elasticsearch/gradle/Jdk.java
View File

@ -20,8 +20,8 @@
package org.elasticsearch.gradle;
import org.gradle.api.Buildable;
import org.gradle.api.Project;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.model.ObjectFactory;
import org.gradle.api.provider.Property;
import org.gradle.api.tasks.TaskDependency;
@ -30,13 +30,15 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class Jdk implements Buildable, Iterable<File> {
private static final List<String> ALLOWED_VENDORS = Collections.unmodifiableList(Arrays.asList("adoptopenjdk", "openjdk"));
static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?");
private static final List<String> ALLOWED_PLATFORMS = Collections.unmodifiableList(Arrays.asList("darwin", "linux", "windows", "mac"));
private static final Pattern VERSION_PATTERN = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?");
private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)\\+(b\\d+?)(@([a-f0-9]{32}))?");
private final String name;
private final Configuration configuration;
@ -44,13 +46,17 @@ public class Jdk implements Buildable, Iterable<File> {
private final Property<String> vendor;
private final Property<String> version;
private final Property<String> platform;
private String baseVersion;
private String major;
private String build;
private String hash;
Jdk(String name, Project project) {
Jdk(String name, Configuration configuration, ObjectFactory objectFactory) {
this.name = name;
this.configuration = project.getConfigurations().create("jdk_" + name);
this.vendor = project.getObjects().property(String.class);
this.version = project.getObjects().property(String.class);
this.platform = project.getObjects().property(String.class);
this.configuration = configuration;
this.vendor = objectFactory.property(String.class);
this.version = objectFactory.property(String.class);
this.platform = objectFactory.property(String.class);
}
public String getName() {
@ -73,9 +79,10 @@ public class Jdk implements Buildable, Iterable<File> {
}
public void setVersion(String version) {
if (VERSION_PATTERN.matcher(version).matches() == false) {
if (VERSION_PATTERN.matcher(version).matches() == false && LEGACY_VERSION_PATTERN.matcher(version).matches() == false) {
throw new IllegalArgumentException("malformed version [" + version + "] for jdk [" + name + "]");
}
parseVersion(version);
this.version.set(version);
}
@ -92,15 +99,30 @@ public class Jdk implements Buildable, Iterable<File> {
this.platform.set(platform);
}
// pkg private, for internal use
Configuration getConfiguration() {
return configuration;
public String getBaseVersion() {
return baseVersion;
}
public String getMajor() {
return major;
}
public String getBuild() {
return build;
}
public String getHash() {
return hash;
}
public String getPath() {
return configuration.getSingleFile().toString();
}
public String getConfigurationName() {
return configuration.getName();
}
@Override
public String toString() {
return getPath();
@ -143,4 +165,23 @@ public class Jdk implements Buildable, Iterable<File> {
return configuration.iterator();
}
private void parseVersion(String version) {
// decompose the bundled jdk version, broken into elements as: [feature, interim, update, build]
// Note the "patch" version is not yet handled here, as it has not yet been used by java.
Matcher jdkVersionMatcher = VERSION_PATTERN.matcher(version);
if (jdkVersionMatcher.matches() == false) {
// Try again with the pre-Java9 version format
jdkVersionMatcher = LEGACY_VERSION_PATTERN.matcher(version);
if (jdkVersionMatcher.matches() == false) {
throw new IllegalArgumentException("Malformed jdk version [" + version + "]");
}
}
baseVersion = jdkVersionMatcher.group(1) + (jdkVersionMatcher.group(2) != null ? (jdkVersionMatcher.group(2)) : "");
major = jdkVersionMatcher.group(1);
build = jdkVersionMatcher.group(3);
hash = jdkVersionMatcher.group(5);
}
}

318
buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java
View File

@ -21,11 +21,11 @@ package org.elasticsearch.gradle;
import org.elasticsearch.gradle.tar.SymbolicLinkPreservingUntarTask;
import org.gradle.api.Action;
import org.gradle.api.GradleException;
import org.gradle.api.NamedDomainObjectContainer;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
import org.gradle.api.Task;
import org.gradle.api.UnknownTaskException;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.artifacts.ConfigurationContainer;
import org.gradle.api.artifacts.dsl.DependencyHandler;
@ -37,44 +37,46 @@ import org.gradle.api.file.FileTree;
import org.gradle.api.file.RelativePath;
import org.gradle.api.provider.Provider;
import org.gradle.api.tasks.Copy;
import org.gradle.api.tasks.TaskProvider;
import java.io.File;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.function.Supplier;
import java.util.regex.Matcher;
import java.util.stream.StreamSupport;
import static org.elasticsearch.gradle.Util.capitalize;
import static org.elasticsearch.gradle.tool.Boilerplate.findByName;
import static org.elasticsearch.gradle.tool.Boilerplate.maybeCreate;
public class JdkDownloadPlugin implements Plugin<Project> {
private static final String REPO_NAME_PREFIX = "jdk_repo_";
private static final String CONTAINER_NAME = "jdks";
private static final String EXTENSION_NAME = "jdks";
@Override
public void apply(Project project) {
NamedDomainObjectContainer<Jdk> jdksContainer = project.container(Jdk.class, name -> new Jdk(name, project));
project.getExtensions().add(CONTAINER_NAME, jdksContainer);
NamedDomainObjectContainer<Jdk> jdksContainer = project.container(
Jdk.class,
name -> new Jdk(name, project.getConfigurations().create("jdk_" + name), project.getObjects())
);
project.getExtensions().add(EXTENSION_NAME, jdksContainer);
project.afterEvaluate(p -> {
for (Jdk jdk : jdksContainer) {
jdk.finalizeValues();
String vendor = jdk.getVendor();
String version = jdk.getVersion();
String platform = jdk.getPlatform();
// depend on the jdk directory "artifact" from the root project
DependencyHandler dependencies = project.getDependencies();
Map<String, Object> depConfig = new HashMap<>();
depConfig.put("path", ":"); // root project
depConfig.put("configuration", configName("extracted_jdk", vendor, version, platform));
dependencies.add(jdk.getConfiguration().getName(), dependencies.project(depConfig));
depConfig.put("configuration", configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform()));
project.getDependencies().add(jdk.getConfigurationName(), dependencies.project(depConfig));
// ensure a root level jdk download task exists
setupRootJdkDownload(project.getRootProject(), platform, vendor, version);
setupRootJdkDownload(project.getRootProject(), jdk);
}
});
@ -91,141 +93,117 @@ public class JdkDownloadPlugin implements Plugin<Project> {
@SuppressWarnings("unchecked")
public static NamedDomainObjectContainer<Jdk> getContainer(Project project) {
return (NamedDomainObjectContainer<Jdk>) project.getExtensions().getByName(CONTAINER_NAME);
return (NamedDomainObjectContainer<Jdk>) project.getExtensions().getByName(EXTENSION_NAME);
}
private static void setupRootJdkDownload(Project rootProject, String platform, String vendor, String version) {
String extractTaskName = "extract" + capitalize(platform) + "Jdk-" + vendor + "-" + version;
// NOTE: this is *horrendous*, but seems to be the only way to check for the existence of a registered task
try {
rootProject.getTasks().named(extractTaskName);
// already setup this version
return;
} catch (UnknownTaskException e) {
// fall through: register the task
}
private static void setupRootJdkDownload(Project rootProject, Jdk jdk) {
String extractTaskName = "extract" + capitalize(jdk.getPlatform()) + "Jdk-" + jdk.getVendor() + "-" + jdk.getVersion();
// decompose the bundled jdk version, broken into elements as: [feature, interim, update, build]
// Note the "patch" version is not yet handled here, as it has not yet been used by java.
Matcher jdkVersionMatcher = Jdk.VERSION_PATTERN.matcher(version);
if (jdkVersionMatcher.matches() == false) {
throw new IllegalArgumentException("Malformed jdk version [" + version + "]");
}
String jdkVersion = jdkVersionMatcher.group(1) + (jdkVersionMatcher.group(2) != null ? (jdkVersionMatcher.group(2)) : "");
String jdkMajor = jdkVersionMatcher.group(1);
String jdkBuild = jdkVersionMatcher.group(3);
String hash = jdkVersionMatcher.group(5);
// Skip setup if we've already configured a JDK for this platform, vendor and version
if (findByName(rootProject.getTasks(), extractTaskName) == null) {
RepositoryHandler repositories = rootProject.getRepositories();
// add fake ivy repo for jdk url
String repoName = REPO_NAME_PREFIX + vendor + "_" + version;
RepositoryHandler repositories = rootProject.getRepositories();
if (rootProject.getRepositories().findByName(repoName) == null) {
if (vendor.equals("adoptopenjdk")) {
if (hash != null) {
throw new IllegalArgumentException("adoptopenjdk versions do not have hashes but was [" + version + "]");
}
repositories.ivy(ivyRepo -> {
ivyRepo.setName(repoName);
ivyRepo.setUrl("https://artifactory.elstc.co/artifactory/oss-jdk-local/");
ivyRepo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
final String pattern = String.format(
Locale.ROOT,
"adoptopenjdk/OpenJDK%sU-jdk_x64_[module]_hotspot_[revision]_%s.[ext]",
jdkMajor,
jdkBuild
);
ivyRepo.patternLayout(layout -> layout.artifact(pattern));
ivyRepo.content(content -> content.includeGroup("adoptopenjdk"));
});
} else {
assert vendor.equals("openjdk") : vendor;
if (hash != null) {
/*
* Define the appropriate repository for the given JDK vendor and version
*
* For AdoptOpenJDK we use a single internally hosted Artifactory repository.
* For Oracle/OpenJDK we define a repository per-version.
*/
String repoName = REPO_NAME_PREFIX + jdk.getVendor() + "_" + jdk.getVersion();
String repoUrl;
String artifactPattern;
if (jdk.getVendor().equals("adoptopenjdk")) {
repoUrl = "https://artifactory.elstc.co/artifactory/oss-jdk-local/";
artifactPattern = String.format(
Locale.ROOT,
"adoptopenjdk/OpenJDK%sU-jdk_x64_[module]_hotspot_[revision]_%s.[ext]",
jdk.getMajor(),
jdk.getBuild()
);
} else if (jdk.getVendor().equals("openjdk")) {
repoUrl = "https://download.oracle.com";
if (jdk.getHash() != null) {
// current pattern since 12.0.1
repositories.ivy(ivyRepo -> {
ivyRepo.setName(repoName);
ivyRepo.setUrl("https://download.oracle.com");
ivyRepo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
ivyRepo.patternLayout(
layout -> layout.artifact(
"java/GA/jdk" + jdkVersion + "/" + hash + "/" + jdkBuild + "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]"
)
);
ivyRepo.content(content -> content.includeGroup("openjdk"));
});
artifactPattern = "java/GA/jdk"
+ jdk.getBaseVersion()
+ "/"
+ jdk.getHash()
+ "/"
+ jdk.getBuild()
+ "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]";
} else {
// simpler legacy pattern from JDK 9 to JDK 12 that we are advocating to Oracle to bring back
repositories.ivy(ivyRepo -> {
ivyRepo.setName(repoName);
ivyRepo.setUrl("https://download.oracle.com");
ivyRepo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
ivyRepo.patternLayout(
layout -> layout.artifact(
"java/GA/jdk" + jdkMajor + "/" + jdkBuild + "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]"
)
);
ivyRepo.content(content -> content.includeGroup("openjdk"));
});
artifactPattern = "java/GA/jdk"
+ jdk.getMajor()
+ "/"
+ jdk.getBuild()
+ "/GPL/openjdk-[revision]_[module]-x64_bin.[ext]";
}
} else {
throw new GradleException("Unknown JDK vendor [" + jdk.getVendor() + "]");
}
// Define the repository if we haven't already
if (rootProject.getRepositories().findByName(repoName) == null) {
repositories.ivy(ivyRepo -> {
ivyRepo.setName(repoName);
ivyRepo.setUrl(repoUrl);
ivyRepo.metadataSources(IvyArtifactRepository.MetadataSources::artifact);
ivyRepo.patternLayout(layout -> layout.artifact(artifactPattern));
ivyRepo.content(content -> content.includeGroup(jdk.getVendor()));
});
}
// Declare a configuration and dependency from which to download the remote JDK
final ConfigurationContainer configurations = rootProject.getConfigurations();
String downloadConfigName = configName(jdk.getVendor(), jdk.getVersion(), jdk.getPlatform());
Configuration downloadConfiguration = maybeCreate(configurations, downloadConfigName);
rootProject.getDependencies().add(downloadConfigName, dependencyNotation(jdk));
// Create JDK extract task
final Provider<Directory> extractPath = rootProject.getLayout()
.getBuildDirectory()
.dir("jdks/" + jdk.getVendor() + "-" + jdk.getBaseVersion() + "_" + jdk.getPlatform());
TaskProvider<?> extractTask = createExtractTask(
extractTaskName,
rootProject,
jdk.getPlatform(),
downloadConfiguration,
extractPath
);
// Declare a configuration for the extracted JDK archive
String artifactConfigName = configName("extracted_jdk", jdk.getVendor(), jdk.getVersion(), jdk.getPlatform());
maybeCreate(configurations, artifactConfigName);
rootProject.getArtifacts().add(artifactConfigName, extractPath, artifact -> artifact.builtBy(extractTask));
}
}
// add the jdk as a "dependency"
final ConfigurationContainer configurations = rootProject.getConfigurations();
String remoteConfigName = configName(vendor, version, platform);
String localConfigName = configName("extracted_jdk", vendor, version, platform);
Configuration jdkConfig = configurations.findByName(remoteConfigName);
if (jdkConfig == null) {
jdkConfig = configurations.create(remoteConfigName);
configurations.create(localConfigName);
}
String platformDep = platform.equals("darwin") || platform.equals("osx")
? (vendor.equals("adoptopenjdk") ? "mac" : "osx")
: platform;
String extension = platform.equals("windows") ? "zip" : "tar.gz";
String jdkDep = vendor + ":" + platformDep + ":" + jdkVersion + "@" + extension;
rootProject.getDependencies().add(configName(vendor, version, platform), jdkDep);
// add task for extraction
final Provider<Directory> extractPath = rootProject.getLayout()
.getBuildDirectory()
.dir("jdks/" + vendor + "-" + jdkVersion + "_" + platform);
// delay resolving jdkConfig until runtime
Supplier<File> jdkArchiveGetter = jdkConfig::getSingleFile;
final Object extractTask;
if (extension.equals("zip")) {
final Callable<FileTree> fileGetter = () -> rootProject.zipTree(jdkArchiveGetter.get());
private static TaskProvider<?> createExtractTask(
String taskName,
Project rootProject,
String platform,
Configuration downloadConfiguration,
Provider<Directory> extractPath
) {
if (platform.equals("windows")) {
final Callable<FileTree> fileGetter = () -> rootProject.zipTree(downloadConfiguration.getSingleFile());
// TODO: look into doing this as an artifact transform, which are cacheable starting in gradle 5.3
Action<CopySpec> removeRootDir = copy -> {
// remove extra unnecessary directory levels
copy.eachFile(details -> {
/*
* We want to remove up to the and including the jdk-.* relative paths. That is a JDK archive is structured as:
* jdk-12.0.1/
* jdk-12.0.1/Contents
* ...
*
* and we want to remove the leading jdk-12.0.1. Note however that there could also be a leading ./ as in
* ./
* ./jdk-12.0.1/
* ./jdk-12.0.1/Contents
*
* so we account for this and search the path components until we find the jdk-12.0.1, and strip the leading components.
*/
String[] pathSegments = details.getRelativePath().getSegments();
int index = 0;
for (; index < pathSegments.length; index++) {
if (pathSegments[index].matches("jdk-.*")) {
break;
}
}
assert index + 1 <= pathSegments.length;
String[] newPathSegments = Arrays.copyOfRange(pathSegments, index + 1, pathSegments.length);
details.setRelativePath(new RelativePath(true, newPathSegments));
Path newPathSegments = trimArchiveExtractPath(details.getRelativePath().getPathString());
String[] segments = StreamSupport.stream(newPathSegments.spliterator(), false)
.map(Path::toString)
.toArray(String[]::new);
details.setRelativePath(new RelativePath(true, segments));
});
copy.setIncludeEmptyDirs(false);
};
extractTask = rootProject.getTasks().register(extractTaskName, Copy.class, copyTask -> {
return rootProject.getTasks().register(taskName, Copy.class, copyTask -> {
copyTask.doFirst(new Action<Task>() {
@Override
public void execute(Task t) {
@ -240,53 +218,53 @@ public class JdkDownloadPlugin implements Plugin<Project> {
* Gradle TarFileTree does not resolve symlinks, so we have to manually extract and preserve the symlinks.
* cf. https://github.com/gradle/gradle/issues/3982 and https://discuss.gradle.org/t/tar-and-untar-losing-symbolic-links/2039
*/
final Configuration jdkConfiguration = jdkConfig;
extractTask = rootProject.getTasks().register(extractTaskName, SymbolicLinkPreservingUntarTask.class, task -> {
task.getTarFile().set(jdkConfiguration.getSingleFile());
return rootProject.getTasks().register(taskName, SymbolicLinkPreservingUntarTask.class, task -> {
task.getTarFile().fileProvider(rootProject.provider(downloadConfiguration::getSingleFile));
task.getExtractPath().set(extractPath);
task.setTransform(name -> {
/*
* We want to remove up to the and including the jdk-.* relative paths. That is a JDK archive is structured as:
* jdk-12.0.1/
* jdk-12.0.1/Contents
* ...
*
* and we want to remove the leading jdk-12.0.1. Note however that there could also be a leading ./ as in
* ./
* ./jdk-12.0.1/
* ./jdk-12.0.1/Contents
*
* so we account for this and search the path components until we find the jdk-12.0.1, and strip the leading
* components.
*/
final Path entryName = Paths.get(name);
int index = 0;
for (; index < entryName.getNameCount(); index++) {
if (entryName.getName(index).toString().matches("jdk-.*")) {
break;
}
}
if (index + 1 >= entryName.getNameCount()) {
// this happens on the top-level directories in the archive, which we are removing
return null;
}
// finally remove the top-level directories from the output path
return entryName.subpath(index + 1, entryName.getNameCount());
});
task.setTransform(JdkDownloadPlugin::trimArchiveExtractPath);
});
}
rootProject.getArtifacts().add(localConfigName, extractPath, artifact -> artifact.builtBy(extractTask));
}
private static String configName(String vendor, String version, String platform) {
return vendor + "_" + version + "_" + platform;
/*
* We want to remove up to the and including the jdk-.* relative paths. That is a JDK archive is structured as:
* jdk-12.0.1/
* jdk-12.0.1/Contents
* ...
*
* and we want to remove the leading jdk-12.0.1. Note however that there could also be a leading ./ as in
* ./
* ./jdk-12.0.1/
* ./jdk-12.0.1/Contents
*
* so we account for this and search the path components until we find the jdk-12.0.1, and strip the leading components.
*/
private static Path trimArchiveExtractPath(String relativePath) {
final Path entryName = Paths.get(relativePath);
int index = 0;
for (; index < entryName.getNameCount(); index++) {
if (entryName.getName(index).toString().matches("jdk-?\\d.*")) {
break;
}
}
if (index + 1 >= entryName.getNameCount()) {
// this happens on the top-level directories in the archive, which we are removing
return null;
}
// finally remove the top-level directories from the output path
return entryName.subpath(index + 1, entryName.getNameCount());
}
private static String configName(String prefix, String vendor, String version, String platform) {
return prefix + "_" + vendor + "_" + version + "_" + platform;
private static String dependencyNotation(Jdk jdk) {
String platformDep = jdk.getPlatform().equals("darwin") || jdk.getPlatform().equals("osx")
? (jdk.getVendor().equals("adoptopenjdk") ? "mac" : "osx")
: jdk.getPlatform();
String extension = jdk.getPlatform().equals("windows") ? "zip" : "tar.gz";
return jdk.getVendor() + ":" + platformDep + ":" + jdk.getBaseVersion() + "@" + extension;
}
private static String capitalize(String s) {
return s.substring(0, 1).toUpperCase(Locale.ROOT) + s.substring(1);
private static String configName(String... parts) {
return String.join("_", parts);
}
}

6
buildSrc/src/main/java/org/elasticsearch/gradle/Util.java
View File

@ -21,6 +21,8 @@ package org.elasticsearch.gradle;
import org.gradle.api.GradleException;
import java.util.Locale;
public class Util {
public static boolean getBooleanProperty(String property, boolean defaultValue) {
@ -36,4 +38,8 @@ public class Util {
throw new GradleException("Sysprop [" + property + "] must be [true] or [false] but was [" + propertyValue + "]");
}
}
public static String capitalize(String s) {
return s.substring(0, 1).toUpperCase(Locale.ROOT) + s.substring(1);
}
}

1
buildSrc/src/main/java/org/elasticsearch/gradle/tool/Boilerplate.java
View File

@ -44,7 +44,6 @@ public abstract class Boilerplate {
public static <T> T maybeCreate(NamedDomainObjectContainer<T> collection, String name) {
return Optional.ofNullable(collection.findByName(name)).orElse(collection.create(name));
}
public static <T> T maybeCreate(NamedDomainObjectContainer<T> collection, String name, Action<T> action) {