Simplify distribution download and extraction (7.x backport) (#61184)
We leverage artifact transforms now when downloading and unpacking elasticsearch distributions. This has the benefit of - handcrafted extract tasks on the root project are not required. - the general tight coupling to the root project has been removed. - the overall required configurations required to handle a distribution have been reduced - ElasticsearchDistribution has been simplified by making Extracted an ordinary Configuration downloaded and unpacked external distributions are reused in later builds by been cached in the gradle user home. DistributionDownloadPlugin functional tests have been extended and ported to DistributionDownloadPluginFuncTest. * Fix java8 compliant Path calculation
This commit is contained in:
Rene Groeschke
GitHub
parent
1ffc983f98
commit
8b7a0a1f64
|
|
@ -92,7 +92,7 @@ dependencies {
|
|||
|
||||
api 'commons-codec:commons-codec:1.12'
|
||||
api 'org.apache.commons:commons-compress:1.19'
|
||||
|
||||
api 'org.apache.ant:ant:1.10.8'
|
||||
api 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3'
|
||||
api 'com.netflix.nebula:nebula-publishing-plugin:4.4.4'
|
||||
api 'com.netflix.nebula:gradle-info-plugin:7.1.3'
|
||||
|
|
|
|||
|
|
@ -0,0 +1,179 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.gradle
|
||||
|
||||
import com.github.tomakehurst.wiremock.WireMockServer
|
||||
import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
|
||||
import org.elasticsearch.gradle.fixtures.WiremockFixture
|
||||
import org.elasticsearch.gradle.transform.SymbolicLinkPreservingUntarTransform
|
||||
import org.gradle.testkit.runner.TaskOutcome
|
||||
import spock.lang.Unroll
|
||||
|
||||
class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
|
||||
|
||||
@Unroll
|
||||
def "#distType version can be resolved"() {
|
||||
given:
|
||||
def mockRepoUrl = urlPath(version, platform)
|
||||
def mockedContent = filebytes(mockRepoUrl)
|
||||
|
||||
buildFile << applyPluginAndSetupDistro(version, platform)
|
||||
|
||||
when:
|
||||
def result = WiremockFixture.withWireMock(mockRepoUrl, mockedContent) { server ->
|
||||
buildFile << repositoryMockSetup(server)
|
||||
gradleRunner('setupDistro').build()
|
||||
}
|
||||
|
||||
then:
|
||||
result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
|
||||
assertExtractedDistroCreated("build/distro")
|
||||
|
||||
where:
|
||||
version | platform | distType
|
||||
VersionProperties.getElasticsearch() | "linux" | "current"
|
||||
"8.1.0-SNAPSHOT" | "linux" | "bwc"
|
||||
"7.0.0" | "windows" | "released"
|
||||
}
|
||||
|
||||
|
||||
def "transformed versions are kept across builds"() {
|
||||
given:
|
||||
def version = VersionProperties.getElasticsearch()
|
||||
def mockRepoUrl = urlPath(version)
|
||||
def mockedContent = filebytes(mockRepoUrl)
|
||||
|
||||
buildFile << applyPluginAndSetupDistro(version, 'linux')
|
||||
buildFile << """
|
||||
apply plugin:'base'
|
||||
"""
|
||||
|
||||
|
||||
when:
|
||||
def result = WiremockFixture.withWireMock(mockRepoUrl, mockedContent) { server ->
|
||||
buildFile << repositoryMockSetup(server)
|
||||
gradleRunner('clean', 'setupDistro').build()
|
||||
gradleRunner('clean', 'setupDistro', '-i').build()
|
||||
}
|
||||
|
||||
then:
|
||||
result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
|
||||
assertOutputContains(result.output, "Skipping ${SymbolicLinkPreservingUntarTransform.class.simpleName}")
|
||||
}
|
||||
|
||||
def "transforms are reused across projects"() {
|
||||
given:
|
||||
def version = VersionProperties.getElasticsearch()
|
||||
def mockRepoUrl = urlPath(version)
|
||||
def mockedContent = filebytes(mockRepoUrl)
|
||||
|
||||
3.times {
|
||||
settingsFile << """
|
||||
include ':sub-$it'
|
||||
"""
|
||||
}
|
||||
buildFile.text = """
|
||||
import org.elasticsearch.gradle.Architecture
|
||||
|
||||
plugins {
|
||||
id 'elasticsearch.distribution-download'
|
||||
}
|
||||
|
||||
subprojects {
|
||||
apply plugin: 'elasticsearch.distribution-download'
|
||||
|
||||
${setupTestDistro(version, 'linux')}
|
||||
${setupDistroTask()}
|
||||
}
|
||||
"""
|
||||
|
||||
when:
|
||||
def result = WiremockFixture.withWireMock(mockRepoUrl, mockedContent) { server ->
|
||||
buildFile << repositoryMockSetup(server)
|
||||
gradleRunner('setupDistro', '-i', '-g', testProjectDir.newFolder().toString()).build()
|
||||
}
|
||||
|
||||
then:
|
||||
result.tasks.size() == 3
|
||||
result.output.count("Unpacking elasticsearch-${version}-linux-x86_64.tar.gz using SymbolicLinkPreservingUntarTransform.") == 1
|
||||
}
|
||||
|
||||
private boolean assertExtractedDistroCreated(String relativePath) {
|
||||
File distroExtracted = new File(testProjectDir.root, relativePath)
|
||||
assert distroExtracted.exists()
|
||||
assert distroExtracted.isDirectory()
|
||||
assert new File(distroExtracted, "elasticsearch-1.2.3/bin/elasticsearch").exists()
|
||||
true
|
||||
}
|
||||
|
||||
private static String urlPath(String version, String platform = 'linux') {
|
||||
String fileType = platform == "linux" ? "tar.gz" : "zip"
|
||||
"/downloads/elasticsearch/elasticsearch-${version}-${platform}-x86_64.$fileType"
|
||||
}
|
||||
|
||||
private static String repositoryMockSetup(WireMockServer server) {
|
||||
"""allprojects{ p ->
|
||||
p.repositories.all { repo ->
|
||||
repo.setUrl('${server.baseUrl()}')
|
||||
}
|
||||
}"""
|
||||
}
|
||||
|
||||
private static byte[] filebytes(String urlPath) throws IOException {
|
||||
String suffix = urlPath.endsWith("zip") ? "zip" : "tar.gz";
|
||||
return DistributionDownloadPluginFuncTest.getResourceAsStream("fake_elasticsearch." + suffix).getBytes()
|
||||
}
|
||||
|
||||
private static String applyPluginAndSetupDistro(String version, String platform) {
|
||||
"""
|
||||
import org.elasticsearch.gradle.Architecture
|
||||
|
||||
plugins {
|
||||
id 'elasticsearch.distribution-download'
|
||||
}
|
||||
|
||||
${setupTestDistro(version, platform)}
|
||||
${setupDistroTask()}
|
||||
|
||||
"""
|
||||
}
|
||||
|
||||
private static String setupTestDistro(String version, String platform) {
|
||||
return """
|
||||
elasticsearch_distributions {
|
||||
test_distro {
|
||||
version = "$version"
|
||||
type = "archive"
|
||||
platform = "$platform"
|
||||
architecture = Architecture.current();
|
||||
}
|
||||
}
|
||||
"""
|
||||
}
|
||||
|
||||
private static String setupDistroTask() {
|
||||
return """
|
||||
tasks.register("setupDistro", Sync) {
|
||||
from(elasticsearch_distributions.test_distro.extracted)
|
||||
into("build/distro")
|
||||
}
|
||||
"""
|
||||
}
|
||||
}
|
||||
|
|
@ -38,7 +38,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
|
|||
"""
|
||||
|
||||
when:
|
||||
def result = gradleRunner("createExtractedTestDistro").buildAndFail()
|
||||
def result = gradleRunner("tasks").buildAndFail()
|
||||
|
||||
then:
|
||||
assertOutputContains(result.output, "Plugin 'elasticsearch.internal-distribution-download' is not supported. " +
|
||||
|
|
@ -61,18 +61,19 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
|
|||
architecture = Architecture.current();
|
||||
}
|
||||
}
|
||||
tasks.register("createExtractedTestDistro") {
|
||||
dependsOn elasticsearch_distributions.test_distro.extracted
|
||||
tasks.register("setupDistro", Sync) {
|
||||
from(elasticsearch_distributions.test_distro.extracted)
|
||||
into("build/distro")
|
||||
}
|
||||
"""
|
||||
|
||||
when:
|
||||
def result = gradleRunner("createExtractedTestDistro").build()
|
||||
def result = gradleRunner("setupDistro", '-g', testProjectDir.newFolder('GUH').path).build()
|
||||
|
||||
then:
|
||||
result.task(":distribution:archives:linux-tar:buildTar").outcome == TaskOutcome.SUCCESS
|
||||
result.task(":extractElasticsearchLinux$distroVersion").outcome == TaskOutcome.SUCCESS
|
||||
assertExtractedDistroIsCreated(distroVersion, 'current-marker.txt')
|
||||
result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
|
||||
assertExtractedDistroIsCreated(distroVersion, "build/distro", 'current-marker.txt')
|
||||
}
|
||||
|
||||
def "resolves bwc versions from source"() {
|
||||
|
|
@ -91,16 +92,18 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
|
|||
architecture = Architecture.current();
|
||||
}
|
||||
}
|
||||
tasks.register("createExtractedTestDistro") {
|
||||
dependsOn elasticsearch_distributions.test_distro.extracted
|
||||
tasks.register("setupDistro", Sync) {
|
||||
from(elasticsearch_distributions.test_distro.extracted)
|
||||
into("build/distro")
|
||||
}
|
||||
"""
|
||||
when:
|
||||
def result = gradleRunner("createExtractedTestDistro").build()
|
||||
|
||||
def result = gradleRunner("setupDistro").build()
|
||||
then:
|
||||
result.task(":distribution:bwc:minor:buildBwcTask").outcome == TaskOutcome.SUCCESS
|
||||
result.task(":extractElasticsearchLinux8.1.0").outcome == TaskOutcome.SUCCESS
|
||||
assertExtractedDistroIsCreated(distroVersion,'bwc-marker.txt')
|
||||
result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
|
||||
assertExtractedDistroIsCreated(distroVersion, "build/distro", 'bwc-marker.txt')
|
||||
}
|
||||
|
||||
def "fails on resolving bwc versions with no bundled jdk"() {
|
||||
|
|
@ -195,8 +198,8 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
|
|||
|
||||
}
|
||||
|
||||
boolean assertExtractedDistroIsCreated(String version, String markerFileName) {
|
||||
File extractedFolder = new File(testProjectDir.root, "build/elasticsearch-distros/extracted_elasticsearch_${version}_archive_linux_default")
|
||||
boolean assertExtractedDistroIsCreated(String version, String relativeDistroPath, String markerFileName) {
|
||||
File extractedFolder = new File(testProjectDir.root, relativeDistroPath)
|
||||
assert extractedFolder.exists()
|
||||
assert new File(extractedFolder, markerFileName).exists()
|
||||
true
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
|
|||
given:
|
||||
def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform)
|
||||
def mockedContent = filebytes(jdkVendor, platform)
|
||||
10.times {
|
||||
3.times {
|
||||
settingsFile << """
|
||||
include ':sub-$it'
|
||||
"""
|
||||
|
|
@ -132,8 +132,8 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
|
|||
}
|
||||
|
||||
then:
|
||||
result.tasks.size() == 10
|
||||
result.output.count("Unpacking linux-12.0.2-x64.tar.gz using SymbolicLinkPreservingUntarTransform.") == 1
|
||||
result.tasks.size() == 3
|
||||
result.output.count("Unpacking linux-12.0.2-x64.tar.gz using ${SymbolicLinkPreservingUntarTransform.simpleName}.") == 1
|
||||
|
||||
where:
|
||||
platform | jdkVendor | jdkVersion | expectedJavaBin
|
||||
|
|
@ -149,7 +149,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
|
|||
plugins {
|
||||
id 'elasticsearch.jdk-download'
|
||||
}
|
||||
|
||||
apply plugin: 'base'
|
||||
apply plugin: 'elasticsearch.jdk-download'
|
||||
|
||||
jdks {
|
||||
|
|
@ -175,9 +175,9 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
|
|||
|
||||
def commonGradleUserHome = testProjectDir.newFolder().toString()
|
||||
// initial run
|
||||
gradleRunner('getJdk', '-g', commonGradleUserHome).build()
|
||||
gradleRunner('clean', 'getJdk', '-g', commonGradleUserHome).build()
|
||||
// run against up-to-date transformations
|
||||
gradleRunner('getJdk', '-i', '-g', commonGradleUserHome).build()
|
||||
gradleRunner('clean', 'getJdk', '-i', '-g', commonGradleUserHome).build()
|
||||
}
|
||||
|
||||
then:
|
||||
|
|
|
|||
|
|
@ -1,188 +0,0 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.gradle;
|
||||
|
||||
import com.github.tomakehurst.wiremock.WireMockServer;
|
||||
import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
|
||||
import org.gradle.testkit.runner.BuildResult;
|
||||
import org.gradle.testkit.runner.GradleRunner;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
|
||||
import static com.github.tomakehurst.wiremock.client.WireMock.get;
|
||||
import static com.github.tomakehurst.wiremock.client.WireMock.head;
|
||||
import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
|
||||
|
||||
public class DistributionDownloadPluginIT extends GradleIntegrationTestCase {
|
||||
|
||||
// TODO: check reuse of root task across projects MOVE TO UNIT TEST
|
||||
// TODO: future: check integ-test-zip to maven, snapshots to snapshot service for external project
|
||||
public void testCurrentExternal() throws Exception {
|
||||
checkService(
|
||||
VersionProperties.getElasticsearch(),
|
||||
"archive",
|
||||
"linux",
|
||||
null,
|
||||
null,
|
||||
"/downloads/elasticsearch/elasticsearch-" + VersionProperties.getElasticsearch() + "-linux-x86_64.tar.gz",
|
||||
"tests.internal",
|
||||
"false"
|
||||
);
|
||||
}
|
||||
|
||||
public void testBwcExternal() throws Exception {
|
||||
checkService(
|
||||
"8.1.0-SNAPSHOT",
|
||||
"archive",
|
||||
"linux",
|
||||
null,
|
||||
null,
|
||||
"/downloads/elasticsearch/elasticsearch-8.1.0-SNAPSHOT-linux-x86_64.tar.gz",
|
||||
"tests.internal",
|
||||
"false",
|
||||
"tests.current_version",
|
||||
"9.0.0"
|
||||
);
|
||||
}
|
||||
|
||||
public void testReleased() throws Exception {
|
||||
checkService("7.0.0", "archive", "windows", null, null, "/downloads/elasticsearch/elasticsearch-7.0.0-windows-x86_64.zip");
|
||||
checkService("6.5.0", "archive", "windows", null, null, "/downloads/elasticsearch/elasticsearch-6.5.0.zip");
|
||||
}
|
||||
|
||||
public void testReleasedExternal() throws Exception {
|
||||
checkService(
|
||||
"7.0.0",
|
||||
"archive",
|
||||
"windows",
|
||||
null,
|
||||
null,
|
||||
"/downloads/elasticsearch/elasticsearch-7.0.0-windows-x86_64.zip",
|
||||
"tests.internal",
|
||||
"false"
|
||||
);
|
||||
checkService(
|
||||
"6.5.0",
|
||||
"archive",
|
||||
"windows",
|
||||
null,
|
||||
null,
|
||||
"/downloads/elasticsearch/elasticsearch-6.5.0.zip",
|
||||
"tests.internal",
|
||||
"false"
|
||||
);
|
||||
}
|
||||
|
||||
private void checkService(
|
||||
String version,
|
||||
String type,
|
||||
String platform,
|
||||
String flavor,
|
||||
Boolean bundledJdk,
|
||||
String urlPath,
|
||||
String... sysProps
|
||||
) throws IOException {
|
||||
String suffix = urlPath.endsWith("zip") ? "zip" : "tar.gz";
|
||||
String sourceFile = "src/testKit/distribution-download/distribution/files/fake_elasticsearch." + suffix;
|
||||
WireMockServer wireMock = new WireMockServer(0);
|
||||
try {
|
||||
final byte[] filebytes;
|
||||
try (InputStream stream = Files.newInputStream(Paths.get(sourceFile))) {
|
||||
filebytes = stream.readAllBytes();
|
||||
}
|
||||
wireMock.stubFor(head(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200)));
|
||||
wireMock.stubFor(get(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200).withBody(filebytes)));
|
||||
wireMock.start();
|
||||
|
||||
List<String> allSysProps = new ArrayList<>();
|
||||
allSysProps.addAll(Arrays.asList(sysProps));
|
||||
allSysProps.add("tests.download_service");
|
||||
allSysProps.add(wireMock.baseUrl());
|
||||
assertExtractedDistro(version, type, platform, flavor, bundledJdk, allSysProps.toArray(new String[0]));
|
||||
} catch (Exception e) {
|
||||
// for debugging
|
||||
System.err.println("missed requests: " + wireMock.findUnmatchedRequests().getRequests());
|
||||
throw e;
|
||||
} finally {
|
||||
wireMock.stop();
|
||||
}
|
||||
}
|
||||
|
||||
private void assertFileDistro(String version, String type, String platform, String flavor, Boolean bundledJdk, String... sysProps)
|
||||
throws IOException {
|
||||
List<String> finalSysProps = new ArrayList<>();
|
||||
addDistroSysProps(finalSysProps, version, type, platform, flavor, bundledJdk);
|
||||
finalSysProps.addAll(Arrays.asList(sysProps));
|
||||
runBuild(":subproj:assertDistroFile", finalSysProps.toArray(new String[0]));
|
||||
}
|
||||
|
||||
private void assertExtractedDistro(String version, String type, String platform, String flavor, Boolean bundledJdk, String... sysProps)
|
||||
throws IOException {
|
||||
List<String> finalSysProps = new ArrayList<>();
|
||||
addDistroSysProps(finalSysProps, version, type, platform, flavor, bundledJdk);
|
||||
finalSysProps.addAll(Arrays.asList(sysProps));
|
||||
runBuild(":subproj:assertDistroExtracted", finalSysProps.toArray(new String[0]));
|
||||
}
|
||||
|
||||
private BuildResult runBuild(String taskname, String... sysProps) throws IOException {
|
||||
assert sysProps.length % 2 == 0;
|
||||
List<String> args = new ArrayList<>();
|
||||
args.add(taskname);
|
||||
for (int i = 0; i < sysProps.length; i += 2) {
|
||||
args.add("-D" + sysProps[i] + "=" + sysProps[i + 1]);
|
||||
}
|
||||
args.add("-i");
|
||||
GradleRunner runner = getGradleRunner("distribution-download").withArguments(args);
|
||||
|
||||
BuildResult result = runner.build();
|
||||
System.out.println(result.getOutput());
|
||||
return result;
|
||||
}
|
||||
|
||||
private void addDistroSysProps(List<String> sysProps, String version, String type, String platform, String flavor, Boolean bundledJdk) {
|
||||
if (version != null) {
|
||||
sysProps.add("tests.distro.version");
|
||||
sysProps.add(version);
|
||||
}
|
||||
if (type != null) {
|
||||
sysProps.add("tests.distro.type");
|
||||
sysProps.add(type);
|
||||
}
|
||||
if (platform != null) {
|
||||
sysProps.add("tests.distro.platform");
|
||||
sysProps.add(platform);
|
||||
}
|
||||
if (flavor != null) {
|
||||
sysProps.add("tests.distro.flavor");
|
||||
sysProps.add(flavor);
|
||||
}
|
||||
if (bundledJdk != null) {
|
||||
sysProps.add("tests.distro.bundledJdk");
|
||||
sysProps.add(bundledJdk.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -24,31 +24,22 @@ import org.elasticsearch.gradle.ElasticsearchDistribution.Platform;
|
|||
import org.elasticsearch.gradle.ElasticsearchDistribution.Type;
|
||||
import org.elasticsearch.gradle.docker.DockerSupportPlugin;
|
||||
import org.elasticsearch.gradle.docker.DockerSupportService;
|
||||
import org.elasticsearch.gradle.transform.SymbolicLinkPreservingUntarTransform;
|
||||
import org.elasticsearch.gradle.transform.UnzipTransform;
|
||||
import org.elasticsearch.gradle.util.GradleUtils;
|
||||
import org.gradle.api.NamedDomainObjectContainer;
|
||||
import org.gradle.api.Plugin;
|
||||
import org.gradle.api.Project;
|
||||
import org.gradle.api.UnknownTaskException;
|
||||
import org.gradle.api.artifacts.Configuration;
|
||||
import org.gradle.api.artifacts.ConfigurationContainer;
|
||||
import org.gradle.api.artifacts.dsl.DependencyHandler;
|
||||
import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
|
||||
import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
|
||||
import org.gradle.api.credentials.HttpHeaderCredentials;
|
||||
import org.gradle.api.file.FileTree;
|
||||
import org.gradle.api.file.RelativePath;
|
||||
import org.gradle.api.internal.artifacts.ArtifactAttributes;
|
||||
import org.gradle.api.provider.Provider;
|
||||
import org.gradle.api.tasks.Sync;
|
||||
import org.gradle.api.tasks.TaskProvider;
|
||||
import org.gradle.authentication.http.HttpHeaderAuthentication;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import static org.elasticsearch.gradle.util.GradleUtils.projectDependency;
|
||||
import static org.elasticsearch.gradle.util.Util.capitalize;
|
||||
|
||||
/**
|
||||
* A plugin to manage getting and extracting distributions of Elasticsearch.
|
||||
|
|
@ -77,6 +68,17 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
|
|||
DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
|
||||
);
|
||||
|
||||
project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> {
|
||||
transformSpec.getFrom().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE);
|
||||
transformSpec.getTo().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
|
||||
});
|
||||
|
||||
ArtifactTypeDefinition tarArtifactTypeDefinition = project.getDependencies().getArtifactTypes().maybeCreate("tar.gz");
|
||||
project.getDependencies().registerTransform(SymbolicLinkPreservingUntarTransform.class, transformSpec -> {
|
||||
transformSpec.getFrom().attribute(ArtifactAttributes.ARTIFACT_FORMAT, tarArtifactTypeDefinition.getName());
|
||||
transformSpec.getTo().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
|
||||
});
|
||||
|
||||
setupResolutionsContainer(project);
|
||||
setupDistributionContainer(project, dockerSupport);
|
||||
setupDownloadServiceRepo(project);
|
||||
|
|
@ -87,6 +89,7 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
|
|||
distributionsContainer = project.container(ElasticsearchDistribution.class, name -> {
|
||||
Configuration fileConfiguration = project.getConfigurations().create("es_distro_file_" + name);
|
||||
Configuration extractedConfiguration = project.getConfigurations().create("es_distro_extracted_" + name);
|
||||
extractedConfiguration.getAttributes().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
|
||||
return new ElasticsearchDistribution(name, project.getObjects(), dockerSupport, fileConfiguration, extractedConfiguration);
|
||||
});
|
||||
project.getExtensions().add(CONTAINER_NAME, distributionsContainer);
|
||||
|
|
@ -115,21 +118,15 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
|
|||
void setupDistributions(Project project) {
|
||||
for (ElasticsearchDistribution distribution : distributionsContainer) {
|
||||
distribution.finalizeValues();
|
||||
|
||||
DependencyHandler dependencies = project.getDependencies();
|
||||
// for the distribution as a file, just depend on the artifact directly
|
||||
dependencies.add(distribution.configuration.getName(), resolveDependencyNotation(project, distribution));
|
||||
|
||||
Object resolvedDependency = resolveDependencyNotation(project, distribution);
|
||||
dependencies.add(distribution.configuration.getName(), resolvedDependency);
|
||||
// no extraction allowed for rpm, deb or docker
|
||||
if (distribution.getType().shouldExtract()) {
|
||||
// for the distribution extracted, add a root level task that does the extraction, and depend on that
|
||||
// extracted configuration as an artifact consisting of the extracted distribution directory
|
||||
dependencies.add(
|
||||
distribution.getExtracted().configuration.getName(),
|
||||
projectDependency(project, ":", configName("extracted_elasticsearch", distribution))
|
||||
);
|
||||
// ensure a root level download task exists
|
||||
setupRootDownload(project.getRootProject(), distribution);
|
||||
// The extracted configuration depends on the artifact directly but has
|
||||
// an artifact transform registered to resolve it as an unpacked folder.
|
||||
dependencies.add(distribution.getExtracted().getName(), resolvedDependency);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -143,59 +140,6 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
|
|||
.orElseGet(() -> dependencyNotation(distribution));
|
||||
}
|
||||
|
||||
private void setupRootDownload(Project rootProject, ElasticsearchDistribution distribution) {
|
||||
String extractTaskName = extractTaskName(distribution);
|
||||
// NOTE: this is *horrendous*, but seems to be the only way to check for the existence of a registered task
|
||||
try {
|
||||
rootProject.getTasks().named(extractTaskName);
|
||||
// already setup this version
|
||||
return;
|
||||
} catch (UnknownTaskException e) {
|
||||
// fall through: register the task
|
||||
}
|
||||
setupDownloadServiceRepo(rootProject);
|
||||
final ConfigurationContainer configurations = rootProject.getConfigurations();
|
||||
String downloadConfigName = configName("elasticsearch", distribution);
|
||||
String extractedConfigName = "extracted_" + downloadConfigName;
|
||||
final Configuration downloadConfig = configurations.create(downloadConfigName);
|
||||
configurations.create(extractedConfigName);
|
||||
rootProject.getDependencies().add(downloadConfigName, resolveDependencyNotation(rootProject, distribution));
|
||||
|
||||
// add task for extraction, delaying resolving config until runtime
|
||||
if (distribution.getType() == Type.ARCHIVE || distribution.getType() == Type.INTEG_TEST_ZIP) {
|
||||
Supplier<File> archiveGetter = downloadConfig::getSingleFile;
|
||||
String extractDir = rootProject.getBuildDir().toPath().resolve("elasticsearch-distros").resolve(extractedConfigName).toString();
|
||||
TaskProvider<Sync> extractTask = rootProject.getTasks().register(extractTaskName, Sync.class, syncTask -> {
|
||||
syncTask.dependsOn(downloadConfig);
|
||||
syncTask.into(extractDir);
|
||||
syncTask.from((Callable<FileTree>) () -> {
|
||||
File archiveFile = archiveGetter.get();
|
||||
String archivePath = archiveFile.toString();
|
||||
if (archivePath.endsWith(".zip")) {
|
||||
return rootProject.zipTree(archiveFile);
|
||||
} else if (archivePath.endsWith(".tar.gz")) {
|
||||
return rootProject.tarTree(rootProject.getResources().gzip(archiveFile));
|
||||
}
|
||||
throw new IllegalStateException("unexpected file extension on [" + archivePath + "]");
|
||||
});
|
||||
|
||||
// Workaround for https://github.com/elastic/elasticsearch/issues/49417
|
||||
syncTask.eachFile(details -> {
|
||||
String[] segments = details.getRelativePath().getSegments();
|
||||
if (segments[0].equals(".")) {
|
||||
details.setRelativePath(new RelativePath(true, Arrays.copyOfRange(segments, 1, segments.length)));
|
||||
}
|
||||
});
|
||||
});
|
||||
rootProject.getArtifacts()
|
||||
.add(
|
||||
extractedConfigName,
|
||||
rootProject.getLayout().getProjectDirectory().dir(extractDir),
|
||||
artifact -> artifact.builtBy(extractTask)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private static void addIvyRepo(Project project, String name, String url, String group) {
|
||||
IvyArtifactRepository ivyRepo = project.getRepositories().ivy(repo -> {
|
||||
repo.setName(name);
|
||||
|
|
@ -263,35 +207,4 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
|
|||
String group = distribution.getVersion().endsWith("-SNAPSHOT") ? FAKE_SNAPSHOT_IVY_GROUP : FAKE_IVY_GROUP;
|
||||
return group + ":elasticsearch" + flavor + ":" + distribution.getVersion() + classifier + "@" + extension;
|
||||
}
|
||||
|
||||
private static String configName(String prefix, ElasticsearchDistribution distribution) {
|
||||
return String.format(
|
||||
"%s_%s_%s_%s%s%s",
|
||||
prefix,
|
||||
distribution.getVersion(),
|
||||
distribution.getType(),
|
||||
distribution.getPlatform() == null ? "" : distribution.getPlatform() + "_",
|
||||
distribution.getFlavor(),
|
||||
distribution.getBundledJdk() ? "" : "_nojdk"
|
||||
);
|
||||
}
|
||||
|
||||
private static String extractTaskName(ElasticsearchDistribution distribution) {
|
||||
String taskName = "extractElasticsearch";
|
||||
if (distribution.getType() != Type.INTEG_TEST_ZIP) {
|
||||
if (distribution.getFlavor() == Flavor.OSS) {
|
||||
taskName += "Oss";
|
||||
}
|
||||
if (distribution.getBundledJdk() == false) {
|
||||
taskName += "NoJdk";
|
||||
}
|
||||
}
|
||||
if (distribution.getType() == Type.ARCHIVE) {
|
||||
taskName += capitalize(distribution.getPlatform().toString());
|
||||
} else if (distribution.getType() != Type.INTEG_TEST_ZIP) {
|
||||
taskName += capitalize(distribution.getType().toString());
|
||||
}
|
||||
taskName += distribution.getVersion();
|
||||
return taskName;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -87,36 +87,10 @@ public class ElasticsearchDistribution implements Buildable, Iterable<File> {
|
|||
.onMac(() -> Platform.DARWIN)
|
||||
.supply();
|
||||
|
||||
public static final class Extracted implements Buildable, Iterable<File> {
|
||||
|
||||
// pkg private so plugin can configure
|
||||
final Configuration configuration;
|
||||
|
||||
private Extracted(Configuration configuration) {
|
||||
this.configuration = configuration;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<File> iterator() {
|
||||
return configuration.iterator();
|
||||
}
|
||||
|
||||
@Override
|
||||
public TaskDependency getBuildDependencies() {
|
||||
return configuration.getBuildDependencies();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return configuration.getSingleFile().toString();
|
||||
}
|
||||
}
|
||||
|
||||
private final String name;
|
||||
private final Provider<DockerSupportService> dockerSupport;
|
||||
// pkg private so plugin can configure
|
||||
final Configuration configuration;
|
||||
private final Extracted extracted;
|
||||
|
||||
private final Property<Architecture> architecture;
|
||||
private final Property<String> version;
|
||||
|
|
@ -125,6 +99,7 @@ public class ElasticsearchDistribution implements Buildable, Iterable<File> {
|
|||
private final Property<Flavor> flavor;
|
||||
private final Property<Boolean> bundledJdk;
|
||||
private final Property<Boolean> failIfUnavailable;
|
||||
private final Configuration extracted;
|
||||
|
||||
ElasticsearchDistribution(
|
||||
String name,
|
||||
|
|
@ -144,7 +119,7 @@ public class ElasticsearchDistribution implements Buildable, Iterable<File> {
|
|||
this.flavor = objectFactory.property(Flavor.class);
|
||||
this.bundledJdk = objectFactory.property(Boolean.class);
|
||||
this.failIfUnavailable = objectFactory.property(Boolean.class).convention(true);
|
||||
this.extracted = new Extracted(extractedConfiguration);
|
||||
this.extracted = extractedConfiguration;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
|
|
@ -213,7 +188,7 @@ public class ElasticsearchDistribution implements Buildable, Iterable<File> {
|
|||
return configuration.getSingleFile().toString();
|
||||
}
|
||||
|
||||
public Extracted getExtracted() {
|
||||
public Configuration getExtracted() {
|
||||
switch (getType()) {
|
||||
case DEB:
|
||||
case DOCKER:
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import org.gradle.api.artifacts.Configuration;
|
|||
import org.gradle.api.artifacts.dsl.RepositoryHandler;
|
||||
import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
|
||||
import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
|
||||
import org.gradle.api.attributes.Attribute;
|
||||
import org.gradle.api.internal.artifacts.ArtifactAttributes;
|
||||
|
||||
public class JdkDownloadPlugin implements Plugin<Project> {
|
||||
|
|
@ -38,25 +39,39 @@ public class JdkDownloadPlugin implements Plugin<Project> {
|
|||
|
||||
private static final String REPO_NAME_PREFIX = "jdk_repo_";
|
||||
private static final String EXTENSION_NAME = "jdks";
|
||||
public static final String JDK_TRIMMED_PREFIX = "jdk-?\\d.*";
|
||||
|
||||
@Override
|
||||
public void apply(Project project) {
|
||||
Attribute<Boolean> jdkAttribute = Attribute.of("jdk", Boolean.class);
|
||||
project.getDependencies().getAttributesSchema().attribute(jdkAttribute);
|
||||
project.getDependencies().getArtifactTypes().maybeCreate(ArtifactTypeDefinition.ZIP_TYPE);
|
||||
project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> {
|
||||
transformSpec.getFrom().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE);
|
||||
transformSpec.getTo().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
|
||||
transformSpec.getFrom()
|
||||
.attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE)
|
||||
.attribute(jdkAttribute, true);
|
||||
transformSpec.getTo()
|
||||
.attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE)
|
||||
.attribute(jdkAttribute, true);
|
||||
transformSpec.parameters(parameters -> parameters.setTrimmedPrefixPattern(JDK_TRIMMED_PREFIX));
|
||||
});
|
||||
|
||||
ArtifactTypeDefinition tarArtifactTypeDefinition = project.getDependencies().getArtifactTypes().maybeCreate("tar.gz");
|
||||
project.getDependencies().registerTransform(SymbolicLinkPreservingUntarTransform.class, transformSpec -> {
|
||||
transformSpec.getFrom().attribute(ArtifactAttributes.ARTIFACT_FORMAT, tarArtifactTypeDefinition.getName());
|
||||
transformSpec.getTo().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
|
||||
transformSpec.getFrom()
|
||||
.attribute(ArtifactAttributes.ARTIFACT_FORMAT, tarArtifactTypeDefinition.getName())
|
||||
.attribute(jdkAttribute, true);
|
||||
transformSpec.getTo()
|
||||
.attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE)
|
||||
.attribute(jdkAttribute, true);
|
||||
transformSpec.parameters(parameters -> parameters.setTrimmedPrefixPattern(JDK_TRIMMED_PREFIX));
|
||||
});
|
||||
|
||||
NamedDomainObjectContainer<Jdk> jdksContainer = project.container(Jdk.class, name -> {
|
||||
Configuration configuration = project.getConfigurations().create("jdk_" + name);
|
||||
configuration.setCanBeConsumed(false);
|
||||
configuration.getAttributes().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
|
||||
configuration.getAttributes().attribute(jdkAttribute, true);
|
||||
Jdk jdk = new Jdk(name, configuration, project.getObjects());
|
||||
configuration.defaultDependencies(dependencies -> {
|
||||
jdk.finalizeValues();
|
||||
|
|
|
|||
|
|
@ -1182,7 +1182,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
|
|||
}
|
||||
|
||||
private Path getExtractedDistributionDir() {
|
||||
return Paths.get(distributions.get(currentDistro).getExtracted().toString());
|
||||
return distributions.get(currentDistro).getExtracted().getSingleFile().toPath();
|
||||
}
|
||||
|
||||
private List<File> getInstalledFileSet(Action<? super PatternFilterable> filter) {
|
||||
|
|
|
|||
|
|
@ -31,22 +31,22 @@ import java.io.IOException;
|
|||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.nio.file.attribute.PosixFileAttributeView;
|
||||
import java.nio.file.attribute.PosixFilePermission;
|
||||
import java.nio.file.attribute.PosixFilePermissions;
|
||||
import java.util.Set;
|
||||
import java.util.function.Function;
|
||||
|
||||
import static org.elasticsearch.gradle.util.PermissionUtils.chmod;
|
||||
|
||||
public abstract class SymbolicLinkPreservingUntarTransform implements UnpackTransform {
|
||||
|
||||
public void unpack(File tarFile, File targetDir) throws IOException {
|
||||
Logging.getLogger(SymbolicLinkPreservingUntarTransform.class)
|
||||
.info("Unpacking " + tarFile.getName() + " using " + SymbolicLinkPreservingUntarTransform.class.getSimpleName() + ".");
|
||||
Function<String, Path> pathModifier = pathResolver();
|
||||
|
||||
TarArchiveInputStream tar = new TarArchiveInputStream(new GzipCompressorInputStream(new FileInputStream(tarFile)));
|
||||
final Path destinationPath = targetDir.toPath();
|
||||
TarArchiveEntry entry = tar.getNextTarEntry();
|
||||
while (entry != null) {
|
||||
final Path relativePath = UnpackTransform.trimArchiveExtractPath(entry.getName());
|
||||
final Path relativePath = pathModifier.apply(entry.getName());
|
||||
if (relativePath == null) {
|
||||
entry = tar.getNextTarEntry();
|
||||
continue;
|
||||
|
|
@ -70,41 +70,10 @@ public abstract class SymbolicLinkPreservingUntarTransform implements UnpackTran
|
|||
}
|
||||
if (entry.isSymbolicLink() == false) {
|
||||
// check if the underlying file system supports POSIX permissions
|
||||
final PosixFileAttributeView view = Files.getFileAttributeView(destination, PosixFileAttributeView.class);
|
||||
if (view != null) {
|
||||
final Set<PosixFilePermission> permissions = PosixFilePermissions.fromString(
|
||||
permissions((entry.getMode() >> 6) & 07) + permissions((entry.getMode() >> 3) & 07) + permissions(
|
||||
(entry.getMode() >> 0) & 07
|
||||
)
|
||||
);
|
||||
Files.setPosixFilePermissions(destination, permissions);
|
||||
}
|
||||
chmod(destination, entry.getMode());
|
||||
}
|
||||
entry = tar.getNextTarEntry();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static String permissions(final int permissions) {
|
||||
if (permissions < 0 || permissions > 7) {
|
||||
throw new IllegalArgumentException("permissions [" + permissions + "] out of range");
|
||||
}
|
||||
final StringBuilder sb = new StringBuilder(3);
|
||||
if ((permissions & 4) == 4) {
|
||||
sb.append('r');
|
||||
} else {
|
||||
sb.append('-');
|
||||
}
|
||||
if ((permissions & 2) == 2) {
|
||||
sb.append('w');
|
||||
} else {
|
||||
sb.append('-');
|
||||
}
|
||||
if ((permissions & 1) == 1) {
|
||||
sb.append('x');
|
||||
} else {
|
||||
sb.append('-');
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,16 +25,28 @@ import org.gradle.api.artifacts.transform.TransformOutputs;
|
|||
import org.gradle.api.artifacts.transform.TransformParameters;
|
||||
import org.gradle.api.file.FileSystemLocation;
|
||||
import org.gradle.api.provider.Provider;
|
||||
import org.gradle.api.tasks.Input;
|
||||
import org.gradle.api.tasks.Optional;
|
||||
import org.gradle.api.tasks.PathSensitive;
|
||||
import org.gradle.api.tasks.PathSensitivity;
|
||||
import org.gradle.internal.UncheckedException;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.FileSystems;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.function.Function;
|
||||
|
||||
public interface UnpackTransform extends TransformAction<TransformParameters.None> {
|
||||
public interface UnpackTransform extends TransformAction<UnpackTransform.Parameters> {
|
||||
|
||||
interface Parameters extends TransformParameters {
|
||||
@Input
|
||||
@Optional
|
||||
String getTrimmedPrefixPattern();
|
||||
|
||||
void setTrimmedPrefixPattern(String pattern);
|
||||
}
|
||||
|
||||
@PathSensitive(PathSensitivity.NAME_ONLY)
|
||||
@InputArtifact
|
||||
|
|
@ -43,9 +55,9 @@ public interface UnpackTransform extends TransformAction<TransformParameters.Non
|
|||
@Override
|
||||
default void transform(TransformOutputs outputs) {
|
||||
File archiveFile = getArchiveFile().get().getAsFile();
|
||||
File unzipDir = outputs.dir(archiveFile.getName());
|
||||
File extractedDir = outputs.dir(archiveFile.getName());
|
||||
try {
|
||||
unpack(archiveFile, unzipDir);
|
||||
unpack(archiveFile, extractedDir);
|
||||
} catch (IOException e) {
|
||||
throw UncheckedException.throwAsUncheckedException(e);
|
||||
}
|
||||
|
|
@ -53,8 +65,17 @@ public interface UnpackTransform extends TransformAction<TransformParameters.Non
|
|||
|
||||
void unpack(File archiveFile, File targetDir) throws IOException;
|
||||
|
||||
default Function<String, Path> pathResolver() {
|
||||
String trimmedPrefixPattern = getParameters().getTrimmedPrefixPattern();
|
||||
return trimmedPrefixPattern != null
|
||||
? (i) -> trimArchiveExtractPath(trimmedPrefixPattern, i)
|
||||
: (i) -> FileSystems.getDefault().getPath(i);
|
||||
}
|
||||
|
||||
/*
|
||||
* We want to remove up to the and including the jdk-.* relative paths. That is a JDK archive is structured as:
|
||||
* We want to be able to trim off certain prefixes when transforming archives.
|
||||
*
|
||||
* E.g We want to remove up to the and including the jdk-.* relative paths. That is a JDK archive is structured as:
|
||||
* jdk-12.0.1/
|
||||
* jdk-12.0.1/Contents
|
||||
* ...
|
||||
|
|
@ -66,11 +87,11 @@ public interface UnpackTransform extends TransformAction<TransformParameters.Non
|
|||
*
|
||||
* so we account for this and search the path components until we find the jdk-12.0.1, and strip the leading components.
|
||||
*/
|
||||
static Path trimArchiveExtractPath(String relativePath) {
|
||||
static Path trimArchiveExtractPath(String ignoredPattern, String relativePath) {
|
||||
final Path entryName = Paths.get(relativePath);
|
||||
int index = 0;
|
||||
for (; index < entryName.getNameCount(); index++) {
|
||||
if (entryName.getName(index).toString().matches("jdk-?\\d.*")) {
|
||||
if (entryName.getName(index).toString().matches(ignoredPattern)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -81,5 +102,4 @@ public interface UnpackTransform extends TransformAction<TransformParameters.Non
|
|||
// finally remove the top-level directories from the output path
|
||||
return entryName.subpath(index + 1, entryName.getNameCount());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,35 +20,48 @@
|
|||
package org.elasticsearch.gradle.transform;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.tools.zip.ZipEntry;
|
||||
import org.apache.tools.zip.ZipFile;
|
||||
import org.gradle.api.logging.Logging;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Enumeration;
|
||||
import java.util.function.Function;
|
||||
|
||||
import static org.elasticsearch.gradle.util.PermissionUtils.chmod;
|
||||
|
||||
public abstract class UnzipTransform implements UnpackTransform {
|
||||
|
||||
public void unpack(File zipFile, File targetDir) throws IOException {
|
||||
Logging.getLogger(UnzipTransform.class)
|
||||
.info("Unpacking " + zipFile.getName() + " using " + UnzipTransform.class.getSimpleName() + ".");
|
||||
|
||||
try (ZipInputStream inputStream = new ZipInputStream(new BufferedInputStream(new FileInputStream(zipFile)))) {
|
||||
ZipEntry entry;
|
||||
while ((entry = inputStream.getNextEntry()) != null) {
|
||||
if (entry.isDirectory()) {
|
||||
Function<String, Path> pathModifier = pathResolver();
|
||||
ZipFile zip = new ZipFile(zipFile);
|
||||
try {
|
||||
Enumeration<ZipEntry> entries = zip.getEntries();
|
||||
while (entries.hasMoreElements()) {
|
||||
ZipEntry zipEntry = entries.nextElement();
|
||||
Path child = pathModifier.apply(zipEntry.getName());
|
||||
if (child == null) {
|
||||
continue;
|
||||
}
|
||||
String child = UnpackTransform.trimArchiveExtractPath(entry.getName()).toString();
|
||||
File outFile = new File(targetDir, child);
|
||||
outFile.getParentFile().mkdirs();
|
||||
try (FileOutputStream outputStream = new FileOutputStream(outFile)) {
|
||||
IOUtils.copyLarge(inputStream, outputStream);
|
||||
Path outputPath = targetDir.toPath().resolve(child);
|
||||
if (zipEntry.isDirectory()) {
|
||||
outputPath.toFile().mkdirs();
|
||||
chmod(outputPath, zipEntry.getUnixMode());
|
||||
continue;
|
||||
}
|
||||
try (FileOutputStream outputStream = new FileOutputStream(outputPath.toFile())) {
|
||||
IOUtils.copyLarge(zip.getInputStream(zipEntry), outputStream);
|
||||
}
|
||||
chmod(outputPath, zipEntry.getUnixMode());
|
||||
}
|
||||
} finally {
|
||||
zip.close();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,69 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.gradle.util;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.attribute.PosixFileAttributeView;
|
||||
import java.nio.file.attribute.PosixFilePermission;
|
||||
import java.nio.file.attribute.PosixFilePermissions;
|
||||
import java.util.Set;
|
||||
|
||||
public class PermissionUtils {
|
||||
|
||||
public static void chmod(Path path, int mode) throws IOException {
|
||||
final PosixFileAttributeView view = Files.getFileAttributeView(path, PosixFileAttributeView.class);
|
||||
if (view != null && (mode != 0)) {
|
||||
final Set<PosixFilePermission> permissions = permissionsFromInt(mode);
|
||||
Files.setPosixFilePermissions(path, permissions);
|
||||
}
|
||||
}
|
||||
|
||||
private static Set<PosixFilePermission> permissionsFromInt(int mode) {
|
||||
return PosixFilePermissions.fromString(
|
||||
permissions((mode >> 6) & 07) + permissions((mode >> 3) & 07) + permissions((mode >> 0) & 07)
|
||||
);
|
||||
}
|
||||
|
||||
private static String permissions(final int permissions) {
|
||||
if (permissions < 0 || permissions > 7) {
|
||||
throw new IllegalArgumentException("permissions [" + permissions + "] out of range");
|
||||
}
|
||||
final StringBuilder sb = new StringBuilder(3);
|
||||
if ((permissions & 4) == 4) {
|
||||
sb.append('r');
|
||||
} else {
|
||||
sb.append('-');
|
||||
}
|
||||
if ((permissions & 2) == 2) {
|
||||
sb.append('w');
|
||||
} else {
|
||||
sb.append('-');
|
||||
}
|
||||
if ((permissions & 1) == 1) {
|
||||
sb.append('x');
|
||||
} else {
|
||||
sb.append('-');
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
import org.elasticsearch.gradle.BwcVersions
|
||||
import org.elasticsearch.gradle.Version
|
||||
import org.elasticsearch.gradle.info.BuildParams
|
||||
|
||||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
plugins {
|
||||
id 'elasticsearch.global-build-info'
|
||||
}
|
||||
boolean internal = Boolean.parseBoolean(System.getProperty("tests.internal", "true"))
|
||||
BuildParams.init { it.setIsInternal(internal) }
|
||||
|
||||
project.gradle.projectsEvaluated {
|
||||
// wire the download service url to wiremock
|
||||
String fakeDownloadService = System.getProperty('tests.download_service')
|
||||
if (fakeDownloadService != null) {
|
||||
IvyArtifactRepository repository = (IvyArtifactRepository) rootProject.repositories.getByName("elasticsearch-downloads")
|
||||
repository.setUrl(fakeDownloadService)
|
||||
repository = (IvyArtifactRepository) project('subproj').repositories.getByName("elasticsearch-downloads")
|
||||
repository.setUrl(fakeDownloadService)
|
||||
if (internal == false) {
|
||||
repository = (IvyArtifactRepository) rootProject.repositories.getByName("elasticsearch-snapshots")
|
||||
repository.setUrl(fakeDownloadService)
|
||||
repository = (IvyArtifactRepository) project('subproj').repositories.getByName("elasticsearch-snapshots")
|
||||
repository.setUrl(fakeDownloadService)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (internal) {
|
||||
Version currentVersion = Version.fromString("9.0.0")
|
||||
BwcVersions versions = new BwcVersions(new TreeSet<>(
|
||||
Arrays.asList(Version.fromString("8.0.0"), Version.fromString("8.0.1"), Version.fromString("8.1.0"), currentVersion)),
|
||||
currentVersion)
|
||||
BuildParams.init { it.setBwcVersions(versions) }
|
||||
}
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
String distroConfig = System.getProperty('tests.local_distro.config')
|
||||
if (distroConfig != null) {
|
||||
// setup the test distribution as an artifact of this project
|
||||
String distroType = System.getProperty('tests.distro.type')
|
||||
|
||||
def buildDistro
|
||||
File buildFile
|
||||
if (['rpm', 'deb'].contains(distroType)) {
|
||||
buildDistro = project.tasks.register("build" + distroType.capitalize(), Copy) {
|
||||
from 'files'
|
||||
into 'build/files'
|
||||
include 'fake_elasticsearch.tar.gz'
|
||||
// this shouldn't be extracted so we just rename the file so it is a dummy package
|
||||
rename { filename -> filename.replace('tar.gz', distroType) }
|
||||
}
|
||||
buildFile = project.file("build/files/fake_elasticsearch." + distroType)
|
||||
} else {
|
||||
String distroPlatform = System.getProperty('tests.distro.platform')
|
||||
String extension = "tar.gz"
|
||||
if (distroType == 'archive' && distroPlatform == 'windows' || distroType == 'integ-test-zip') {
|
||||
extension = "zip"
|
||||
}
|
||||
// copy file as is
|
||||
buildDistro = project.tasks.register("buildArchive", Copy) {
|
||||
from 'files'
|
||||
include "fake_elasticsearch.${extension}"
|
||||
into 'build/files'
|
||||
}
|
||||
buildFile = project.file("build/files/fake_elasticsearch.${extension}")
|
||||
}
|
||||
|
||||
configurations.create(distroConfig)
|
||||
artifacts.add(distroConfig, [file: buildFile, builtBy: buildDistro])
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
include 'subproj'
|
||||
|
||||
String distroProject = System.getProperty('tests.local_distro.project')
|
||||
System.out.println("local distro project: " + distroProject);
|
||||
if (distroProject != null) {
|
||||
include distroProject
|
||||
project(distroProject).projectDir = new File(rootDir, 'distribution')
|
||||
}
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
import org.elasticsearch.gradle.Architecture
|
||||
|
||||
plugins {
|
||||
id 'elasticsearch.distribution-download'
|
||||
}
|
||||
|
||||
String distroVersion = System.getProperty('tests.distro.version')
|
||||
String distroType = System.getProperty('tests.distro.type')
|
||||
String distroPlatform = System.getProperty('tests.distro.platform')
|
||||
String distroFlavor = System.getProperty('tests.distro.flavor')
|
||||
String distroBundledJdk = System.getProperty('tests.distro.bundledJdk')
|
||||
String distroArch = System.getProperty('tests.distro.arch');
|
||||
|
||||
elasticsearch_distributions {
|
||||
test_distro {
|
||||
if (distroVersion != null) {
|
||||
version = distroVersion
|
||||
}
|
||||
if (distroType != null) {
|
||||
type = distroType
|
||||
}
|
||||
if (distroPlatform != null) {
|
||||
platform = distroPlatform
|
||||
}
|
||||
if (distroFlavor != null) {
|
||||
flavor = distroFlavor
|
||||
}
|
||||
if (distroBundledJdk != null) {
|
||||
bundledJdk = Boolean.parseBoolean(distroBundledJdk)
|
||||
}
|
||||
if (distroArch != null) {
|
||||
architecture = Architecture.valueOf(distroArch);
|
||||
} else {
|
||||
architecture = Architecture.current();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tasks.register("assertDistroFile") {
|
||||
dependsOn elasticsearch_distributions.test_distro
|
||||
doLast {
|
||||
File distroFile = new File(elasticsearch_distributions.test_distro.toString())
|
||||
if (distroFile.exists() == false) {
|
||||
throw new GradleException("distro file does not exist: ${distroFile}")
|
||||
}
|
||||
if (distroFile.isFile() == false) {
|
||||
throw new GradleException("distro file is not a regular file: ${distroFile}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (['rpm', 'deb'].contains(distroType) == false) {
|
||||
tasks.register("assertDistroExtracted") {
|
||||
dependsOn elasticsearch_distributions.test_distro.extracted, assertDistroFile
|
||||
doLast {
|
||||
File distroExtracted = new File(elasticsearch_distributions.test_distro.extracted.toString())
|
||||
if (distroExtracted.exists() == false) {
|
||||
throw new GradleException("extracted does not exist: ${distroExtracted}")
|
||||
}
|
||||
if (distroExtracted.isDirectory() == false) {
|
||||
throw new GradleException("extracted distro is not a directory: ${distroExtracted}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -316,7 +316,7 @@ BuildParams.bwcVersions.forPreviousUnreleased { BwcVersions.UnreleasedVersionInf
|
|||
File artifactFile = e.value
|
||||
String artifactFileName = artifactFile.name
|
||||
String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
|
||||
String suffix = artifactFile.toString()[-3..-1]
|
||||
String suffix = artifactFileName.endsWith("tar.gz") ? "tar.gz" : artifactFileName[-3..-1]
|
||||
int archIndex = artifactFileName.indexOf('x86_64')
|
||||
String classifier = ''
|
||||
if (archIndex != -1) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue