Revert "Simplify distribution download and extraction (7.x backport) (#61184)"
This reverts commit 8b7a0a1f64.
This commit is contained in:
Mark Vieira
parent
a3f357c8a5
commit
d26a3e144e
|
|
@ -92,7 +92,7 @@ dependencies {
|
|||
|
||||
api 'commons-codec:commons-codec:1.12'
|
||||
api 'org.apache.commons:commons-compress:1.19'
|
||||
api 'org.apache.ant:ant:1.10.8'
|
||||
|
||||
api 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3'
|
||||
api 'com.netflix.nebula:nebula-publishing-plugin:4.4.4'
|
||||
api 'com.netflix.nebula:gradle-info-plugin:7.1.3'
|
||||
|
|
|
|||
|
|
@ -1,179 +0,0 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.gradle
|
||||
|
||||
import com.github.tomakehurst.wiremock.WireMockServer
|
||||
import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
|
||||
import org.elasticsearch.gradle.fixtures.WiremockFixture
|
||||
import org.elasticsearch.gradle.transform.SymbolicLinkPreservingUntarTransform
|
||||
import org.gradle.testkit.runner.TaskOutcome
|
||||
import spock.lang.Unroll
|
||||
|
||||
class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
|
||||
|
||||
@Unroll
|
||||
def "#distType version can be resolved"() {
|
||||
given:
|
||||
def mockRepoUrl = urlPath(version, platform)
|
||||
def mockedContent = filebytes(mockRepoUrl)
|
||||
|
||||
buildFile << applyPluginAndSetupDistro(version, platform)
|
||||
|
||||
when:
|
||||
def result = WiremockFixture.withWireMock(mockRepoUrl, mockedContent) { server ->
|
||||
buildFile << repositoryMockSetup(server)
|
||||
gradleRunner('setupDistro').build()
|
||||
}
|
||||
|
||||
then:
|
||||
result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
|
||||
assertExtractedDistroCreated("build/distro")
|
||||
|
||||
where:
|
||||
version | platform | distType
|
||||
VersionProperties.getElasticsearch() | "linux" | "current"
|
||||
"8.1.0-SNAPSHOT" | "linux" | "bwc"
|
||||
"7.0.0" | "windows" | "released"
|
||||
}
|
||||
|
||||
|
||||
def "transformed versions are kept across builds"() {
|
||||
given:
|
||||
def version = VersionProperties.getElasticsearch()
|
||||
def mockRepoUrl = urlPath(version)
|
||||
def mockedContent = filebytes(mockRepoUrl)
|
||||
|
||||
buildFile << applyPluginAndSetupDistro(version, 'linux')
|
||||
buildFile << """
|
||||
apply plugin:'base'
|
||||
"""
|
||||
|
||||
|
||||
when:
|
||||
def result = WiremockFixture.withWireMock(mockRepoUrl, mockedContent) { server ->
|
||||
buildFile << repositoryMockSetup(server)
|
||||
gradleRunner('clean', 'setupDistro').build()
|
||||
gradleRunner('clean', 'setupDistro', '-i').build()
|
||||
}
|
||||
|
||||
then:
|
||||
result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
|
||||
assertOutputContains(result.output, "Skipping ${SymbolicLinkPreservingUntarTransform.class.simpleName}")
|
||||
}
|
||||
|
||||
def "transforms are reused across projects"() {
|
||||
given:
|
||||
def version = VersionProperties.getElasticsearch()
|
||||
def mockRepoUrl = urlPath(version)
|
||||
def mockedContent = filebytes(mockRepoUrl)
|
||||
|
||||
3.times {
|
||||
settingsFile << """
|
||||
include ':sub-$it'
|
||||
"""
|
||||
}
|
||||
buildFile.text = """
|
||||
import org.elasticsearch.gradle.Architecture
|
||||
|
||||
plugins {
|
||||
id 'elasticsearch.distribution-download'
|
||||
}
|
||||
|
||||
subprojects {
|
||||
apply plugin: 'elasticsearch.distribution-download'
|
||||
|
||||
${setupTestDistro(version, 'linux')}
|
||||
${setupDistroTask()}
|
||||
}
|
||||
"""
|
||||
|
||||
when:
|
||||
def result = WiremockFixture.withWireMock(mockRepoUrl, mockedContent) { server ->
|
||||
buildFile << repositoryMockSetup(server)
|
||||
gradleRunner('setupDistro', '-i', '-g', testProjectDir.newFolder().toString()).build()
|
||||
}
|
||||
|
||||
then:
|
||||
result.tasks.size() == 3
|
||||
result.output.count("Unpacking elasticsearch-${version}-linux-x86_64.tar.gz using SymbolicLinkPreservingUntarTransform.") == 1
|
||||
}
|
||||
|
||||
private boolean assertExtractedDistroCreated(String relativePath) {
|
||||
File distroExtracted = new File(testProjectDir.root, relativePath)
|
||||
assert distroExtracted.exists()
|
||||
assert distroExtracted.isDirectory()
|
||||
assert new File(distroExtracted, "elasticsearch-1.2.3/bin/elasticsearch").exists()
|
||||
true
|
||||
}
|
||||
|
||||
private static String urlPath(String version, String platform = 'linux') {
|
||||
String fileType = platform == "linux" ? "tar.gz" : "zip"
|
||||
"/downloads/elasticsearch/elasticsearch-${version}-${platform}-x86_64.$fileType"
|
||||
}
|
||||
|
||||
private static String repositoryMockSetup(WireMockServer server) {
|
||||
"""allprojects{ p ->
|
||||
p.repositories.all { repo ->
|
||||
repo.setUrl('${server.baseUrl()}')
|
||||
}
|
||||
}"""
|
||||
}
|
||||
|
||||
private static byte[] filebytes(String urlPath) throws IOException {
|
||||
String suffix = urlPath.endsWith("zip") ? "zip" : "tar.gz";
|
||||
return DistributionDownloadPluginFuncTest.getResourceAsStream("fake_elasticsearch." + suffix).getBytes()
|
||||
}
|
||||
|
||||
private static String applyPluginAndSetupDistro(String version, String platform) {
|
||||
"""
|
||||
import org.elasticsearch.gradle.Architecture
|
||||
|
||||
plugins {
|
||||
id 'elasticsearch.distribution-download'
|
||||
}
|
||||
|
||||
${setupTestDistro(version, platform)}
|
||||
${setupDistroTask()}
|
||||
|
||||
"""
|
||||
}
|
||||
|
||||
private static String setupTestDistro(String version, String platform) {
|
||||
return """
|
||||
elasticsearch_distributions {
|
||||
test_distro {
|
||||
version = "$version"
|
||||
type = "archive"
|
||||
platform = "$platform"
|
||||
architecture = Architecture.current();
|
||||
}
|
||||
}
|
||||
"""
|
||||
}
|
||||
|
||||
private static String setupDistroTask() {
|
||||
return """
|
||||
tasks.register("setupDistro", Sync) {
|
||||
from(elasticsearch_distributions.test_distro.extracted)
|
||||
into("build/distro")
|
||||
}
|
||||
"""
|
||||
}
|
||||
}
|
||||
|
|
@ -38,7 +38,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
|
|||
"""
|
||||
|
||||
when:
|
||||
def result = gradleRunner("tasks").buildAndFail()
|
||||
def result = gradleRunner("createExtractedTestDistro").buildAndFail()
|
||||
|
||||
then:
|
||||
assertOutputContains(result.output, "Plugin 'elasticsearch.internal-distribution-download' is not supported. " +
|
||||
|
|
@ -61,19 +61,18 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
|
|||
architecture = Architecture.current();
|
||||
}
|
||||
}
|
||||
tasks.register("setupDistro", Sync) {
|
||||
from(elasticsearch_distributions.test_distro.extracted)
|
||||
into("build/distro")
|
||||
tasks.register("createExtractedTestDistro") {
|
||||
dependsOn elasticsearch_distributions.test_distro.extracted
|
||||
}
|
||||
"""
|
||||
|
||||
when:
|
||||
def result = gradleRunner("setupDistro", '-g', testProjectDir.newFolder('GUH').path).build()
|
||||
def result = gradleRunner("createExtractedTestDistro").build()
|
||||
|
||||
then:
|
||||
result.task(":distribution:archives:linux-tar:buildTar").outcome == TaskOutcome.SUCCESS
|
||||
result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
|
||||
assertExtractedDistroIsCreated(distroVersion, "build/distro", 'current-marker.txt')
|
||||
result.task(":extractElasticsearchLinux$distroVersion").outcome == TaskOutcome.SUCCESS
|
||||
assertExtractedDistroIsCreated(distroVersion, 'current-marker.txt')
|
||||
}
|
||||
|
||||
def "resolves bwc versions from source"() {
|
||||
|
|
@ -92,18 +91,16 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
|
|||
architecture = Architecture.current();
|
||||
}
|
||||
}
|
||||
tasks.register("setupDistro", Sync) {
|
||||
from(elasticsearch_distributions.test_distro.extracted)
|
||||
into("build/distro")
|
||||
tasks.register("createExtractedTestDistro") {
|
||||
dependsOn elasticsearch_distributions.test_distro.extracted
|
||||
}
|
||||
"""
|
||||
when:
|
||||
|
||||
def result = gradleRunner("setupDistro").build()
|
||||
def result = gradleRunner("createExtractedTestDistro").build()
|
||||
then:
|
||||
result.task(":distribution:bwc:minor:buildBwcTask").outcome == TaskOutcome.SUCCESS
|
||||
result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
|
||||
assertExtractedDistroIsCreated(distroVersion, "build/distro", 'bwc-marker.txt')
|
||||
result.task(":extractElasticsearchLinux8.1.0").outcome == TaskOutcome.SUCCESS
|
||||
assertExtractedDistroIsCreated(distroVersion,'bwc-marker.txt')
|
||||
}
|
||||
|
||||
def "fails on resolving bwc versions with no bundled jdk"() {
|
||||
|
|
@ -198,8 +195,8 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
|
|||
|
||||
}
|
||||
|
||||
boolean assertExtractedDistroIsCreated(String version, String relativeDistroPath, String markerFileName) {
|
||||
File extractedFolder = new File(testProjectDir.root, relativeDistroPath)
|
||||
boolean assertExtractedDistroIsCreated(String version, String markerFileName) {
|
||||
File extractedFolder = new File(testProjectDir.root, "build/elasticsearch-distros/extracted_elasticsearch_${version}_archive_linux_default")
|
||||
assert extractedFolder.exists()
|
||||
assert new File(extractedFolder, markerFileName).exists()
|
||||
true
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
|
|||
given:
|
||||
def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform)
|
||||
def mockedContent = filebytes(jdkVendor, platform)
|
||||
3.times {
|
||||
10.times {
|
||||
settingsFile << """
|
||||
include ':sub-$it'
|
||||
"""
|
||||
|
|
@ -132,8 +132,8 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
|
|||
}
|
||||
|
||||
then:
|
||||
result.tasks.size() == 3
|
||||
result.output.count("Unpacking linux-12.0.2-x64.tar.gz using ${SymbolicLinkPreservingUntarTransform.simpleName}.") == 1
|
||||
result.tasks.size() == 10
|
||||
result.output.count("Unpacking linux-12.0.2-x64.tar.gz using SymbolicLinkPreservingUntarTransform.") == 1
|
||||
|
||||
where:
|
||||
platform | jdkVendor | jdkVersion | expectedJavaBin
|
||||
|
|
@ -149,7 +149,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
|
|||
plugins {
|
||||
id 'elasticsearch.jdk-download'
|
||||
}
|
||||
apply plugin: 'base'
|
||||
|
||||
apply plugin: 'elasticsearch.jdk-download'
|
||||
|
||||
jdks {
|
||||
|
|
@ -175,9 +175,9 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
|
|||
|
||||
def commonGradleUserHome = testProjectDir.newFolder().toString()
|
||||
// initial run
|
||||
gradleRunner('clean', 'getJdk', '-g', commonGradleUserHome).build()
|
||||
gradleRunner('getJdk', '-g', commonGradleUserHome).build()
|
||||
// run against up-to-date transformations
|
||||
gradleRunner('clean', 'getJdk', '-i', '-g', commonGradleUserHome).build()
|
||||
gradleRunner('getJdk', '-i', '-g', commonGradleUserHome).build()
|
||||
}
|
||||
|
||||
then:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,188 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.gradle;
|
||||
|
||||
import com.github.tomakehurst.wiremock.WireMockServer;
|
||||
import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
|
||||
import org.gradle.testkit.runner.BuildResult;
|
||||
import org.gradle.testkit.runner.GradleRunner;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
|
||||
import static com.github.tomakehurst.wiremock.client.WireMock.get;
|
||||
import static com.github.tomakehurst.wiremock.client.WireMock.head;
|
||||
import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
|
||||
|
||||
public class DistributionDownloadPluginIT extends GradleIntegrationTestCase {
|
||||
|
||||
// TODO: check reuse of root task across projects MOVE TO UNIT TEST
|
||||
// TODO: future: check integ-test-zip to maven, snapshots to snapshot service for external project
|
||||
public void testCurrentExternal() throws Exception {
|
||||
checkService(
|
||||
VersionProperties.getElasticsearch(),
|
||||
"archive",
|
||||
"linux",
|
||||
null,
|
||||
null,
|
||||
"/downloads/elasticsearch/elasticsearch-" + VersionProperties.getElasticsearch() + "-linux-x86_64.tar.gz",
|
||||
"tests.internal",
|
||||
"false"
|
||||
);
|
||||
}
|
||||
|
||||
public void testBwcExternal() throws Exception {
|
||||
checkService(
|
||||
"8.1.0-SNAPSHOT",
|
||||
"archive",
|
||||
"linux",
|
||||
null,
|
||||
null,
|
||||
"/downloads/elasticsearch/elasticsearch-8.1.0-SNAPSHOT-linux-x86_64.tar.gz",
|
||||
"tests.internal",
|
||||
"false",
|
||||
"tests.current_version",
|
||||
"9.0.0"
|
||||
);
|
||||
}
|
||||
|
||||
public void testReleased() throws Exception {
|
||||
checkService("7.0.0", "archive", "windows", null, null, "/downloads/elasticsearch/elasticsearch-7.0.0-windows-x86_64.zip");
|
||||
checkService("6.5.0", "archive", "windows", null, null, "/downloads/elasticsearch/elasticsearch-6.5.0.zip");
|
||||
}
|
||||
|
||||
public void testReleasedExternal() throws Exception {
|
||||
checkService(
|
||||
"7.0.0",
|
||||
"archive",
|
||||
"windows",
|
||||
null,
|
||||
null,
|
||||
"/downloads/elasticsearch/elasticsearch-7.0.0-windows-x86_64.zip",
|
||||
"tests.internal",
|
||||
"false"
|
||||
);
|
||||
checkService(
|
||||
"6.5.0",
|
||||
"archive",
|
||||
"windows",
|
||||
null,
|
||||
null,
|
||||
"/downloads/elasticsearch/elasticsearch-6.5.0.zip",
|
||||
"tests.internal",
|
||||
"false"
|
||||
);
|
||||
}
|
||||
|
||||
private void checkService(
|
||||
String version,
|
||||
String type,
|
||||
String platform,
|
||||
String flavor,
|
||||
Boolean bundledJdk,
|
||||
String urlPath,
|
||||
String... sysProps
|
||||
) throws IOException {
|
||||
String suffix = urlPath.endsWith("zip") ? "zip" : "tar.gz";
|
||||
String sourceFile = "src/testKit/distribution-download/distribution/files/fake_elasticsearch." + suffix;
|
||||
WireMockServer wireMock = new WireMockServer(0);
|
||||
try {
|
||||
final byte[] filebytes;
|
||||
try (InputStream stream = Files.newInputStream(Paths.get(sourceFile))) {
|
||||
filebytes = stream.readAllBytes();
|
||||
}
|
||||
wireMock.stubFor(head(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200)));
|
||||
wireMock.stubFor(get(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200).withBody(filebytes)));
|
||||
wireMock.start();
|
||||
|
||||
List<String> allSysProps = new ArrayList<>();
|
||||
allSysProps.addAll(Arrays.asList(sysProps));
|
||||
allSysProps.add("tests.download_service");
|
||||
allSysProps.add(wireMock.baseUrl());
|
||||
assertExtractedDistro(version, type, platform, flavor, bundledJdk, allSysProps.toArray(new String[0]));
|
||||
} catch (Exception e) {
|
||||
// for debugging
|
||||
System.err.println("missed requests: " + wireMock.findUnmatchedRequests().getRequests());
|
||||
throw e;
|
||||
} finally {
|
||||
wireMock.stop();
|
||||
}
|
||||
}
|
||||
|
||||
private void assertFileDistro(String version, String type, String platform, String flavor, Boolean bundledJdk, String... sysProps)
|
||||
throws IOException {
|
||||
List<String> finalSysProps = new ArrayList<>();
|
||||
addDistroSysProps(finalSysProps, version, type, platform, flavor, bundledJdk);
|
||||
finalSysProps.addAll(Arrays.asList(sysProps));
|
||||
runBuild(":subproj:assertDistroFile", finalSysProps.toArray(new String[0]));
|
||||
}
|
||||
|
||||
private void assertExtractedDistro(String version, String type, String platform, String flavor, Boolean bundledJdk, String... sysProps)
|
||||
throws IOException {
|
||||
List<String> finalSysProps = new ArrayList<>();
|
||||
addDistroSysProps(finalSysProps, version, type, platform, flavor, bundledJdk);
|
||||
finalSysProps.addAll(Arrays.asList(sysProps));
|
||||
runBuild(":subproj:assertDistroExtracted", finalSysProps.toArray(new String[0]));
|
||||
}
|
||||
|
||||
private BuildResult runBuild(String taskname, String... sysProps) throws IOException {
|
||||
assert sysProps.length % 2 == 0;
|
||||
List<String> args = new ArrayList<>();
|
||||
args.add(taskname);
|
||||
for (int i = 0; i < sysProps.length; i += 2) {
|
||||
args.add("-D" + sysProps[i] + "=" + sysProps[i + 1]);
|
||||
}
|
||||
args.add("-i");
|
||||
GradleRunner runner = getGradleRunner("distribution-download").withArguments(args);
|
||||
|
||||
BuildResult result = runner.build();
|
||||
System.out.println(result.getOutput());
|
||||
return result;
|
||||
}
|
||||
|
||||
private void addDistroSysProps(List<String> sysProps, String version, String type, String platform, String flavor, Boolean bundledJdk) {
|
||||
if (version != null) {
|
||||
sysProps.add("tests.distro.version");
|
||||
sysProps.add(version);
|
||||
}
|
||||
if (type != null) {
|
||||
sysProps.add("tests.distro.type");
|
||||
sysProps.add(type);
|
||||
}
|
||||
if (platform != null) {
|
||||
sysProps.add("tests.distro.platform");
|
||||
sysProps.add(platform);
|
||||
}
|
||||
if (flavor != null) {
|
||||
sysProps.add("tests.distro.flavor");
|
||||
sysProps.add(flavor);
|
||||
}
|
||||
if (bundledJdk != null) {
|
||||
sysProps.add("tests.distro.bundledJdk");
|
||||
sysProps.add(bundledJdk.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -24,22 +24,31 @@ import org.elasticsearch.gradle.ElasticsearchDistribution.Platform;
|
|||
import org.elasticsearch.gradle.ElasticsearchDistribution.Type;
|
||||
import org.elasticsearch.gradle.docker.DockerSupportPlugin;
|
||||
import org.elasticsearch.gradle.docker.DockerSupportService;
|
||||
import org.elasticsearch.gradle.transform.SymbolicLinkPreservingUntarTransform;
|
||||
import org.elasticsearch.gradle.transform.UnzipTransform;
|
||||
import org.elasticsearch.gradle.util.GradleUtils;
|
||||
import org.gradle.api.NamedDomainObjectContainer;
|
||||
import org.gradle.api.Plugin;
|
||||
import org.gradle.api.Project;
|
||||
import org.gradle.api.UnknownTaskException;
|
||||
import org.gradle.api.artifacts.Configuration;
|
||||
import org.gradle.api.artifacts.ConfigurationContainer;
|
||||
import org.gradle.api.artifacts.dsl.DependencyHandler;
|
||||
import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
|
||||
import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
|
||||
import org.gradle.api.credentials.HttpHeaderCredentials;
|
||||
import org.gradle.api.internal.artifacts.ArtifactAttributes;
|
||||
import org.gradle.api.file.FileTree;
|
||||
import org.gradle.api.file.RelativePath;
|
||||
import org.gradle.api.provider.Provider;
|
||||
import org.gradle.api.tasks.Sync;
|
||||
import org.gradle.api.tasks.TaskProvider;
|
||||
import org.gradle.authentication.http.HttpHeaderAuthentication;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import static org.elasticsearch.gradle.util.GradleUtils.projectDependency;
|
||||
import static org.elasticsearch.gradle.util.Util.capitalize;
|
||||
|
||||
/**
|
||||
* A plugin to manage getting and extracting distributions of Elasticsearch.
|
||||
|
|
@ -68,17 +77,6 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
|
|||
DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
|
||||
);
|
||||
|
||||
project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> {
|
||||
transformSpec.getFrom().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE);
|
||||
transformSpec.getTo().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
|
||||
});
|
||||
|
||||
ArtifactTypeDefinition tarArtifactTypeDefinition = project.getDependencies().getArtifactTypes().maybeCreate("tar.gz");
|
||||
project.getDependencies().registerTransform(SymbolicLinkPreservingUntarTransform.class, transformSpec -> {
|
||||
transformSpec.getFrom().attribute(ArtifactAttributes.ARTIFACT_FORMAT, tarArtifactTypeDefinition.getName());
|
||||
transformSpec.getTo().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
|
||||
});
|
||||
|
||||
setupResolutionsContainer(project);
|
||||
setupDistributionContainer(project, dockerSupport);
|
||||
setupDownloadServiceRepo(project);
|
||||
|
|
@ -89,7 +87,6 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
|
|||
distributionsContainer = project.container(ElasticsearchDistribution.class, name -> {
|
||||
Configuration fileConfiguration = project.getConfigurations().create("es_distro_file_" + name);
|
||||
Configuration extractedConfiguration = project.getConfigurations().create("es_distro_extracted_" + name);
|
||||
extractedConfiguration.getAttributes().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
|
||||
return new ElasticsearchDistribution(name, project.getObjects(), dockerSupport, fileConfiguration, extractedConfiguration);
|
||||
});
|
||||
project.getExtensions().add(CONTAINER_NAME, distributionsContainer);
|
||||
|
|
@ -118,15 +115,21 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
|
|||
void setupDistributions(Project project) {
|
||||
for (ElasticsearchDistribution distribution : distributionsContainer) {
|
||||
distribution.finalizeValues();
|
||||
|
||||
DependencyHandler dependencies = project.getDependencies();
|
||||
// for the distribution as a file, just depend on the artifact directly
|
||||
Object resolvedDependency = resolveDependencyNotation(project, distribution);
|
||||
dependencies.add(distribution.configuration.getName(), resolvedDependency);
|
||||
dependencies.add(distribution.configuration.getName(), resolveDependencyNotation(project, distribution));
|
||||
|
||||
// no extraction allowed for rpm, deb or docker
|
||||
if (distribution.getType().shouldExtract()) {
|
||||
// The extracted configuration depends on the artifact directly but has
|
||||
// an artifact transform registered to resolve it as an unpacked folder.
|
||||
dependencies.add(distribution.getExtracted().getName(), resolvedDependency);
|
||||
// for the distribution extracted, add a root level task that does the extraction, and depend on that
|
||||
// extracted configuration as an artifact consisting of the extracted distribution directory
|
||||
dependencies.add(
|
||||
distribution.getExtracted().configuration.getName(),
|
||||
projectDependency(project, ":", configName("extracted_elasticsearch", distribution))
|
||||
);
|
||||
// ensure a root level download task exists
|
||||
setupRootDownload(project.getRootProject(), distribution);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -140,6 +143,59 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
|
|||
.orElseGet(() -> dependencyNotation(distribution));
|
||||
}
|
||||
|
||||
private void setupRootDownload(Project rootProject, ElasticsearchDistribution distribution) {
|
||||
String extractTaskName = extractTaskName(distribution);
|
||||
// NOTE: this is *horrendous*, but seems to be the only way to check for the existence of a registered task
|
||||
try {
|
||||
rootProject.getTasks().named(extractTaskName);
|
||||
// already setup this version
|
||||
return;
|
||||
} catch (UnknownTaskException e) {
|
||||
// fall through: register the task
|
||||
}
|
||||
setupDownloadServiceRepo(rootProject);
|
||||
final ConfigurationContainer configurations = rootProject.getConfigurations();
|
||||
String downloadConfigName = configName("elasticsearch", distribution);
|
||||
String extractedConfigName = "extracted_" + downloadConfigName;
|
||||
final Configuration downloadConfig = configurations.create(downloadConfigName);
|
||||
configurations.create(extractedConfigName);
|
||||
rootProject.getDependencies().add(downloadConfigName, resolveDependencyNotation(rootProject, distribution));
|
||||
|
||||
// add task for extraction, delaying resolving config until runtime
|
||||
if (distribution.getType() == Type.ARCHIVE || distribution.getType() == Type.INTEG_TEST_ZIP) {
|
||||
Supplier<File> archiveGetter = downloadConfig::getSingleFile;
|
||||
String extractDir = rootProject.getBuildDir().toPath().resolve("elasticsearch-distros").resolve(extractedConfigName).toString();
|
||||
TaskProvider<Sync> extractTask = rootProject.getTasks().register(extractTaskName, Sync.class, syncTask -> {
|
||||
syncTask.dependsOn(downloadConfig);
|
||||
syncTask.into(extractDir);
|
||||
syncTask.from((Callable<FileTree>) () -> {
|
||||
File archiveFile = archiveGetter.get();
|
||||
String archivePath = archiveFile.toString();
|
||||
if (archivePath.endsWith(".zip")) {
|
||||
return rootProject.zipTree(archiveFile);
|
||||
} else if (archivePath.endsWith(".tar.gz")) {
|
||||
return rootProject.tarTree(rootProject.getResources().gzip(archiveFile));
|
||||
}
|
||||
throw new IllegalStateException("unexpected file extension on [" + archivePath + "]");
|
||||
});
|
||||
|
||||
// Workaround for https://github.com/elastic/elasticsearch/issues/49417
|
||||
syncTask.eachFile(details -> {
|
||||
String[] segments = details.getRelativePath().getSegments();
|
||||
if (segments[0].equals(".")) {
|
||||
details.setRelativePath(new RelativePath(true, Arrays.copyOfRange(segments, 1, segments.length)));
|
||||
}
|
||||
});
|
||||
});
|
||||
rootProject.getArtifacts()
|
||||
.add(
|
||||
extractedConfigName,
|
||||
rootProject.getLayout().getProjectDirectory().dir(extractDir),
|
||||
artifact -> artifact.builtBy(extractTask)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private static void addIvyRepo(Project project, String name, String url, String group) {
|
||||
IvyArtifactRepository ivyRepo = project.getRepositories().ivy(repo -> {
|
||||
repo.setName(name);
|
||||
|
|
@ -207,4 +263,35 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
|
|||
String group = distribution.getVersion().endsWith("-SNAPSHOT") ? FAKE_SNAPSHOT_IVY_GROUP : FAKE_IVY_GROUP;
|
||||
return group + ":elasticsearch" + flavor + ":" + distribution.getVersion() + classifier + "@" + extension;
|
||||
}
|
||||
|
||||
private static String configName(String prefix, ElasticsearchDistribution distribution) {
|
||||
return String.format(
|
||||
"%s_%s_%s_%s%s%s",
|
||||
prefix,
|
||||
distribution.getVersion(),
|
||||
distribution.getType(),
|
||||
distribution.getPlatform() == null ? "" : distribution.getPlatform() + "_",
|
||||
distribution.getFlavor(),
|
||||
distribution.getBundledJdk() ? "" : "_nojdk"
|
||||
);
|
||||
}
|
||||
|
||||
private static String extractTaskName(ElasticsearchDistribution distribution) {
|
||||
String taskName = "extractElasticsearch";
|
||||
if (distribution.getType() != Type.INTEG_TEST_ZIP) {
|
||||
if (distribution.getFlavor() == Flavor.OSS) {
|
||||
taskName += "Oss";
|
||||
}
|
||||
if (distribution.getBundledJdk() == false) {
|
||||
taskName += "NoJdk";
|
||||
}
|
||||
}
|
||||
if (distribution.getType() == Type.ARCHIVE) {
|
||||
taskName += capitalize(distribution.getPlatform().toString());
|
||||
} else if (distribution.getType() != Type.INTEG_TEST_ZIP) {
|
||||
taskName += capitalize(distribution.getType().toString());
|
||||
}
|
||||
taskName += distribution.getVersion();
|
||||
return taskName;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -87,10 +87,36 @@ public class ElasticsearchDistribution implements Buildable, Iterable<File> {
|
|||
.onMac(() -> Platform.DARWIN)
|
||||
.supply();
|
||||
|
||||
public static final class Extracted implements Buildable, Iterable<File> {
|
||||
|
||||
// pkg private so plugin can configure
|
||||
final Configuration configuration;
|
||||
|
||||
private Extracted(Configuration configuration) {
|
||||
this.configuration = configuration;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iterator<File> iterator() {
|
||||
return configuration.iterator();
|
||||
}
|
||||
|
||||
@Override
|
||||
public TaskDependency getBuildDependencies() {
|
||||
return configuration.getBuildDependencies();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return configuration.getSingleFile().toString();
|
||||
}
|
||||
}
|
||||
|
||||
private final String name;
|
||||
private final Provider<DockerSupportService> dockerSupport;
|
||||
// pkg private so plugin can configure
|
||||
final Configuration configuration;
|
||||
private final Extracted extracted;
|
||||
|
||||
private final Property<Architecture> architecture;
|
||||
private final Property<String> version;
|
||||
|
|
@ -99,7 +125,6 @@ public class ElasticsearchDistribution implements Buildable, Iterable<File> {
|
|||
private final Property<Flavor> flavor;
|
||||
private final Property<Boolean> bundledJdk;
|
||||
private final Property<Boolean> failIfUnavailable;
|
||||
private final Configuration extracted;
|
||||
|
||||
ElasticsearchDistribution(
|
||||
String name,
|
||||
|
|
@ -119,7 +144,7 @@ public class ElasticsearchDistribution implements Buildable, Iterable<File> {
|
|||
this.flavor = objectFactory.property(Flavor.class);
|
||||
this.bundledJdk = objectFactory.property(Boolean.class);
|
||||
this.failIfUnavailable = objectFactory.property(Boolean.class).convention(true);
|
||||
this.extracted = extractedConfiguration;
|
||||
this.extracted = new Extracted(extractedConfiguration);
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
|
|
@ -188,7 +213,7 @@ public class ElasticsearchDistribution implements Buildable, Iterable<File> {
|
|||
return configuration.getSingleFile().toString();
|
||||
}
|
||||
|
||||
public Configuration getExtracted() {
|
||||
public Extracted getExtracted() {
|
||||
switch (getType()) {
|
||||
case DEB:
|
||||
case DOCKER:
|
||||
|
|
|
|||
|
|
@ -29,7 +29,6 @@ import org.gradle.api.artifacts.Configuration;
|
|||
import org.gradle.api.artifacts.dsl.RepositoryHandler;
|
||||
import org.gradle.api.artifacts.repositories.IvyArtifactRepository;
|
||||
import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
|
||||
import org.gradle.api.attributes.Attribute;
|
||||
import org.gradle.api.internal.artifacts.ArtifactAttributes;
|
||||
|
||||
public class JdkDownloadPlugin implements Plugin<Project> {
|
||||
|
|
@ -39,39 +38,25 @@ public class JdkDownloadPlugin implements Plugin<Project> {
|
|||
|
||||
private static final String REPO_NAME_PREFIX = "jdk_repo_";
|
||||
private static final String EXTENSION_NAME = "jdks";
|
||||
public static final String JDK_TRIMMED_PREFIX = "jdk-?\\d.*";
|
||||
|
||||
@Override
|
||||
public void apply(Project project) {
|
||||
Attribute<Boolean> jdkAttribute = Attribute.of("jdk", Boolean.class);
|
||||
project.getDependencies().getAttributesSchema().attribute(jdkAttribute);
|
||||
project.getDependencies().getArtifactTypes().maybeCreate(ArtifactTypeDefinition.ZIP_TYPE);
|
||||
project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> {
|
||||
transformSpec.getFrom()
|
||||
.attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE)
|
||||
.attribute(jdkAttribute, true);
|
||||
transformSpec.getTo()
|
||||
.attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE)
|
||||
.attribute(jdkAttribute, true);
|
||||
transformSpec.parameters(parameters -> parameters.setTrimmedPrefixPattern(JDK_TRIMMED_PREFIX));
|
||||
transformSpec.getFrom().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE);
|
||||
transformSpec.getTo().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
|
||||
});
|
||||
|
||||
ArtifactTypeDefinition tarArtifactTypeDefinition = project.getDependencies().getArtifactTypes().maybeCreate("tar.gz");
|
||||
project.getDependencies().registerTransform(SymbolicLinkPreservingUntarTransform.class, transformSpec -> {
|
||||
transformSpec.getFrom()
|
||||
.attribute(ArtifactAttributes.ARTIFACT_FORMAT, tarArtifactTypeDefinition.getName())
|
||||
.attribute(jdkAttribute, true);
|
||||
transformSpec.getTo()
|
||||
.attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE)
|
||||
.attribute(jdkAttribute, true);
|
||||
transformSpec.parameters(parameters -> parameters.setTrimmedPrefixPattern(JDK_TRIMMED_PREFIX));
|
||||
transformSpec.getFrom().attribute(ArtifactAttributes.ARTIFACT_FORMAT, tarArtifactTypeDefinition.getName());
|
||||
transformSpec.getTo().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
|
||||
});
|
||||
|
||||
NamedDomainObjectContainer<Jdk> jdksContainer = project.container(Jdk.class, name -> {
|
||||
Configuration configuration = project.getConfigurations().create("jdk_" + name);
|
||||
configuration.setCanBeConsumed(false);
|
||||
configuration.getAttributes().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
|
||||
configuration.getAttributes().attribute(jdkAttribute, true);
|
||||
Jdk jdk = new Jdk(name, configuration, project.getObjects());
|
||||
configuration.defaultDependencies(dependencies -> {
|
||||
jdk.finalizeValues();
|
||||
|
|
|
|||
|
|
@ -1182,7 +1182,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
|
|||
}
|
||||
|
||||
private Path getExtractedDistributionDir() {
|
||||
return distributions.get(currentDistro).getExtracted().getSingleFile().toPath();
|
||||
return Paths.get(distributions.get(currentDistro).getExtracted().toString());
|
||||
}
|
||||
|
||||
private List<File> getInstalledFileSet(Action<? super PatternFilterable> filter) {
|
||||
|
|
|
|||
|
|
@ -31,22 +31,22 @@ import java.io.IOException;
|
|||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.function.Function;
|
||||
|
||||
import static org.elasticsearch.gradle.util.PermissionUtils.chmod;
|
||||
import java.nio.file.attribute.PosixFileAttributeView;
|
||||
import java.nio.file.attribute.PosixFilePermission;
|
||||
import java.nio.file.attribute.PosixFilePermissions;
|
||||
import java.util.Set;
|
||||
|
||||
public abstract class SymbolicLinkPreservingUntarTransform implements UnpackTransform {
|
||||
|
||||
public void unpack(File tarFile, File targetDir) throws IOException {
|
||||
Logging.getLogger(SymbolicLinkPreservingUntarTransform.class)
|
||||
.info("Unpacking " + tarFile.getName() + " using " + SymbolicLinkPreservingUntarTransform.class.getSimpleName() + ".");
|
||||
Function<String, Path> pathModifier = pathResolver();
|
||||
|
||||
TarArchiveInputStream tar = new TarArchiveInputStream(new GzipCompressorInputStream(new FileInputStream(tarFile)));
|
||||
final Path destinationPath = targetDir.toPath();
|
||||
TarArchiveEntry entry = tar.getNextTarEntry();
|
||||
while (entry != null) {
|
||||
final Path relativePath = pathModifier.apply(entry.getName());
|
||||
final Path relativePath = UnpackTransform.trimArchiveExtractPath(entry.getName());
|
||||
if (relativePath == null) {
|
||||
entry = tar.getNextTarEntry();
|
||||
continue;
|
||||
|
|
@ -70,10 +70,41 @@ public abstract class SymbolicLinkPreservingUntarTransform implements UnpackTran
|
|||
}
|
||||
if (entry.isSymbolicLink() == false) {
|
||||
// check if the underlying file system supports POSIX permissions
|
||||
chmod(destination, entry.getMode());
|
||||
final PosixFileAttributeView view = Files.getFileAttributeView(destination, PosixFileAttributeView.class);
|
||||
if (view != null) {
|
||||
final Set<PosixFilePermission> permissions = PosixFilePermissions.fromString(
|
||||
permissions((entry.getMode() >> 6) & 07) + permissions((entry.getMode() >> 3) & 07) + permissions(
|
||||
(entry.getMode() >> 0) & 07
|
||||
)
|
||||
);
|
||||
Files.setPosixFilePermissions(destination, permissions);
|
||||
}
|
||||
}
|
||||
entry = tar.getNextTarEntry();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private static String permissions(final int permissions) {
|
||||
if (permissions < 0 || permissions > 7) {
|
||||
throw new IllegalArgumentException("permissions [" + permissions + "] out of range");
|
||||
}
|
||||
final StringBuilder sb = new StringBuilder(3);
|
||||
if ((permissions & 4) == 4) {
|
||||
sb.append('r');
|
||||
} else {
|
||||
sb.append('-');
|
||||
}
|
||||
if ((permissions & 2) == 2) {
|
||||
sb.append('w');
|
||||
} else {
|
||||
sb.append('-');
|
||||
}
|
||||
if ((permissions & 1) == 1) {
|
||||
sb.append('x');
|
||||
} else {
|
||||
sb.append('-');
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,28 +25,16 @@ import org.gradle.api.artifacts.transform.TransformOutputs;
|
|||
import org.gradle.api.artifacts.transform.TransformParameters;
|
||||
import org.gradle.api.file.FileSystemLocation;
|
||||
import org.gradle.api.provider.Provider;
|
||||
import org.gradle.api.tasks.Input;
|
||||
import org.gradle.api.tasks.Optional;
|
||||
import org.gradle.api.tasks.PathSensitive;
|
||||
import org.gradle.api.tasks.PathSensitivity;
|
||||
import org.gradle.internal.UncheckedException;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.FileSystems;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.function.Function;
|
||||
|
||||
public interface UnpackTransform extends TransformAction<UnpackTransform.Parameters> {
|
||||
|
||||
interface Parameters extends TransformParameters {
|
||||
@Input
|
||||
@Optional
|
||||
String getTrimmedPrefixPattern();
|
||||
|
||||
void setTrimmedPrefixPattern(String pattern);
|
||||
}
|
||||
public interface UnpackTransform extends TransformAction<TransformParameters.None> {
|
||||
|
||||
@PathSensitive(PathSensitivity.NAME_ONLY)
|
||||
@InputArtifact
|
||||
|
|
@ -55,9 +43,9 @@ public interface UnpackTransform extends TransformAction<UnpackTransform.Paramet
|
|||
@Override
|
||||
default void transform(TransformOutputs outputs) {
|
||||
File archiveFile = getArchiveFile().get().getAsFile();
|
||||
File extractedDir = outputs.dir(archiveFile.getName());
|
||||
File unzipDir = outputs.dir(archiveFile.getName());
|
||||
try {
|
||||
unpack(archiveFile, extractedDir);
|
||||
unpack(archiveFile, unzipDir);
|
||||
} catch (IOException e) {
|
||||
throw UncheckedException.throwAsUncheckedException(e);
|
||||
}
|
||||
|
|
@ -65,17 +53,8 @@ public interface UnpackTransform extends TransformAction<UnpackTransform.Paramet
|
|||
|
||||
void unpack(File archiveFile, File targetDir) throws IOException;
|
||||
|
||||
default Function<String, Path> pathResolver() {
|
||||
String trimmedPrefixPattern = getParameters().getTrimmedPrefixPattern();
|
||||
return trimmedPrefixPattern != null
|
||||
? (i) -> trimArchiveExtractPath(trimmedPrefixPattern, i)
|
||||
: (i) -> FileSystems.getDefault().getPath(i);
|
||||
}
|
||||
|
||||
/*
|
||||
* We want to be able to trim off certain prefixes when transforming archives.
|
||||
*
|
||||
* E.g We want to remove up to the and including the jdk-.* relative paths. That is a JDK archive is structured as:
|
||||
* We want to remove up to the and including the jdk-.* relative paths. That is a JDK archive is structured as:
|
||||
* jdk-12.0.1/
|
||||
* jdk-12.0.1/Contents
|
||||
* ...
|
||||
|
|
@ -87,11 +66,11 @@ public interface UnpackTransform extends TransformAction<UnpackTransform.Paramet
|
|||
*
|
||||
* so we account for this and search the path components until we find the jdk-12.0.1, and strip the leading components.
|
||||
*/
|
||||
static Path trimArchiveExtractPath(String ignoredPattern, String relativePath) {
|
||||
static Path trimArchiveExtractPath(String relativePath) {
|
||||
final Path entryName = Paths.get(relativePath);
|
||||
int index = 0;
|
||||
for (; index < entryName.getNameCount(); index++) {
|
||||
if (entryName.getName(index).toString().matches(ignoredPattern)) {
|
||||
if (entryName.getName(index).toString().matches("jdk-?\\d.*")) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -102,4 +81,5 @@ public interface UnpackTransform extends TransformAction<UnpackTransform.Paramet
|
|||
// finally remove the top-level directories from the output path
|
||||
return entryName.subpath(index + 1, entryName.getNameCount());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,48 +20,35 @@
|
|||
package org.elasticsearch.gradle.transform;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.tools.zip.ZipEntry;
|
||||
import org.apache.tools.zip.ZipFile;
|
||||
import org.gradle.api.logging.Logging;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Enumeration;
|
||||
import java.util.function.Function;
|
||||
|
||||
import static org.elasticsearch.gradle.util.PermissionUtils.chmod;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
|
||||
public abstract class UnzipTransform implements UnpackTransform {
|
||||
|
||||
public void unpack(File zipFile, File targetDir) throws IOException {
|
||||
Logging.getLogger(UnzipTransform.class)
|
||||
.info("Unpacking " + zipFile.getName() + " using " + UnzipTransform.class.getSimpleName() + ".");
|
||||
Function<String, Path> pathModifier = pathResolver();
|
||||
ZipFile zip = new ZipFile(zipFile);
|
||||
try {
|
||||
Enumeration<ZipEntry> entries = zip.getEntries();
|
||||
while (entries.hasMoreElements()) {
|
||||
ZipEntry zipEntry = entries.nextElement();
|
||||
Path child = pathModifier.apply(zipEntry.getName());
|
||||
if (child == null) {
|
||||
|
||||
try (ZipInputStream inputStream = new ZipInputStream(new BufferedInputStream(new FileInputStream(zipFile)))) {
|
||||
ZipEntry entry;
|
||||
while ((entry = inputStream.getNextEntry()) != null) {
|
||||
if (entry.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
Path outputPath = targetDir.toPath().resolve(child);
|
||||
if (zipEntry.isDirectory()) {
|
||||
outputPath.toFile().mkdirs();
|
||||
chmod(outputPath, zipEntry.getUnixMode());
|
||||
continue;
|
||||
String child = UnpackTransform.trimArchiveExtractPath(entry.getName()).toString();
|
||||
File outFile = new File(targetDir, child);
|
||||
outFile.getParentFile().mkdirs();
|
||||
try (FileOutputStream outputStream = new FileOutputStream(outFile)) {
|
||||
IOUtils.copyLarge(inputStream, outputStream);
|
||||
}
|
||||
try (FileOutputStream outputStream = new FileOutputStream(outputPath.toFile())) {
|
||||
IOUtils.copyLarge(zip.getInputStream(zipEntry), outputStream);
|
||||
}
|
||||
chmod(outputPath, zipEntry.getUnixMode());
|
||||
}
|
||||
} finally {
|
||||
zip.close();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,69 +0,0 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.gradle.util;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.attribute.PosixFileAttributeView;
|
||||
import java.nio.file.attribute.PosixFilePermission;
|
||||
import java.nio.file.attribute.PosixFilePermissions;
|
||||
import java.util.Set;
|
||||
|
||||
public class PermissionUtils {
|
||||
|
||||
public static void chmod(Path path, int mode) throws IOException {
|
||||
final PosixFileAttributeView view = Files.getFileAttributeView(path, PosixFileAttributeView.class);
|
||||
if (view != null && (mode != 0)) {
|
||||
final Set<PosixFilePermission> permissions = permissionsFromInt(mode);
|
||||
Files.setPosixFilePermissions(path, permissions);
|
||||
}
|
||||
}
|
||||
|
||||
private static Set<PosixFilePermission> permissionsFromInt(int mode) {
|
||||
return PosixFilePermissions.fromString(
|
||||
permissions((mode >> 6) & 07) + permissions((mode >> 3) & 07) + permissions((mode >> 0) & 07)
|
||||
);
|
||||
}
|
||||
|
||||
private static String permissions(final int permissions) {
|
||||
if (permissions < 0 || permissions > 7) {
|
||||
throw new IllegalArgumentException("permissions [" + permissions + "] out of range");
|
||||
}
|
||||
final StringBuilder sb = new StringBuilder(3);
|
||||
if ((permissions & 4) == 4) {
|
||||
sb.append('r');
|
||||
} else {
|
||||
sb.append('-');
|
||||
}
|
||||
if ((permissions & 2) == 2) {
|
||||
sb.append('w');
|
||||
} else {
|
||||
sb.append('-');
|
||||
}
|
||||
if ((permissions & 1) == 1) {
|
||||
sb.append('x');
|
||||
} else {
|
||||
sb.append('-');
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
import org.elasticsearch.gradle.BwcVersions
|
||||
import org.elasticsearch.gradle.Version
|
||||
import org.elasticsearch.gradle.info.BuildParams
|
||||
|
||||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
plugins {
|
||||
id 'elasticsearch.global-build-info'
|
||||
}
|
||||
boolean internal = Boolean.parseBoolean(System.getProperty("tests.internal", "true"))
|
||||
BuildParams.init { it.setIsInternal(internal) }
|
||||
|
||||
project.gradle.projectsEvaluated {
|
||||
// wire the download service url to wiremock
|
||||
String fakeDownloadService = System.getProperty('tests.download_service')
|
||||
if (fakeDownloadService != null) {
|
||||
IvyArtifactRepository repository = (IvyArtifactRepository) rootProject.repositories.getByName("elasticsearch-downloads")
|
||||
repository.setUrl(fakeDownloadService)
|
||||
repository = (IvyArtifactRepository) project('subproj').repositories.getByName("elasticsearch-downloads")
|
||||
repository.setUrl(fakeDownloadService)
|
||||
if (internal == false) {
|
||||
repository = (IvyArtifactRepository) rootProject.repositories.getByName("elasticsearch-snapshots")
|
||||
repository.setUrl(fakeDownloadService)
|
||||
repository = (IvyArtifactRepository) project('subproj').repositories.getByName("elasticsearch-snapshots")
|
||||
repository.setUrl(fakeDownloadService)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (internal) {
|
||||
Version currentVersion = Version.fromString("9.0.0")
|
||||
BwcVersions versions = new BwcVersions(new TreeSet<>(
|
||||
Arrays.asList(Version.fromString("8.0.0"), Version.fromString("8.0.1"), Version.fromString("8.1.0"), currentVersion)),
|
||||
currentVersion)
|
||||
BuildParams.init { it.setBwcVersions(versions) }
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
String distroConfig = System.getProperty('tests.local_distro.config')
|
||||
if (distroConfig != null) {
|
||||
// setup the test distribution as an artifact of this project
|
||||
String distroType = System.getProperty('tests.distro.type')
|
||||
|
||||
def buildDistro
|
||||
File buildFile
|
||||
if (['rpm', 'deb'].contains(distroType)) {
|
||||
buildDistro = project.tasks.register("build" + distroType.capitalize(), Copy) {
|
||||
from 'files'
|
||||
into 'build/files'
|
||||
include 'fake_elasticsearch.tar.gz'
|
||||
// this shouldn't be extracted so we just rename the file so it is a dummy package
|
||||
rename { filename -> filename.replace('tar.gz', distroType) }
|
||||
}
|
||||
buildFile = project.file("build/files/fake_elasticsearch." + distroType)
|
||||
} else {
|
||||
String distroPlatform = System.getProperty('tests.distro.platform')
|
||||
String extension = "tar.gz"
|
||||
if (distroType == 'archive' && distroPlatform == 'windows' || distroType == 'integ-test-zip') {
|
||||
extension = "zip"
|
||||
}
|
||||
// copy file as is
|
||||
buildDistro = project.tasks.register("buildArchive", Copy) {
|
||||
from 'files'
|
||||
include "fake_elasticsearch.${extension}"
|
||||
into 'build/files'
|
||||
}
|
||||
buildFile = project.file("build/files/fake_elasticsearch.${extension}")
|
||||
}
|
||||
|
||||
configurations.create(distroConfig)
|
||||
artifacts.add(distroConfig, [file: buildFile, builtBy: buildDistro])
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
include 'subproj'
|
||||
|
||||
String distroProject = System.getProperty('tests.local_distro.project')
|
||||
System.out.println("local distro project: " + distroProject);
|
||||
if (distroProject != null) {
|
||||
include distroProject
|
||||
project(distroProject).projectDir = new File(rootDir, 'distribution')
|
||||
}
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
import org.elasticsearch.gradle.Architecture
|
||||
|
||||
plugins {
|
||||
id 'elasticsearch.distribution-download'
|
||||
}
|
||||
|
||||
String distroVersion = System.getProperty('tests.distro.version')
|
||||
String distroType = System.getProperty('tests.distro.type')
|
||||
String distroPlatform = System.getProperty('tests.distro.platform')
|
||||
String distroFlavor = System.getProperty('tests.distro.flavor')
|
||||
String distroBundledJdk = System.getProperty('tests.distro.bundledJdk')
|
||||
String distroArch = System.getProperty('tests.distro.arch');
|
||||
|
||||
elasticsearch_distributions {
|
||||
test_distro {
|
||||
if (distroVersion != null) {
|
||||
version = distroVersion
|
||||
}
|
||||
if (distroType != null) {
|
||||
type = distroType
|
||||
}
|
||||
if (distroPlatform != null) {
|
||||
platform = distroPlatform
|
||||
}
|
||||
if (distroFlavor != null) {
|
||||
flavor = distroFlavor
|
||||
}
|
||||
if (distroBundledJdk != null) {
|
||||
bundledJdk = Boolean.parseBoolean(distroBundledJdk)
|
||||
}
|
||||
if (distroArch != null) {
|
||||
architecture = Architecture.valueOf(distroArch);
|
||||
} else {
|
||||
architecture = Architecture.current();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tasks.register("assertDistroFile") {
|
||||
dependsOn elasticsearch_distributions.test_distro
|
||||
doLast {
|
||||
File distroFile = new File(elasticsearch_distributions.test_distro.toString())
|
||||
if (distroFile.exists() == false) {
|
||||
throw new GradleException("distro file does not exist: ${distroFile}")
|
||||
}
|
||||
if (distroFile.isFile() == false) {
|
||||
throw new GradleException("distro file is not a regular file: ${distroFile}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (['rpm', 'deb'].contains(distroType) == false) {
|
||||
tasks.register("assertDistroExtracted") {
|
||||
dependsOn elasticsearch_distributions.test_distro.extracted, assertDistroFile
|
||||
doLast {
|
||||
File distroExtracted = new File(elasticsearch_distributions.test_distro.extracted.toString())
|
||||
if (distroExtracted.exists() == false) {
|
||||
throw new GradleException("extracted does not exist: ${distroExtracted}")
|
||||
}
|
||||
if (distroExtracted.isDirectory() == false) {
|
||||
throw new GradleException("extracted distro is not a directory: ${distroExtracted}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -316,7 +316,7 @@ BuildParams.bwcVersions.forPreviousUnreleased { BwcVersions.UnreleasedVersionInf
|
|||
File artifactFile = e.value
|
||||
String artifactFileName = artifactFile.name
|
||||
String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
|
||||
String suffix = artifactFileName.endsWith("tar.gz") ? "tar.gz" : artifactFileName[-3..-1]
|
||||
String suffix = artifactFile.toString()[-3..-1]
|
||||
int archIndex = artifactFileName.indexOf('x86_64')
|
||||
String classifier = ''
|
||||
if (archIndex != -1) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue