From f08981c4da8ea65db175d1ea5e8ea3ee858a5bd5 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Mon, 31 Aug 2020 14:36:42 +0200 Subject: [PATCH] Simplify distribution download configurations (7.x backport) (#61708) We leverage artifact transforms now when downloading and unpacking elasticsearch distributions. This has the benefit of - handcrafted extract tasks on the root project are not required. - The general tight coupling to the root project has been removed. - The overall required configurations required to handle a distribution have been reduced - ElasticsearchDistribution has been simplified by making Extracted an ordinary Configuration downloaded and unpacked external distributions are reused in later builds by been cached in the gradle user home. DistributionDownloadPlugin functional tests have been extended and ported to DistributionDownloadPluginFuncTest. * Fix ElasticsearchNode#getDistributionFiles (#61219) Fixes #61647 --- buildSrc/build.gradle | 2 +- .../DistributionDownloadPluginFuncTest.groovy | 158 +++++++++++++++ .../gradle/JdkDownloadPluginFuncTest.groovy | 12 +- .../gradle/TestClustersPluginFuncTest.groovy | 11 +- .../DistributionDownloadFixture.groovy | 17 +- ...lDistributionDownloadPluginFuncTest.groovy | 29 +-- .../gradle/DistributionDownloadPluginIT.java | 188 ------------------ .../gradle/DistributionDownloadPlugin.java | 97 ++------- .../gradle/ElasticsearchDistribution.java | 31 +-- .../gradle/JdkDownloadPlugin.java | 23 ++- .../testclusters/ElasticsearchNode.java | 15 +- .../SymbolicLinkPreservingUntarTransform.java | 43 +--- .../gradle/transform/UnpackTransform.java | 31 ++- .../gradle/transform/UnzipTransform.java | 41 ++-- .../gradle/util/PermissionUtils.java | 69 +++++++ .../distribution-download/build.gradle | 53 ----- .../distribution/build.gradle | 34 ---- .../files/fake_elasticsearch.tar.gz | Bin 1139 -> 0 bytes .../distribution/files/fake_elasticsearch.zip | Bin 568 -> 0 bytes .../distribution-download/settings.gradle | 8 - .../subproj/build.gradle | 65 ------ distribution/bwc/build.gradle | 2 +- 22 files changed, 375 insertions(+), 554 deletions(-) create mode 100644 buildSrc/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy delete mode 100644 buildSrc/src/integTest/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/util/PermissionUtils.java delete mode 100644 buildSrc/src/testKit/distribution-download/build.gradle delete mode 100644 buildSrc/src/testKit/distribution-download/distribution/build.gradle delete mode 100644 buildSrc/src/testKit/distribution-download/distribution/files/fake_elasticsearch.tar.gz delete mode 100644 buildSrc/src/testKit/distribution-download/distribution/files/fake_elasticsearch.zip delete mode 100644 buildSrc/src/testKit/distribution-download/settings.gradle delete mode 100644 buildSrc/src/testKit/distribution-download/subproj/build.gradle diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 144a927cc4f..4bae2afd18a 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -92,7 +92,7 @@ dependencies { api 'commons-codec:commons-codec:1.12' api 'org.apache.commons:commons-compress:1.19' - + api 'org.apache.ant:ant:1.10.8' api 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3' api 'com.netflix.nebula:nebula-publishing-plugin:4.4.4' api 'com.netflix.nebula:gradle-info-plugin:7.1.3' diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy new file mode 100644 index 00000000000..c4c803787c5 --- /dev/null +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy @@ -0,0 +1,158 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle + +import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest +import org.elasticsearch.gradle.transform.SymbolicLinkPreservingUntarTransform +import org.gradle.testkit.runner.TaskOutcome +import spock.lang.Unroll + +import static org.elasticsearch.gradle.fixtures.DistributionDownloadFixture.withMockedDistributionDownload + +class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest { + + @Unroll + def "#distType version can be resolved"() { + given: + buildFile << applyPluginAndSetupDistro(version, platform) + + when: + def result = withMockedDistributionDownload(version, platform, gradleRunner('setupDistro', '-i')) { + build() + } + + then: + result.task(":setupDistro").outcome == TaskOutcome.SUCCESS + assertExtractedDistroCreated("build/distro") + + where: + version | platform | distType + VersionProperties.getElasticsearch() | ElasticsearchDistribution.Platform.LINUX | "current" + "8.1.0-SNAPSHOT" | ElasticsearchDistribution.Platform.LINUX | "bwc" + "7.0.0" | ElasticsearchDistribution.Platform.WINDOWS | "released" + } + + + def "transformed versions are kept across builds"() { + given: + def version = VersionProperties.getElasticsearch() + def platform = ElasticsearchDistribution.Platform.LINUX + + buildFile << applyPluginAndSetupDistro(version, platform) + buildFile << """ + apply plugin:'base' + """ + + when: + def runner = gradleRunner('clean', 'setupDistro', '-i') + def result = withMockedDistributionDownload(version, platform, runner) { + // initial run + build() + // 2nd invocation + build() + } + + then: + result.task(":setupDistro").outcome == TaskOutcome.SUCCESS + assertOutputContains(result.output, "Skipping ${SymbolicLinkPreservingUntarTransform.class.simpleName}") + } + + def "transforms are reused across projects"() { + given: + def version = VersionProperties.getElasticsearch() + def platform = ElasticsearchDistribution.Platform.LINUX + + 3.times { + settingsFile << """ + include ':sub-$it' + """ + } + buildFile.text = """ + import org.elasticsearch.gradle.Architecture + + plugins { + id 'elasticsearch.distribution-download' + } + + subprojects { + apply plugin: 'elasticsearch.distribution-download' + + ${setupTestDistro(version, platform)} + ${setupDistroTask()} + } + """ + + when: + def customGradleUserHome = testProjectDir.newFolder().absolutePath; + def runner = gradleRunner('setupDistro', '-i', '-g', customGradleUserHome) + def result = withMockedDistributionDownload(version, platform, runner) { + build() + } + + then: + result.tasks.size() == 3 + result.output.count("Unpacking elasticsearch-${version}-linux-x86_64.tar.gz " + + "using SymbolicLinkPreservingUntarTransform.") == 1 + } + + private boolean assertExtractedDistroCreated(String relativePath) { + File distroExtracted = new File(testProjectDir.root, relativePath) + assert distroExtracted.exists() + assert distroExtracted.isDirectory() + assert new File(distroExtracted, "elasticsearch-1.2.3/bin/elasticsearch").exists() + true + } + + private static String applyPluginAndSetupDistro(String version, ElasticsearchDistribution.Platform platform) { + """ + import org.elasticsearch.gradle.Architecture + + plugins { + id 'elasticsearch.distribution-download' + } + + ${setupTestDistro(version, platform)} + ${setupDistroTask()} + + """ + } + + private static String setupTestDistro(String version, ElasticsearchDistribution.Platform platform) { + return """ + elasticsearch_distributions { + test_distro { + version = "$version" + type = "archive" + platform = "$platform" + architecture = Architecture.current(); + } + } + """ + } + + private static String setupDistroTask() { + return """ + tasks.register("setupDistro", Sync) { + from(elasticsearch_distributions.test_distro.extracted) + into("build/distro") + } + """ + } +} \ No newline at end of file diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/JdkDownloadPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/JdkDownloadPluginFuncTest.groovy index 090feece450..fc5dc456fdb 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/JdkDownloadPluginFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/JdkDownloadPluginFuncTest.groovy @@ -95,7 +95,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest { given: def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform) def mockedContent = filebytes(jdkVendor, platform) - 10.times { + 3.times { settingsFile << """ include ':sub-$it' """ @@ -132,8 +132,8 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest { } then: - result.tasks.size() == 10 - result.output.count("Unpacking linux-12.0.2-x64.tar.gz using SymbolicLinkPreservingUntarTransform.") == 1 + result.tasks.size() == 3 + result.output.count("Unpacking linux-12.0.2-x64.tar.gz using ${SymbolicLinkPreservingUntarTransform.simpleName}.") == 1 where: platform | jdkVendor | jdkVersion | expectedJavaBin @@ -149,7 +149,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest { plugins { id 'elasticsearch.jdk-download' } - + apply plugin: 'base' apply plugin: 'elasticsearch.jdk-download' jdks { @@ -175,9 +175,9 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest { def commonGradleUserHome = testProjectDir.newFolder().toString() // initial run - gradleRunner('getJdk', '-g', commonGradleUserHome).build() + gradleRunner('clean', 'getJdk', '-g', commonGradleUserHome).build() // run against up-to-date transformations - gradleRunner('getJdk', '-i', '-g', commonGradleUserHome).build() + gradleRunner('clean', 'getJdk', '-i', '-g', commonGradleUserHome).build() } then: diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy index 71e67b3cc99..bde1f70a457 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy @@ -64,8 +64,8 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest { """ when: - def result = withMockedDistributionDownload(gradleRunner("myTask", '-i')) { GradleRunner runner -> - return runner.build() + def result = withMockedDistributionDownload(gradleRunner("myTask", '-i')) { + build() } then: @@ -91,8 +91,8 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest { """ when: - def result = withMockedDistributionDownload(gradleRunner("myTask", '-i')) { GradleRunner runner -> - return runner.build() + def result = withMockedDistributionDownload(gradleRunner("myTask", '-i')) { + build() } then: @@ -103,7 +103,8 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest { } boolean assertEsStdoutContains(String testCluster, String expectedOutput) { - assert new File(testProjectDir.root, "build/testclusters/${testCluster}-0/logs/es.stdout.log").text.contains(expectedOutput) + assert new File(testProjectDir.root, + "build/testclusters/${testCluster}-0/logs/es.stdout.log").text.contains(expectedOutput) true } diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/DistributionDownloadFixture.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/DistributionDownloadFixture.groovy index 4301a18faa7..e053d51cd68 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/DistributionDownloadFixture.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/DistributionDownloadFixture.groovy @@ -29,7 +29,13 @@ class DistributionDownloadFixture { public static final String INIT_SCRIPT = "repositories-init.gradle" static BuildResult withMockedDistributionDownload(GradleRunner gradleRunner, Closure buildRunClosure) { - String urlPath = urlPath(); + return withMockedDistributionDownload(VersionProperties.getElasticsearch(), ElasticsearchDistribution.CURRENT_PLATFORM, + gradleRunner, buildRunClosure) + } + + static BuildResult withMockedDistributionDownload(String version, ElasticsearchDistribution.Platform platform, + GradleRunner gradleRunner, Closure buildRunClosure) { + String urlPath = urlPath(version, platform); return WiremockFixture.withWireMock(urlPath, filebytes(urlPath)) { server -> File initFile = new File(gradleRunner.getProjectDir(), INIT_SCRIPT) initFile.text = """allprojects { p -> @@ -40,15 +46,14 @@ class DistributionDownloadFixture { }""" List givenArguments = gradleRunner.getArguments() GradleRunner effectiveRunner = gradleRunner.withArguments(givenArguments + ['-I', initFile.getAbsolutePath()]) + buildRunClosure.delegate = effectiveRunner return buildRunClosure.call(effectiveRunner) } } - private static String urlPath() { - String version = VersionProperties.getElasticsearch() - ElasticsearchDistribution.Platform platform = ElasticsearchDistribution.CURRENT_PLATFORM - String fileType = ((ElasticsearchDistribution.CURRENT_PLATFORM == ElasticsearchDistribution.Platform.LINUX || - ElasticsearchDistribution.CURRENT_PLATFORM == ElasticsearchDistribution.Platform.DARWIN)) ? "tar.gz" : "zip" + private static String urlPath(String version,ElasticsearchDistribution.Platform platform) { + String fileType = ((platform == ElasticsearchDistribution.Platform.LINUX || + platform == ElasticsearchDistribution.Platform.DARWIN)) ? "tar.gz" : "zip" "/downloads/elasticsearch/elasticsearch-${version}-${platform}-x86_64.$fileType" } diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy index b1221650630..0d082256fe5 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy @@ -39,7 +39,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest """ when: - def result = gradleRunner("createExtractedTestDistro").buildAndFail() + def result = gradleRunner("tasks").buildAndFail() then: assertOutputContains(result.output, "Plugin 'elasticsearch.internal-distribution-download' is not supported. " + @@ -62,18 +62,19 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest architecture = Architecture.current(); } } - tasks.register("createExtractedTestDistro") { - dependsOn elasticsearch_distributions.test_distro.extracted + tasks.register("setupDistro", Sync) { + from(elasticsearch_distributions.test_distro.extracted) + into("build/distro") } """ when: - def result = gradleRunner("createExtractedTestDistro").build() + def result = gradleRunner("setupDistro", '-g', testProjectDir.newFolder('GUH').path).build() then: result.task(":distribution:archives:linux-tar:buildTar").outcome == TaskOutcome.SUCCESS - result.task(":extractElasticsearchLinux$distroVersion").outcome == TaskOutcome.SUCCESS - assertExtractedDistroIsCreated(distroVersion, 'current-marker.txt') + result.task(":setupDistro").outcome == TaskOutcome.SUCCESS + assertExtractedDistroIsCreated(distroVersion, "build/distro", 'current-marker.txt') } def "resolves bwc versions from source"() { @@ -92,16 +93,18 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest architecture = Architecture.current(); } } - tasks.register("createExtractedTestDistro") { - dependsOn elasticsearch_distributions.test_distro.extracted + tasks.register("setupDistro", Sync) { + from(elasticsearch_distributions.test_distro.extracted) + into("build/distro") } """ when: - def result = gradleRunner("createExtractedTestDistro").build() + + def result = gradleRunner("setupDistro").build() then: result.task(":distribution:bwc:minor:buildBwcTask").outcome == TaskOutcome.SUCCESS - result.task(":extractElasticsearchLinux8.1.0").outcome == TaskOutcome.SUCCESS - assertExtractedDistroIsCreated(distroVersion,'bwc-marker.txt') + result.task(":setupDistro").outcome == TaskOutcome.SUCCESS + assertExtractedDistroIsCreated(distroVersion, "build/distro", 'bwc-marker.txt') } def "fails on resolving bwc versions with no bundled jdk"() { @@ -195,8 +198,8 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest } - boolean assertExtractedDistroIsCreated(String version, String markerFileName) { - File extractedFolder = new File(testProjectDir.root, "build/elasticsearch-distros/extracted_elasticsearch_${version}_archive_linux_default") + boolean assertExtractedDistroIsCreated(String version, String relativeDistroPath, String markerFileName) { + File extractedFolder = new File(testProjectDir.root, relativeDistroPath) assert extractedFolder.exists() assert new File(extractedFolder, markerFileName).exists() true diff --git a/buildSrc/src/integTest/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java b/buildSrc/src/integTest/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java deleted file mode 100644 index aada8dfd2a1..00000000000 --- a/buildSrc/src/integTest/java/org/elasticsearch/gradle/DistributionDownloadPluginIT.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.gradle; - -import com.github.tomakehurst.wiremock.WireMockServer; -import org.elasticsearch.gradle.test.GradleIntegrationTestCase; -import org.gradle.testkit.runner.BuildResult; -import org.gradle.testkit.runner.GradleRunner; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; -import static com.github.tomakehurst.wiremock.client.WireMock.get; -import static com.github.tomakehurst.wiremock.client.WireMock.head; -import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; - -public class DistributionDownloadPluginIT extends GradleIntegrationTestCase { - - // TODO: check reuse of root task across projects MOVE TO UNIT TEST - // TODO: future: check integ-test-zip to maven, snapshots to snapshot service for external project - public void testCurrentExternal() throws Exception { - checkService( - VersionProperties.getElasticsearch(), - "archive", - "linux", - null, - null, - "/downloads/elasticsearch/elasticsearch-" + VersionProperties.getElasticsearch() + "-linux-x86_64.tar.gz", - "tests.internal", - "false" - ); - } - - public void testBwcExternal() throws Exception { - checkService( - "8.1.0-SNAPSHOT", - "archive", - "linux", - null, - null, - "/downloads/elasticsearch/elasticsearch-8.1.0-SNAPSHOT-linux-x86_64.tar.gz", - "tests.internal", - "false", - "tests.current_version", - "9.0.0" - ); - } - - public void testReleased() throws Exception { - checkService("7.0.0", "archive", "windows", null, null, "/downloads/elasticsearch/elasticsearch-7.0.0-windows-x86_64.zip"); - checkService("6.5.0", "archive", "windows", null, null, "/downloads/elasticsearch/elasticsearch-6.5.0.zip"); - } - - public void testReleasedExternal() throws Exception { - checkService( - "7.0.0", - "archive", - "windows", - null, - null, - "/downloads/elasticsearch/elasticsearch-7.0.0-windows-x86_64.zip", - "tests.internal", - "false" - ); - checkService( - "6.5.0", - "archive", - "windows", - null, - null, - "/downloads/elasticsearch/elasticsearch-6.5.0.zip", - "tests.internal", - "false" - ); - } - - private void checkService( - String version, - String type, - String platform, - String flavor, - Boolean bundledJdk, - String urlPath, - String... sysProps - ) throws IOException { - String suffix = urlPath.endsWith("zip") ? "zip" : "tar.gz"; - String sourceFile = "src/testKit/distribution-download/distribution/files/fake_elasticsearch." + suffix; - WireMockServer wireMock = new WireMockServer(0); - try { - final byte[] filebytes; - try (InputStream stream = Files.newInputStream(Paths.get(sourceFile))) { - filebytes = stream.readAllBytes(); - } - wireMock.stubFor(head(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200))); - wireMock.stubFor(get(urlEqualTo(urlPath)).willReturn(aResponse().withStatus(200).withBody(filebytes))); - wireMock.start(); - - List allSysProps = new ArrayList<>(); - allSysProps.addAll(Arrays.asList(sysProps)); - allSysProps.add("tests.download_service"); - allSysProps.add(wireMock.baseUrl()); - assertExtractedDistro(version, type, platform, flavor, bundledJdk, allSysProps.toArray(new String[0])); - } catch (Exception e) { - // for debugging - System.err.println("missed requests: " + wireMock.findUnmatchedRequests().getRequests()); - throw e; - } finally { - wireMock.stop(); - } - } - - private void assertFileDistro(String version, String type, String platform, String flavor, Boolean bundledJdk, String... sysProps) - throws IOException { - List finalSysProps = new ArrayList<>(); - addDistroSysProps(finalSysProps, version, type, platform, flavor, bundledJdk); - finalSysProps.addAll(Arrays.asList(sysProps)); - runBuild(":subproj:assertDistroFile", finalSysProps.toArray(new String[0])); - } - - private void assertExtractedDistro(String version, String type, String platform, String flavor, Boolean bundledJdk, String... sysProps) - throws IOException { - List finalSysProps = new ArrayList<>(); - addDistroSysProps(finalSysProps, version, type, platform, flavor, bundledJdk); - finalSysProps.addAll(Arrays.asList(sysProps)); - runBuild(":subproj:assertDistroExtracted", finalSysProps.toArray(new String[0])); - } - - private BuildResult runBuild(String taskname, String... sysProps) throws IOException { - assert sysProps.length % 2 == 0; - List args = new ArrayList<>(); - args.add(taskname); - for (int i = 0; i < sysProps.length; i += 2) { - args.add("-D" + sysProps[i] + "=" + sysProps[i + 1]); - } - args.add("-i"); - GradleRunner runner = getGradleRunner("distribution-download").withArguments(args); - - BuildResult result = runner.build(); - System.out.println(result.getOutput()); - return result; - } - - private void addDistroSysProps(List sysProps, String version, String type, String platform, String flavor, Boolean bundledJdk) { - if (version != null) { - sysProps.add("tests.distro.version"); - sysProps.add(version); - } - if (type != null) { - sysProps.add("tests.distro.type"); - sysProps.add(type); - } - if (platform != null) { - sysProps.add("tests.distro.platform"); - sysProps.add(platform); - } - if (flavor != null) { - sysProps.add("tests.distro.flavor"); - sysProps.add(flavor); - } - if (bundledJdk != null) { - sysProps.add("tests.distro.bundledJdk"); - sysProps.add(bundledJdk.toString()); - } - } -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java index 3922337b5e6..7ee4729e08b 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java @@ -24,30 +24,22 @@ import org.elasticsearch.gradle.ElasticsearchDistribution.Platform; import org.elasticsearch.gradle.ElasticsearchDistribution.Type; import org.elasticsearch.gradle.docker.DockerSupportPlugin; import org.elasticsearch.gradle.docker.DockerSupportService; +import org.elasticsearch.gradle.transform.SymbolicLinkPreservingUntarTransform; +import org.elasticsearch.gradle.transform.UnzipTransform; import org.elasticsearch.gradle.util.GradleUtils; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Plugin; import org.gradle.api.Project; -import org.gradle.api.UnknownTaskException; import org.gradle.api.artifacts.Configuration; -import org.gradle.api.artifacts.ConfigurationContainer; import org.gradle.api.artifacts.dsl.DependencyHandler; import org.gradle.api.artifacts.repositories.IvyArtifactRepository; +import org.gradle.api.artifacts.type.ArtifactTypeDefinition; import org.gradle.api.credentials.HttpHeaderCredentials; -import org.gradle.api.file.FileTree; -import org.gradle.api.file.RelativePath; +import org.gradle.api.internal.artifacts.ArtifactAttributes; import org.gradle.api.provider.Provider; -import org.gradle.api.tasks.Sync; -import org.gradle.api.tasks.TaskProvider; import org.gradle.authentication.http.HttpHeaderAuthentication; -import java.io.File; -import java.util.Arrays; import java.util.Comparator; -import java.util.concurrent.Callable; -import java.util.function.Supplier; - -import static org.elasticsearch.gradle.util.GradleUtils.projectDependency; import static org.elasticsearch.gradle.util.Util.capitalize; /** @@ -77,6 +69,17 @@ public class DistributionDownloadPlugin implements Plugin { DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME ); + project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> { + transformSpec.getFrom().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE); + transformSpec.getTo().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE); + }); + + ArtifactTypeDefinition tarArtifactTypeDefinition = project.getDependencies().getArtifactTypes().maybeCreate("tar.gz"); + project.getDependencies().registerTransform(SymbolicLinkPreservingUntarTransform.class, transformSpec -> { + transformSpec.getFrom().attribute(ArtifactAttributes.ARTIFACT_FORMAT, tarArtifactTypeDefinition.getName()); + transformSpec.getTo().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE); + }); + setupResolutionsContainer(project); setupDistributionContainer(project, dockerSupport); setupDownloadServiceRepo(project); @@ -87,6 +90,7 @@ public class DistributionDownloadPlugin implements Plugin { distributionsContainer = project.container(ElasticsearchDistribution.class, name -> { Configuration fileConfiguration = project.getConfigurations().create("es_distro_file_" + name); Configuration extractedConfiguration = project.getConfigurations().create("es_distro_extracted_" + name); + extractedConfiguration.getAttributes().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE); return new ElasticsearchDistribution(name, project.getObjects(), dockerSupport, fileConfiguration, extractedConfiguration); }); project.getExtensions().add(CONTAINER_NAME, distributionsContainer); @@ -115,21 +119,15 @@ public class DistributionDownloadPlugin implements Plugin { void setupDistributions(Project project) { for (ElasticsearchDistribution distribution : distributionsContainer) { distribution.finalizeValues(); - DependencyHandler dependencies = project.getDependencies(); // for the distribution as a file, just depend on the artifact directly - dependencies.add(distribution.configuration.getName(), resolveDependencyNotation(project, distribution)); - + Object resolvedDependency = resolveDependencyNotation(project, distribution); + dependencies.add(distribution.configuration.getName(), resolvedDependency); // no extraction allowed for rpm, deb or docker if (distribution.getType().shouldExtract()) { - // for the distribution extracted, add a root level task that does the extraction, and depend on that - // extracted configuration as an artifact consisting of the extracted distribution directory - dependencies.add( - distribution.getExtracted().configuration.getName(), - projectDependency(project, ":", configName("extracted_elasticsearch", distribution)) - ); - // ensure a root level download task exists - setupRootDownload(project.getRootProject(), distribution); + // The extracted configuration depends on the artifact directly but has + // an artifact transform registered to resolve it as an unpacked folder. + dependencies.add(distribution.getExtracted().getName(), resolvedDependency); } } } @@ -143,59 +141,6 @@ public class DistributionDownloadPlugin implements Plugin { .orElseGet(() -> dependencyNotation(distribution)); } - private void setupRootDownload(Project rootProject, ElasticsearchDistribution distribution) { - String extractTaskName = extractTaskName(distribution); - // NOTE: this is *horrendous*, but seems to be the only way to check for the existence of a registered task - try { - rootProject.getTasks().named(extractTaskName); - // already setup this version - return; - } catch (UnknownTaskException e) { - // fall through: register the task - } - setupDownloadServiceRepo(rootProject); - final ConfigurationContainer configurations = rootProject.getConfigurations(); - String downloadConfigName = configName("elasticsearch", distribution); - String extractedConfigName = "extracted_" + downloadConfigName; - final Configuration downloadConfig = configurations.create(downloadConfigName); - configurations.create(extractedConfigName); - rootProject.getDependencies().add(downloadConfigName, resolveDependencyNotation(rootProject, distribution)); - - // add task for extraction, delaying resolving config until runtime - if (distribution.getType() == Type.ARCHIVE || distribution.getType() == Type.INTEG_TEST_ZIP) { - Supplier archiveGetter = downloadConfig::getSingleFile; - String extractDir = rootProject.getBuildDir().toPath().resolve("elasticsearch-distros").resolve(extractedConfigName).toString(); - TaskProvider extractTask = rootProject.getTasks().register(extractTaskName, Sync.class, syncTask -> { - syncTask.dependsOn(downloadConfig); - syncTask.into(extractDir); - syncTask.from((Callable) () -> { - File archiveFile = archiveGetter.get(); - String archivePath = archiveFile.toString(); - if (archivePath.endsWith(".zip")) { - return rootProject.zipTree(archiveFile); - } else if (archivePath.endsWith(".tar.gz")) { - return rootProject.tarTree(rootProject.getResources().gzip(archiveFile)); - } - throw new IllegalStateException("unexpected file extension on [" + archivePath + "]"); - }); - - // Workaround for https://github.com/elastic/elasticsearch/issues/49417 - syncTask.eachFile(details -> { - String[] segments = details.getRelativePath().getSegments(); - if (segments[0].equals(".")) { - details.setRelativePath(new RelativePath(true, Arrays.copyOfRange(segments, 1, segments.length))); - } - }); - }); - rootProject.getArtifacts() - .add( - extractedConfigName, - rootProject.getLayout().getProjectDirectory().dir(extractDir), - artifact -> artifact.builtBy(extractTask) - ); - } - } - private static void addIvyRepo(Project project, String name, String url, String group) { IvyArtifactRepository ivyRepo = project.getRepositories().ivy(repo -> { repo.setName(name); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java index b93535759cf..ba7b2a68e77 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java @@ -90,36 +90,10 @@ public class ElasticsearchDistribution implements Buildable, Iterable { .onMac(() -> Platform.DARWIN) .supply(); - public static final class Extracted implements Buildable, Iterable { - - // pkg private so plugin can configure - final Configuration configuration; - - private Extracted(Configuration configuration) { - this.configuration = configuration; - } - - @Override - public Iterator iterator() { - return configuration.iterator(); - } - - @Override - public TaskDependency getBuildDependencies() { - return configuration.getBuildDependencies(); - } - - @Override - public String toString() { - return configuration.getSingleFile().toString(); - } - } - private final String name; private final Provider dockerSupport; // pkg private so plugin can configure final Configuration configuration; - private final Extracted extracted; private final Property architecture; private final Property version; @@ -128,6 +102,7 @@ public class ElasticsearchDistribution implements Buildable, Iterable { private final Property flavor; private final Property bundledJdk; private final Property failIfUnavailable; + private final Configuration extracted; ElasticsearchDistribution( String name, @@ -147,7 +122,7 @@ public class ElasticsearchDistribution implements Buildable, Iterable { this.flavor = objectFactory.property(Flavor.class); this.bundledJdk = objectFactory.property(Boolean.class); this.failIfUnavailable = objectFactory.property(Boolean.class).convention(true); - this.extracted = new Extracted(extractedConfiguration); + this.extracted = extractedConfiguration; } public String getName() { @@ -221,7 +196,7 @@ public class ElasticsearchDistribution implements Buildable, Iterable { return configuration.getSingleFile().toString(); } - public Extracted getExtracted() { + public Configuration getExtracted() { switch (getType()) { case DEB: case DOCKER: diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java index 9194aa1bd18..031fa2a88c0 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/JdkDownloadPlugin.java @@ -29,6 +29,7 @@ import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.dsl.RepositoryHandler; import org.gradle.api.artifacts.repositories.IvyArtifactRepository; import org.gradle.api.artifacts.type.ArtifactTypeDefinition; +import org.gradle.api.attributes.Attribute; import org.gradle.api.internal.artifacts.ArtifactAttributes; public class JdkDownloadPlugin implements Plugin { @@ -38,25 +39,39 @@ public class JdkDownloadPlugin implements Plugin { private static final String REPO_NAME_PREFIX = "jdk_repo_"; private static final String EXTENSION_NAME = "jdks"; + public static final String JDK_TRIMMED_PREFIX = "jdk-?\\d.*"; @Override public void apply(Project project) { + Attribute jdkAttribute = Attribute.of("jdk", Boolean.class); + project.getDependencies().getAttributesSchema().attribute(jdkAttribute); project.getDependencies().getArtifactTypes().maybeCreate(ArtifactTypeDefinition.ZIP_TYPE); project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> { - transformSpec.getFrom().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE); - transformSpec.getTo().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE); + transformSpec.getFrom() + .attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.ZIP_TYPE) + .attribute(jdkAttribute, true); + transformSpec.getTo() + .attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE) + .attribute(jdkAttribute, true); + transformSpec.parameters(parameters -> parameters.setTrimmedPrefixPattern(JDK_TRIMMED_PREFIX)); }); ArtifactTypeDefinition tarArtifactTypeDefinition = project.getDependencies().getArtifactTypes().maybeCreate("tar.gz"); project.getDependencies().registerTransform(SymbolicLinkPreservingUntarTransform.class, transformSpec -> { - transformSpec.getFrom().attribute(ArtifactAttributes.ARTIFACT_FORMAT, tarArtifactTypeDefinition.getName()); - transformSpec.getTo().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE); + transformSpec.getFrom() + .attribute(ArtifactAttributes.ARTIFACT_FORMAT, tarArtifactTypeDefinition.getName()) + .attribute(jdkAttribute, true); + transformSpec.getTo() + .attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE) + .attribute(jdkAttribute, true); + transformSpec.parameters(parameters -> parameters.setTrimmedPrefixPattern(JDK_TRIMMED_PREFIX)); }); NamedDomainObjectContainer jdksContainer = project.container(Jdk.class, name -> { Configuration configuration = project.getConfigurations().create("jdk_" + name); configuration.setCanBeConsumed(false); configuration.getAttributes().attribute(ArtifactAttributes.ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE); + configuration.getAttributes().attribute(jdkAttribute, true); Jdk jdk = new Jdk(name, configuration, project.getObjects()); configuration.defaultDependencies(dependencies -> { jdk.finalizeValues(); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index 6fe952738b9..66f5df632d2 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -38,6 +38,7 @@ import org.gradle.api.Action; import org.gradle.api.Named; import org.gradle.api.NamedDomainObjectContainer; import org.gradle.api.Project; +import org.gradle.api.file.FileTree; import org.gradle.api.file.RegularFile; import org.gradle.api.file.RegularFileProperty; import org.gradle.api.logging.Logger; @@ -65,7 +66,6 @@ import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.time.Instant; @@ -81,7 +81,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.TreeSet; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -1260,7 +1259,7 @@ public class ElasticsearchNode implements TestClusterConfiguration { } private Path getExtractedDistributionDir() { - return Paths.get(distributions.get(currentDistro).getExtracted().toString()); + return distributions.get(currentDistro).getExtracted().getSingleFile().toPath(); } private List getInstalledFileSet(Action filter) { @@ -1298,20 +1297,20 @@ public class ElasticsearchNode implements TestClusterConfiguration { } @Classpath - public Set getDistributionClasspath() { + public List getDistributionClasspath() { return getDistributionFiles(filter -> filter.include("**/*.jar")); } @InputFiles @PathSensitive(PathSensitivity.RELATIVE) - public Set getDistributionFiles() { + public List getDistributionFiles() { return getDistributionFiles(filter -> filter.exclude("**/*.jar")); } - private Set getDistributionFiles(Action patternFilter) { - Set files = new TreeSet<>(); + private List getDistributionFiles(Action patternFilter) { + List files = new ArrayList<>(); for (ElasticsearchDistribution distribution : distributions) { - files.addAll(project.fileTree(Paths.get(distribution.getExtracted().toString())).matching(patternFilter).getFiles()); + files.add(distribution.getExtracted().getAsFileTree().matching(patternFilter)); } return files; } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/transform/SymbolicLinkPreservingUntarTransform.java b/buildSrc/src/main/java/org/elasticsearch/gradle/transform/SymbolicLinkPreservingUntarTransform.java index 20754d077c7..9a2bc8bce19 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/transform/SymbolicLinkPreservingUntarTransform.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/transform/SymbolicLinkPreservingUntarTransform.java @@ -31,22 +31,22 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.attribute.PosixFileAttributeView; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; -import java.util.Set; +import java.util.function.Function; + +import static org.elasticsearch.gradle.util.PermissionUtils.chmod; public abstract class SymbolicLinkPreservingUntarTransform implements UnpackTransform { public void unpack(File tarFile, File targetDir) throws IOException { Logging.getLogger(SymbolicLinkPreservingUntarTransform.class) .info("Unpacking " + tarFile.getName() + " using " + SymbolicLinkPreservingUntarTransform.class.getSimpleName() + "."); + Function pathModifier = pathResolver(); TarArchiveInputStream tar = new TarArchiveInputStream(new GzipCompressorInputStream(new FileInputStream(tarFile))); final Path destinationPath = targetDir.toPath(); TarArchiveEntry entry = tar.getNextTarEntry(); while (entry != null) { - final Path relativePath = UnpackTransform.trimArchiveExtractPath(entry.getName()); + final Path relativePath = pathModifier.apply(entry.getName()); if (relativePath == null) { entry = tar.getNextTarEntry(); continue; @@ -70,41 +70,10 @@ public abstract class SymbolicLinkPreservingUntarTransform implements UnpackTran } if (entry.isSymbolicLink() == false) { // check if the underlying file system supports POSIX permissions - final PosixFileAttributeView view = Files.getFileAttributeView(destination, PosixFileAttributeView.class); - if (view != null) { - final Set permissions = PosixFilePermissions.fromString( - permissions((entry.getMode() >> 6) & 07) + permissions((entry.getMode() >> 3) & 07) + permissions( - (entry.getMode() >> 0) & 07 - ) - ); - Files.setPosixFilePermissions(destination, permissions); - } + chmod(destination, entry.getMode()); } entry = tar.getNextTarEntry(); } } - - private static String permissions(final int permissions) { - if (permissions < 0 || permissions > 7) { - throw new IllegalArgumentException("permissions [" + permissions + "] out of range"); - } - final StringBuilder sb = new StringBuilder(3); - if ((permissions & 4) == 4) { - sb.append('r'); - } else { - sb.append('-'); - } - if ((permissions & 2) == 2) { - sb.append('w'); - } else { - sb.append('-'); - } - if ((permissions & 1) == 1) { - sb.append('x'); - } else { - sb.append('-'); - } - return sb.toString(); - } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/transform/UnpackTransform.java b/buildSrc/src/main/java/org/elasticsearch/gradle/transform/UnpackTransform.java index 1b7b6ce4d42..9c407c08505 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/transform/UnpackTransform.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/transform/UnpackTransform.java @@ -25,6 +25,8 @@ import org.gradle.api.artifacts.transform.TransformOutputs; import org.gradle.api.artifacts.transform.TransformParameters; import org.gradle.api.file.FileSystemLocation; import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.Optional; import org.gradle.api.tasks.PathSensitive; import org.gradle.api.tasks.PathSensitivity; import org.gradle.internal.UncheckedException; @@ -33,8 +35,17 @@ import java.io.File; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.function.Function; -public interface UnpackTransform extends TransformAction { +public interface UnpackTransform extends TransformAction { + + interface Parameters extends TransformParameters { + @Input + @Optional + String getTrimmedPrefixPattern(); + + void setTrimmedPrefixPattern(String pattern); + } @PathSensitive(PathSensitivity.NAME_ONLY) @InputArtifact @@ -43,9 +54,9 @@ public interface UnpackTransform extends TransformAction pathResolver() { + String trimmedPrefixPattern = getParameters().getTrimmedPrefixPattern(); + return trimmedPrefixPattern != null ? (i) -> trimArchiveExtractPath(trimmedPrefixPattern, i) : (i) -> Paths.get(i); + } + /* - * We want to remove up to the and including the jdk-.* relative paths. That is a JDK archive is structured as: + * We want to be able to trim off certain prefixes when transforming archives. + * + * E.g We want to remove up to the and including the jdk-.* relative paths. That is a JDK archive is structured as: * jdk-12.0.1/ * jdk-12.0.1/Contents * ... @@ -66,11 +84,11 @@ public interface UnpackTransform extends TransformAction pathModifier = pathResolver(); + ZipFile zip = new ZipFile(zipFile); + try { + Enumeration entries = zip.getEntries(); + while (entries.hasMoreElements()) { + ZipEntry zipEntry = entries.nextElement(); + Path child = pathModifier.apply(zipEntry.getName()); + if (child == null) { continue; } - String child = UnpackTransform.trimArchiveExtractPath(entry.getName()).toString(); - File outFile = new File(targetDir, child); - outFile.getParentFile().mkdirs(); - try (FileOutputStream outputStream = new FileOutputStream(outFile)) { - IOUtils.copyLarge(inputStream, outputStream); + Path outputPath = targetDir.toPath().resolve(child); + if (zipEntry.isDirectory()) { + outputPath.toFile().mkdirs(); + chmod(outputPath, zipEntry.getUnixMode()); + continue; } + try (FileOutputStream outputStream = new FileOutputStream(outputPath.toFile())) { + IOUtils.copyLarge(zip.getInputStream(zipEntry), outputStream); + } + chmod(outputPath, zipEntry.getUnixMode()); } + } finally { + zip.close(); } } + } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/util/PermissionUtils.java b/buildSrc/src/main/java/org/elasticsearch/gradle/util/PermissionUtils.java new file mode 100644 index 00000000000..c0ee454b4c6 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/util/PermissionUtils.java @@ -0,0 +1,69 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.util; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Set; + +public class PermissionUtils { + + public static void chmod(Path path, int mode) throws IOException { + final PosixFileAttributeView view = Files.getFileAttributeView(path, PosixFileAttributeView.class); + if (view != null && (mode != 0)) { + final Set permissions = permissionsFromInt(mode); + Files.setPosixFilePermissions(path, permissions); + } + } + + private static Set permissionsFromInt(int mode) { + return PosixFilePermissions.fromString( + permissions((mode >> 6) & 07) + permissions((mode >> 3) & 07) + permissions((mode >> 0) & 07) + ); + } + + private static String permissions(final int permissions) { + if (permissions < 0 || permissions > 7) { + throw new IllegalArgumentException("permissions [" + permissions + "] out of range"); + } + final StringBuilder sb = new StringBuilder(3); + if ((permissions & 4) == 4) { + sb.append('r'); + } else { + sb.append('-'); + } + if ((permissions & 2) == 2) { + sb.append('w'); + } else { + sb.append('-'); + } + if ((permissions & 1) == 1) { + sb.append('x'); + } else { + sb.append('-'); + } + return sb.toString(); + } + +} diff --git a/buildSrc/src/testKit/distribution-download/build.gradle b/buildSrc/src/testKit/distribution-download/build.gradle deleted file mode 100644 index d408184be67..00000000000 --- a/buildSrc/src/testKit/distribution-download/build.gradle +++ /dev/null @@ -1,53 +0,0 @@ -import org.elasticsearch.gradle.BwcVersions -import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.info.BuildParams - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -plugins { - id 'elasticsearch.global-build-info' -} -boolean internal = Boolean.parseBoolean(System.getProperty("tests.internal", "true")) -BuildParams.init { it.setIsInternal(internal) } - -project.gradle.projectsEvaluated { - // wire the download service url to wiremock - String fakeDownloadService = System.getProperty('tests.download_service') - if (fakeDownloadService != null) { - IvyArtifactRepository repository = (IvyArtifactRepository) rootProject.repositories.getByName("elasticsearch-downloads") - repository.setUrl(fakeDownloadService) - repository = (IvyArtifactRepository) project('subproj').repositories.getByName("elasticsearch-downloads") - repository.setUrl(fakeDownloadService) - if (internal == false) { - repository = (IvyArtifactRepository) rootProject.repositories.getByName("elasticsearch-snapshots") - repository.setUrl(fakeDownloadService) - repository = (IvyArtifactRepository) project('subproj').repositories.getByName("elasticsearch-snapshots") - repository.setUrl(fakeDownloadService) - } - } -} - -if (internal) { - Version currentVersion = Version.fromString("9.0.0") - BwcVersions versions = new BwcVersions(new TreeSet<>( - Arrays.asList(Version.fromString("8.0.0"), Version.fromString("8.0.1"), Version.fromString("8.1.0"), currentVersion)), - currentVersion) - BuildParams.init { it.setBwcVersions(versions) } -} diff --git a/buildSrc/src/testKit/distribution-download/distribution/build.gradle b/buildSrc/src/testKit/distribution-download/distribution/build.gradle deleted file mode 100644 index 0d618d4fbe1..00000000000 --- a/buildSrc/src/testKit/distribution-download/distribution/build.gradle +++ /dev/null @@ -1,34 +0,0 @@ -String distroConfig = System.getProperty('tests.local_distro.config') -if (distroConfig != null) { - // setup the test distribution as an artifact of this project - String distroType = System.getProperty('tests.distro.type') - - def buildDistro - File buildFile - if (['rpm', 'deb'].contains(distroType)) { - buildDistro = project.tasks.register("build" + distroType.capitalize(), Copy) { - from 'files' - into 'build/files' - include 'fake_elasticsearch.tar.gz' - // this shouldn't be extracted so we just rename the file so it is a dummy package - rename { filename -> filename.replace('tar.gz', distroType) } - } - buildFile = project.file("build/files/fake_elasticsearch." + distroType) - } else { - String distroPlatform = System.getProperty('tests.distro.platform') - String extension = "tar.gz" - if (distroType == 'archive' && distroPlatform == 'windows' || distroType == 'integ-test-zip') { - extension = "zip" - } - // copy file as is - buildDistro = project.tasks.register("buildArchive", Copy) { - from 'files' - include "fake_elasticsearch.${extension}" - into 'build/files' - } - buildFile = project.file("build/files/fake_elasticsearch.${extension}") - } - - configurations.create(distroConfig) - artifacts.add(distroConfig, [file: buildFile, builtBy: buildDistro]) -} diff --git a/buildSrc/src/testKit/distribution-download/distribution/files/fake_elasticsearch.tar.gz b/buildSrc/src/testKit/distribution-download/distribution/files/fake_elasticsearch.tar.gz deleted file mode 100644 index 498b53ab68467ec291e816169f4bde24b3c17b64..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1139 zcmV-(1dRJ1iwFR4eLr6S1MOICZz4AkzR&y$pJ-L72g@!jP*L&WnzzeUlqgA0>8HqN zW*1*}*DKr0ORN6(&UndkM1*?@jrzhUioN!DW<1Y4!vZASSSbu}EvC)DYkQs6i#?0) z`~7a0xv^J{emgA3m9U`G5Bj}+7=!`ygD~v$Ua;=I1KB2pv0P&)8Zy|WhiXx@V}Esz zYW$Z7Y{CD%%=TIXUx0rX|0Dk0cH2MT{~WRnf1$ES#``OQFTp=-`-Fer4-fc1hwQ-r z!z}ewZl%i1(|lk)dSUn|{++(x-^9P|2W`aP-_I3y%l{nzdY!#}H)18^h2{iztyX`c zpaFJ1x)@(xzy9TV;5~BD}xFrxL)N3vj0v4^LsXWm=?< zpQ=&Lf|!N{4tpuGlSK~N5-?U~vDsH`y-t$KCVLNwn&TBERhXdMGmcmwWeg&!VL~Th zGZ2p*tL!^0N31H)Y%cATl9aL{OTbW^Y>D+-WM}|83l{5q%B@=?+Y^OG_JnYqogF>k z_YcjWGicu23|{?e;7Da+#P!|P93VeCT5leWuFhR0(2d4I6gkht@>Fw|gJHk>z|L0n zah5F0AOZ~IXd{d|F{?1(_8OQBq0$uV$(2(E3!KaqOJoMd@wvz++=APkVcG}?{KLl@ zQ6xc>&@(=3Bw=|9?PE&yT0L^7-yZ*rN2=(XTe9$Q`|-;KVYA8sU8Gt#3oel!dIlGu z1zu~|?pAwiV{eckmBD+>j_-0t&C5cXE6w$iO{B*1Q+ioz9)I97?n&j@G{@!PH-DU5 z-CmC_j?XXubSE9{CPkVX;&_aoZJJWY*Be-HKy4D6XQG4yghg7VWrEOrsTy&aV7Y!| zuq=KJXm5h8!s*YIzN6e%Ay%1wH>$bh&^3?o_&6IEb{vfW``PkDYo#eblY@}avJA(9 z&M=-Cu$&`psQInE|HvRSg)XaL?G6^wSXU(!D@UdlQ6z<=YAU-AMR6a*1=<~AE%DFs zO&~;^dKpl-EK%5J$eN4(R323#6WhxeAU{F|OS_%y`*|0n|PD7}I3Z&hj2kN!MUYORt`t$#>?|+-? zcd#^==bbM|kIN_D|9jQ@U$5Jt_rIV=-~SHpe}BK!UplzWv<70T*ml^N5nARJSq7p& z+Q0nzdOOHJOFoPLPxSFg{&#yH@t=PG>*E)}1OJ~xcJco}`-2?hAP4#H%3sXmo)rKb F007M8ItKs% diff --git a/buildSrc/src/testKit/distribution-download/distribution/files/fake_elasticsearch.zip b/buildSrc/src/testKit/distribution-download/distribution/files/fake_elasticsearch.zip deleted file mode 100644 index 42b572e37f3fbfbb90dee93ecbff745d37988879..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 568 zcmWIWW@h1H00E1zJ$_&Yln`N%VMxtMEH24RE>2A>O3u(V)HBjE)(;KgWMJM?@;2u7 zG^W_n3T_5QmamKq3@jo*eF1ROtk6u8z+qZaW*)@E1EiX$ipNA0cY_@aGJYa92QxCs zG2;pV320CVFuZjHF;R?#2L~%8I52_&VKQc5Ae#&db!;X>f(B?h$n7MUuF3#2oq=IV e;{rUUqu3f6g3BsaHlP^{%s{9NjOhO$4g&zPL2_UK diff --git a/buildSrc/src/testKit/distribution-download/settings.gradle b/buildSrc/src/testKit/distribution-download/settings.gradle deleted file mode 100644 index 1a64261480d..00000000000 --- a/buildSrc/src/testKit/distribution-download/settings.gradle +++ /dev/null @@ -1,8 +0,0 @@ -include 'subproj' - -String distroProject = System.getProperty('tests.local_distro.project') -System.out.println("local distro project: " + distroProject); -if (distroProject != null) { - include distroProject - project(distroProject).projectDir = new File(rootDir, 'distribution') -} diff --git a/buildSrc/src/testKit/distribution-download/subproj/build.gradle b/buildSrc/src/testKit/distribution-download/subproj/build.gradle deleted file mode 100644 index 4f7b1195d20..00000000000 --- a/buildSrc/src/testKit/distribution-download/subproj/build.gradle +++ /dev/null @@ -1,65 +0,0 @@ -import org.elasticsearch.gradle.Architecture - -plugins { - id 'elasticsearch.distribution-download' -} - -String distroVersion = System.getProperty('tests.distro.version') -String distroType = System.getProperty('tests.distro.type') -String distroPlatform = System.getProperty('tests.distro.platform') -String distroFlavor = System.getProperty('tests.distro.flavor') -String distroBundledJdk = System.getProperty('tests.distro.bundledJdk') -String distroArch = System.getProperty('tests.distro.arch'); - -elasticsearch_distributions { - test_distro { - if (distroVersion != null) { - version = distroVersion - } - if (distroType != null) { - type = distroType - } - if (distroPlatform != null) { - platform = distroPlatform - } - if (distroFlavor != null) { - flavor = distroFlavor - } - if (distroBundledJdk != null) { - bundledJdk = Boolean.parseBoolean(distroBundledJdk) - } - if (distroArch != null) { - architecture = Architecture.valueOf(distroArch); - } else { - architecture = Architecture.current(); - } - } -} - -tasks.register("assertDistroFile") { - dependsOn elasticsearch_distributions.test_distro - doLast { - File distroFile = new File(elasticsearch_distributions.test_distro.toString()) - if (distroFile.exists() == false) { - throw new GradleException("distro file does not exist: ${distroFile}") - } - if (distroFile.isFile() == false) { - throw new GradleException("distro file is not a regular file: ${distroFile}") - } - } -} - -if (['rpm', 'deb'].contains(distroType) == false) { - tasks.register("assertDistroExtracted") { - dependsOn elasticsearch_distributions.test_distro.extracted, assertDistroFile - doLast { - File distroExtracted = new File(elasticsearch_distributions.test_distro.extracted.toString()) - if (distroExtracted.exists() == false) { - throw new GradleException("extracted does not exist: ${distroExtracted}") - } - if (distroExtracted.isDirectory() == false) { - throw new GradleException("extracted distro is not a directory: ${distroExtracted}") - } - } - } -} diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle index 0ebd18123f5..8ad9a63527c 100644 --- a/distribution/bwc/build.gradle +++ b/distribution/bwc/build.gradle @@ -316,7 +316,7 @@ BuildParams.bwcVersions.forPreviousUnreleased { BwcVersions.UnreleasedVersionInf File artifactFile = e.value String artifactFileName = artifactFile.name String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch' - String suffix = artifactFile.toString()[-3..-1] + String suffix = artifactFileName.endsWith("tar.gz") ? "tar.gz" : artifactFileName[-3..-1] int archIndex = artifactFileName.indexOf('x86_64') String classifier = '' if (archIndex != -1) {