diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy index 3b441478e29..90734ac4fef 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy @@ -43,9 +43,13 @@ abstract class AbstractGradleFuncTest extends Specification { } GradleRunner gradleRunner(String... arguments) { + return gradleRunner(testProjectDir.root, arguments) + } + + GradleRunner gradleRunner(File projectDir, String... arguments) { GradleRunner.create() .withDebug(ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0) - .withProjectDir(testProjectDir.root) + .withProjectDir(projectDir) .withArguments(arguments) .withPluginClasspath() .forwardOutput() @@ -85,4 +89,31 @@ abstract class AbstractGradleFuncTest extends Specification { return jarFile; } + + File internalBuild(File buildScript = buildFile) { + buildScript << """plugins { + id 'elasticsearch.global-build-info' + } + import org.elasticsearch.gradle.Architecture + import org.elasticsearch.gradle.info.BuildParams + + BuildParams.init { it.setIsInternal(true) } + + import org.elasticsearch.gradle.BwcVersions + import org.elasticsearch.gradle.Version + + Version currentVersion = Version.fromString("9.0.0") + BwcVersions versions = new BwcVersions(new TreeSet<>( + Arrays.asList(Version.fromString("8.0.0"), Version.fromString("8.0.1"), Version.fromString("8.1.0"), currentVersion)), + currentVersion) + + BuildParams.init { it.setBwcVersions(versions) } + """ + } + + void setupLocalGitRepo() { + "git init".execute(Collections.emptyList(), testProjectDir.root).waitFor() + "git add .".execute(Collections.emptyList(), testProjectDir.root).waitFor() + 'git commit -m "Initial"'.execute(Collections.emptyList(), testProjectDir.root).waitFor() + } } diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalBwcGitPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalBwcGitPluginFuncTest.groovy new file mode 100644 index 00000000000..5cb73be3b46 --- /dev/null +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalBwcGitPluginFuncTest.groovy @@ -0,0 +1,51 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.internal + +import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest +import org.gradle.testkit.runner.TaskOutcome + +class InternalBwcGitPluginFuncTest extends AbstractGradleFuncTest { + + def setup() { + setupLocalGitRepo() + } + + def "current repository can be cloned"() { + given: + internalBuild(); + buildFile << """ + import org.elasticsearch.gradle.Version; + apply plugin: org.elasticsearch.gradle.internal.InternalBwcGitPlugin + + bwcGitConfig { + bwcVersion = project.provider { Version.fromString("7.10.0") } + bwcBranch = project.provider { "7.x" } + checkoutDir = project.provider{file("build/checkout")} + } + """ + when: + def result = gradleRunner("createClone", '--stacktrace').build() + then: + result.task(":createClone").outcome == TaskOutcome.SUCCESS + file("build/checkout/build.gradle").exists() + file("build/checkout/settings.gradle").exists() + } +} diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy new file mode 100644 index 00000000000..295d3a38f32 --- /dev/null +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPluginFuncTest.groovy @@ -0,0 +1,151 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.internal + +import org.apache.commons.io.FileUtils +import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest +import org.gradle.testkit.runner.TaskOutcome +import org.junit.Rule +import org.junit.rules.TemporaryFolder + +class InternalDistributionBwcSetupPluginFuncTest extends AbstractGradleFuncTest { + + @Rule + TemporaryFolder remoteRepoDirs = new TemporaryFolder() + + File remoteGitRepo + + def setup() { + remoteGitRepo = new File(setupGitRemote(), '.git') + + "git clone ${remoteGitRepo.absolutePath}".execute(Collections.emptyList(), testProjectDir.root).waitFor() + File buildScript = new File(testProjectDir.root, 'remote/build.gradle') + internalBuild(buildScript) + buildScript << """ + apply plugin: 'elasticsearch.internal-distribution-bwc-setup' + """ + } + + def "builds distribution from branches via archives assemble"() { + when: + def result = gradleRunner(new File(testProjectDir.root, "remote"), + ":distribution:bwc:bugfix:buildBwcDarwinTar", + ":distribution:bwc:bugfix:buildBwcOssDarwinTar", + "-DtestRemoteRepo=" + remoteGitRepo, + "-Dbwc.remote=origin") + .build() + then: + result.task(":distribution:bwc:bugfix:buildBwcDarwinTar").outcome == TaskOutcome.SUCCESS + result.task(":distribution:bwc:bugfix:buildBwcOssDarwinTar").outcome == TaskOutcome.SUCCESS + + and: "assemble task triggered" + result.output.contains("[8.0.1] > Task :distribution:archives:darwin-tar:assemble") + result.output.contains("[8.0.1] > Task :distribution:archives:oss-darwin-tar:assemble") + } + + def "bwc distribution archives can be resolved as bwc project artifact"() { + setup: + new File(testProjectDir.root, 'remote/build.gradle') << """ + + configurations { + dists + } + + dependencies { + dists project(path: ":distribution:bwc:bugfix", configuration:"darwin-tar") + } + + tasks.register("resolveDistributionArchive") { + inputs.files(configurations.dists) + doLast { + configurations.dists.files.each { + println "distfile " + (it.absolutePath - project.rootDir.absolutePath) + } + } + } + """ + when: + def result = gradleRunner(new File(testProjectDir.root, "remote"), + ":resolveDistributionArchive", + "-DtestRemoteRepo=" + remoteGitRepo, + "-Dbwc.remote=origin") + .build() + then: + result.task(":resolveDistributionArchive").outcome == TaskOutcome.SUCCESS + result.task(":distribution:bwc:bugfix:buildBwcDarwinTar").outcome == TaskOutcome.SUCCESS + + and: "assemble task triggered" + result.output.contains("[8.0.1] > Task :distribution:archives:darwin-tar:assemble") + normalizedOutput(result.output) + .contains("distfile /distribution/bwc/bugfix/build/bwc/checkout-8.0/distribution/archives/darwin-tar/" + + "build/distributions/elasticsearch-8.0.1-SNAPSHOT-darwin-x86_64.tar.gz") + } + + def "bwc expanded distribution folder can be resolved as bwc project artifact"() { + setup: + new File(testProjectDir.root, 'remote/build.gradle') << """ + + configurations { + expandedDist + } + + dependencies { + expandedDist project(path: ":distribution:bwc:bugfix", configuration:"expanded-darwin-tar") + } + + tasks.register("resolveExpandedDistribution") { + inputs.files(configurations.expandedDist) + doLast { + configurations.expandedDist.files.each { + println "distfile " + (it.absolutePath - project.rootDir.absolutePath) + } + } + } + """ + when: + def result = gradleRunner(new File(testProjectDir.root, "remote"), + ":resolveExpandedDistribution", + "-DtestRemoteRepo=" + remoteGitRepo, + "-Dbwc.remote=origin") + .build() + then: + result.task(":resolveExpandedDistribution").outcome == TaskOutcome.SUCCESS + result.task(":distribution:bwc:bugfix:buildBwcDarwinTar").outcome == TaskOutcome.SUCCESS + + and: "assemble task triggered" + result.output.contains("[8.0.1] > Task :distribution:archives:darwin-tar:assemble") + normalizedOutput(result.output) + .contains("distfile /distribution/bwc/bugfix/build/bwc/checkout-8.0/" + + "distribution/archives/darwin-tar/build/install") + } + + File setupGitRemote() { + URL fakeRemote = getClass().getResource("fake_git/remote") + File workingRemoteGit = new File(remoteRepoDirs.root, 'remote') + FileUtils.copyDirectory(new File(fakeRemote.file), workingRemoteGit) + fakeRemote.file + "/.git" + gradleRunner(workingRemoteGit, "wrapper").build() + "git init".execute(Collections.emptyList(), workingRemoteGit).waitFor() + "git add .".execute(Collections.emptyList(), workingRemoteGit).waitFor() + 'git commit -m"Initial"'.execute(Collections.emptyList(), workingRemoteGit).waitFor() + "git checkout -b origin/8.0".execute(Collections.emptyList(), workingRemoteGit).waitFor() + return workingRemoteGit; + } +} diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy index 88ede9dab0a..823ef749696 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy @@ -72,16 +72,15 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest def result = gradleRunner("setupDistro", '-g', testProjectDir.newFolder('GUH').path).build() then: - result.task(":distribution:archives:linux-tar:buildExploded").outcome == TaskOutcome.SUCCESS + result.task(":distribution:archives:linux-tar:buildExpanded").outcome == TaskOutcome.SUCCESS result.task(":setupDistro").outcome == TaskOutcome.SUCCESS - assertExtractedDistroIsCreated(distroVersion, "build/distro", 'current-marker.txt') + assertExtractedDistroIsCreated("build/distro", 'current-marker.txt') } - def "resolves bwc versions from source"() { + def "resolves expanded bwc versions from source"() { given: internalBuild() bwcMinorProjectSetup() - def distroVersion = "8.1.0" buildFile << """ apply plugin: 'elasticsearch.internal-distribution-download' @@ -102,9 +101,10 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest def result = gradleRunner("setupDistro").build() then: - result.task(":distribution:bwc:minor:buildBwcTask").outcome == TaskOutcome.SUCCESS + result.task(":distribution:bwc:minor:buildBwcExpandedTask").outcome == TaskOutcome.SUCCESS result.task(":setupDistro").outcome == TaskOutcome.SUCCESS - assertExtractedDistroIsCreated(distroVersion, "build/distro", 'bwc-marker.txt') + assertExtractedDistroIsCreated("distribution/bwc/minor/build/install/elastic-distro", + 'bwc-marker.txt') } def "fails on resolving bwc versions with no bundled jdk"() { @@ -134,28 +134,6 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest "without a bundled JDK is not supported.") } - private File internalBuild() { - buildFile << """plugins { - id 'elasticsearch.global-build-info' - } - import org.elasticsearch.gradle.Architecture - import org.elasticsearch.gradle.info.BuildParams - - BuildParams.init { it.setIsInternal(true) } - - import org.elasticsearch.gradle.BwcVersions - import org.elasticsearch.gradle.Version - - Version currentVersion = Version.fromString("9.0.0") - BwcVersions versions = new BwcVersions(new TreeSet<>( - Arrays.asList(Version.fromString("8.0.0"), Version.fromString("8.0.1"), Version.fromString("8.1.0"), currentVersion)), - currentVersion) - - BuildParams.init { it.setBwcVersions(versions) } - """ - } - - private void bwcMinorProjectSetup() { settingsFile << """ include ':distribution:bwc:minor' @@ -164,6 +142,8 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest new File(bwcSubProjectFolder, 'bwc-marker.txt') << "bwc=minor" new File(bwcSubProjectFolder, 'build.gradle') << """ apply plugin:'base' + + // packed distro configurations.create("linux-tar") tasks.register("buildBwcTask", Tar) { from('bwc-marker.txt') @@ -173,6 +153,19 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest artifacts { it.add("linux-tar", buildBwcTask) } + + // expanded distro + configurations.create("expanded-linux-tar") + def expandedTask = tasks.register("buildBwcExpandedTask", Copy) { + from('bwc-marker.txt') + into('build/install/elastic-distro') + } + artifacts { + it.add("expanded-linux-tar", file('build/install')) { + builtBy expandedTask + type = 'directory' + } + } """ } @@ -192,7 +185,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest archiveExtension = "tar.gz" compression = Compression.GZIP } - def buildExploded = tasks.register("buildExploded", Copy) { + def buildExpanded = tasks.register("buildExpanded", Copy) { from('current-marker.txt') into("build/local") } @@ -205,15 +198,14 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest } artifacts { it.add("default", buildTar) - it.add("extracted", buildExploded) + it.add("extracted", buildExpanded) } """ buildFile << """ """ - } - boolean assertExtractedDistroIsCreated(String version, String relativeDistroPath, String markerFileName) { + boolean assertExtractedDistroIsCreated(String relativeDistroPath, String markerFileName) { File extractedFolder = new File(testProjectDir.root, relativeDistroPath) assert extractedFolder.exists() assert new File(extractedFolder, markerFileName).exists() diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/.ci/java-versions.properties b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/.ci/java-versions.properties new file mode 100644 index 00000000000..1337f4013e6 --- /dev/null +++ b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/.ci/java-versions.properties @@ -0,0 +1,21 @@ +# +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +ES_BUILD_JAVA=openjdk14 +ES_RUNTIME_JAVA=openjdk14 +GRADLE_TASK=build diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/build.gradle b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/build.gradle b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/build.gradle new file mode 100644 index 00000000000..d73e32b6b12 --- /dev/null +++ b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/build.gradle @@ -0,0 +1,33 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +subprojects { + apply plugin:'base' + + tasks.register('tar', Tar) { + from('.') + destinationDirectory.set(file('build/distributions')) + archiveBaseName.set("elasticsearch${project.name.startsWith('oss')?'-oss':''}") + archiveVersion.set("8.0.1-SNAPSHOT") + archiveClassifier.set("darwin-x86_64") + archiveExtension.set('tar.gz') + } + + assemble.dependsOn('tar') +} diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/darwin-tar/build.gradle b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/darwin-tar/build.gradle new file mode 100644 index 00000000000..b03b6202c79 --- /dev/null +++ b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/darwin-tar/build.gradle @@ -0,0 +1,18 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/oss-darwin-tar/build.gradle b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/archives/oss-darwin-tar/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/bwc/bugfix/build.gradle b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/bwc/bugfix/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/bwc/minor/build.gradle b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/distribution/bwc/minor/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/settings.gradle b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/settings.gradle new file mode 100644 index 00000000000..24d61623821 --- /dev/null +++ b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/settings.gradle @@ -0,0 +1,23 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +include ":distribution:bwc:bugfix" +include ":distribution:bwc:minor" +include ":distribution:archives:darwin-tar" +include ":distribution:archives:oss-darwin-tar" diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java index f720d35498d..2a2670e91ce 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java @@ -444,7 +444,7 @@ public class GlobalBuildInfoPlugin implements Plugin { return firstLine; } - private static class GitInfo { + public static class GitInfo { private final String revision; private final String origin; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/BwcGitExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/BwcGitExtension.java new file mode 100644 index 00000000000..96997762fe2 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/BwcGitExtension.java @@ -0,0 +1,56 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.internal; + +import org.elasticsearch.gradle.Version; +import org.gradle.api.provider.Provider; + +import java.io.File; + +public class BwcGitExtension { + + private Provider bwcVersion; + private Provider bwcBranch; + private Provider checkoutDir; + + public Provider getBwcVersion() { + return bwcVersion; + } + + public void setBwcVersion(Provider bwcVersion) { + this.bwcVersion = bwcVersion; + } + + public Provider getBwcBranch() { + return bwcBranch; + } + + public void setBwcBranch(Provider bwcBranch) { + this.bwcBranch = bwcBranch; + } + + public Provider getCheckoutDir() { + return checkoutDir; + } + + public void setCheckoutDir(Provider checkoutDir) { + this.checkoutDir = checkoutDir; + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java new file mode 100644 index 00000000000..e110e84fcbc --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java @@ -0,0 +1,177 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.internal; + +import org.apache.commons.io.FileUtils; +import org.apache.tools.ant.taskdefs.condition.Os; +import org.elasticsearch.gradle.BwcVersions; +import org.elasticsearch.gradle.LoggedExec; +import org.gradle.api.Action; +import org.gradle.api.GradleException; +import org.gradle.api.Project; +import org.gradle.api.logging.LogLevel; +import org.gradle.api.provider.Provider; +import org.gradle.api.tasks.TaskProvider; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; + +import static org.elasticsearch.gradle.util.JavaUtil.getJavaHome; + +/** + * By registering bwc tasks via this extension we can support declaring custom bwc tasks from the build script + * without relying on groovy closures and sharing common logic for tasks created by the BwcSetup plugin already. + * */ +public class BwcSetupExtension { + + private final Project project; + + private final Provider unreleasedVersionInfo; + private Provider checkoutDir; + + public BwcSetupExtension( + Project project, + Provider unreleasedVersionInfo, + Provider checkoutDir + ) { + this.project = project; + this.unreleasedVersionInfo = unreleasedVersionInfo; + this.checkoutDir = checkoutDir; + } + + TaskProvider bwcTask(String name, Action configuration) { + return createRunBwcGradleTask(project, name, configuration); + } + + private TaskProvider createRunBwcGradleTask(Project project, String name, Action configAction) { + return project.getTasks().register(name, LoggedExec.class, loggedExec -> { + // TODO revisit + loggedExec.dependsOn("checkoutBwcBranch"); + loggedExec.setSpoolOutput(true); + loggedExec.setWorkingDir(checkoutDir.get()); + loggedExec.doFirst(t -> { + // Execution time so that the checkouts are available + String javaVersionsString = readFromFile(new File(checkoutDir.get(), ".ci/java-versions.properties")); + loggedExec.environment( + "JAVA_HOME", + getJavaHome( + Integer.parseInt( + Arrays.asList(javaVersionsString.split("\n")) + .stream() + .filter(l -> l.trim().startsWith("ES_BUILD_JAVA=")) + .map(l -> l.replace("ES_BUILD_JAVA=java", "").trim()) + .map(l -> l.replace("ES_BUILD_JAVA=openjdk", "").trim()) + .collect(Collectors.joining("!!")) + ) + ) + ); + loggedExec.environment( + "RUNTIME_JAVA_HOME", + getJavaHome( + Integer.parseInt( + Arrays.asList(javaVersionsString.split("\n")) + .stream() + .filter(l -> l.trim().startsWith("ES_RUNTIME_JAVA=")) + .map(l -> l.replace("ES_RUNTIME_JAVA=java", "").trim()) + .map(l -> l.replace("ES_RUNTIME_JAVA=openjdk", "").trim()) + .collect(Collectors.joining("!!")) + ) + ) + ); + }); + + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + loggedExec.executable("cmd"); + loggedExec.args("/C", "call", new File(checkoutDir.get(), "gradlew").toString()); + } else { + loggedExec.executable(new File(checkoutDir.get(), "gradlew").toString()); + } + if (project.getGradle().getStartParameter().isOffline()) { + loggedExec.args("--offline"); + } + // TODO resolve + String buildCacheUrl = System.getProperty("org.elasticsearch.build.cache.url"); + if (buildCacheUrl != null) { + loggedExec.args("-Dorg.elasticsearch.build.cache.url=" + buildCacheUrl); + } + + loggedExec.args("-Dbuild.snapshot=true"); + loggedExec.args("-Dscan.tag.NESTED"); + final LogLevel logLevel = project.getGradle().getStartParameter().getLogLevel(); + List nonDefaultLogLevels = Arrays.asList(LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG); + if (nonDefaultLogLevels.contains(logLevel)) { + loggedExec.args("--" + logLevel.name().toLowerCase(Locale.ENGLISH)); + } + final String showStacktraceName = project.getGradle().getStartParameter().getShowStacktrace().name(); + assert Arrays.asList("INTERNAL_EXCEPTIONS", "ALWAYS", "ALWAYS_FULL").contains(showStacktraceName); + if (showStacktraceName.equals("ALWAYS")) { + loggedExec.args("--stacktrace"); + } else if (showStacktraceName.equals("ALWAYS_FULL")) { + loggedExec.args("--full-stacktrace"); + } + if (project.getGradle().getStartParameter().isParallelProjectExecutionEnabled()) { + loggedExec.args("--parallel"); + } + loggedExec.setStandardOutput(new IndentingOutputStream(System.out, unreleasedVersionInfo.get().version)); + loggedExec.setErrorOutput(new IndentingOutputStream(System.err, unreleasedVersionInfo.get().version)); + configAction.execute(loggedExec); + }); + } + + private static class IndentingOutputStream extends OutputStream { + + public final byte[] indent; + private final OutputStream delegate; + + IndentingOutputStream(OutputStream delegate, Object version) { + this.delegate = delegate; + indent = (" [" + version + "] ").getBytes(StandardCharsets.UTF_8); + } + + @Override + public void write(int b) throws IOException { + int[] arr = { b }; + write(arr, 0, 1); + } + + public void write(int[] bytes, int offset, int length) throws IOException { + for (int i = 0; i < bytes.length; i++) { + delegate.write(bytes[i]); + if (bytes[i] == '\n') { + delegate.write(indent); + } + } + } + } + + private static String readFromFile(File file) { + try { + return FileUtils.readFileToString(file).trim(); + } catch (IOException ioException) { + throw new GradleException("Cannot read java properties file.", ioException); + } + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/DistributionArchive.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/DistributionArchive.java index 6299f479b58..cad21e74289 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/DistributionArchive.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/DistributionArchive.java @@ -30,12 +30,12 @@ import java.util.function.Supplier; public class DistributionArchive implements Named { private TaskProvider archiveTask; - private TaskProvider explodedDistTask; + private TaskProvider expandedDistTask; private final String name; - public DistributionArchive(TaskProvider archiveTask, TaskProvider explodedDistTask, String name) { + public DistributionArchive(TaskProvider archiveTask, TaskProvider expandedDistTask, String name) { this.archiveTask = archiveTask; - this.explodedDistTask = explodedDistTask; + this.expandedDistTask = expandedDistTask; this.name = name; } @@ -45,7 +45,7 @@ public class DistributionArchive implements Named { public void content(Supplier p) { this.archiveTask.configure(t -> t.with(p.get())); - this.explodedDistTask.configure(t -> t.with(p.get())); + this.expandedDistTask.configure(t -> t.with(p.get())); } @Override @@ -57,7 +57,7 @@ public class DistributionArchive implements Named { return archiveTask; } - public TaskProvider getExplodedArchiveTask() { - return explodedDistTask; + public TaskProvider getExpandedDistTask() { + return expandedDistTask; } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java new file mode 100644 index 00000000000..faec6cfb24c --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalBwcGitPlugin.java @@ -0,0 +1,209 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.internal; + +import org.apache.commons.io.FileUtils; +import org.elasticsearch.gradle.LoggedExec; +import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin; +import org.gradle.api.Action; +import org.gradle.api.GradleException; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.logging.Logger; +import org.gradle.api.plugins.ExtraPropertiesExtension; +import org.gradle.api.provider.Provider; +import org.gradle.api.provider.ProviderFactory; +import org.gradle.api.tasks.TaskContainer; +import org.gradle.api.tasks.TaskProvider; +import org.gradle.process.ExecOperations; +import org.gradle.process.ExecResult; +import org.gradle.process.ExecSpec; + +import javax.inject.Inject; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import static java.util.Arrays.asList; + +public class InternalBwcGitPlugin implements Plugin { + + private final ProviderFactory providerFactory; + private final ExecOperations execOperations; + + private BwcGitExtension gitExtension; + private Project project; + + @Inject + public InternalBwcGitPlugin(ProviderFactory providerFactory, ExecOperations execOperations) { + this.providerFactory = providerFactory; + this.execOperations = execOperations; + } + + @Override + public void apply(Project project) { + this.project = project; + this.gitExtension = project.getExtensions().create("bwcGitConfig", BwcGitExtension.class); + Provider remote = providerFactory.systemProperty("bwc.remote").forUseAtConfigurationTime().orElse("elastic"); + + TaskContainer tasks = project.getTasks(); + TaskProvider createCloneTaskProvider = tasks.register("createClone", LoggedExec.class, createClone -> { + createClone.onlyIf(task -> this.gitExtension.getCheckoutDir().get().exists() == false); + createClone.setCommandLine(asList("git", "clone", project.getRootDir(), gitExtension.getCheckoutDir().get())); + }); + + TaskProvider findRemoteTaskProvider = tasks.register("findRemote", LoggedExec.class, findRemote -> { + findRemote.dependsOn(createCloneTaskProvider); + // TODO Gradle should provide property based configuration here + findRemote.setWorkingDir(gitExtension.getCheckoutDir().get()); + + findRemote.setCommandLine(asList("git", "remote", "-v")); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + findRemote.setStandardOutput(output); + findRemote.doLast(t -> { + ExtraPropertiesExtension extraProperties = project.getExtensions().getExtraProperties(); + extraProperties.set("remoteExists", isRemoteAvailable(remote, output)); + }); + }); + + TaskProvider addRemoteTaskProvider = tasks.register("addRemote", LoggedExec.class, addRemote -> { + addRemote.dependsOn(findRemoteTaskProvider); + addRemote.onlyIf(task -> ((boolean) project.getExtensions().getExtraProperties().get("remoteExists")) == false); + addRemote.setWorkingDir(gitExtension.getCheckoutDir().get()); + String remoteRepo = remote.get(); + // for testing only we can override the base remote url + String remoteRepoUrl = providerFactory.systemProperty("testRemoteRepo") + .forUseAtConfigurationTime() + .getOrElse("https://github.com/" + remoteRepo + "/elasticsearch.git"); + addRemote.setCommandLine(asList("git", "remote", "add", remoteRepo, remoteRepoUrl)); + }); + + TaskProvider fetchLatestTaskProvider = tasks.register("fetchLatest", LoggedExec.class, fetchLatest -> { + var gitFetchLatest = project.getProviders() + .systemProperty("tests.bwc.git_fetch_latest") + .forUseAtConfigurationTime() + .orElse("true") + .map(fetchProp -> { + if ("true".equals(fetchProp)) { + return true; + } + if ("false".equals(fetchProp)) { + return false; + } + throw new GradleException("tests.bwc.git_fetch_latest must be [true] or [false] but was [" + fetchProp + "]"); + }); + fetchLatest.onlyIf(t -> project.getGradle().getStartParameter().isOffline() == false && gitFetchLatest.get()); + fetchLatest.dependsOn(addRemoteTaskProvider); + fetchLatest.setWorkingDir(gitExtension.getCheckoutDir().get()); + fetchLatest.setCommandLine(asList("git", "fetch", "--all")); + }); + + tasks.register("checkoutBwcBranch", checkoutBwcBranch -> { + checkoutBwcBranch.dependsOn(fetchLatestTaskProvider); + checkoutBwcBranch.doLast(t -> { + Logger logger = project.getLogger(); + + String bwcBranch = this.gitExtension.getBwcBranch().get(); + final String refspec = providerFactory.systemProperty("bwc.refspec." + bwcBranch) + .orElse(providerFactory.systemProperty("tests.bwc.refspec." + bwcBranch)) + .getOrElse(remote.get() + "/" + bwcBranch); + + String effectiveRefSpec = maybeAlignedRefSpec(logger, refspec); + + logger.lifecycle("Performing checkout of {}...", refspec); + LoggedExec.exec(project, spec -> { + spec.workingDir(gitExtension.getCheckoutDir()); + spec.commandLine("git", "checkout", effectiveRefSpec); + }); + + String checkoutHash = GlobalBuildInfoPlugin.gitInfo(gitExtension.getCheckoutDir().get()).getRevision(); + logger.lifecycle("Checkout hash for {} is {}", project.getPath(), checkoutHash); + writeFile(new File(project.getBuildDir(), "refspec"), checkoutHash); + }); + }); + } + + public BwcGitExtension getGitExtension() { + return gitExtension; + } + + /** + * We use a time based approach to make the bwc versions built deterministic and compatible with the current hash. + * Most of the time we want to test against latest, but when running delayed exhaustive tests or wanting + * reproducible builds we want this to be deterministic by using a hash that was the latest when the current + * commit was made. + *

+ * This approach doesn't work with merge commits as these can introduce commits in the chronological order + * after the fact e.x. a merge done today can add commits dated with yesterday so the result will no longer be + * deterministic. + *

+ * We don't use merge commits, but for additional safety we check that no such commits exist in the time period + * we are interested in. + *

+ * Timestamps are at seconds resolution. rev-parse --before and --after are inclusive w.r.t the second + * passed as input. This means the results might not be deterministic in the current second, but this + * should not matter in practice. + */ + private String maybeAlignedRefSpec(Logger logger, String defaultRefSpec) { + if (providerFactory.systemProperty("bwc.checkout.align").isPresent() == false) { + return defaultRefSpec; + } + + String timeOfCurrent = execInCheckoutDir(execSpec -> { + execSpec.commandLine(asList("git", "show", "--no-patch", "--no-notes", "--pretty='%cD'")); + execSpec.workingDir(project.getRootDir()); + }); + + logger.lifecycle("Commit date of current: {}", timeOfCurrent); + + String mergeCommits = execInCheckoutDir( + spec -> spec.commandLine(asList("git", "rev-list", defaultRefSpec, "--after", timeOfCurrent, "--merges")) + ); + if (mergeCommits.isEmpty() == false) { + throw new IllegalStateException("Found the following merge commits which prevent determining bwc commits: " + mergeCommits); + } + return execInCheckoutDir( + spec -> spec.commandLine(asList("git", "rev-list", defaultRefSpec, "-n", "1", "--before", timeOfCurrent, "--date-order")) + ); + } + + private void writeFile(File file, String content) { + try { + FileUtils.writeStringToFile(file, content); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private String execInCheckoutDir(Action execSpecConfig) { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + ExecResult exec = execOperations.exec(execSpec -> { + execSpec.setStandardOutput(os); + execSpec.workingDir(gitExtension.getCheckoutDir().get()); + execSpecConfig.execute(execSpec); + }); + exec.assertNormalExitValue(); + return os.toString().trim(); + } + + private static boolean isRemoteAvailable(Provider remote, ByteArrayOutputStream output) { + return new String(output.toByteArray()).contains(remote.get() + "\t"); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java index 5bbf98d7904..54c23bc5770 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java @@ -88,7 +88,7 @@ public class InternalDistributionArchiveSetupPlugin implements Plugin { var extractedConfiguration = sub.getConfigurations().create("extracted"); extractedConfiguration.setCanBeResolved(false); extractedConfiguration.getAttributes().attribute(ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE); - sub.getArtifacts().add(EXTRACTED_CONFIGURATION_NAME, distributionArchive.getExplodedArchiveTask()); + sub.getArtifacts().add(EXTRACTED_CONFIGURATION_NAME, distributionArchive.getExpandedDistTask()); }); }); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java new file mode 100644 index 00000000000..a6080685673 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java @@ -0,0 +1,268 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.internal; + +import org.elasticsearch.gradle.BwcVersions; +import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.info.BuildParams; +import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin; +import org.gradle.api.InvalidUserDataException; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.Task; +import org.gradle.api.provider.Provider; +import org.gradle.api.provider.ProviderFactory; +import org.gradle.api.tasks.TaskProvider; +import org.gradle.language.base.plugins.LifecycleBasePlugin; + +import javax.inject.Inject; +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; + +import static java.util.Arrays.asList; +import static java.util.Arrays.stream; + +/** + * We want to be able to do BWC tests for unreleased versions without relying on and waiting for snapshots. + * For this we need to check out and build the unreleased versions. + * Since these depend on the current version, we can't name the Gradle projects statically, and don't know what the + * unreleased versions are when Gradle projects are set up, so we use "build-unreleased-version-*" as placeholders + * and configure them to build various versions here. + */ +public class InternalDistributionBwcSetupPlugin implements Plugin { + + private ProviderFactory providerFactory; + + @Inject + public InternalDistributionBwcSetupPlugin(ProviderFactory providerFactory) { + this.providerFactory = providerFactory; + } + + @Override + public void apply(Project project) { + project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class); + BuildParams.getBwcVersions() + .forPreviousUnreleased( + (BwcVersions.UnreleasedVersionInfo unreleasedVersion) -> { + configureBwcProject(project.project(unreleasedVersion.gradleProjectPath), unreleasedVersion); + } + ); + } + + private void configureBwcProject(Project project, BwcVersions.UnreleasedVersionInfo versionInfo) { + Provider versionInfoProvider = providerFactory.provider(() -> versionInfo); + Provider checkoutDir = versionInfoProvider.map(info -> new File(project.getBuildDir(), "bwc/checkout-" + info.branch)); + BwcSetupExtension bwcSetupExtension = project.getExtensions() + .create("bwcSetup", BwcSetupExtension.class, project, versionInfoProvider, checkoutDir); + BwcGitExtension gitExtension = project.getPlugins().apply(InternalBwcGitPlugin.class).getGitExtension(); + Provider bwcVersion = versionInfoProvider.map(info -> info.version); + gitExtension.setBwcVersion(versionInfoProvider.map(info -> info.version)); + gitExtension.setBwcBranch(versionInfoProvider.map(info -> info.branch)); + gitExtension.setCheckoutDir(checkoutDir); + + // we want basic lifecycle tasks like `clean` here. + project.getPlugins().apply(LifecycleBasePlugin.class); + + TaskProvider buildBwcTaskProvider = project.getTasks().register("buildBwc"); + List distributionProjects = resolveArchiveProjects(checkoutDir.get(), bwcVersion.get()); + + for (DistributionProject distributionProject : distributionProjects) { + createBuildBwcTask( + bwcSetupExtension, + project, + bwcVersion, + distributionProject.name, + distributionProject.getProjectPath(), + distributionProject.getDistFile(), + buildBwcTaskProvider + ); + + registerBwcArtifacts(project, distributionProject); + } + + // Create build tasks for the JDBC driver used for compatibility testing + String jdbcProjectDir = "x-pack/plugin/sql/jdbc"; + + File jdbcProjectArtifact = new File( + checkoutDir.get(), + jdbcProjectDir + "/build/distributions/x-pack-sql-jdbc-" + bwcVersion.get() + "-SNAPSHOT.jar" + ); + + createBuildBwcTask(bwcSetupExtension, project, bwcVersion, "jdbc", jdbcProjectDir, jdbcProjectArtifact, buildBwcTaskProvider); + } + + private void registerBwcArtifacts(Project bwcProject, DistributionProject distributionProject) { + String projectName = distributionProject.name; + String buildBwcTask = buildBwcTaskName(projectName); + + registerDistributionArchiveArtifact(bwcProject, distributionProject, buildBwcTask); + if (distributionProject.getExpandedDistDirectory() != null) { + String expandedDistConfiguration = "expanded-" + projectName; + bwcProject.getConfigurations().create(expandedDistConfiguration); + bwcProject.getArtifacts().add(expandedDistConfiguration, distributionProject.getExpandedDistDirectory(), artifact -> { + artifact.setName("elasticsearch"); + artifact.builtBy(buildBwcTask); + artifact.setType("directory"); + }); + } + } + + private void registerDistributionArchiveArtifact(Project bwcProject, DistributionProject distributionProject, String buildBwcTask) { + String artifactFileName = distributionProject.getDistFile().getName(); + String artifactName = artifactFileName.contains("oss") ? "elasticsearch-oss" : "elasticsearch"; + + String suffix = artifactFileName.endsWith("tar.gz") ? "tar.gz" : artifactFileName.substring(artifactFileName.length() - 3); + int archIndex = artifactFileName.indexOf("x86_64"); + + bwcProject.getConfigurations().create(distributionProject.name); + bwcProject.getArtifacts().add(distributionProject.name, distributionProject.getDistFile(), artifact -> { + artifact.setName(artifactName); + artifact.builtBy(buildBwcTask); + artifact.setType(suffix); + + String classifier = ""; + if (archIndex != -1) { + int osIndex = artifactFileName.lastIndexOf('-', archIndex - 2); + classifier = "-" + artifactFileName.substring(osIndex + 1, archIndex - 1) + "-x86_64"; + } + artifact.setClassifier(classifier); + }); + } + + private static List resolveArchiveProjects(File checkoutDir, Version bwcVersion) { + List projects = new ArrayList<>(); + projects.addAll(asList("deb", "rpm")); + if (bwcVersion.onOrAfter("7.0.0")) { + projects.addAll(asList("oss-windows-zip", "windows-zip", "oss-darwin-tar", "darwin-tar", "oss-linux-tar", "linux-tar")); + } else { + projects.addAll(asList("oss-zip", "zip", "oss-deb", "oss-rpm")); + } + + return projects.stream().map(name -> { + String baseDir = "distribution" + (name.endsWith("zip") || name.endsWith("tar") ? "/archives" : "/packages"); + String classifier = ""; + String extension = name; + if (bwcVersion.onOrAfter("7.0.0")) { + if (name.contains("zip") || name.contains("tar")) { + int index = name.lastIndexOf('-'); + String baseName = name.startsWith("oss-") ? name.substring(4, index) : name.substring(0, index); + classifier = "-" + baseName + "-x86_64"; + extension = name.substring(index + 1); + if (extension.equals("tar")) { + extension += ".gz"; + } + } else if (name.contains("deb")) { + classifier = "-amd64"; + } else if (name.contains("rpm")) { + classifier = "-x86_64"; + } + } else if (name.contains("oss-")) { + extension = name.substring(4); + } + return new DistributionProject(name, baseDir, bwcVersion, classifier, extension, checkoutDir); + }).collect(Collectors.toList()); + } + + private static String buildBwcTaskName(String projectName) { + return "buildBwc" + + stream(projectName.split("-")).map(i -> i.substring(0, 1).toUpperCase(Locale.ROOT) + i.substring(1)) + .collect(Collectors.joining()); + } + + static void createBuildBwcTask( + BwcSetupExtension bwcSetupExtension, + Project project, + Provider bwcVersion, + String projectName, + String projectPath, + File projectArtifact, + TaskProvider bwcTaskProvider + ) { + String bwcTaskName = buildBwcTaskName(projectName); + bwcSetupExtension.bwcTask(bwcTaskName, c -> { + c.getInputs().file(new File(project.getBuildDir(), "refspec")); + c.getOutputs().files(projectArtifact); + c.getOutputs().cacheIf("BWC distribution caching is disabled on 'master' branch", task -> { + String gitBranch = System.getenv("GIT_BRANCH"); + return BuildParams.isCi() && (gitBranch == null || gitBranch.endsWith("master") == false); + }); + c.args(projectPath.replace('/', ':') + ":assemble"); + if (project.getGradle().getStartParameter().isBuildCacheEnabled()) { + c.args("--build-cache"); + } + c.doLast(task -> { + if (projectArtifact.exists() == false) { + throw new InvalidUserDataException( + "Building " + bwcVersion.get() + " didn't generate expected file " + projectArtifact + ); + } + }); + }); + bwcTaskProvider.configure(t -> t.dependsOn(bwcTaskName)); + } + + /** + * Represents an archive project (distribution/archives/*) + * we build from a bwc Version in a cloned repository + */ + private static class DistributionProject { + private final String name; + private String projectPath; + private File distFile; + private File expandedDistDir; + + DistributionProject(String name, String baseDir, Version version, String classifier, String extension, File checkoutDir) { + this.name = name; + this.projectPath = baseDir + "/" + name; + this.distFile = new File( + checkoutDir, + baseDir + + "/" + + name + + "/build/distributions/elasticsearch-" + + (name.startsWith("oss") ? "oss-" : "") + + version + + "-SNAPSHOT" + + classifier + + "." + + extension + ); + // we only ported this down to the 7.x branch. + if (version.onOrAfter("7.10.0") && (name.endsWith("zip") || name.endsWith("tar"))) { + this.expandedDistDir = new File(checkoutDir, baseDir + "/" + name + "/build/install"); + } + } + + public String getProjectPath() { + return projectPath; + } + + public File getDistFile() { + return distFile; + } + + public File getExpandedDistDirectory() { + return expandedDistDir; + } + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java index 276b7ab233f..00f622ad64a 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java @@ -94,14 +94,25 @@ public class InternalDistributionDownloadPlugin implements Plugin { + "without a bundled JDK is not supported." ); } + String distributionProjectName = distributionProjectName(distribution); + String projectConfig = getProjectConfig(distributionProjectName, unreleasedInfo); return new ProjectBasedDistributionDependency( - (config) -> projectDependency(project, unreleasedInfo.gradleProjectPath, distributionProjectName(distribution)) + (config) -> projectDependency(project, unreleasedInfo.gradleProjectPath, projectConfig) ); } return null; })); } + /** + * Will be removed once this is backported to all unreleased branches. + * */ + private static String getProjectConfig(String distributionProjectName, BwcVersions.UnreleasedVersionInfo info) { + return (info.gradleProjectPath.equals(":distribution") || info.version.before("7.10.0")) + ? distributionProjectName + : "expanded-" + distributionProjectName; + } + private static String distributionProjectPath(ElasticsearchDistribution distribution) { String projectPath = ":distribution"; switch (distribution.getType()) { @@ -171,7 +182,7 @@ public class InternalDistributionDownloadPlugin implements Plugin { return projectName; } - private class ProjectBasedDistributionDependency implements DistributionDependency { + private static class ProjectBasedDistributionDependency implements DistributionDependency { private Function function; diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/util/JavaUtil.java b/buildSrc/src/main/java/org/elasticsearch/gradle/util/JavaUtil.java index 4157bd5bb9b..9d532768c5e 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/util/JavaUtil.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/util/JavaUtil.java @@ -29,7 +29,7 @@ import java.util.Optional; public class JavaUtil { /** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */ - static String getJavaHome(final int version) { + public static String getJavaHome(final int version) { List javaHomes = BuildParams.getJavaVersions(); Optional java = javaHomes.stream().filter(j -> j.getVersion() == version).findFirst(); return java.orElseThrow(() -> new GradleException("JAVA" + version + "_HOME required")).getJavaHome().get().getAbsolutePath(); diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-distribution-bwc-setup.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-distribution-bwc-setup.properties new file mode 100644 index 00000000000..c082a0171d7 --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.internal-distribution-bwc-setup.properties @@ -0,0 +1,20 @@ +# +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +implementation-class=org.elasticsearch.gradle.internal.InternalDistributionBwcSetupPlugin diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle index 8ad9a63527c..826cdc1c610 100644 --- a/distribution/bwc/build.gradle +++ b/distribution/bwc/build.gradle @@ -17,345 +17,23 @@ * under the License. */ -import org.apache.tools.ant.taskdefs.condition.Os -import org.elasticsearch.gradle.LoggedExec +apply plugin:"elasticsearch.internal-distribution-bwc-setup" + import org.elasticsearch.gradle.Version -import org.elasticsearch.gradle.BwcVersions import org.elasticsearch.gradle.info.BuildParams -import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin -import org.gradle.util.GradleVersion -import java.nio.charset.StandardCharsets -import static org.elasticsearch.gradle.util.JavaUtil.getJavaHome - -/** - * We want to be able to do BWC tests for unreleased versions without relying on and waiting for snapshots. - * For this we need to check out and build the unreleased versions. - * Since These depend on the current version, we can't name the Gradle projects statically, and don't know what the - * unreleased versions are when Gradle projects are set up, so we use "build-unreleased-version-*" as placeholders - * and configure them to build various versions here. - */ -BuildParams.bwcVersions.forPreviousUnreleased { BwcVersions.UnreleasedVersionInfo unreleasedVersion -> - project("${unreleasedVersion.gradleProjectPath}") { - Version bwcVersion = unreleasedVersion.version - String bwcBranch = unreleasedVersion.branch - apply plugin: 'distribution' - // Not published so no need to assemble - tasks.named("assemble").configure { - enabled = false - } - File checkoutDir = file("${buildDir}/bwc/checkout-${bwcBranch}") - - final String remote = System.getProperty("bwc.remote", "elastic") - - boolean gitFetchLatest - final String gitFetchLatestProperty = System.getProperty("tests.bwc.git_fetch_latest", "true") - if ("true".equals(gitFetchLatestProperty)) { - gitFetchLatest = true - } else if ("false".equals(gitFetchLatestProperty)) { - gitFetchLatest = false - } else { - throw new GradleException("tests.bwc.git_fetch_latest must be [true] or [false] but was [" + gitFetchLatestProperty + "]") - } - - tasks.register("createClone", LoggedExec) { - onlyIf { checkoutDir.exists() == false } - commandLine = ['git', 'clone', rootDir, checkoutDir] - } - - tasks.register("findRemote", LoggedExec) { - dependsOn "createClone" - workingDir = checkoutDir - commandLine = ['git', 'remote', '-v'] - ByteArrayOutputStream output = new ByteArrayOutputStream() - standardOutput = output - doLast { - project.ext.remoteExists = false - output.toString('UTF-8').eachLine { - if (it.contains("${remote}\t")) { - project.ext.remoteExists = true - } +BuildParams.getBwcVersions().forPreviousUnreleased { unreleasedVersion -> + project(unreleasedVersion.gradleProjectPath) { + Version currentVersion = Version.fromString(version) + TaskProvider resolveAllBwcDepsTaskProvider = bwcSetup.bwcTask("resolveAllBwcDependencies") { + t -> t.args("resolveAllDependencies") } - } - } - - tasks.register("addRemote", LoggedExec) { - dependsOn findRemote - onlyIf { project.ext.remoteExists == false } - workingDir = checkoutDir - commandLine = ['git', 'remote', 'add', "${remote}", "https://github.com/${remote}/elasticsearch.git"] - } - - tasks.register("fetchLatest", LoggedExec) { - onlyIf { project.gradle.startParameter.isOffline() == false && gitFetchLatest } - dependsOn("addRemote") - workingDir = checkoutDir - commandLine = ['git', 'fetch', '--all'] - } - - Closure execGit = { Action action -> - new ByteArrayOutputStream().withStream { os -> - ExecResult result = project.exec { spec -> - workingDir = checkoutDir - standardOutput os - action.execute(spec) + if (currentVersion.getMinor() == 0 && currentVersion.getRevision() == 0) { + // We only want to resolve dependencies for live versions of master, without cascading this to older versions + tasks.named("resolveAllDependencies").configure { + dependsOn(resolveAllBwcDepsTaskProvider) + } } - result.assertNormalExitValue() - return os.toString().trim() - } } - tasks.register("checkoutBwcBranch") { - dependsOn("fetchLatest") - doLast { - def refspec = System.getProperty("bwc.refspec." + bwcBranch) ?: System.getProperty("tests.bwc.refspec." + bwcBranch) ?: "${remote}/${bwcBranch}" - if (System.getProperty("bwc.checkout.align") != null) { - /* - We use a time based approach to make the bwc versions built deterministic and compatible with the current hash. - Most of the time we want to test against latest, but when running delayed exhaustive tests or wanting - reproducible builds we want this to be deterministic by using a hash that was the latest when the current - commit was made. - - This approach doesn't work with merge commits as these can introduce commits in the chronological order - after the fact e.x. a merge done today can add commits dated with yesterday so the result will no longer be - deterministic. - - We don't use merge commits, but for additional safety we check that no such commits exist in the time period - we are interested in. - - Timestamps are at seconds resolution. rev-parse --before and --after are inclusive w.r.t the second - passed as input. This means the results might not be deterministic in the current second, but this - should not matter in practice. - */ - String timeOfCurrent = execGit { spec -> - spec.commandLine 'git', 'show', '--no-patch', '--no-notes', "--pretty='%cD'" - spec.workingDir project.rootDir - } - logger.lifecycle("Commit date of current: {}", timeOfCurrent) - String mergeCommits = execGit { spec -> - spec.commandLine "git", "rev-list", refspec, "--after", timeOfCurrent, "--merges" - } - if (mergeCommits.isEmpty() == false) { - throw new IllegalStateException( - "Found the following merge commits which prevent determining bwc commits: " + mergeCommits - ) - } - refspec = execGit { spec -> - spec.commandLine "git", "rev-list", refspec, "-n", "1", "--before", timeOfCurrent, "--date-order" - } - } - - logger.lifecycle("Performing checkout of ${refspec}...") - LoggedExec.exec(project) { spec -> - spec.workingDir = checkoutDir - spec.commandLine "git", "checkout", refspec - } - String checkoutHash = GlobalBuildInfoPlugin.gitInfo(checkoutDir).revision - logger.lifecycle("Checkout hash for ${project.path} is ${checkoutHash}") - file("${project.buildDir}/refspec").text = checkoutHash - } - } - - - Closure createRunBwcGradleTask = { name, extraConfig -> - return tasks.register("$name", LoggedExec) { - dependsOn "checkoutBwcBranch" - spoolOutput = true - workingDir = checkoutDir - doFirst { - // Execution time so that the checkouts are available - List lines = file("${checkoutDir}/.ci/java-versions.properties").readLines() - environment( - 'JAVA_HOME', - getJavaHome(Integer.parseInt( - lines - .findAll({ it.startsWith("ES_BUILD_JAVA=") }) - .collect({ it.replace("ES_BUILD_JAVA=java", "").trim() }) - .collect({ it.replace("ES_BUILD_JAVA=openjdk", "").trim() }) - .join("!!") - )) - ) - environment( - 'RUNTIME_JAVA_HOME', - getJavaHome(Integer.parseInt( - lines - .findAll({ it.startsWith("ES_RUNTIME_JAVA=java") }) - .collect({ it.replace("ES_RUNTIME_JAVA=java", "").trim() }) - .join("!!") - )) - ) - } - - if (Os.isFamily(Os.FAMILY_WINDOWS)) { - executable 'cmd' - args '/C', 'call', new File(checkoutDir, 'gradlew').toString() - } else { - executable new File(checkoutDir, 'gradlew').toString() - } - if (gradle.startParameter.isOffline()) { - args "--offline" - } - String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url') - if (buildCacheUrl) { - args "-Dorg.elasticsearch.build.cache.url=${buildCacheUrl}" - } - - args "-Dbuild.snapshot=true" - args "-Dscan.tag.NESTED" - final LogLevel logLevel = gradle.startParameter.logLevel - if ([LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG].contains(logLevel)) { - args "--${logLevel.name().toLowerCase(Locale.ENGLISH)}" - } - final String showStacktraceName = gradle.startParameter.showStacktrace.name() - assert ["INTERNAL_EXCEPTIONS", "ALWAYS", "ALWAYS_FULL"].contains(showStacktraceName) - if (showStacktraceName.equals("ALWAYS")) { - args "--stacktrace" - } else if (showStacktraceName.equals("ALWAYS_FULL")) { - args "--full-stacktrace" - } - if (gradle.getStartParameter().isParallelProjectExecutionEnabled()) { - args "--parallel" - } - standardOutput = new IndentingOutputStream(System.out, bwcVersion) - errorOutput = new IndentingOutputStream(System.err, bwcVersion) - configure extraConfig - } - } - - Closure buildBwcTaskName = { projectName -> - return "buildBwc${projectName.replaceAll(/-\w/) { it[1].toUpperCase() }.capitalize()}" - } - - def buildBwc = tasks.register("buildBwc"); - - Closure createBuildBwcTask = { projectName, projectDir, projectArtifact -> - def bwcTaskName = buildBwcTaskName(projectName) - createRunBwcGradleTask(bwcTaskName) { - inputs.file("${project.buildDir}/refspec") - outputs.files(projectArtifact) - outputs.cacheIf("BWC distribution caching is disabled on 'master' branch") { - // Don't bother caching in 'master' since the BWC branches move too quickly to make this cost worthwhile - BuildParams.ci && System.getenv('GIT_BRANCH')?.endsWith("master") == false - } - args ":${projectDir.replace('/', ':')}:assemble" - if (project.gradle.startParameter.buildCacheEnabled) { - args "--build-cache" - } - doLast { - if (projectArtifact.exists() == false) { - throw new InvalidUserDataException("Building ${bwcVersion} didn't generate expected file ${projectArtifact}") - } - } - } - buildBwc.configure { - dependsOn(bwcTaskName) - } - } - - Map artifactFiles = [:] - List projectDirs = [] - List projects = ['deb', 'rpm'] - if (bwcVersion.onOrAfter('7.0.0')) { - projects.addAll(['windows-zip', 'darwin-tar', 'linux-tar']) - } else { - projects.add('zip') - } - - for (String projectName : projects) { - String baseDir = "distribution" - String classifier = "" - String extension = projectName - if (bwcVersion.onOrAfter('7.0.0') && (projectName.contains('zip') || projectName.contains('tar'))) { - int index = projectName.indexOf('-') - classifier = "-${projectName.substring(0, index)}-x86_64" - extension = projectName.substring(index + 1) - if (extension.equals('tar')) { - extension += '.gz' - } - } - if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('deb')) { - classifier = "-amd64" - } - if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('rpm')) { - classifier = "-x86_64" - } - if (bwcVersion.onOrAfter('6.3.0')) { - baseDir += projectName.endsWith('zip') || projectName.endsWith('tar') ? '/archives' : '/packages' - // add oss variant first - projectDirs.add("${baseDir}/oss-${projectName}") - File ossProjectArtifact = file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT${classifier}.${extension}") - artifactFiles.put("oss-" + projectName, ossProjectArtifact) - createBuildBwcTask("oss-${projectName}", "${baseDir}/oss-${projectName}", ossProjectArtifact) - } - projectDirs.add("${baseDir}/${projectName}") - File projectArtifact = file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT${classifier}.${extension}") - artifactFiles.put(projectName, projectArtifact) - - createBuildBwcTask(projectName, "${baseDir}/${projectName}", projectArtifact) - } - - // Create build tasks for the JDBC driver used for compatibility testing - String jdbcProjectDir = 'x-pack/plugin/sql/jdbc' - File jdbcProjectArtifact = file("${checkoutDir}/${jdbcProjectDir}/build/distributions/x-pack-sql-jdbc-${bwcVersion}-SNAPSHOT.jar") - createBuildBwcTask('jdbc', jdbcProjectDir, jdbcProjectArtifact) - - createRunBwcGradleTask("resolveAllBwcDependencies") { - args 'resolveAllDependencies' - } - Version currentVersion = Version.fromString(version) - if (currentVersion.getMinor() == 0 && currentVersion.getRevision() == 0) { - // We only want to resolve dependencies for live versions of master, without cascading this to older versions - tasks.named("resolveAllDependencies").configure { - dependsOn("resolveAllBwcDependencies") - } - } - - for (e in artifactFiles) { - String projectName = e.key - String buildBwcTask = buildBwcTaskName(projectName) - File artifactFile = e.value - String artifactFileName = artifactFile.name - String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch' - String suffix = artifactFileName.endsWith("tar.gz") ? "tar.gz" : artifactFileName[-3..-1] - int archIndex = artifactFileName.indexOf('x86_64') - String classifier = '' - if (archIndex != -1) { - int osIndex = artifactFileName.lastIndexOf('-', archIndex - 2) - classifier = "${artifactFileName.substring(osIndex + 1, archIndex - 1)}-x86_64" - } - configurations.create(projectName) - artifacts { - it.add(projectName, [file: artifactFile, name: artifactName, classifier: classifier, type: suffix, builtBy: buildBwcTask]) - } - } - // make sure no dependencies were added to assemble; we want it to be a no-op - tasks.named("assemble").configure { - dependsOn = [] - } - } -} - -class IndentingOutputStream extends OutputStream { - - public final byte[] indent - private final OutputStream delegate - - public IndentingOutputStream(OutputStream delegate, Object version) { - this.delegate = delegate - indent = " [${version}] ".getBytes(StandardCharsets.UTF_8) - } - - @Override - public void write(int b) { - write([b] as int[], 0, 1) - } - - public void write(int[] bytes, int offset, int length) { - for (int i = 0; i < bytes.length; i++) { - delegate.write(bytes[i]) - if (bytes[i] == '\n') { - delegate.write(indent) - } - } - } -} +} \ No newline at end of file