Build local unreleased bwc versions more efficient for tests (7.x backport) (#63188)
* Wire local unreleased bwc versions more efficient for tests (#62473) For testing against the local distribution we already avoid the packaging/unpackaging cycle of es distributions when setting up test clusters. This PR adopts the usage of the expanded created distributions for unreleased bwc versions (versions that are checkout from a branch and build from source in the :distribution:bwc:minor / :distribution:bwc:bugfix). This makes the setup of bwc based cross version tests a bit faster by avoiding the unpackaging overhead. We still assemble both in the bwcBuild tasks atm which will be addressed in a later issue. This reworks the :distribution:bwc project: - Convert all the custom logic from build script logic (groovy) into gradle binary plugins (java) - Tried to make the bwc setup logic a bit more readable - Add basic functional test coverage for the bwc logic this PR tweaked. - Extracted a general internal BWC Git plugin out of the bwc setup plugin to improve maintenance and testability - Changed the InternalDistributionPlugin to resolve the extracted distro instead on relying on unpacking the distribution archive * Fix java8 incompatibility * Fix extension calculation for 6.8.* distribution
This commit is contained in:
Rene Groeschke
GitHub
parent
abf9b885b4
commit
8144106ace
|
|
@ -43,9 +43,13 @@ abstract class AbstractGradleFuncTest extends Specification {
|
|||
}
|
||||
|
||||
GradleRunner gradleRunner(String... arguments) {
|
||||
return gradleRunner(testProjectDir.root, arguments)
|
||||
}
|
||||
|
||||
GradleRunner gradleRunner(File projectDir, String... arguments) {
|
||||
GradleRunner.create()
|
||||
.withDebug(ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0)
|
||||
.withProjectDir(testProjectDir.root)
|
||||
.withProjectDir(projectDir)
|
||||
.withArguments(arguments)
|
||||
.withPluginClasspath()
|
||||
.forwardOutput()
|
||||
|
|
@ -85,4 +89,31 @@ abstract class AbstractGradleFuncTest extends Specification {
|
|||
|
||||
return jarFile;
|
||||
}
|
||||
|
||||
File internalBuild(File buildScript = buildFile) {
|
||||
buildScript << """plugins {
|
||||
id 'elasticsearch.global-build-info'
|
||||
}
|
||||
import org.elasticsearch.gradle.Architecture
|
||||
import org.elasticsearch.gradle.info.BuildParams
|
||||
|
||||
BuildParams.init { it.setIsInternal(true) }
|
||||
|
||||
import org.elasticsearch.gradle.BwcVersions
|
||||
import org.elasticsearch.gradle.Version
|
||||
|
||||
Version currentVersion = Version.fromString("9.0.0")
|
||||
BwcVersions versions = new BwcVersions(new TreeSet<>(
|
||||
Arrays.asList(Version.fromString("8.0.0"), Version.fromString("8.0.1"), Version.fromString("8.1.0"), currentVersion)),
|
||||
currentVersion)
|
||||
|
||||
BuildParams.init { it.setBwcVersions(versions) }
|
||||
"""
|
||||
}
|
||||
|
||||
void setupLocalGitRepo() {
|
||||
"git init".execute(Collections.emptyList(), testProjectDir.root).waitFor()
|
||||
"git add .".execute(Collections.emptyList(), testProjectDir.root).waitFor()
|
||||
'git commit -m "Initial"'.execute(Collections.emptyList(), testProjectDir.root).waitFor()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,51 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.gradle.internal
|
||||
|
||||
import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
|
||||
import org.gradle.testkit.runner.TaskOutcome
|
||||
|
||||
class InternalBwcGitPluginFuncTest extends AbstractGradleFuncTest {
|
||||
|
||||
def setup() {
|
||||
setupLocalGitRepo()
|
||||
}
|
||||
|
||||
def "current repository can be cloned"() {
|
||||
given:
|
||||
internalBuild();
|
||||
buildFile << """
|
||||
import org.elasticsearch.gradle.Version;
|
||||
apply plugin: org.elasticsearch.gradle.internal.InternalBwcGitPlugin
|
||||
|
||||
bwcGitConfig {
|
||||
bwcVersion = project.provider { Version.fromString("7.10.0") }
|
||||
bwcBranch = project.provider { "7.x" }
|
||||
checkoutDir = project.provider{file("build/checkout")}
|
||||
}
|
||||
"""
|
||||
when:
|
||||
def result = gradleRunner("createClone", '--stacktrace').build()
|
||||
then:
|
||||
result.task(":createClone").outcome == TaskOutcome.SUCCESS
|
||||
file("build/checkout/build.gradle").exists()
|
||||
file("build/checkout/settings.gradle").exists()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,151 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.gradle.internal
|
||||
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
|
||||
import org.gradle.testkit.runner.TaskOutcome
|
||||
import org.junit.Rule
|
||||
import org.junit.rules.TemporaryFolder
|
||||
|
||||
class InternalDistributionBwcSetupPluginFuncTest extends AbstractGradleFuncTest {
|
||||
|
||||
@Rule
|
||||
TemporaryFolder remoteRepoDirs = new TemporaryFolder()
|
||||
|
||||
File remoteGitRepo
|
||||
|
||||
def setup() {
|
||||
remoteGitRepo = new File(setupGitRemote(), '.git')
|
||||
|
||||
"git clone ${remoteGitRepo.absolutePath}".execute(Collections.emptyList(), testProjectDir.root).waitFor()
|
||||
File buildScript = new File(testProjectDir.root, 'remote/build.gradle')
|
||||
internalBuild(buildScript)
|
||||
buildScript << """
|
||||
apply plugin: 'elasticsearch.internal-distribution-bwc-setup'
|
||||
"""
|
||||
}
|
||||
|
||||
def "builds distribution from branches via archives assemble"() {
|
||||
when:
|
||||
def result = gradleRunner(new File(testProjectDir.root, "remote"),
|
||||
":distribution:bwc:bugfix:buildBwcDarwinTar",
|
||||
":distribution:bwc:bugfix:buildBwcOssDarwinTar",
|
||||
"-DtestRemoteRepo=" + remoteGitRepo,
|
||||
"-Dbwc.remote=origin")
|
||||
.build()
|
||||
then:
|
||||
result.task(":distribution:bwc:bugfix:buildBwcDarwinTar").outcome == TaskOutcome.SUCCESS
|
||||
result.task(":distribution:bwc:bugfix:buildBwcOssDarwinTar").outcome == TaskOutcome.SUCCESS
|
||||
|
||||
and: "assemble task triggered"
|
||||
result.output.contains("[8.0.1] > Task :distribution:archives:darwin-tar:assemble")
|
||||
result.output.contains("[8.0.1] > Task :distribution:archives:oss-darwin-tar:assemble")
|
||||
}
|
||||
|
||||
def "bwc distribution archives can be resolved as bwc project artifact"() {
|
||||
setup:
|
||||
new File(testProjectDir.root, 'remote/build.gradle') << """
|
||||
|
||||
configurations {
|
||||
dists
|
||||
}
|
||||
|
||||
dependencies {
|
||||
dists project(path: ":distribution:bwc:bugfix", configuration:"darwin-tar")
|
||||
}
|
||||
|
||||
tasks.register("resolveDistributionArchive") {
|
||||
inputs.files(configurations.dists)
|
||||
doLast {
|
||||
configurations.dists.files.each {
|
||||
println "distfile " + (it.absolutePath - project.rootDir.absolutePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
when:
|
||||
def result = gradleRunner(new File(testProjectDir.root, "remote"),
|
||||
":resolveDistributionArchive",
|
||||
"-DtestRemoteRepo=" + remoteGitRepo,
|
||||
"-Dbwc.remote=origin")
|
||||
.build()
|
||||
then:
|
||||
result.task(":resolveDistributionArchive").outcome == TaskOutcome.SUCCESS
|
||||
result.task(":distribution:bwc:bugfix:buildBwcDarwinTar").outcome == TaskOutcome.SUCCESS
|
||||
|
||||
and: "assemble task triggered"
|
||||
result.output.contains("[8.0.1] > Task :distribution:archives:darwin-tar:assemble")
|
||||
normalizedOutput(result.output)
|
||||
.contains("distfile /distribution/bwc/bugfix/build/bwc/checkout-8.0/distribution/archives/darwin-tar/" +
|
||||
"build/distributions/elasticsearch-8.0.1-SNAPSHOT-darwin-x86_64.tar.gz")
|
||||
}
|
||||
|
||||
def "bwc expanded distribution folder can be resolved as bwc project artifact"() {
|
||||
setup:
|
||||
new File(testProjectDir.root, 'remote/build.gradle') << """
|
||||
|
||||
configurations {
|
||||
expandedDist
|
||||
}
|
||||
|
||||
dependencies {
|
||||
expandedDist project(path: ":distribution:bwc:bugfix", configuration:"expanded-darwin-tar")
|
||||
}
|
||||
|
||||
tasks.register("resolveExpandedDistribution") {
|
||||
inputs.files(configurations.expandedDist)
|
||||
doLast {
|
||||
configurations.expandedDist.files.each {
|
||||
println "distfile " + (it.absolutePath - project.rootDir.absolutePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
when:
|
||||
def result = gradleRunner(new File(testProjectDir.root, "remote"),
|
||||
":resolveExpandedDistribution",
|
||||
"-DtestRemoteRepo=" + remoteGitRepo,
|
||||
"-Dbwc.remote=origin")
|
||||
.build()
|
||||
then:
|
||||
result.task(":resolveExpandedDistribution").outcome == TaskOutcome.SUCCESS
|
||||
result.task(":distribution:bwc:bugfix:buildBwcDarwinTar").outcome == TaskOutcome.SUCCESS
|
||||
|
||||
and: "assemble task triggered"
|
||||
result.output.contains("[8.0.1] > Task :distribution:archives:darwin-tar:assemble")
|
||||
normalizedOutput(result.output)
|
||||
.contains("distfile /distribution/bwc/bugfix/build/bwc/checkout-8.0/" +
|
||||
"distribution/archives/darwin-tar/build/install")
|
||||
}
|
||||
|
||||
File setupGitRemote() {
|
||||
URL fakeRemote = getClass().getResource("fake_git/remote")
|
||||
File workingRemoteGit = new File(remoteRepoDirs.root, 'remote')
|
||||
FileUtils.copyDirectory(new File(fakeRemote.file), workingRemoteGit)
|
||||
fakeRemote.file + "/.git"
|
||||
gradleRunner(workingRemoteGit, "wrapper").build()
|
||||
"git init".execute(Collections.emptyList(), workingRemoteGit).waitFor()
|
||||
"git add .".execute(Collections.emptyList(), workingRemoteGit).waitFor()
|
||||
'git commit -m"Initial"'.execute(Collections.emptyList(), workingRemoteGit).waitFor()
|
||||
"git checkout -b origin/8.0".execute(Collections.emptyList(), workingRemoteGit).waitFor()
|
||||
return workingRemoteGit;
|
||||
}
|
||||
}
|
||||
|
|
@ -72,16 +72,15 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
|
|||
def result = gradleRunner("setupDistro", '-g', testProjectDir.newFolder('GUH').path).build()
|
||||
|
||||
then:
|
||||
result.task(":distribution:archives:linux-tar:buildExploded").outcome == TaskOutcome.SUCCESS
|
||||
result.task(":distribution:archives:linux-tar:buildExpanded").outcome == TaskOutcome.SUCCESS
|
||||
result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
|
||||
assertExtractedDistroIsCreated(distroVersion, "build/distro", 'current-marker.txt')
|
||||
assertExtractedDistroIsCreated("build/distro", 'current-marker.txt')
|
||||
}
|
||||
|
||||
def "resolves bwc versions from source"() {
|
||||
def "resolves expanded bwc versions from source"() {
|
||||
given:
|
||||
internalBuild()
|
||||
bwcMinorProjectSetup()
|
||||
def distroVersion = "8.1.0"
|
||||
buildFile << """
|
||||
apply plugin: 'elasticsearch.internal-distribution-download'
|
||||
|
||||
|
|
@ -102,9 +101,10 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
|
|||
|
||||
def result = gradleRunner("setupDistro").build()
|
||||
then:
|
||||
result.task(":distribution:bwc:minor:buildBwcTask").outcome == TaskOutcome.SUCCESS
|
||||
result.task(":distribution:bwc:minor:buildBwcExpandedTask").outcome == TaskOutcome.SUCCESS
|
||||
result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
|
||||
assertExtractedDistroIsCreated(distroVersion, "build/distro", 'bwc-marker.txt')
|
||||
assertExtractedDistroIsCreated("distribution/bwc/minor/build/install/elastic-distro",
|
||||
'bwc-marker.txt')
|
||||
}
|
||||
|
||||
def "fails on resolving bwc versions with no bundled jdk"() {
|
||||
|
|
@ -134,28 +134,6 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
|
|||
"without a bundled JDK is not supported.")
|
||||
}
|
||||
|
||||
private File internalBuild() {
|
||||
buildFile << """plugins {
|
||||
id 'elasticsearch.global-build-info'
|
||||
}
|
||||
import org.elasticsearch.gradle.Architecture
|
||||
import org.elasticsearch.gradle.info.BuildParams
|
||||
|
||||
BuildParams.init { it.setIsInternal(true) }
|
||||
|
||||
import org.elasticsearch.gradle.BwcVersions
|
||||
import org.elasticsearch.gradle.Version
|
||||
|
||||
Version currentVersion = Version.fromString("9.0.0")
|
||||
BwcVersions versions = new BwcVersions(new TreeSet<>(
|
||||
Arrays.asList(Version.fromString("8.0.0"), Version.fromString("8.0.1"), Version.fromString("8.1.0"), currentVersion)),
|
||||
currentVersion)
|
||||
|
||||
BuildParams.init { it.setBwcVersions(versions) }
|
||||
"""
|
||||
}
|
||||
|
||||
|
||||
private void bwcMinorProjectSetup() {
|
||||
settingsFile << """
|
||||
include ':distribution:bwc:minor'
|
||||
|
|
@ -164,6 +142,8 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
|
|||
new File(bwcSubProjectFolder, 'bwc-marker.txt') << "bwc=minor"
|
||||
new File(bwcSubProjectFolder, 'build.gradle') << """
|
||||
apply plugin:'base'
|
||||
|
||||
// packed distro
|
||||
configurations.create("linux-tar")
|
||||
tasks.register("buildBwcTask", Tar) {
|
||||
from('bwc-marker.txt')
|
||||
|
|
@ -173,6 +153,19 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
|
|||
artifacts {
|
||||
it.add("linux-tar", buildBwcTask)
|
||||
}
|
||||
|
||||
// expanded distro
|
||||
configurations.create("expanded-linux-tar")
|
||||
def expandedTask = tasks.register("buildBwcExpandedTask", Copy) {
|
||||
from('bwc-marker.txt')
|
||||
into('build/install/elastic-distro')
|
||||
}
|
||||
artifacts {
|
||||
it.add("expanded-linux-tar", file('build/install')) {
|
||||
builtBy expandedTask
|
||||
type = 'directory'
|
||||
}
|
||||
}
|
||||
"""
|
||||
}
|
||||
|
||||
|
|
@ -192,7 +185,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
|
|||
archiveExtension = "tar.gz"
|
||||
compression = Compression.GZIP
|
||||
}
|
||||
def buildExploded = tasks.register("buildExploded", Copy) {
|
||||
def buildExpanded = tasks.register("buildExpanded", Copy) {
|
||||
from('current-marker.txt')
|
||||
into("build/local")
|
||||
}
|
||||
|
|
@ -205,15 +198,14 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
|
|||
}
|
||||
artifacts {
|
||||
it.add("default", buildTar)
|
||||
it.add("extracted", buildExploded)
|
||||
it.add("extracted", buildExpanded)
|
||||
}
|
||||
"""
|
||||
buildFile << """
|
||||
"""
|
||||
|
||||
}
|
||||
|
||||
boolean assertExtractedDistroIsCreated(String version, String relativeDistroPath, String markerFileName) {
|
||||
boolean assertExtractedDistroIsCreated(String relativeDistroPath, String markerFileName) {
|
||||
File extractedFolder = new File(testProjectDir.root, relativeDistroPath)
|
||||
assert extractedFolder.exists()
|
||||
assert new File(extractedFolder, markerFileName).exists()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,21 @@
|
|||
#
|
||||
# Licensed to Elasticsearch under one or more contributor
|
||||
# license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright
|
||||
# ownership. Elasticsearch licenses this file to you under
|
||||
# the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
ES_BUILD_JAVA=openjdk14
|
||||
ES_RUNTIME_JAVA=openjdk14
|
||||
GRADLE_TASK=build
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
subprojects {
|
||||
apply plugin:'base'
|
||||
|
||||
tasks.register('tar', Tar) {
|
||||
from('.')
|
||||
destinationDirectory.set(file('build/distributions'))
|
||||
archiveBaseName.set("elasticsearch${project.name.startsWith('oss')?'-oss':''}")
|
||||
archiveVersion.set("8.0.1-SNAPSHOT")
|
||||
archiveClassifier.set("darwin-x86_64")
|
||||
archiveExtension.set('tar.gz')
|
||||
}
|
||||
|
||||
assemble.dependsOn('tar')
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
include ":distribution:bwc:bugfix"
|
||||
include ":distribution:bwc:minor"
|
||||
include ":distribution:archives:darwin-tar"
|
||||
include ":distribution:archives:oss-darwin-tar"
|
||||
|
|
@ -444,7 +444,7 @@ public class GlobalBuildInfoPlugin implements Plugin<Project> {
|
|||
return firstLine;
|
||||
}
|
||||
|
||||
private static class GitInfo {
|
||||
public static class GitInfo {
|
||||
private final String revision;
|
||||
private final String origin;
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.gradle.internal;
|
||||
|
||||
import org.elasticsearch.gradle.Version;
|
||||
import org.gradle.api.provider.Provider;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
public class BwcGitExtension {
|
||||
|
||||
private Provider<Version> bwcVersion;
|
||||
private Provider<String> bwcBranch;
|
||||
private Provider<File> checkoutDir;
|
||||
|
||||
public Provider<Version> getBwcVersion() {
|
||||
return bwcVersion;
|
||||
}
|
||||
|
||||
public void setBwcVersion(Provider<Version> bwcVersion) {
|
||||
this.bwcVersion = bwcVersion;
|
||||
}
|
||||
|
||||
public Provider<String> getBwcBranch() {
|
||||
return bwcBranch;
|
||||
}
|
||||
|
||||
public void setBwcBranch(Provider<String> bwcBranch) {
|
||||
this.bwcBranch = bwcBranch;
|
||||
}
|
||||
|
||||
public Provider<File> getCheckoutDir() {
|
||||
return checkoutDir;
|
||||
}
|
||||
|
||||
public void setCheckoutDir(Provider<File> checkoutDir) {
|
||||
this.checkoutDir = checkoutDir;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,177 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.gradle.internal;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.tools.ant.taskdefs.condition.Os;
|
||||
import org.elasticsearch.gradle.BwcVersions;
|
||||
import org.elasticsearch.gradle.LoggedExec;
|
||||
import org.gradle.api.Action;
|
||||
import org.gradle.api.GradleException;
|
||||
import org.gradle.api.Project;
|
||||
import org.gradle.api.logging.LogLevel;
|
||||
import org.gradle.api.provider.Provider;
|
||||
import org.gradle.api.tasks.TaskProvider;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.elasticsearch.gradle.util.JavaUtil.getJavaHome;
|
||||
|
||||
/**
|
||||
* By registering bwc tasks via this extension we can support declaring custom bwc tasks from the build script
|
||||
* without relying on groovy closures and sharing common logic for tasks created by the BwcSetup plugin already.
|
||||
* */
|
||||
public class BwcSetupExtension {
|
||||
|
||||
private final Project project;
|
||||
|
||||
private final Provider<BwcVersions.UnreleasedVersionInfo> unreleasedVersionInfo;
|
||||
private Provider<File> checkoutDir;
|
||||
|
||||
public BwcSetupExtension(
|
||||
Project project,
|
||||
Provider<BwcVersions.UnreleasedVersionInfo> unreleasedVersionInfo,
|
||||
Provider<File> checkoutDir
|
||||
) {
|
||||
this.project = project;
|
||||
this.unreleasedVersionInfo = unreleasedVersionInfo;
|
||||
this.checkoutDir = checkoutDir;
|
||||
}
|
||||
|
||||
TaskProvider<LoggedExec> bwcTask(String name, Action<LoggedExec> configuration) {
|
||||
return createRunBwcGradleTask(project, name, configuration);
|
||||
}
|
||||
|
||||
private TaskProvider<LoggedExec> createRunBwcGradleTask(Project project, String name, Action<LoggedExec> configAction) {
|
||||
return project.getTasks().register(name, LoggedExec.class, loggedExec -> {
|
||||
// TODO revisit
|
||||
loggedExec.dependsOn("checkoutBwcBranch");
|
||||
loggedExec.setSpoolOutput(true);
|
||||
loggedExec.setWorkingDir(checkoutDir.get());
|
||||
loggedExec.doFirst(t -> {
|
||||
// Execution time so that the checkouts are available
|
||||
String javaVersionsString = readFromFile(new File(checkoutDir.get(), ".ci/java-versions.properties"));
|
||||
loggedExec.environment(
|
||||
"JAVA_HOME",
|
||||
getJavaHome(
|
||||
Integer.parseInt(
|
||||
Arrays.asList(javaVersionsString.split("\n"))
|
||||
.stream()
|
||||
.filter(l -> l.trim().startsWith("ES_BUILD_JAVA="))
|
||||
.map(l -> l.replace("ES_BUILD_JAVA=java", "").trim())
|
||||
.map(l -> l.replace("ES_BUILD_JAVA=openjdk", "").trim())
|
||||
.collect(Collectors.joining("!!"))
|
||||
)
|
||||
)
|
||||
);
|
||||
loggedExec.environment(
|
||||
"RUNTIME_JAVA_HOME",
|
||||
getJavaHome(
|
||||
Integer.parseInt(
|
||||
Arrays.asList(javaVersionsString.split("\n"))
|
||||
.stream()
|
||||
.filter(l -> l.trim().startsWith("ES_RUNTIME_JAVA="))
|
||||
.map(l -> l.replace("ES_RUNTIME_JAVA=java", "").trim())
|
||||
.map(l -> l.replace("ES_RUNTIME_JAVA=openjdk", "").trim())
|
||||
.collect(Collectors.joining("!!"))
|
||||
)
|
||||
)
|
||||
);
|
||||
});
|
||||
|
||||
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
|
||||
loggedExec.executable("cmd");
|
||||
loggedExec.args("/C", "call", new File(checkoutDir.get(), "gradlew").toString());
|
||||
} else {
|
||||
loggedExec.executable(new File(checkoutDir.get(), "gradlew").toString());
|
||||
}
|
||||
if (project.getGradle().getStartParameter().isOffline()) {
|
||||
loggedExec.args("--offline");
|
||||
}
|
||||
// TODO resolve
|
||||
String buildCacheUrl = System.getProperty("org.elasticsearch.build.cache.url");
|
||||
if (buildCacheUrl != null) {
|
||||
loggedExec.args("-Dorg.elasticsearch.build.cache.url=" + buildCacheUrl);
|
||||
}
|
||||
|
||||
loggedExec.args("-Dbuild.snapshot=true");
|
||||
loggedExec.args("-Dscan.tag.NESTED");
|
||||
final LogLevel logLevel = project.getGradle().getStartParameter().getLogLevel();
|
||||
List<LogLevel> nonDefaultLogLevels = Arrays.asList(LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG);
|
||||
if (nonDefaultLogLevels.contains(logLevel)) {
|
||||
loggedExec.args("--" + logLevel.name().toLowerCase(Locale.ENGLISH));
|
||||
}
|
||||
final String showStacktraceName = project.getGradle().getStartParameter().getShowStacktrace().name();
|
||||
assert Arrays.asList("INTERNAL_EXCEPTIONS", "ALWAYS", "ALWAYS_FULL").contains(showStacktraceName);
|
||||
if (showStacktraceName.equals("ALWAYS")) {
|
||||
loggedExec.args("--stacktrace");
|
||||
} else if (showStacktraceName.equals("ALWAYS_FULL")) {
|
||||
loggedExec.args("--full-stacktrace");
|
||||
}
|
||||
if (project.getGradle().getStartParameter().isParallelProjectExecutionEnabled()) {
|
||||
loggedExec.args("--parallel");
|
||||
}
|
||||
loggedExec.setStandardOutput(new IndentingOutputStream(System.out, unreleasedVersionInfo.get().version));
|
||||
loggedExec.setErrorOutput(new IndentingOutputStream(System.err, unreleasedVersionInfo.get().version));
|
||||
configAction.execute(loggedExec);
|
||||
});
|
||||
}
|
||||
|
||||
private static class IndentingOutputStream extends OutputStream {
|
||||
|
||||
public final byte[] indent;
|
||||
private final OutputStream delegate;
|
||||
|
||||
IndentingOutputStream(OutputStream delegate, Object version) {
|
||||
this.delegate = delegate;
|
||||
indent = (" [" + version + "] ").getBytes(StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(int b) throws IOException {
|
||||
int[] arr = { b };
|
||||
write(arr, 0, 1);
|
||||
}
|
||||
|
||||
public void write(int[] bytes, int offset, int length) throws IOException {
|
||||
for (int i = 0; i < bytes.length; i++) {
|
||||
delegate.write(bytes[i]);
|
||||
if (bytes[i] == '\n') {
|
||||
delegate.write(indent);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static String readFromFile(File file) {
|
||||
try {
|
||||
return FileUtils.readFileToString(file).trim();
|
||||
} catch (IOException ioException) {
|
||||
throw new GradleException("Cannot read java properties file.", ioException);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -30,12 +30,12 @@ import java.util.function.Supplier;
|
|||
public class DistributionArchive implements Named {
|
||||
|
||||
private TaskProvider<? extends AbstractArchiveTask> archiveTask;
|
||||
private TaskProvider<Copy> explodedDistTask;
|
||||
private TaskProvider<Copy> expandedDistTask;
|
||||
private final String name;
|
||||
|
||||
public DistributionArchive(TaskProvider<? extends AbstractArchiveTask> archiveTask, TaskProvider<Copy> explodedDistTask, String name) {
|
||||
public DistributionArchive(TaskProvider<? extends AbstractArchiveTask> archiveTask, TaskProvider<Copy> expandedDistTask, String name) {
|
||||
this.archiveTask = archiveTask;
|
||||
this.explodedDistTask = explodedDistTask;
|
||||
this.expandedDistTask = expandedDistTask;
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
|
|
@ -45,7 +45,7 @@ public class DistributionArchive implements Named {
|
|||
|
||||
public void content(Supplier<CopySpec> p) {
|
||||
this.archiveTask.configure(t -> t.with(p.get()));
|
||||
this.explodedDistTask.configure(t -> t.with(p.get()));
|
||||
this.expandedDistTask.configure(t -> t.with(p.get()));
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -57,7 +57,7 @@ public class DistributionArchive implements Named {
|
|||
return archiveTask;
|
||||
}
|
||||
|
||||
public TaskProvider<Copy> getExplodedArchiveTask() {
|
||||
return explodedDistTask;
|
||||
public TaskProvider<Copy> getExpandedDistTask() {
|
||||
return expandedDistTask;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,209 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.gradle.internal;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.elasticsearch.gradle.LoggedExec;
|
||||
import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin;
|
||||
import org.gradle.api.Action;
|
||||
import org.gradle.api.GradleException;
|
||||
import org.gradle.api.Plugin;
|
||||
import org.gradle.api.Project;
|
||||
import org.gradle.api.logging.Logger;
|
||||
import org.gradle.api.plugins.ExtraPropertiesExtension;
|
||||
import org.gradle.api.provider.Provider;
|
||||
import org.gradle.api.provider.ProviderFactory;
|
||||
import org.gradle.api.tasks.TaskContainer;
|
||||
import org.gradle.api.tasks.TaskProvider;
|
||||
import org.gradle.process.ExecOperations;
|
||||
import org.gradle.process.ExecResult;
|
||||
import org.gradle.process.ExecSpec;
|
||||
|
||||
import javax.inject.Inject;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import static java.util.Arrays.asList;
|
||||
|
||||
public class InternalBwcGitPlugin implements Plugin<Project> {
|
||||
|
||||
private final ProviderFactory providerFactory;
|
||||
private final ExecOperations execOperations;
|
||||
|
||||
private BwcGitExtension gitExtension;
|
||||
private Project project;
|
||||
|
||||
@Inject
|
||||
public InternalBwcGitPlugin(ProviderFactory providerFactory, ExecOperations execOperations) {
|
||||
this.providerFactory = providerFactory;
|
||||
this.execOperations = execOperations;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void apply(Project project) {
|
||||
this.project = project;
|
||||
this.gitExtension = project.getExtensions().create("bwcGitConfig", BwcGitExtension.class);
|
||||
Provider<String> remote = providerFactory.systemProperty("bwc.remote").forUseAtConfigurationTime().orElse("elastic");
|
||||
|
||||
TaskContainer tasks = project.getTasks();
|
||||
TaskProvider<LoggedExec> createCloneTaskProvider = tasks.register("createClone", LoggedExec.class, createClone -> {
|
||||
createClone.onlyIf(task -> this.gitExtension.getCheckoutDir().get().exists() == false);
|
||||
createClone.setCommandLine(asList("git", "clone", project.getRootDir(), gitExtension.getCheckoutDir().get()));
|
||||
});
|
||||
|
||||
TaskProvider<LoggedExec> findRemoteTaskProvider = tasks.register("findRemote", LoggedExec.class, findRemote -> {
|
||||
findRemote.dependsOn(createCloneTaskProvider);
|
||||
// TODO Gradle should provide property based configuration here
|
||||
findRemote.setWorkingDir(gitExtension.getCheckoutDir().get());
|
||||
|
||||
findRemote.setCommandLine(asList("git", "remote", "-v"));
|
||||
ByteArrayOutputStream output = new ByteArrayOutputStream();
|
||||
findRemote.setStandardOutput(output);
|
||||
findRemote.doLast(t -> {
|
||||
ExtraPropertiesExtension extraProperties = project.getExtensions().getExtraProperties();
|
||||
extraProperties.set("remoteExists", isRemoteAvailable(remote, output));
|
||||
});
|
||||
});
|
||||
|
||||
TaskProvider<LoggedExec> addRemoteTaskProvider = tasks.register("addRemote", LoggedExec.class, addRemote -> {
|
||||
addRemote.dependsOn(findRemoteTaskProvider);
|
||||
addRemote.onlyIf(task -> ((boolean) project.getExtensions().getExtraProperties().get("remoteExists")) == false);
|
||||
addRemote.setWorkingDir(gitExtension.getCheckoutDir().get());
|
||||
String remoteRepo = remote.get();
|
||||
// for testing only we can override the base remote url
|
||||
String remoteRepoUrl = providerFactory.systemProperty("testRemoteRepo")
|
||||
.forUseAtConfigurationTime()
|
||||
.getOrElse("https://github.com/" + remoteRepo + "/elasticsearch.git");
|
||||
addRemote.setCommandLine(asList("git", "remote", "add", remoteRepo, remoteRepoUrl));
|
||||
});
|
||||
|
||||
TaskProvider<LoggedExec> fetchLatestTaskProvider = tasks.register("fetchLatest", LoggedExec.class, fetchLatest -> {
|
||||
var gitFetchLatest = project.getProviders()
|
||||
.systemProperty("tests.bwc.git_fetch_latest")
|
||||
.forUseAtConfigurationTime()
|
||||
.orElse("true")
|
||||
.map(fetchProp -> {
|
||||
if ("true".equals(fetchProp)) {
|
||||
return true;
|
||||
}
|
||||
if ("false".equals(fetchProp)) {
|
||||
return false;
|
||||
}
|
||||
throw new GradleException("tests.bwc.git_fetch_latest must be [true] or [false] but was [" + fetchProp + "]");
|
||||
});
|
||||
fetchLatest.onlyIf(t -> project.getGradle().getStartParameter().isOffline() == false && gitFetchLatest.get());
|
||||
fetchLatest.dependsOn(addRemoteTaskProvider);
|
||||
fetchLatest.setWorkingDir(gitExtension.getCheckoutDir().get());
|
||||
fetchLatest.setCommandLine(asList("git", "fetch", "--all"));
|
||||
});
|
||||
|
||||
tasks.register("checkoutBwcBranch", checkoutBwcBranch -> {
|
||||
checkoutBwcBranch.dependsOn(fetchLatestTaskProvider);
|
||||
checkoutBwcBranch.doLast(t -> {
|
||||
Logger logger = project.getLogger();
|
||||
|
||||
String bwcBranch = this.gitExtension.getBwcBranch().get();
|
||||
final String refspec = providerFactory.systemProperty("bwc.refspec." + bwcBranch)
|
||||
.orElse(providerFactory.systemProperty("tests.bwc.refspec." + bwcBranch))
|
||||
.getOrElse(remote.get() + "/" + bwcBranch);
|
||||
|
||||
String effectiveRefSpec = maybeAlignedRefSpec(logger, refspec);
|
||||
|
||||
logger.lifecycle("Performing checkout of {}...", refspec);
|
||||
LoggedExec.exec(project, spec -> {
|
||||
spec.workingDir(gitExtension.getCheckoutDir());
|
||||
spec.commandLine("git", "checkout", effectiveRefSpec);
|
||||
});
|
||||
|
||||
String checkoutHash = GlobalBuildInfoPlugin.gitInfo(gitExtension.getCheckoutDir().get()).getRevision();
|
||||
logger.lifecycle("Checkout hash for {} is {}", project.getPath(), checkoutHash);
|
||||
writeFile(new File(project.getBuildDir(), "refspec"), checkoutHash);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
public BwcGitExtension getGitExtension() {
|
||||
return gitExtension;
|
||||
}
|
||||
|
||||
/**
|
||||
* We use a time based approach to make the bwc versions built deterministic and compatible with the current hash.
|
||||
* Most of the time we want to test against latest, but when running delayed exhaustive tests or wanting
|
||||
* reproducible builds we want this to be deterministic by using a hash that was the latest when the current
|
||||
* commit was made.
|
||||
* <p>
|
||||
* This approach doesn't work with merge commits as these can introduce commits in the chronological order
|
||||
* after the fact e.x. a merge done today can add commits dated with yesterday so the result will no longer be
|
||||
* deterministic.
|
||||
* <p>
|
||||
* We don't use merge commits, but for additional safety we check that no such commits exist in the time period
|
||||
* we are interested in.
|
||||
* <p>
|
||||
* Timestamps are at seconds resolution. rev-parse --before and --after are inclusive w.r.t the second
|
||||
* passed as input. This means the results might not be deterministic in the current second, but this
|
||||
* should not matter in practice.
|
||||
*/
|
||||
private String maybeAlignedRefSpec(Logger logger, String defaultRefSpec) {
|
||||
if (providerFactory.systemProperty("bwc.checkout.align").isPresent() == false) {
|
||||
return defaultRefSpec;
|
||||
}
|
||||
|
||||
String timeOfCurrent = execInCheckoutDir(execSpec -> {
|
||||
execSpec.commandLine(asList("git", "show", "--no-patch", "--no-notes", "--pretty='%cD'"));
|
||||
execSpec.workingDir(project.getRootDir());
|
||||
});
|
||||
|
||||
logger.lifecycle("Commit date of current: {}", timeOfCurrent);
|
||||
|
||||
String mergeCommits = execInCheckoutDir(
|
||||
spec -> spec.commandLine(asList("git", "rev-list", defaultRefSpec, "--after", timeOfCurrent, "--merges"))
|
||||
);
|
||||
if (mergeCommits.isEmpty() == false) {
|
||||
throw new IllegalStateException("Found the following merge commits which prevent determining bwc commits: " + mergeCommits);
|
||||
}
|
||||
return execInCheckoutDir(
|
||||
spec -> spec.commandLine(asList("git", "rev-list", defaultRefSpec, "-n", "1", "--before", timeOfCurrent, "--date-order"))
|
||||
);
|
||||
}
|
||||
|
||||
private void writeFile(File file, String content) {
|
||||
try {
|
||||
FileUtils.writeStringToFile(file, content);
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private String execInCheckoutDir(Action<ExecSpec> execSpecConfig) {
|
||||
ByteArrayOutputStream os = new ByteArrayOutputStream();
|
||||
ExecResult exec = execOperations.exec(execSpec -> {
|
||||
execSpec.setStandardOutput(os);
|
||||
execSpec.workingDir(gitExtension.getCheckoutDir().get());
|
||||
execSpecConfig.execute(execSpec);
|
||||
});
|
||||
exec.assertNormalExitValue();
|
||||
return os.toString().trim();
|
||||
}
|
||||
|
||||
private static boolean isRemoteAvailable(Provider<String> remote, ByteArrayOutputStream output) {
|
||||
return new String(output.toByteArray()).contains(remote.get() + "\t");
|
||||
}
|
||||
}
|
||||
|
|
@ -88,7 +88,7 @@ public class InternalDistributionArchiveSetupPlugin implements Plugin<Project> {
|
|||
var extractedConfiguration = sub.getConfigurations().create("extracted");
|
||||
extractedConfiguration.setCanBeResolved(false);
|
||||
extractedConfiguration.getAttributes().attribute(ARTIFACT_FORMAT, ArtifactTypeDefinition.DIRECTORY_TYPE);
|
||||
sub.getArtifacts().add(EXTRACTED_CONFIGURATION_NAME, distributionArchive.getExplodedArchiveTask());
|
||||
sub.getArtifacts().add(EXTRACTED_CONFIGURATION_NAME, distributionArchive.getExpandedDistTask());
|
||||
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,268 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.gradle.internal;
|
||||
|
||||
import org.elasticsearch.gradle.BwcVersions;
|
||||
import org.elasticsearch.gradle.Version;
|
||||
import org.elasticsearch.gradle.info.BuildParams;
|
||||
import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin;
|
||||
import org.gradle.api.InvalidUserDataException;
|
||||
import org.gradle.api.Plugin;
|
||||
import org.gradle.api.Project;
|
||||
import org.gradle.api.Task;
|
||||
import org.gradle.api.provider.Provider;
|
||||
import org.gradle.api.provider.ProviderFactory;
|
||||
import org.gradle.api.tasks.TaskProvider;
|
||||
import org.gradle.language.base.plugins.LifecycleBasePlugin;
|
||||
|
||||
import javax.inject.Inject;
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static java.util.Arrays.asList;
|
||||
import static java.util.Arrays.stream;
|
||||
|
||||
/**
|
||||
* We want to be able to do BWC tests for unreleased versions without relying on and waiting for snapshots.
|
||||
* For this we need to check out and build the unreleased versions.
|
||||
* Since these depend on the current version, we can't name the Gradle projects statically, and don't know what the
|
||||
* unreleased versions are when Gradle projects are set up, so we use "build-unreleased-version-*" as placeholders
|
||||
* and configure them to build various versions here.
|
||||
*/
|
||||
public class InternalDistributionBwcSetupPlugin implements Plugin<Project> {
|
||||
|
||||
private ProviderFactory providerFactory;
|
||||
|
||||
@Inject
|
||||
public InternalDistributionBwcSetupPlugin(ProviderFactory providerFactory) {
|
||||
this.providerFactory = providerFactory;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void apply(Project project) {
|
||||
project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class);
|
||||
BuildParams.getBwcVersions()
|
||||
.forPreviousUnreleased(
|
||||
(BwcVersions.UnreleasedVersionInfo unreleasedVersion) -> {
|
||||
configureBwcProject(project.project(unreleasedVersion.gradleProjectPath), unreleasedVersion);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
private void configureBwcProject(Project project, BwcVersions.UnreleasedVersionInfo versionInfo) {
|
||||
Provider<BwcVersions.UnreleasedVersionInfo> versionInfoProvider = providerFactory.provider(() -> versionInfo);
|
||||
Provider<File> checkoutDir = versionInfoProvider.map(info -> new File(project.getBuildDir(), "bwc/checkout-" + info.branch));
|
||||
BwcSetupExtension bwcSetupExtension = project.getExtensions()
|
||||
.create("bwcSetup", BwcSetupExtension.class, project, versionInfoProvider, checkoutDir);
|
||||
BwcGitExtension gitExtension = project.getPlugins().apply(InternalBwcGitPlugin.class).getGitExtension();
|
||||
Provider<Version> bwcVersion = versionInfoProvider.map(info -> info.version);
|
||||
gitExtension.setBwcVersion(versionInfoProvider.map(info -> info.version));
|
||||
gitExtension.setBwcBranch(versionInfoProvider.map(info -> info.branch));
|
||||
gitExtension.setCheckoutDir(checkoutDir);
|
||||
|
||||
// we want basic lifecycle tasks like `clean` here.
|
||||
project.getPlugins().apply(LifecycleBasePlugin.class);
|
||||
|
||||
TaskProvider<Task> buildBwcTaskProvider = project.getTasks().register("buildBwc");
|
||||
List<DistributionProject> distributionProjects = resolveArchiveProjects(checkoutDir.get(), bwcVersion.get());
|
||||
|
||||
for (DistributionProject distributionProject : distributionProjects) {
|
||||
createBuildBwcTask(
|
||||
bwcSetupExtension,
|
||||
project,
|
||||
bwcVersion,
|
||||
distributionProject.name,
|
||||
distributionProject.getProjectPath(),
|
||||
distributionProject.getDistFile(),
|
||||
buildBwcTaskProvider
|
||||
);
|
||||
|
||||
registerBwcArtifacts(project, distributionProject);
|
||||
}
|
||||
|
||||
// Create build tasks for the JDBC driver used for compatibility testing
|
||||
String jdbcProjectDir = "x-pack/plugin/sql/jdbc";
|
||||
|
||||
File jdbcProjectArtifact = new File(
|
||||
checkoutDir.get(),
|
||||
jdbcProjectDir + "/build/distributions/x-pack-sql-jdbc-" + bwcVersion.get() + "-SNAPSHOT.jar"
|
||||
);
|
||||
|
||||
createBuildBwcTask(bwcSetupExtension, project, bwcVersion, "jdbc", jdbcProjectDir, jdbcProjectArtifact, buildBwcTaskProvider);
|
||||
}
|
||||
|
||||
private void registerBwcArtifacts(Project bwcProject, DistributionProject distributionProject) {
|
||||
String projectName = distributionProject.name;
|
||||
String buildBwcTask = buildBwcTaskName(projectName);
|
||||
|
||||
registerDistributionArchiveArtifact(bwcProject, distributionProject, buildBwcTask);
|
||||
if (distributionProject.getExpandedDistDirectory() != null) {
|
||||
String expandedDistConfiguration = "expanded-" + projectName;
|
||||
bwcProject.getConfigurations().create(expandedDistConfiguration);
|
||||
bwcProject.getArtifacts().add(expandedDistConfiguration, distributionProject.getExpandedDistDirectory(), artifact -> {
|
||||
artifact.setName("elasticsearch");
|
||||
artifact.builtBy(buildBwcTask);
|
||||
artifact.setType("directory");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private void registerDistributionArchiveArtifact(Project bwcProject, DistributionProject distributionProject, String buildBwcTask) {
|
||||
String artifactFileName = distributionProject.getDistFile().getName();
|
||||
String artifactName = artifactFileName.contains("oss") ? "elasticsearch-oss" : "elasticsearch";
|
||||
|
||||
String suffix = artifactFileName.endsWith("tar.gz") ? "tar.gz" : artifactFileName.substring(artifactFileName.length() - 3);
|
||||
int archIndex = artifactFileName.indexOf("x86_64");
|
||||
|
||||
bwcProject.getConfigurations().create(distributionProject.name);
|
||||
bwcProject.getArtifacts().add(distributionProject.name, distributionProject.getDistFile(), artifact -> {
|
||||
artifact.setName(artifactName);
|
||||
artifact.builtBy(buildBwcTask);
|
||||
artifact.setType(suffix);
|
||||
|
||||
String classifier = "";
|
||||
if (archIndex != -1) {
|
||||
int osIndex = artifactFileName.lastIndexOf('-', archIndex - 2);
|
||||
classifier = "-" + artifactFileName.substring(osIndex + 1, archIndex - 1) + "-x86_64";
|
||||
}
|
||||
artifact.setClassifier(classifier);
|
||||
});
|
||||
}
|
||||
|
||||
private static List<DistributionProject> resolveArchiveProjects(File checkoutDir, Version bwcVersion) {
|
||||
List<String> projects = new ArrayList<>();
|
||||
projects.addAll(asList("deb", "rpm"));
|
||||
if (bwcVersion.onOrAfter("7.0.0")) {
|
||||
projects.addAll(asList("oss-windows-zip", "windows-zip", "oss-darwin-tar", "darwin-tar", "oss-linux-tar", "linux-tar"));
|
||||
} else {
|
||||
projects.addAll(asList("oss-zip", "zip", "oss-deb", "oss-rpm"));
|
||||
}
|
||||
|
||||
return projects.stream().map(name -> {
|
||||
String baseDir = "distribution" + (name.endsWith("zip") || name.endsWith("tar") ? "/archives" : "/packages");
|
||||
String classifier = "";
|
||||
String extension = name;
|
||||
if (bwcVersion.onOrAfter("7.0.0")) {
|
||||
if (name.contains("zip") || name.contains("tar")) {
|
||||
int index = name.lastIndexOf('-');
|
||||
String baseName = name.startsWith("oss-") ? name.substring(4, index) : name.substring(0, index);
|
||||
classifier = "-" + baseName + "-x86_64";
|
||||
extension = name.substring(index + 1);
|
||||
if (extension.equals("tar")) {
|
||||
extension += ".gz";
|
||||
}
|
||||
} else if (name.contains("deb")) {
|
||||
classifier = "-amd64";
|
||||
} else if (name.contains("rpm")) {
|
||||
classifier = "-x86_64";
|
||||
}
|
||||
} else if (name.contains("oss-")) {
|
||||
extension = name.substring(4);
|
||||
}
|
||||
return new DistributionProject(name, baseDir, bwcVersion, classifier, extension, checkoutDir);
|
||||
}).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
private static String buildBwcTaskName(String projectName) {
|
||||
return "buildBwc"
|
||||
+ stream(projectName.split("-")).map(i -> i.substring(0, 1).toUpperCase(Locale.ROOT) + i.substring(1))
|
||||
.collect(Collectors.joining());
|
||||
}
|
||||
|
||||
static void createBuildBwcTask(
|
||||
BwcSetupExtension bwcSetupExtension,
|
||||
Project project,
|
||||
Provider<Version> bwcVersion,
|
||||
String projectName,
|
||||
String projectPath,
|
||||
File projectArtifact,
|
||||
TaskProvider<Task> bwcTaskProvider
|
||||
) {
|
||||
String bwcTaskName = buildBwcTaskName(projectName);
|
||||
bwcSetupExtension.bwcTask(bwcTaskName, c -> {
|
||||
c.getInputs().file(new File(project.getBuildDir(), "refspec"));
|
||||
c.getOutputs().files(projectArtifact);
|
||||
c.getOutputs().cacheIf("BWC distribution caching is disabled on 'master' branch", task -> {
|
||||
String gitBranch = System.getenv("GIT_BRANCH");
|
||||
return BuildParams.isCi() && (gitBranch == null || gitBranch.endsWith("master") == false);
|
||||
});
|
||||
c.args(projectPath.replace('/', ':') + ":assemble");
|
||||
if (project.getGradle().getStartParameter().isBuildCacheEnabled()) {
|
||||
c.args("--build-cache");
|
||||
}
|
||||
c.doLast(task -> {
|
||||
if (projectArtifact.exists() == false) {
|
||||
throw new InvalidUserDataException(
|
||||
"Building " + bwcVersion.get() + " didn't generate expected file " + projectArtifact
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
bwcTaskProvider.configure(t -> t.dependsOn(bwcTaskName));
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents an archive project (distribution/archives/*)
|
||||
* we build from a bwc Version in a cloned repository
|
||||
*/
|
||||
private static class DistributionProject {
|
||||
private final String name;
|
||||
private String projectPath;
|
||||
private File distFile;
|
||||
private File expandedDistDir;
|
||||
|
||||
DistributionProject(String name, String baseDir, Version version, String classifier, String extension, File checkoutDir) {
|
||||
this.name = name;
|
||||
this.projectPath = baseDir + "/" + name;
|
||||
this.distFile = new File(
|
||||
checkoutDir,
|
||||
baseDir
|
||||
+ "/"
|
||||
+ name
|
||||
+ "/build/distributions/elasticsearch-"
|
||||
+ (name.startsWith("oss") ? "oss-" : "")
|
||||
+ version
|
||||
+ "-SNAPSHOT"
|
||||
+ classifier
|
||||
+ "."
|
||||
+ extension
|
||||
);
|
||||
// we only ported this down to the 7.x branch.
|
||||
if (version.onOrAfter("7.10.0") && (name.endsWith("zip") || name.endsWith("tar"))) {
|
||||
this.expandedDistDir = new File(checkoutDir, baseDir + "/" + name + "/build/install");
|
||||
}
|
||||
}
|
||||
|
||||
public String getProjectPath() {
|
||||
return projectPath;
|
||||
}
|
||||
|
||||
public File getDistFile() {
|
||||
return distFile;
|
||||
}
|
||||
|
||||
public File getExpandedDistDirectory() {
|
||||
return expandedDistDir;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -94,14 +94,25 @@ public class InternalDistributionDownloadPlugin implements Plugin<Project> {
|
|||
+ "without a bundled JDK is not supported."
|
||||
);
|
||||
}
|
||||
String distributionProjectName = distributionProjectName(distribution);
|
||||
String projectConfig = getProjectConfig(distributionProjectName, unreleasedInfo);
|
||||
return new ProjectBasedDistributionDependency(
|
||||
(config) -> projectDependency(project, unreleasedInfo.gradleProjectPath, distributionProjectName(distribution))
|
||||
(config) -> projectDependency(project, unreleasedInfo.gradleProjectPath, projectConfig)
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Will be removed once this is backported to all unreleased branches.
|
||||
* */
|
||||
private static String getProjectConfig(String distributionProjectName, BwcVersions.UnreleasedVersionInfo info) {
|
||||
return (info.gradleProjectPath.equals(":distribution") || info.version.before("7.10.0"))
|
||||
? distributionProjectName
|
||||
: "expanded-" + distributionProjectName;
|
||||
}
|
||||
|
||||
private static String distributionProjectPath(ElasticsearchDistribution distribution) {
|
||||
String projectPath = ":distribution";
|
||||
switch (distribution.getType()) {
|
||||
|
|
@ -171,7 +182,7 @@ public class InternalDistributionDownloadPlugin implements Plugin<Project> {
|
|||
return projectName;
|
||||
}
|
||||
|
||||
private class ProjectBasedDistributionDependency implements DistributionDependency {
|
||||
private static class ProjectBasedDistributionDependency implements DistributionDependency {
|
||||
|
||||
private Function<String, Dependency> function;
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ import java.util.Optional;
|
|||
public class JavaUtil {
|
||||
|
||||
/** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */
|
||||
static String getJavaHome(final int version) {
|
||||
public static String getJavaHome(final int version) {
|
||||
List<JavaHome> javaHomes = BuildParams.getJavaVersions();
|
||||
Optional<JavaHome> java = javaHomes.stream().filter(j -> j.getVersion() == version).findFirst();
|
||||
return java.orElseThrow(() -> new GradleException("JAVA" + version + "_HOME required")).getJavaHome().get().getAbsolutePath();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
#
|
||||
# Licensed to Elasticsearch under one or more contributor
|
||||
# license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright
|
||||
# ownership. Elasticsearch licenses this file to you under
|
||||
# the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
|
||||
implementation-class=org.elasticsearch.gradle.internal.InternalDistributionBwcSetupPlugin
|
||||
|
|
@ -17,345 +17,23 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
import org.apache.tools.ant.taskdefs.condition.Os
|
||||
import org.elasticsearch.gradle.LoggedExec
|
||||
apply plugin:"elasticsearch.internal-distribution-bwc-setup"
|
||||
|
||||
import org.elasticsearch.gradle.Version
|
||||
import org.elasticsearch.gradle.BwcVersions
|
||||
import org.elasticsearch.gradle.info.BuildParams
|
||||
import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
|
||||
import org.gradle.util.GradleVersion
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
|
||||
import static org.elasticsearch.gradle.util.JavaUtil.getJavaHome
|
||||
|
||||
/**
|
||||
* We want to be able to do BWC tests for unreleased versions without relying on and waiting for snapshots.
|
||||
* For this we need to check out and build the unreleased versions.
|
||||
* Since These depend on the current version, we can't name the Gradle projects statically, and don't know what the
|
||||
* unreleased versions are when Gradle projects are set up, so we use "build-unreleased-version-*" as placeholders
|
||||
* and configure them to build various versions here.
|
||||
*/
|
||||
BuildParams.bwcVersions.forPreviousUnreleased { BwcVersions.UnreleasedVersionInfo unreleasedVersion ->
|
||||
project("${unreleasedVersion.gradleProjectPath}") {
|
||||
Version bwcVersion = unreleasedVersion.version
|
||||
String bwcBranch = unreleasedVersion.branch
|
||||
apply plugin: 'distribution'
|
||||
// Not published so no need to assemble
|
||||
tasks.named("assemble").configure {
|
||||
enabled = false
|
||||
}
|
||||
File checkoutDir = file("${buildDir}/bwc/checkout-${bwcBranch}")
|
||||
|
||||
final String remote = System.getProperty("bwc.remote", "elastic")
|
||||
|
||||
boolean gitFetchLatest
|
||||
final String gitFetchLatestProperty = System.getProperty("tests.bwc.git_fetch_latest", "true")
|
||||
if ("true".equals(gitFetchLatestProperty)) {
|
||||
gitFetchLatest = true
|
||||
} else if ("false".equals(gitFetchLatestProperty)) {
|
||||
gitFetchLatest = false
|
||||
} else {
|
||||
throw new GradleException("tests.bwc.git_fetch_latest must be [true] or [false] but was [" + gitFetchLatestProperty + "]")
|
||||
}
|
||||
|
||||
tasks.register("createClone", LoggedExec) {
|
||||
onlyIf { checkoutDir.exists() == false }
|
||||
commandLine = ['git', 'clone', rootDir, checkoutDir]
|
||||
}
|
||||
|
||||
tasks.register("findRemote", LoggedExec) {
|
||||
dependsOn "createClone"
|
||||
workingDir = checkoutDir
|
||||
commandLine = ['git', 'remote', '-v']
|
||||
ByteArrayOutputStream output = new ByteArrayOutputStream()
|
||||
standardOutput = output
|
||||
doLast {
|
||||
project.ext.remoteExists = false
|
||||
output.toString('UTF-8').eachLine {
|
||||
if (it.contains("${remote}\t")) {
|
||||
project.ext.remoteExists = true
|
||||
}
|
||||
BuildParams.getBwcVersions().forPreviousUnreleased { unreleasedVersion ->
|
||||
project(unreleasedVersion.gradleProjectPath) {
|
||||
Version currentVersion = Version.fromString(version)
|
||||
TaskProvider<Task> resolveAllBwcDepsTaskProvider = bwcSetup.bwcTask("resolveAllBwcDependencies") {
|
||||
t -> t.args("resolveAllDependencies")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tasks.register("addRemote", LoggedExec) {
|
||||
dependsOn findRemote
|
||||
onlyIf { project.ext.remoteExists == false }
|
||||
workingDir = checkoutDir
|
||||
commandLine = ['git', 'remote', 'add', "${remote}", "https://github.com/${remote}/elasticsearch.git"]
|
||||
}
|
||||
|
||||
tasks.register("fetchLatest", LoggedExec) {
|
||||
onlyIf { project.gradle.startParameter.isOffline() == false && gitFetchLatest }
|
||||
dependsOn("addRemote")
|
||||
workingDir = checkoutDir
|
||||
commandLine = ['git', 'fetch', '--all']
|
||||
}
|
||||
|
||||
Closure execGit = { Action<ExecSpec> action ->
|
||||
new ByteArrayOutputStream().withStream { os ->
|
||||
ExecResult result = project.exec { spec ->
|
||||
workingDir = checkoutDir
|
||||
standardOutput os
|
||||
action.execute(spec)
|
||||
if (currentVersion.getMinor() == 0 && currentVersion.getRevision() == 0) {
|
||||
// We only want to resolve dependencies for live versions of master, without cascading this to older versions
|
||||
tasks.named("resolveAllDependencies").configure {
|
||||
dependsOn(resolveAllBwcDepsTaskProvider)
|
||||
}
|
||||
}
|
||||
result.assertNormalExitValue()
|
||||
return os.toString().trim()
|
||||
}
|
||||
}
|
||||
tasks.register("checkoutBwcBranch") {
|
||||
dependsOn("fetchLatest")
|
||||
doLast {
|
||||
def refspec = System.getProperty("bwc.refspec." + bwcBranch) ?: System.getProperty("tests.bwc.refspec." + bwcBranch) ?: "${remote}/${bwcBranch}"
|
||||
if (System.getProperty("bwc.checkout.align") != null) {
|
||||
/*
|
||||
We use a time based approach to make the bwc versions built deterministic and compatible with the current hash.
|
||||
Most of the time we want to test against latest, but when running delayed exhaustive tests or wanting
|
||||
reproducible builds we want this to be deterministic by using a hash that was the latest when the current
|
||||
commit was made.
|
||||
|
||||
This approach doesn't work with merge commits as these can introduce commits in the chronological order
|
||||
after the fact e.x. a merge done today can add commits dated with yesterday so the result will no longer be
|
||||
deterministic.
|
||||
|
||||
We don't use merge commits, but for additional safety we check that no such commits exist in the time period
|
||||
we are interested in.
|
||||
|
||||
Timestamps are at seconds resolution. rev-parse --before and --after are inclusive w.r.t the second
|
||||
passed as input. This means the results might not be deterministic in the current second, but this
|
||||
should not matter in practice.
|
||||
*/
|
||||
String timeOfCurrent = execGit { spec ->
|
||||
spec.commandLine 'git', 'show', '--no-patch', '--no-notes', "--pretty='%cD'"
|
||||
spec.workingDir project.rootDir
|
||||
}
|
||||
logger.lifecycle("Commit date of current: {}", timeOfCurrent)
|
||||
String mergeCommits = execGit { spec ->
|
||||
spec.commandLine "git", "rev-list", refspec, "--after", timeOfCurrent, "--merges"
|
||||
}
|
||||
if (mergeCommits.isEmpty() == false) {
|
||||
throw new IllegalStateException(
|
||||
"Found the following merge commits which prevent determining bwc commits: " + mergeCommits
|
||||
)
|
||||
}
|
||||
refspec = execGit { spec ->
|
||||
spec.commandLine "git", "rev-list", refspec, "-n", "1", "--before", timeOfCurrent, "--date-order"
|
||||
}
|
||||
}
|
||||
|
||||
logger.lifecycle("Performing checkout of ${refspec}...")
|
||||
LoggedExec.exec(project) { spec ->
|
||||
spec.workingDir = checkoutDir
|
||||
spec.commandLine "git", "checkout", refspec
|
||||
}
|
||||
String checkoutHash = GlobalBuildInfoPlugin.gitInfo(checkoutDir).revision
|
||||
logger.lifecycle("Checkout hash for ${project.path} is ${checkoutHash}")
|
||||
file("${project.buildDir}/refspec").text = checkoutHash
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Closure<TaskProvider> createRunBwcGradleTask = { name, extraConfig ->
|
||||
return tasks.register("$name", LoggedExec) {
|
||||
dependsOn "checkoutBwcBranch"
|
||||
spoolOutput = true
|
||||
workingDir = checkoutDir
|
||||
doFirst {
|
||||
// Execution time so that the checkouts are available
|
||||
List<String> lines = file("${checkoutDir}/.ci/java-versions.properties").readLines()
|
||||
environment(
|
||||
'JAVA_HOME',
|
||||
getJavaHome(Integer.parseInt(
|
||||
lines
|
||||
.findAll({ it.startsWith("ES_BUILD_JAVA=") })
|
||||
.collect({ it.replace("ES_BUILD_JAVA=java", "").trim() })
|
||||
.collect({ it.replace("ES_BUILD_JAVA=openjdk", "").trim() })
|
||||
.join("!!")
|
||||
))
|
||||
)
|
||||
environment(
|
||||
'RUNTIME_JAVA_HOME',
|
||||
getJavaHome(Integer.parseInt(
|
||||
lines
|
||||
.findAll({ it.startsWith("ES_RUNTIME_JAVA=java") })
|
||||
.collect({ it.replace("ES_RUNTIME_JAVA=java", "").trim() })
|
||||
.join("!!")
|
||||
))
|
||||
)
|
||||
}
|
||||
|
||||
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
|
||||
executable 'cmd'
|
||||
args '/C', 'call', new File(checkoutDir, 'gradlew').toString()
|
||||
} else {
|
||||
executable new File(checkoutDir, 'gradlew').toString()
|
||||
}
|
||||
if (gradle.startParameter.isOffline()) {
|
||||
args "--offline"
|
||||
}
|
||||
String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url')
|
||||
if (buildCacheUrl) {
|
||||
args "-Dorg.elasticsearch.build.cache.url=${buildCacheUrl}"
|
||||
}
|
||||
|
||||
args "-Dbuild.snapshot=true"
|
||||
args "-Dscan.tag.NESTED"
|
||||
final LogLevel logLevel = gradle.startParameter.logLevel
|
||||
if ([LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG].contains(logLevel)) {
|
||||
args "--${logLevel.name().toLowerCase(Locale.ENGLISH)}"
|
||||
}
|
||||
final String showStacktraceName = gradle.startParameter.showStacktrace.name()
|
||||
assert ["INTERNAL_EXCEPTIONS", "ALWAYS", "ALWAYS_FULL"].contains(showStacktraceName)
|
||||
if (showStacktraceName.equals("ALWAYS")) {
|
||||
args "--stacktrace"
|
||||
} else if (showStacktraceName.equals("ALWAYS_FULL")) {
|
||||
args "--full-stacktrace"
|
||||
}
|
||||
if (gradle.getStartParameter().isParallelProjectExecutionEnabled()) {
|
||||
args "--parallel"
|
||||
}
|
||||
standardOutput = new IndentingOutputStream(System.out, bwcVersion)
|
||||
errorOutput = new IndentingOutputStream(System.err, bwcVersion)
|
||||
configure extraConfig
|
||||
}
|
||||
}
|
||||
|
||||
Closure buildBwcTaskName = { projectName ->
|
||||
return "buildBwc${projectName.replaceAll(/-\w/) { it[1].toUpperCase() }.capitalize()}"
|
||||
}
|
||||
|
||||
def buildBwc = tasks.register("buildBwc");
|
||||
|
||||
Closure createBuildBwcTask = { projectName, projectDir, projectArtifact ->
|
||||
def bwcTaskName = buildBwcTaskName(projectName)
|
||||
createRunBwcGradleTask(bwcTaskName) {
|
||||
inputs.file("${project.buildDir}/refspec")
|
||||
outputs.files(projectArtifact)
|
||||
outputs.cacheIf("BWC distribution caching is disabled on 'master' branch") {
|
||||
// Don't bother caching in 'master' since the BWC branches move too quickly to make this cost worthwhile
|
||||
BuildParams.ci && System.getenv('GIT_BRANCH')?.endsWith("master") == false
|
||||
}
|
||||
args ":${projectDir.replace('/', ':')}:assemble"
|
||||
if (project.gradle.startParameter.buildCacheEnabled) {
|
||||
args "--build-cache"
|
||||
}
|
||||
doLast {
|
||||
if (projectArtifact.exists() == false) {
|
||||
throw new InvalidUserDataException("Building ${bwcVersion} didn't generate expected file ${projectArtifact}")
|
||||
}
|
||||
}
|
||||
}
|
||||
buildBwc.configure {
|
||||
dependsOn(bwcTaskName)
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, File> artifactFiles = [:]
|
||||
List<String> projectDirs = []
|
||||
List<String> projects = ['deb', 'rpm']
|
||||
if (bwcVersion.onOrAfter('7.0.0')) {
|
||||
projects.addAll(['windows-zip', 'darwin-tar', 'linux-tar'])
|
||||
} else {
|
||||
projects.add('zip')
|
||||
}
|
||||
|
||||
for (String projectName : projects) {
|
||||
String baseDir = "distribution"
|
||||
String classifier = ""
|
||||
String extension = projectName
|
||||
if (bwcVersion.onOrAfter('7.0.0') && (projectName.contains('zip') || projectName.contains('tar'))) {
|
||||
int index = projectName.indexOf('-')
|
||||
classifier = "-${projectName.substring(0, index)}-x86_64"
|
||||
extension = projectName.substring(index + 1)
|
||||
if (extension.equals('tar')) {
|
||||
extension += '.gz'
|
||||
}
|
||||
}
|
||||
if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('deb')) {
|
||||
classifier = "-amd64"
|
||||
}
|
||||
if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('rpm')) {
|
||||
classifier = "-x86_64"
|
||||
}
|
||||
if (bwcVersion.onOrAfter('6.3.0')) {
|
||||
baseDir += projectName.endsWith('zip') || projectName.endsWith('tar') ? '/archives' : '/packages'
|
||||
// add oss variant first
|
||||
projectDirs.add("${baseDir}/oss-${projectName}")
|
||||
File ossProjectArtifact = file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT${classifier}.${extension}")
|
||||
artifactFiles.put("oss-" + projectName, ossProjectArtifact)
|
||||
createBuildBwcTask("oss-${projectName}", "${baseDir}/oss-${projectName}", ossProjectArtifact)
|
||||
}
|
||||
projectDirs.add("${baseDir}/${projectName}")
|
||||
File projectArtifact = file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT${classifier}.${extension}")
|
||||
artifactFiles.put(projectName, projectArtifact)
|
||||
|
||||
createBuildBwcTask(projectName, "${baseDir}/${projectName}", projectArtifact)
|
||||
}
|
||||
|
||||
// Create build tasks for the JDBC driver used for compatibility testing
|
||||
String jdbcProjectDir = 'x-pack/plugin/sql/jdbc'
|
||||
File jdbcProjectArtifact = file("${checkoutDir}/${jdbcProjectDir}/build/distributions/x-pack-sql-jdbc-${bwcVersion}-SNAPSHOT.jar")
|
||||
createBuildBwcTask('jdbc', jdbcProjectDir, jdbcProjectArtifact)
|
||||
|
||||
createRunBwcGradleTask("resolveAllBwcDependencies") {
|
||||
args 'resolveAllDependencies'
|
||||
}
|
||||
Version currentVersion = Version.fromString(version)
|
||||
if (currentVersion.getMinor() == 0 && currentVersion.getRevision() == 0) {
|
||||
// We only want to resolve dependencies for live versions of master, without cascading this to older versions
|
||||
tasks.named("resolveAllDependencies").configure {
|
||||
dependsOn("resolveAllBwcDependencies")
|
||||
}
|
||||
}
|
||||
|
||||
for (e in artifactFiles) {
|
||||
String projectName = e.key
|
||||
String buildBwcTask = buildBwcTaskName(projectName)
|
||||
File artifactFile = e.value
|
||||
String artifactFileName = artifactFile.name
|
||||
String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
|
||||
String suffix = artifactFileName.endsWith("tar.gz") ? "tar.gz" : artifactFileName[-3..-1]
|
||||
int archIndex = artifactFileName.indexOf('x86_64')
|
||||
String classifier = ''
|
||||
if (archIndex != -1) {
|
||||
int osIndex = artifactFileName.lastIndexOf('-', archIndex - 2)
|
||||
classifier = "${artifactFileName.substring(osIndex + 1, archIndex - 1)}-x86_64"
|
||||
}
|
||||
configurations.create(projectName)
|
||||
artifacts {
|
||||
it.add(projectName, [file: artifactFile, name: artifactName, classifier: classifier, type: suffix, builtBy: buildBwcTask])
|
||||
}
|
||||
}
|
||||
// make sure no dependencies were added to assemble; we want it to be a no-op
|
||||
tasks.named("assemble").configure {
|
||||
dependsOn = []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class IndentingOutputStream extends OutputStream {
|
||||
|
||||
public final byte[] indent
|
||||
private final OutputStream delegate
|
||||
|
||||
public IndentingOutputStream(OutputStream delegate, Object version) {
|
||||
this.delegate = delegate
|
||||
indent = " [${version}] ".getBytes(StandardCharsets.UTF_8)
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(int b) {
|
||||
write([b] as int[], 0, 1)
|
||||
}
|
||||
|
||||
public void write(int[] bytes, int offset, int length) {
|
||||
for (int i = 0; i < bytes.length; i++) {
|
||||
delegate.write(bytes[i])
|
||||
if (bytes[i] == '\n') {
|
||||
delegate.write(indent)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue