352 lines
14 KiB
Groovy
352 lines
14 KiB
Groovy
/*
|
|
* Licensed to Elasticsearch under one or more contributor
|
|
* license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright
|
|
* ownership. Elasticsearch licenses this file to you under
|
|
* the Apache License, Version 2.0 (the "License"); you may
|
|
* not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing,
|
|
* software distributed under the License is distributed on an
|
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
* KIND, either express or implied. See the License for the
|
|
* specific language governing permissions and limitations
|
|
* under the License.
|
|
*/
|
|
|
|
import org.apache.tools.ant.taskdefs.condition.Os
|
|
import org.elasticsearch.gradle.LoggedExec
|
|
import org.elasticsearch.gradle.Version
|
|
import org.elasticsearch.gradle.BwcVersions
|
|
import org.elasticsearch.gradle.info.BuildParams
|
|
import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
|
|
import org.gradle.util.GradleVersion
|
|
|
|
import java.nio.charset.StandardCharsets
|
|
|
|
import static org.elasticsearch.gradle.util.JavaUtil.getJavaHome
|
|
|
|
/**
|
|
* We want to be able to do BWC tests for unreleased versions without relying on and waiting for snapshots.
|
|
* For this we need to check out and build the unreleased versions.
|
|
* Since These depend on the current version, we can't name the Gradle projects statically, and don't know what the
|
|
* unreleased versions are when Gradle projects are set up, so we use "build-unreleased-version-*" as placeholders
|
|
* and configure them to build various versions here.
|
|
*/
|
|
BuildParams.bwcVersions.forPreviousUnreleased { BwcVersions.UnreleasedVersionInfo unreleasedVersion ->
|
|
project("${unreleasedVersion.gradleProjectPath}") {
|
|
Version bwcVersion = unreleasedVersion.version
|
|
String bwcBranch = unreleasedVersion.branch
|
|
apply plugin: 'distribution'
|
|
// Not published so no need to assemble
|
|
assemble.enabled = false
|
|
|
|
File checkoutDir = file("${buildDir}/bwc/checkout-${bwcBranch}")
|
|
|
|
final String remote = System.getProperty("bwc.remote", "elastic")
|
|
|
|
boolean gitFetchLatest
|
|
final String gitFetchLatestProperty = System.getProperty("tests.bwc.git_fetch_latest", "true")
|
|
if ("true".equals(gitFetchLatestProperty)) {
|
|
gitFetchLatest = true
|
|
} else if ("false".equals(gitFetchLatestProperty)) {
|
|
gitFetchLatest = false
|
|
} else {
|
|
throw new GradleException("tests.bwc.git_fetch_latest must be [true] or [false] but was [" + gitFetchLatestProperty + "]")
|
|
}
|
|
|
|
task createClone(type: LoggedExec) {
|
|
onlyIf { checkoutDir.exists() == false }
|
|
commandLine = ['git', 'clone', rootDir, checkoutDir]
|
|
}
|
|
|
|
task findRemote(type: LoggedExec) {
|
|
dependsOn createClone
|
|
workingDir = checkoutDir
|
|
commandLine = ['git', 'remote', '-v']
|
|
ByteArrayOutputStream output = new ByteArrayOutputStream()
|
|
standardOutput = output
|
|
doLast {
|
|
project.ext.remoteExists = false
|
|
output.toString('UTF-8').eachLine {
|
|
if (it.contains("${remote}\t")) {
|
|
project.ext.remoteExists = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
task addRemote(type: LoggedExec) {
|
|
dependsOn findRemote
|
|
onlyIf { project.ext.remoteExists == false }
|
|
workingDir = checkoutDir
|
|
commandLine = ['git', 'remote', 'add', "${remote}", "https://github.com/${remote}/elasticsearch.git"]
|
|
}
|
|
|
|
task fetchLatest(type: LoggedExec) {
|
|
onlyIf { project.gradle.startParameter.isOffline() == false && gitFetchLatest }
|
|
dependsOn addRemote
|
|
workingDir = checkoutDir
|
|
commandLine = ['git', 'fetch', '--all']
|
|
}
|
|
|
|
Closure execGit = { Action<ExecSpec> action ->
|
|
new ByteArrayOutputStream().withStream { os ->
|
|
ExecResult result = project.exec { spec ->
|
|
workingDir = checkoutDir
|
|
standardOutput os
|
|
action.execute(spec)
|
|
}
|
|
result.assertNormalExitValue()
|
|
return os.toString().trim()
|
|
}
|
|
}
|
|
task checkoutBwcBranch() {
|
|
dependsOn fetchLatest
|
|
doLast {
|
|
def refspec = System.getProperty("bwc.refspec." + bwcBranch) ?: System.getProperty("tests.bwc.refspec." + bwcBranch) ?: "${remote}/${bwcBranch}"
|
|
if (System.getProperty("bwc.checkout.align") != null) {
|
|
/*
|
|
We use a time based approach to make the bwc versions built deterministic and compatible with the current hash.
|
|
Most of the time we want to test against latest, but when running delayed exhaustive tests or wanting
|
|
reproducible builds we want this to be deterministic by using a hash that was the latest when the current
|
|
commit was made.
|
|
|
|
This approach doesn't work with merge commits as these can introduce commits in the chronological order
|
|
after the fact e.x. a merge done today can add commits dated with yesterday so the result will no longer be
|
|
deterministic.
|
|
|
|
We don't use merge commits, but for additional safety we check that no such commits exist in the time period
|
|
we are interested in.
|
|
|
|
Timestamps are at seconds resolution. rev-parse --before and --after are inclusive w.r.t the second
|
|
passed as input. This means the results might not be deterministic in the current second, but this
|
|
should not matter in practice.
|
|
*/
|
|
String timeOfCurrent = execGit { spec ->
|
|
spec.commandLine 'git', 'show', '--no-patch', '--no-notes', "--pretty='%cD'"
|
|
spec.workingDir project.rootDir
|
|
}
|
|
logger.lifecycle("Commit date of current: {}", timeOfCurrent)
|
|
String mergeCommits = execGit { spec ->
|
|
spec.commandLine "git", "rev-list", refspec, "--after", timeOfCurrent, "--merges"
|
|
}
|
|
if (mergeCommits.isEmpty() == false) {
|
|
throw new IllegalStateException(
|
|
"Found the following merge commits which prevent determining bwc commits: " + mergeCommits
|
|
)
|
|
}
|
|
refspec = execGit { spec ->
|
|
spec.commandLine "git", "rev-list", refspec, "-n", "1", "--before", timeOfCurrent, "--date-order"
|
|
}
|
|
}
|
|
|
|
logger.lifecycle("Performing checkout of ${refspec}...")
|
|
LoggedExec.exec(project) { spec ->
|
|
spec.workingDir = checkoutDir
|
|
spec.commandLine "git", "checkout", refspec
|
|
}
|
|
String checkoutHash = GlobalBuildInfoPlugin.gitInfo(checkoutDir).revision
|
|
logger.lifecycle("Checkout hash for ${project.path} is ${checkoutHash}")
|
|
file("${project.buildDir}/refspec").text = checkoutHash
|
|
}
|
|
}
|
|
|
|
|
|
Closure createRunBwcGradleTask = { name, extraConfig ->
|
|
return tasks.register("$name", LoggedExec) {
|
|
dependsOn checkoutBwcBranch
|
|
spoolOutput = true
|
|
workingDir = checkoutDir
|
|
doFirst {
|
|
// Execution time so that the checkouts are available
|
|
List<String> lines = file("${checkoutDir}/.ci/java-versions.properties").readLines()
|
|
environment(
|
|
'JAVA_HOME',
|
|
getJavaHome(Integer.parseInt(
|
|
lines
|
|
.findAll({ it.startsWith("ES_BUILD_JAVA=") })
|
|
.collect({ it.replace("ES_BUILD_JAVA=java", "").trim() })
|
|
.collect({ it.replace("ES_BUILD_JAVA=openjdk", "").trim() })
|
|
.join("!!")
|
|
))
|
|
)
|
|
environment(
|
|
'RUNTIME_JAVA_HOME',
|
|
getJavaHome(Integer.parseInt(
|
|
lines
|
|
.findAll({ it.startsWith("ES_RUNTIME_JAVA=java") })
|
|
.collect({ it.replace("ES_RUNTIME_JAVA=java", "").trim() })
|
|
.join("!!")
|
|
))
|
|
)
|
|
}
|
|
|
|
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
|
|
executable 'cmd'
|
|
args '/C', 'call', new File(checkoutDir, 'gradlew').toString()
|
|
} else {
|
|
executable new File(checkoutDir, 'gradlew').toString()
|
|
}
|
|
if (gradle.startParameter.isOffline()) {
|
|
args "--offline"
|
|
}
|
|
String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url')
|
|
if (buildCacheUrl) {
|
|
args "-Dorg.elasticsearch.build.cache.url=${buildCacheUrl}"
|
|
}
|
|
|
|
args "-Dbuild.snapshot=true"
|
|
args "-Dscan.tag.NESTED"
|
|
final LogLevel logLevel = gradle.startParameter.logLevel
|
|
if ([LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG].contains(logLevel)) {
|
|
args "--${logLevel.name().toLowerCase(Locale.ENGLISH)}"
|
|
}
|
|
final String showStacktraceName = gradle.startParameter.showStacktrace.name()
|
|
assert ["INTERNAL_EXCEPTIONS", "ALWAYS", "ALWAYS_FULL"].contains(showStacktraceName)
|
|
if (showStacktraceName.equals("ALWAYS")) {
|
|
args "--stacktrace"
|
|
} else if (showStacktraceName.equals("ALWAYS_FULL")) {
|
|
args "--full-stacktrace"
|
|
}
|
|
if (gradle.getStartParameter().isParallelProjectExecutionEnabled()) {
|
|
args "--parallel"
|
|
}
|
|
standardOutput = new IndentingOutputStream(System.out, bwcVersion)
|
|
errorOutput = new IndentingOutputStream(System.err, bwcVersion)
|
|
configure extraConfig
|
|
}
|
|
}
|
|
|
|
Closure buildBwcTaskName = { projectName ->
|
|
return "buildBwc${projectName.replaceAll(/-\w/) { it[1].toUpperCase() }.capitalize()}"
|
|
}
|
|
|
|
def buildBwc = tasks.register("buildBwc");
|
|
|
|
Closure createBuildBwcTask = { projectName, projectDir, projectArtifact ->
|
|
def bwcTaskName = buildBwcTaskName(projectName)
|
|
createRunBwcGradleTask(bwcTaskName) {
|
|
inputs.file("${project.buildDir}/refspec")
|
|
outputs.files(projectArtifact)
|
|
outputs.cacheIf("BWC distribution caching is disabled on 'master' branch") {
|
|
// Don't bother caching in 'master' since the BWC branches move too quickly to make this cost worthwhile
|
|
BuildParams.ci && System.getenv('GIT_BRANCH')?.endsWith("master") == false
|
|
}
|
|
args ":${projectDir.replace('/', ':')}:assemble"
|
|
if (project.gradle.startParameter.buildCacheEnabled) {
|
|
args "--build-cache"
|
|
}
|
|
doLast {
|
|
if (projectArtifact.exists() == false) {
|
|
throw new InvalidUserDataException("Building ${bwcVersion} didn't generate expected file ${projectArtifact}")
|
|
}
|
|
}
|
|
}
|
|
buildBwc.configure {
|
|
dependsOn(bwcTaskName)
|
|
}
|
|
}
|
|
|
|
Map<String, File> artifactFiles = [:]
|
|
List<String> projectDirs = []
|
|
List<String> projects = ['deb', 'rpm']
|
|
if (bwcVersion.onOrAfter('7.0.0')) {
|
|
projects.addAll(['windows-zip', 'darwin-tar', 'linux-tar'])
|
|
} else {
|
|
projects.add('zip')
|
|
}
|
|
|
|
for (String projectName : projects) {
|
|
String baseDir = "distribution"
|
|
String classifier = ""
|
|
String extension = projectName
|
|
if (bwcVersion.onOrAfter('7.0.0') && (projectName.contains('zip') || projectName.contains('tar'))) {
|
|
int index = projectName.indexOf('-')
|
|
classifier = "-${projectName.substring(0, index)}-x86_64"
|
|
extension = projectName.substring(index + 1)
|
|
if (extension.equals('tar')) {
|
|
extension += '.gz'
|
|
}
|
|
}
|
|
if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('deb')) {
|
|
classifier = "-amd64"
|
|
}
|
|
if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('rpm')) {
|
|
classifier = "-x86_64"
|
|
}
|
|
if (bwcVersion.onOrAfter('6.3.0')) {
|
|
baseDir += projectName.endsWith('zip') || projectName.endsWith('tar') ? '/archives' : '/packages'
|
|
// add oss variant first
|
|
projectDirs.add("${baseDir}/oss-${projectName}")
|
|
File ossProjectArtifact = file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT${classifier}.${extension}")
|
|
artifactFiles.put("oss-" + projectName, ossProjectArtifact)
|
|
createBuildBwcTask("oss-${projectName}", "${baseDir}/oss-${projectName}", ossProjectArtifact)
|
|
}
|
|
projectDirs.add("${baseDir}/${projectName}")
|
|
File projectArtifact = file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT${classifier}.${extension}")
|
|
artifactFiles.put(projectName, projectArtifact)
|
|
|
|
createBuildBwcTask(projectName, "${baseDir}/${projectName}", projectArtifact)
|
|
}
|
|
|
|
createRunBwcGradleTask("resolveAllBwcDependencies") {
|
|
args 'resolveAllDependencies'
|
|
}
|
|
Version currentVersion = Version.fromString(version)
|
|
if (currentVersion.getMinor() == 0 && currentVersion.getRevision() == 0) {
|
|
// We only want to resolve dependencies for live versions of master, without cascading this to older versions
|
|
resolveAllDependencies.dependsOn resolveAllBwcDependencies
|
|
}
|
|
|
|
for (e in artifactFiles) {
|
|
String projectName = e.key
|
|
String buildBwcTask = buildBwcTaskName(projectName)
|
|
File artifactFile = e.value
|
|
String artifactFileName = artifactFile.name
|
|
String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
|
|
String suffix = artifactFile.toString()[-3..-1]
|
|
int archIndex = artifactFileName.indexOf('x86_64')
|
|
String classifier = ''
|
|
if (archIndex != -1) {
|
|
int osIndex = artifactFileName.lastIndexOf('-', archIndex - 2)
|
|
classifier = "${artifactFileName.substring(osIndex + 1, archIndex - 1)}-x86_64"
|
|
}
|
|
configurations.create(projectName)
|
|
artifacts {
|
|
it.add(projectName, [file: artifactFile, name: artifactName, classifier: classifier, type: suffix, builtBy: buildBwcTask])
|
|
}
|
|
}
|
|
// make sure no dependencies were added to assemble; we want it to be a no-op
|
|
assemble.dependsOn = []
|
|
}
|
|
}
|
|
|
|
class IndentingOutputStream extends OutputStream {
|
|
|
|
public final byte[] indent
|
|
private final OutputStream delegate
|
|
|
|
public IndentingOutputStream(OutputStream delegate, Object version) {
|
|
this.delegate = delegate
|
|
indent = " [${version}] ".getBytes(StandardCharsets.UTF_8)
|
|
}
|
|
|
|
@Override
|
|
public void write(int b) {
|
|
write([b] as int[], 0, 1)
|
|
}
|
|
|
|
public void write(int[] bytes, int offset, int length) {
|
|
for (int i = 0; i < bytes.length; i++) {
|
|
delegate.write(bytes[i])
|
|
if (bytes[i] == '\n') {
|
|
delegate.write(indent)
|
|
}
|
|
}
|
|
}
|
|
}
|