mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-02-06 21:18:31 +00:00
507d879b70
This commit disables caching of BWC snapshot distributions in the "trunk" (aka master) branch. Since the previous major release branches move quickly we rarely get cache hits for these tasks, and the artifacts themselves are very large. This means the overhead here is high and savings basically zero. We conditionally disable task output caching in this scenario in CI to avoid excessive build cache overhead as well as causing too much turn in the cache itself which would lead to lots of cache entry evictions.
340 lines
15 KiB
Groovy
340 lines
15 KiB
Groovy
/*
|
|
* Licensed to Elasticsearch under one or more contributor
|
|
* license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright
|
|
* ownership. Elasticsearch licenses this file to you under
|
|
* the Apache License, Version 2.0 (the "License"); you may
|
|
* not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing,
|
|
* software distributed under the License is distributed on an
|
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
* KIND, either express or implied. See the License for the
|
|
* specific language governing permissions and limitations
|
|
* under the License.
|
|
*/
|
|
|
|
import org.apache.tools.ant.taskdefs.condition.Os
|
|
import org.elasticsearch.gradle.LoggedExec
|
|
import org.elasticsearch.gradle.Version
|
|
import org.elasticsearch.gradle.BwcVersions
|
|
import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
|
|
|
|
import java.nio.charset.StandardCharsets
|
|
|
|
import static org.elasticsearch.gradle.BuildPlugin.getJavaHome
|
|
|
|
/**
|
|
* We want to be able to do BWC tests for unreleased versions without relying on and waiting for snapshots.
|
|
* For this we need to check out and build the unreleased versions.
|
|
* Since These depend on the current version, we can't name the Gradle projects statically, and don't know what the
|
|
* unreleased versions are when Gradle projects are set up, so we use "build-unreleased-version-*" as placeholders
|
|
* and configure them to build various versions here.
|
|
*/
|
|
bwcVersions.forPreviousUnreleased { BwcVersions.UnreleasedVersionInfo unreleasedVersion -> project("${unreleasedVersion.gradleProjectPath}") {
|
|
Version bwcVersion = unreleasedVersion.version
|
|
String bwcBranch = unreleasedVersion.branch
|
|
apply plugin: 'distribution'
|
|
// Not published so no need to assemble
|
|
assemble.enabled = false
|
|
|
|
File checkoutDir = file("${buildDir}/bwc/checkout-${bwcBranch}")
|
|
|
|
final String remote = System.getProperty("bwc.remote", "elastic")
|
|
|
|
boolean gitFetchLatest
|
|
final String gitFetchLatestProperty = System.getProperty("tests.bwc.git_fetch_latest", "true")
|
|
if ("true".equals(gitFetchLatestProperty)) {
|
|
gitFetchLatest = true
|
|
} else if ("false".equals(gitFetchLatestProperty)) {
|
|
gitFetchLatest = false
|
|
} else {
|
|
throw new GradleException("tests.bwc.git_fetch_latest must be [true] or [false] but was [" + gitFetchLatestProperty + "]")
|
|
}
|
|
|
|
task createClone(type: LoggedExec) {
|
|
onlyIf { checkoutDir.exists() == false }
|
|
commandLine = ['git', 'clone', rootDir, checkoutDir]
|
|
}
|
|
|
|
task findRemote(type: LoggedExec) {
|
|
dependsOn createClone
|
|
workingDir = checkoutDir
|
|
commandLine = ['git', 'remote', '-v']
|
|
ByteArrayOutputStream output = new ByteArrayOutputStream()
|
|
standardOutput = output
|
|
doLast {
|
|
project.ext.remoteExists = false
|
|
output.toString('UTF-8').eachLine {
|
|
if (it.contains("${remote}\t")) {
|
|
project.ext.remoteExists = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
task addRemote(type: LoggedExec) {
|
|
dependsOn findRemote
|
|
onlyIf { project.ext.remoteExists == false }
|
|
workingDir = checkoutDir
|
|
commandLine = ['git', 'remote', 'add', "${remote}", "https://github.com/${remote}/elasticsearch.git"]
|
|
}
|
|
|
|
task fetchLatest(type: LoggedExec) {
|
|
onlyIf { project.gradle.startParameter.isOffline() == false && gitFetchLatest }
|
|
dependsOn addRemote
|
|
workingDir = checkoutDir
|
|
commandLine = ['git', 'fetch', '--all']
|
|
}
|
|
|
|
Closure execGit = { Action<ExecSpec> action ->
|
|
new ByteArrayOutputStream().withStream { os ->
|
|
ExecResult result = project.exec { spec ->
|
|
workingDir = checkoutDir
|
|
standardOutput os
|
|
action.execute(spec)
|
|
}
|
|
result.assertNormalExitValue()
|
|
return os.toString().trim()
|
|
}
|
|
}
|
|
task checkoutBwcBranch() {
|
|
dependsOn fetchLatest
|
|
doLast {
|
|
String refspec = System.getProperty("bwc.refspec.${bwcBranch}") ?: System.getProperty("tests.bwc.refspec.${bwcBranch}") ?: "${remote}/${bwcBranch}"
|
|
if (System.getProperty("bwc.checkout.align") != null) {
|
|
/*
|
|
We use a time based approach to make the bwc versions built deterministic and compatible with the current hash.
|
|
Most of the time we want to test against latest, but when running delayed exhaustive tests or wanting
|
|
reproducible builds we want this to be deterministic by using a hash that was the latest when the current
|
|
commit was made.
|
|
|
|
This approach doesn't work with merge commits as these can introduce commits in the chronological order
|
|
after the fact e.x. a merge done today can add commits dated with yesterday so the result will no longer be
|
|
deterministic.
|
|
|
|
We don't use merge commits, but for additional safety we check that no such commits exist in the time period
|
|
we are interested in.
|
|
|
|
Timestamps are at seconds resolution. rev-parse --before and --after are inclusive w.r.t the second
|
|
passed as input. This means the results might not be deterministic in the current second, but this
|
|
should not matter in practice.
|
|
*/
|
|
String timeOfCurrent = execGit { spec ->
|
|
spec.commandLine 'git', 'show', '--no-patch', '--no-notes', "--pretty='%cD'"
|
|
spec.workingDir project.rootDir
|
|
}
|
|
logger.lifecycle("Commit date of current: {}", timeOfCurrent)
|
|
String mergeCommits = execGit { spec ->
|
|
spec.commandLine "git", "rev-list", refspec, "--after", timeOfCurrent, "--merges"
|
|
}
|
|
if (mergeCommits.isEmpty() == false) {
|
|
throw new IllegalStateException(
|
|
"Found the following merge commits which prevent determining bwc commits: " + mergeCommits
|
|
)
|
|
}
|
|
refspec = execGit { spec ->
|
|
spec.commandLine "git", "rev-list", refspec, "-n", "1", "--before", timeOfCurrent, "--date-order"
|
|
}
|
|
}
|
|
|
|
logger.lifecycle("Performing checkout of ${refspec}...")
|
|
LoggedExec.exec(project) { spec ->
|
|
spec.workingDir = checkoutDir
|
|
spec.commandLine "git", "checkout", refspec
|
|
}
|
|
String checkoutHash = GlobalBuildInfoPlugin.gitRevision(checkoutDir)
|
|
logger.lifecycle("Checkout hash for ${project.path} is ${checkoutHash}")
|
|
file("${project.buildDir}/refspec").text = checkoutHash
|
|
}
|
|
}
|
|
|
|
|
|
Closure createRunBwcGradleTask = { name, extraConfig ->
|
|
return tasks.create(name: "$name", type: LoggedExec) {
|
|
dependsOn checkoutBwcBranch
|
|
spoolOutput = true
|
|
workingDir = checkoutDir
|
|
doFirst {
|
|
// Execution time so that the checkouts are available
|
|
List<String> lines = file("${checkoutDir}/.ci/java-versions.properties").readLines()
|
|
environment(
|
|
'JAVA_HOME',
|
|
getJavaHome(it, Integer.parseInt(
|
|
lines
|
|
.findAll({ it.startsWith("ES_BUILD_JAVA=") })
|
|
.collect({ it.replace("ES_BUILD_JAVA=java", "").trim() })
|
|
.collect({ it.replace("ES_BUILD_JAVA=openjdk", "").trim() })
|
|
.join("!!")
|
|
))
|
|
)
|
|
environment(
|
|
'RUNTIME_JAVA_HOME',
|
|
getJavaHome(it, Integer.parseInt(
|
|
lines
|
|
.findAll({ it.startsWith("ES_RUNTIME_JAVA=java") })
|
|
.collect({ it.replace("ES_RUNTIME_JAVA=java", "").trim() })
|
|
.join("!!")
|
|
))
|
|
)
|
|
}
|
|
|
|
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
|
|
executable 'cmd'
|
|
args '/C', 'call', new File(checkoutDir, 'gradlew').toString()
|
|
} else {
|
|
executable new File(checkoutDir, 'gradlew').toString()
|
|
}
|
|
if (gradle.startParameter.isOffline()) {
|
|
args "--offline"
|
|
}
|
|
|
|
args "-Dbuild.snapshot=true"
|
|
final LogLevel logLevel = gradle.startParameter.logLevel
|
|
if ([LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG].contains(logLevel)) {
|
|
args "--${logLevel.name().toLowerCase(Locale.ENGLISH)}"
|
|
}
|
|
final String showStacktraceName = gradle.startParameter.showStacktrace.name()
|
|
assert ["INTERNAL_EXCEPTIONS", "ALWAYS", "ALWAYS_FULL"].contains(showStacktraceName)
|
|
if (showStacktraceName.equals("ALWAYS")) {
|
|
args "--stacktrace"
|
|
} else if (showStacktraceName.equals("ALWAYS_FULL")) {
|
|
args "--full-stacktrace"
|
|
}
|
|
if (gradle.getStartParameter().isParallelProjectExecutionEnabled()) {
|
|
args "--parallel"
|
|
}
|
|
standardOutput = new IndentingOutputStream(System.out, bwcVersion)
|
|
errorOutput = new IndentingOutputStream(System.err, bwcVersion)
|
|
configure extraConfig
|
|
}
|
|
}
|
|
|
|
Closure buildBwcTaskName = { projectName ->
|
|
return "buildBwc${projectName.replaceAll(/-\w/){ it[1].toUpperCase() }.capitalize()}"
|
|
}
|
|
|
|
task buildBwc {}
|
|
|
|
Closure createBuildBwcTask = { projectName, projectDir, projectArtifact ->
|
|
Task bwcTask = createRunBwcGradleTask(buildBwcTaskName(projectName)) {
|
|
inputs.file("${project.buildDir}/refspec")
|
|
outputs.files(projectArtifact)
|
|
outputs.cacheIf("BWC distribution caching is disabled on 'master' branch") {
|
|
// Don't bother caching in 'master' since the BWC branches move too quickly to make this cost worthwhile
|
|
project.ext.isCi && System.getenv('GIT_BRANCH')?.endsWith("master") == false
|
|
}
|
|
args ":${projectDir.replace('/', ':')}:assemble"
|
|
if (project.gradle.startParameter.buildCacheEnabled) {
|
|
args "--build-cache"
|
|
}
|
|
doLast {
|
|
if (projectArtifact.exists() == false) {
|
|
throw new InvalidUserDataException("Building ${bwcVersion} didn't generate expected file ${projectArtifact}")
|
|
}
|
|
}
|
|
}
|
|
buildBwc.dependsOn bwcTask
|
|
}
|
|
|
|
Map<String, File> artifactFiles = [:]
|
|
List<String> projectDirs = []
|
|
List<String> projects = ['deb', 'rpm']
|
|
if (bwcVersion.onOrAfter('7.0.0')) {
|
|
projects.addAll(['windows-zip', 'darwin-tar', 'linux-tar'])
|
|
} else {
|
|
projects.add('zip')
|
|
}
|
|
|
|
for (String projectName : projects) {
|
|
String baseDir = "distribution"
|
|
String classifier = ""
|
|
String extension = projectName
|
|
if (bwcVersion.onOrAfter('7.0.0') && (projectName.contains('zip') || projectName.contains('tar'))) {
|
|
int index = projectName.indexOf('-')
|
|
classifier = "-${projectName.substring(0, index)}-x86_64"
|
|
extension = projectName.substring(index + 1)
|
|
if (extension.equals('tar')) {
|
|
extension += '.gz'
|
|
}
|
|
}
|
|
if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('deb')) {
|
|
classifier = "-amd64"
|
|
}
|
|
if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('rpm')) {
|
|
classifier = "-x86_64"
|
|
}
|
|
if (bwcVersion.onOrAfter('6.3.0')) {
|
|
baseDir += projectName.endsWith('zip') || projectName.endsWith('tar') ? '/archives' : '/packages'
|
|
// add oss variant first
|
|
projectDirs.add("${baseDir}/oss-${projectName}")
|
|
File ossProjectArtifact = file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT${classifier}.${extension}")
|
|
artifactFiles.put("oss-" + projectName, ossProjectArtifact)
|
|
createBuildBwcTask("oss-${projectName}", "${baseDir}/oss-${projectName}", ossProjectArtifact)
|
|
}
|
|
projectDirs.add("${baseDir}/${projectName}")
|
|
File projectArtifact = file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT${classifier}.${extension}")
|
|
artifactFiles.put(projectName, projectArtifact)
|
|
|
|
createBuildBwcTask(projectName, "${baseDir}/${projectName}", projectArtifact)
|
|
}
|
|
|
|
createRunBwcGradleTask("resolveAllBwcDependencies") {
|
|
args 'resolveAllDependencies'
|
|
}
|
|
Version currentVersion = Version.fromString(version)
|
|
if (currentVersion.getMinor() == 0 && currentVersion.getRevision() == 0) {
|
|
// We only want to resolve dependencies for live versions of master, without cascading this to older versions
|
|
resolveAllDependencies.dependsOn resolveAllBwcDependencies
|
|
}
|
|
|
|
for (e in artifactFiles) {
|
|
String projectName = e.key
|
|
String buildBwcTask = buildBwcTaskName(projectName)
|
|
File artifactFile = e.value
|
|
String artifactFileName = artifactFile.name
|
|
String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
|
|
String suffix = artifactFile.toString()[-3..-1]
|
|
int archIndex = artifactFileName.indexOf('x86_64')
|
|
String classifier = ''
|
|
if (archIndex != -1) {
|
|
int osIndex = artifactFileName.lastIndexOf('-', archIndex - 2)
|
|
classifier = "${artifactFileName.substring(osIndex + 1, archIndex - 1)}-x86_64"
|
|
}
|
|
configurations.create(projectName)
|
|
artifacts {
|
|
it.add(projectName, [file: artifactFile, name: artifactName, classifier: classifier, type: suffix, builtBy: buildBwcTask])
|
|
}
|
|
}
|
|
// make sure no dependencies were added to assemble; we want it to be a no-op
|
|
assemble.dependsOn = []
|
|
}}
|
|
|
|
class IndentingOutputStream extends OutputStream {
|
|
|
|
public final byte[] indent
|
|
private final OutputStream delegate
|
|
|
|
public IndentingOutputStream(OutputStream delegate, Object version) {
|
|
this.delegate = delegate
|
|
indent = " [${version}] ".getBytes(StandardCharsets.UTF_8)
|
|
}
|
|
|
|
@Override
|
|
public void write(int b) {
|
|
write([b] as int[], 0, 1)
|
|
}
|
|
|
|
public void write(int[] bytes, int offset, int length) {
|
|
for (int i = 0; i < bytes.length; i++) {
|
|
delegate.write(bytes[i])
|
|
if (bytes[i] == '\n') {
|
|
delegate.write(indent)
|
|
}
|
|
}
|
|
}
|
|
}
|