mirror of
https://github.com/honeymoose/OpenSearch.git
synced 2025-02-06 13:08:29 +00:00
4dbf6c0df9
Before this change we would recurse to cache bwc versions. This proved to be problematic due to the number of steps it was generating taking too long. Also this required tricky maintenance to break the recursion for old branches we don't really care about. With this change we now cache specific branches only.
298 lines
12 KiB
Groovy
298 lines
12 KiB
Groovy
/*
|
|
* Licensed to Elasticsearch under one or more contributor
|
|
* license agreements. See the NOTICE file distributed with
|
|
* this work for additional information regarding copyright
|
|
* ownership. Elasticsearch licenses this file to you under
|
|
* the Apache License, Version 2.0 (the "License"); you may
|
|
* not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing,
|
|
* software distributed under the License is distributed on an
|
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
* KIND, either express or implied. See the License for the
|
|
* specific language governing permissions and limitations
|
|
* under the License.
|
|
*/
|
|
|
|
import org.apache.tools.ant.taskdefs.condition.Os
|
|
import org.elasticsearch.gradle.LoggedExec
|
|
import org.elasticsearch.gradle.Version
|
|
import org.elasticsearch.gradle.BwcVersions
|
|
|
|
import java.nio.charset.StandardCharsets
|
|
|
|
import static org.elasticsearch.gradle.BuildPlugin.getJavaHome
|
|
|
|
/**
|
|
* We want to be able to do BWC tests for unreleased versions without relying on and waiting for snapshots.
|
|
* For this we need to check out and build the unreleased versions.
|
|
* Since These depend on the current version, we can't name the Gradle projects statically, and don't know what the
|
|
* unreleased versions are when Gradle projects are set up, so we use "build-unreleased-version-*" as placeholders
|
|
* and configure them to build various versions here.
|
|
*/
|
|
bwcVersions.forPreviousUnreleased { BwcVersions.UnreleasedVersionInfo unreleasedVersion -> project("${unreleasedVersion.gradleProjectPath}") {
|
|
Version bwcVersion = unreleasedVersion.version
|
|
String bwcBranch = unreleasedVersion.branch
|
|
apply plugin: 'distribution'
|
|
// Not published so no need to assemble
|
|
assemble.enabled = false
|
|
|
|
File checkoutDir = file("${buildDir}/bwc/checkout-${bwcBranch}")
|
|
|
|
final String remote = System.getProperty("tests.bwc.remote", "elastic")
|
|
|
|
boolean gitFetchLatest
|
|
final String gitFetchLatestProperty = System.getProperty("tests.bwc.git_fetch_latest", "true")
|
|
if ("true".equals(gitFetchLatestProperty)) {
|
|
gitFetchLatest = true
|
|
} else if ("false".equals(gitFetchLatestProperty)) {
|
|
gitFetchLatest = false
|
|
} else {
|
|
throw new GradleException("tests.bwc.git_fetch_latest must be [true] or [false] but was [" + gitFetchLatestProperty + "]")
|
|
}
|
|
|
|
task createClone(type: LoggedExec) {
|
|
onlyIf { checkoutDir.exists() == false }
|
|
commandLine = ['git', 'clone', rootDir, checkoutDir]
|
|
}
|
|
|
|
task findRemote(type: LoggedExec) {
|
|
dependsOn createClone
|
|
workingDir = checkoutDir
|
|
commandLine = ['git', 'remote', '-v']
|
|
ByteArrayOutputStream output = new ByteArrayOutputStream()
|
|
standardOutput = output
|
|
doLast {
|
|
project.ext.remoteExists = false
|
|
output.toString('UTF-8').eachLine {
|
|
if (it.contains("${remote}\t")) {
|
|
project.ext.remoteExists = true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
task addRemote(type: LoggedExec) {
|
|
dependsOn findRemote
|
|
onlyIf { project.ext.remoteExists == false }
|
|
workingDir = checkoutDir
|
|
commandLine = ['git', 'remote', 'add', "${remote}", "https://github.com/${remote}/elasticsearch.git"]
|
|
}
|
|
|
|
task fetchLatest(type: LoggedExec) {
|
|
onlyIf { project.gradle.startParameter.isOffline() == false && gitFetchLatest }
|
|
dependsOn addRemote
|
|
workingDir = checkoutDir
|
|
commandLine = ['git', 'fetch', '--all']
|
|
}
|
|
|
|
String buildMetadataKey = "bwc_refspec_${project.path.substring(1)}"
|
|
task checkoutBwcBranch(type: LoggedExec) {
|
|
String refspec = System.getProperty("tests.bwc.refspec.${bwcBranch}", buildMetadata.get(buildMetadataKey, "${remote}/${bwcBranch}"))
|
|
dependsOn fetchLatest
|
|
workingDir = checkoutDir
|
|
commandLine = ['git', 'checkout', refspec]
|
|
doFirst {
|
|
println "Checking out elasticsearch ${refspec} for branch ${bwcBranch}"
|
|
}
|
|
}
|
|
|
|
File buildMetadataFile = project.file("build/${project.name}/build_metadata")
|
|
task writeBuildMetadata(type: LoggedExec) {
|
|
dependsOn checkoutBwcBranch
|
|
workingDir = checkoutDir
|
|
commandLine = ['git', 'rev-parse', 'HEAD']
|
|
ignoreExitValue = true
|
|
ByteArrayOutputStream output = new ByteArrayOutputStream()
|
|
standardOutput = output
|
|
doLast {
|
|
if (execResult.exitValue != 0) {
|
|
output.toString('UTF-8').eachLine { line -> logger.error(line) }
|
|
execResult.assertNormalExitValue()
|
|
}
|
|
project.mkdir(buildMetadataFile.parent)
|
|
String commit = output.toString('UTF-8')
|
|
buildMetadataFile.setText("${buildMetadataKey}=${commit}", 'UTF-8')
|
|
println "Checked out elasticsearch commit ${commit}"
|
|
}
|
|
}
|
|
|
|
Closure createRunBwcGradleTask = { name, extraConfig ->
|
|
return tasks.create(name: "$name", type: LoggedExec) {
|
|
dependsOn checkoutBwcBranch, writeBuildMetadata
|
|
spoolOutput = true
|
|
workingDir = checkoutDir
|
|
doFirst {
|
|
// Execution time so that the checkouts are available
|
|
List<String> lines = file("${checkoutDir}/.ci/java-versions.properties").readLines()
|
|
environment(
|
|
'JAVA_HOME',
|
|
getJavaHome(it, Integer.parseInt(
|
|
lines
|
|
.findAll({ it.startsWith("ES_BUILD_JAVA=") })
|
|
.collect({ it.replace("ES_BUILD_JAVA=java", "").trim() })
|
|
.collect({ it.replace("ES_BUILD_JAVA=openjdk", "").trim() })
|
|
.join("!!")
|
|
))
|
|
)
|
|
environment(
|
|
'RUNTIME_JAVA_HOME',
|
|
getJavaHome(it, Integer.parseInt(
|
|
lines
|
|
.findAll({ it.startsWith("ES_RUNTIME_JAVA=java") })
|
|
.collect({ it.replace("ES_RUNTIME_JAVA=java", "").trim() })
|
|
.join("!!")
|
|
))
|
|
)
|
|
}
|
|
|
|
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
|
|
executable 'cmd'
|
|
args '/C', 'call', new File(checkoutDir, 'gradlew').toString()
|
|
} else {
|
|
executable new File(checkoutDir, 'gradlew').toString()
|
|
}
|
|
if (gradle.startParameter.isOffline()) {
|
|
args "--offline"
|
|
}
|
|
|
|
args "-Dbuild.snapshot=true"
|
|
final LogLevel logLevel = gradle.startParameter.logLevel
|
|
if ([LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG].contains(logLevel)) {
|
|
args "--${logLevel.name().toLowerCase(Locale.ENGLISH)}"
|
|
}
|
|
final String showStacktraceName = gradle.startParameter.showStacktrace.name()
|
|
assert ["INTERNAL_EXCEPTIONS", "ALWAYS", "ALWAYS_FULL"].contains(showStacktraceName)
|
|
if (showStacktraceName.equals("ALWAYS")) {
|
|
args "--stacktrace"
|
|
} else if (showStacktraceName.equals("ALWAYS_FULL")) {
|
|
args "--full-stacktrace"
|
|
}
|
|
if (gradle.getStartParameter().isParallelProjectExecutionEnabled()) {
|
|
args "--parallel"
|
|
}
|
|
standardOutput = new IndentingOutputStream(System.out, bwcVersion)
|
|
errorOutput = new IndentingOutputStream(System.err, bwcVersion)
|
|
configure extraConfig
|
|
}
|
|
}
|
|
|
|
Closure buildBwcTaskName = { projectName ->
|
|
return "buildBwc${projectName.replaceAll(/-\w/){ it[1].toUpperCase() }.capitalize()}"
|
|
}
|
|
|
|
task buildBwc {}
|
|
|
|
Closure createBuildBwcTask = { projectName, projectDir, projectArtifact ->
|
|
Task bwcTask = createRunBwcGradleTask(buildBwcTaskName(projectName)) {
|
|
args ":${projectDir.replace('/', ':')}:assemble"
|
|
doLast {
|
|
if (projectArtifact.exists() == false) {
|
|
throw new InvalidUserDataException("Building ${bwcVersion} didn't generate expected file ${projectArtifact}")
|
|
}
|
|
}
|
|
}
|
|
buildBwc.dependsOn bwcTask
|
|
}
|
|
|
|
Map<String, File> artifactFiles = [:]
|
|
List<String> projectDirs = []
|
|
List<String> projects = ['deb', 'rpm']
|
|
if (bwcVersion.onOrAfter('7.0.0')) {
|
|
projects.addAll(['windows-zip', 'darwin-tar', 'linux-tar'])
|
|
} else {
|
|
projects.add('zip')
|
|
}
|
|
|
|
for (String projectName : projects) {
|
|
String baseDir = "distribution"
|
|
String classifier = ""
|
|
String extension = projectName
|
|
if (bwcVersion.onOrAfter('7.0.0') && (projectName.contains('zip') || projectName.contains('tar'))) {
|
|
int index = projectName.indexOf('-')
|
|
classifier = "-${projectName.substring(0, index)}-x86_64"
|
|
extension = projectName.substring(index + 1)
|
|
if (extension.equals('tar')) {
|
|
extension += '.gz'
|
|
}
|
|
}
|
|
if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('deb')) {
|
|
classifier = "-amd64"
|
|
}
|
|
if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('rpm')) {
|
|
classifier = "-x86_64"
|
|
}
|
|
if (bwcVersion.onOrAfter('6.3.0')) {
|
|
baseDir += projectName.endsWith('zip') || projectName.endsWith('tar') ? '/archives' : '/packages'
|
|
// add oss variant first
|
|
projectDirs.add("${baseDir}/oss-${projectName}")
|
|
File ossProjectArtifact = file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT${classifier}.${extension}")
|
|
artifactFiles.put("oss-" + projectName, ossProjectArtifact)
|
|
createBuildBwcTask("oss-${projectName}", "${baseDir}/oss-${projectName}", ossProjectArtifact)
|
|
}
|
|
projectDirs.add("${baseDir}/${projectName}")
|
|
File projectArtifact = file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT${classifier}.${extension}")
|
|
artifactFiles.put(projectName, projectArtifact)
|
|
|
|
createBuildBwcTask(projectName, "${baseDir}/${projectName}", projectArtifact)
|
|
}
|
|
|
|
createRunBwcGradleTask("resolveAllBwcDependencies") {
|
|
args 'resolveAllDependencies'
|
|
}
|
|
Version currentVersion = Version.fromString(version)
|
|
if (currentVersion.getMinor() == 0 && currentVersion.getRevision() == 0) {
|
|
// We only want to resolve dependencies for live versions of master, without cascading this to older versions
|
|
resolveAllDependencies.dependsOn resolveAllBwcDependencies
|
|
}
|
|
|
|
for (e in artifactFiles) {
|
|
String projectName = e.key
|
|
String buildBwcTask = buildBwcTaskName(projectName)
|
|
File artifactFile = e.value
|
|
String artifactFileName = artifactFile.name
|
|
String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
|
|
String suffix = artifactFile.toString()[-3..-1]
|
|
int archIndex = artifactFileName.indexOf('x86_64')
|
|
String classifier = ''
|
|
if (archIndex != -1) {
|
|
int osIndex = artifactFileName.lastIndexOf('-', archIndex - 2)
|
|
classifier = "${artifactFileName.substring(osIndex + 1, archIndex - 1)}-x86_64"
|
|
}
|
|
configurations.create(projectName)
|
|
artifacts {
|
|
it.add(projectName, [file: artifactFile, name: artifactName, classifier: classifier, type: suffix, builtBy: buildBwcTask])
|
|
}
|
|
}
|
|
// make sure no dependencies were added to assemble; we want it to be a no-op
|
|
assemble.dependsOn = []
|
|
}}
|
|
|
|
class IndentingOutputStream extends OutputStream {
|
|
|
|
public final byte[] indent
|
|
private final OutputStream delegate
|
|
|
|
public IndentingOutputStream(OutputStream delegate, Object version) {
|
|
this.delegate = delegate
|
|
indent = " [${version}] ".getBytes(StandardCharsets.UTF_8)
|
|
}
|
|
|
|
@Override
|
|
public void write(int b) {
|
|
write([b] as int[], 0, 1)
|
|
}
|
|
|
|
public void write(int[] bytes, int offset, int length) {
|
|
for (int i = 0; i < bytes.length; i++) {
|
|
delegate.write(bytes[i])
|
|
if (bytes[i] == '\n') {
|
|
delegate.write(indent)
|
|
}
|
|
}
|
|
}
|
|
}
|