/* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright * ownership. Elasticsearch licenses this file to you under * the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ import org.apache.tools.ant.taskdefs.condition.Os import org.elasticsearch.gradle.LoggedExec import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.BwcVersions import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin import org.gradle.util.GradleVersion import java.nio.charset.StandardCharsets import static org.elasticsearch.gradle.util.JavaUtil.getJavaHome /** * We want to be able to do BWC tests for unreleased versions without relying on and waiting for snapshots. * For this we need to check out and build the unreleased versions. * Since These depend on the current version, we can't name the Gradle projects statically, and don't know what the * unreleased versions are when Gradle projects are set up, so we use "build-unreleased-version-*" as placeholders * and configure them to build various versions here. */ bwcVersions.forPreviousUnreleased { BwcVersions.UnreleasedVersionInfo unreleasedVersion -> project("${unreleasedVersion.gradleProjectPath}") { Version bwcVersion = unreleasedVersion.version String bwcBranch = unreleasedVersion.branch apply plugin: 'distribution' // Not published so no need to assemble assemble.enabled = false File checkoutDir = file("${buildDir}/bwc/checkout-${bwcBranch}") final String remote = System.getProperty("bwc.remote", "elastic") boolean gitFetchLatest final String gitFetchLatestProperty = System.getProperty("tests.bwc.git_fetch_latest", "true") if ("true".equals(gitFetchLatestProperty)) { gitFetchLatest = true } else if ("false".equals(gitFetchLatestProperty)) { gitFetchLatest = false } else { throw new GradleException("tests.bwc.git_fetch_latest must be [true] or [false] but was [" + gitFetchLatestProperty + "]") } task createClone(type: LoggedExec) { onlyIf { checkoutDir.exists() == false } commandLine = ['git', 'clone', rootDir, checkoutDir] } task findRemote(type: LoggedExec) { dependsOn createClone workingDir = checkoutDir commandLine = ['git', 'remote', '-v'] ByteArrayOutputStream output = new ByteArrayOutputStream() standardOutput = output doLast { project.ext.remoteExists = false output.toString('UTF-8').eachLine { if (it.contains("${remote}\t")) { project.ext.remoteExists = true } } } } task addRemote(type: LoggedExec) { dependsOn findRemote onlyIf { project.ext.remoteExists == false } workingDir = checkoutDir commandLine = ['git', 'remote', 'add', "${remote}", "https://github.com/${remote}/elasticsearch.git"] } task fetchLatest(type: LoggedExec) { onlyIf { project.gradle.startParameter.isOffline() == false && gitFetchLatest } dependsOn addRemote workingDir = checkoutDir commandLine = ['git', 'fetch', '--all'] } Closure execGit = { Action action -> new ByteArrayOutputStream().withStream { os -> ExecResult result = project.exec { spec -> workingDir = checkoutDir standardOutput os action.execute(spec) } result.assertNormalExitValue() return os.toString().trim() } } task checkoutBwcBranch() { dependsOn fetchLatest doLast { String refspec = System.getProperty("bwc.refspec.${bwcBranch}") ?: System.getProperty("tests.bwc.refspec.${bwcBranch}") ?: "${remote}/${bwcBranch}" if (System.getProperty("bwc.checkout.align") != null) { /* We use a time based approach to make the bwc versions built deterministic and compatible with the current hash. Most of the time we want to test against latest, but when running delayed exhaustive tests or wanting reproducible builds we want this to be deterministic by using a hash that was the latest when the current commit was made. This approach doesn't work with merge commits as these can introduce commits in the chronological order after the fact e.x. a merge done today can add commits dated with yesterday so the result will no longer be deterministic. We don't use merge commits, but for additional safety we check that no such commits exist in the time period we are interested in. Timestamps are at seconds resolution. rev-parse --before and --after are inclusive w.r.t the second passed as input. This means the results might not be deterministic in the current second, but this should not matter in practice. */ String timeOfCurrent = execGit { spec -> spec.commandLine 'git', 'show', '--no-patch', '--no-notes', "--pretty='%cD'" spec.workingDir project.rootDir } logger.lifecycle("Commit date of current: {}", timeOfCurrent) String mergeCommits = execGit { spec -> spec.commandLine "git", "rev-list", refspec, "--after", timeOfCurrent, "--merges" } if (mergeCommits.isEmpty() == false) { throw new IllegalStateException( "Found the following merge commits which prevent determining bwc commits: " + mergeCommits ) } refspec = execGit { spec -> spec.commandLine "git", "rev-list", refspec, "-n", "1", "--before", timeOfCurrent, "--date-order" } } logger.lifecycle("Performing checkout of ${refspec}...") LoggedExec.exec(project) { spec -> spec.workingDir = checkoutDir spec.commandLine "git", "checkout", refspec } String checkoutHash = GlobalBuildInfoPlugin.gitInfo(checkoutDir).revision logger.lifecycle("Checkout hash for ${project.path} is ${checkoutHash}") file("${project.buildDir}/refspec").text = checkoutHash } } Closure createRunBwcGradleTask = { name, extraConfig -> return tasks.create(name: "$name", type: LoggedExec) { dependsOn checkoutBwcBranch spoolOutput = true workingDir = checkoutDir doFirst { // Execution time so that the checkouts are available List lines = file("${checkoutDir}/.ci/java-versions.properties").readLines() environment( 'JAVA_HOME', getJavaHome(Integer.parseInt( lines .findAll({ it.startsWith("ES_BUILD_JAVA=") }) .collect({ it.replace("ES_BUILD_JAVA=java", "").trim() }) .collect({ it.replace("ES_BUILD_JAVA=openjdk", "").trim() }) .join("!!") )) ) environment( 'RUNTIME_JAVA_HOME', getJavaHome(Integer.parseInt( lines .findAll({ it.startsWith("ES_RUNTIME_JAVA=java") }) .collect({ it.replace("ES_RUNTIME_JAVA=java", "").trim() }) .join("!!") )) ) } if (Os.isFamily(Os.FAMILY_WINDOWS)) { executable 'cmd' args '/C', 'call', new File(checkoutDir, 'gradlew').toString() } else { executable new File(checkoutDir, 'gradlew').toString() } if (gradle.startParameter.isOffline()) { args "--offline" } String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url') if (buildCacheUrl) { args "-Dorg.elasticsearch.build.cache.url=${buildCacheUrl}" } args "-Dbuild.snapshot=true" args "-Dscan.tag.NESTED" final LogLevel logLevel = gradle.startParameter.logLevel if ([LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG].contains(logLevel)) { args "--${logLevel.name().toLowerCase(Locale.ENGLISH)}" } final String showStacktraceName = gradle.startParameter.showStacktrace.name() assert ["INTERNAL_EXCEPTIONS", "ALWAYS", "ALWAYS_FULL"].contains(showStacktraceName) if (showStacktraceName.equals("ALWAYS")) { args "--stacktrace" } else if (showStacktraceName.equals("ALWAYS_FULL")) { args "--full-stacktrace" } if (gradle.getStartParameter().isParallelProjectExecutionEnabled()) { args "--parallel" } standardOutput = new IndentingOutputStream(System.out, bwcVersion) errorOutput = new IndentingOutputStream(System.err, bwcVersion) configure extraConfig } } Closure buildBwcTaskName = { projectName -> return "buildBwc${projectName.replaceAll(/-\w/) { it[1].toUpperCase() }.capitalize()}" } task buildBwc {} Closure createBuildBwcTask = { projectName, projectDir, projectArtifact -> Task bwcTask = createRunBwcGradleTask(buildBwcTaskName(projectName)) { inputs.file("${project.buildDir}/refspec") outputs.files(projectArtifact) outputs.cacheIf("BWC distribution caching is disabled on 'master' branch") { // Don't bother caching in 'master' since the BWC branches move too quickly to make this cost worthwhile BuildParams.ci && System.getenv('GIT_BRANCH')?.endsWith("master") == false } args ":${projectDir.replace('/', ':')}:assemble" if (project.gradle.startParameter.buildCacheEnabled) { args "--build-cache" } doLast { if (projectArtifact.exists() == false) { throw new InvalidUserDataException("Building ${bwcVersion} didn't generate expected file ${projectArtifact}") } } } buildBwc.dependsOn bwcTask } Map artifactFiles = [:] List projectDirs = [] List projects = ['deb', 'rpm'] if (bwcVersion.onOrAfter('7.0.0')) { projects.addAll(['windows-zip', 'darwin-tar', 'linux-tar']) } else { projects.add('zip') } for (String projectName : projects) { String baseDir = "distribution" String classifier = "" String extension = projectName if (bwcVersion.onOrAfter('7.0.0') && (projectName.contains('zip') || projectName.contains('tar'))) { int index = projectName.indexOf('-') classifier = "-${projectName.substring(0, index)}-x86_64" extension = projectName.substring(index + 1) if (extension.equals('tar')) { extension += '.gz' } } if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('deb')) { classifier = "-amd64" } if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('rpm')) { classifier = "-x86_64" } if (bwcVersion.onOrAfter('6.3.0')) { baseDir += projectName.endsWith('zip') || projectName.endsWith('tar') ? '/archives' : '/packages' // add oss variant first projectDirs.add("${baseDir}/oss-${projectName}") File ossProjectArtifact = file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT${classifier}.${extension}") artifactFiles.put("oss-" + projectName, ossProjectArtifact) createBuildBwcTask("oss-${projectName}", "${baseDir}/oss-${projectName}", ossProjectArtifact) } projectDirs.add("${baseDir}/${projectName}") File projectArtifact = file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT${classifier}.${extension}") artifactFiles.put(projectName, projectArtifact) createBuildBwcTask(projectName, "${baseDir}/${projectName}", projectArtifact) } createRunBwcGradleTask("resolveAllBwcDependencies") { args 'resolveAllDependencies' } Version currentVersion = Version.fromString(version) if (currentVersion.getMinor() == 0 && currentVersion.getRevision() == 0) { // We only want to resolve dependencies for live versions of master, without cascading this to older versions resolveAllDependencies.dependsOn resolveAllBwcDependencies } for (e in artifactFiles) { String projectName = e.key String buildBwcTask = buildBwcTaskName(projectName) File artifactFile = e.value String artifactFileName = artifactFile.name String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch' String suffix = artifactFile.toString()[-3..-1] int archIndex = artifactFileName.indexOf('x86_64') String classifier = '' if (archIndex != -1) { int osIndex = artifactFileName.lastIndexOf('-', archIndex - 2) classifier = "${artifactFileName.substring(osIndex + 1, archIndex - 1)}-x86_64" } configurations.create(projectName) artifacts { it.add(projectName, [file: artifactFile, name: artifactName, classifier: classifier, type: suffix, builtBy: buildBwcTask]) } } // make sure no dependencies were added to assemble; we want it to be a no-op assemble.dependsOn = [] } } class IndentingOutputStream extends OutputStream { public final byte[] indent private final OutputStream delegate public IndentingOutputStream(OutputStream delegate, Object version) { this.delegate = delegate indent = " [${version}] ".getBytes(StandardCharsets.UTF_8) } @Override public void write(int b) { write([b] as int[], 0, 1) } public void write(int[] bytes, int offset, int length) { for (int i = 0; i < bytes.length; i++) { delegate.write(bytes[i]) if (bytes[i] == '\n') { delegate.write(indent) } } } }