OpenSearch/distribution/bwc/build.gradle

/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.BwcVersions
import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin

import java.nio.charset.StandardCharsets

import static org.elasticsearch.gradle.BuildPlugin.getJavaHome

/**
 *  We want to be able to do BWC tests for unreleased versions without relying on and waiting for snapshots.
 *  For this we need to check out and build the unreleased versions.
 *  Since These depend on the current version, we can't name the Gradle projects statically, and don't know what the
 *  unreleased versions are when Gradle projects are set up, so we use "build-unreleased-version-*" as placeholders
 *  and configure them to build various versions here.
 */
bwcVersions.forPreviousUnreleased { BwcVersions.UnreleasedVersionInfo unreleasedVersion -> project("${unreleasedVersion.gradleProjectPath}") {
    Version bwcVersion = unreleasedVersion.version
    String bwcBranch = unreleasedVersion.branch
    apply plugin: 'distribution'
    // Not published so no need to assemble
    assemble.enabled = false

    File checkoutDir = file("${buildDir}/bwc/checkout-${bwcBranch}")

    final String remote = System.getProperty("bwc.remote", "elastic")

    boolean gitFetchLatest
    final String gitFetchLatestProperty = System.getProperty("tests.bwc.git_fetch_latest", "true")
    if ("true".equals(gitFetchLatestProperty)) {
        gitFetchLatest = true
    } else if ("false".equals(gitFetchLatestProperty)) {
        gitFetchLatest = false
    } else {
        throw new GradleException("tests.bwc.git_fetch_latest must be [true] or [false] but was [" + gitFetchLatestProperty + "]")
    }

    task createClone(type: LoggedExec) {
        onlyIf { checkoutDir.exists() == false }
        commandLine = ['git', 'clone', rootDir, checkoutDir]
    }

    task findRemote(type: LoggedExec) {
        dependsOn createClone
        workingDir = checkoutDir
        commandLine = ['git', 'remote', '-v']
        ByteArrayOutputStream output = new ByteArrayOutputStream()
        standardOutput = output
        doLast {
            project.ext.remoteExists = false
            output.toString('UTF-8').eachLine {
                if (it.contains("${remote}\t")) {
                    project.ext.remoteExists = true
                }
            }
        }
    }

    task addRemote(type: LoggedExec) {
        dependsOn findRemote
        onlyIf { project.ext.remoteExists == false }
        workingDir = checkoutDir
        commandLine = ['git', 'remote', 'add', "${remote}", "https://github.com/${remote}/elasticsearch.git"]
    }

    task fetchLatest(type: LoggedExec) {
        onlyIf { project.gradle.startParameter.isOffline() == false && gitFetchLatest }
        dependsOn addRemote
        workingDir = checkoutDir
        commandLine = ['git', 'fetch', '--all']
    }

    Closure execGit = {   Action<ExecSpec> action ->
        new ByteArrayOutputStream().withStream { os ->
            ExecResult result = project.exec { spec ->
                workingDir = checkoutDir
                standardOutput os
                action.execute(spec)
            }
            result.assertNormalExitValue()
            return os.toString().trim()
        }
    }
    task checkoutBwcBranch() {
        dependsOn fetchLatest
        doLast {
            String refspec = System.getProperty("bwc.refspec.${bwcBranch}") ?: System.getProperty("tests.bwc.refspec.${bwcBranch}") ?: "${remote}/${bwcBranch}"
            if (System.getProperty("bwc.checkout.align") != null) {
                /*
                    We use a time based approach to make the bwc versions built deterministic and compatible with the current hash.
                    Most of the time we want to test against latest, but when running delayed exhaustive tests or wanting
                    reproducible builds we want this to be deterministic by using a hash that was the latest when the current
                    commit was made.

                    This approach doesn't work with merge commits as these can introduce commits in the chronological order
                    after the fact e.x. a merge done today can add commits dated with yesterday so the result will no longer be
                    deterministic.

                    We don't use merge commits, but for additional safety we check that no such commits exist in the time period
                    we are interested in.

                    Timestamps are at seconds resolution. rev-parse --before and --after are inclusive w.r.t the second
                    passed as input. This means the results might not be deterministic in the current second, but this
                    should not matter in practice.
                */
                String timeOfCurrent = execGit { spec ->
                    spec.commandLine 'git', 'show', '--no-patch', '--no-notes', "--pretty='%cD'"
                    spec.workingDir project.rootDir
                }
                logger.lifecycle("Commit date of current: {}", timeOfCurrent)
                String mergeCommits = execGit { spec ->
                    spec.commandLine "git", "rev-list", refspec, "--after", timeOfCurrent, "--merges"
                }
                if (mergeCommits.isEmpty() == false) {
                    throw new IllegalStateException(
                            "Found the following merge commits which prevent determining bwc commits: " + mergeCommits
                    )
                }
                refspec = execGit { spec ->
                    spec.commandLine "git", "rev-list", refspec, "-n", "1", "--before", timeOfCurrent, "--date-order"
                }
            }

            logger.lifecycle("Performing checkout of ${refspec}...")
            LoggedExec.exec(project) { spec ->
                spec.workingDir = checkoutDir
                spec.commandLine "git", "checkout", refspec
            }
            String checkoutHash = GlobalBuildInfoPlugin.gitRevision(checkoutDir)
            logger.lifecycle("Checkout hash for ${project.path} is ${checkoutHash}")
            file("${project.buildDir}/refspec").text = checkoutHash
        }
    }


    Closure createRunBwcGradleTask = { name, extraConfig ->
        return tasks.create(name: "$name", type: LoggedExec) {
            dependsOn checkoutBwcBranch
            spoolOutput = true
            workingDir = checkoutDir
            doFirst {
                // Execution time so that the checkouts are available
                List<String> lines = file("${checkoutDir}/.ci/java-versions.properties").readLines()
                environment(
                        'JAVA_HOME',
                        getJavaHome(it, Integer.parseInt(
                                lines
                                        .findAll({ it.startsWith("ES_BUILD_JAVA=") })
                                        .collect({ it.replace("ES_BUILD_JAVA=java", "").trim() })
                                        .collect({ it.replace("ES_BUILD_JAVA=openjdk", "").trim() })
                                        .join("!!")
                        ))
                )
                environment(
                        'RUNTIME_JAVA_HOME',
                        getJavaHome(it, Integer.parseInt(
                                lines
                                        .findAll({ it.startsWith("ES_RUNTIME_JAVA=java") })
                                        .collect({ it.replace("ES_RUNTIME_JAVA=java", "").trim() })
                                        .join("!!")
                        ))
                )
            }

            if (Os.isFamily(Os.FAMILY_WINDOWS)) {
                executable 'cmd'
                args '/C', 'call', new File(checkoutDir, 'gradlew').toString()
            } else {
                executable new File(checkoutDir, 'gradlew').toString()
            }
            if (gradle.startParameter.isOffline()) {
                args "--offline"
            }

            args "-Dbuild.snapshot=true"
            final LogLevel logLevel = gradle.startParameter.logLevel
            if ([LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG].contains(logLevel)) {
                args "--${logLevel.name().toLowerCase(Locale.ENGLISH)}"
            }
            final String showStacktraceName = gradle.startParameter.showStacktrace.name()
            assert ["INTERNAL_EXCEPTIONS", "ALWAYS", "ALWAYS_FULL"].contains(showStacktraceName)
            if (showStacktraceName.equals("ALWAYS")) {
                args "--stacktrace"
            } else if (showStacktraceName.equals("ALWAYS_FULL")) {
                args "--full-stacktrace"
            }
            if (gradle.getStartParameter().isParallelProjectExecutionEnabled()) {
                args "--parallel"
            }
            standardOutput = new IndentingOutputStream(System.out, bwcVersion)
            errorOutput = new IndentingOutputStream(System.err, bwcVersion)
            configure extraConfig
        }
    }

    Closure buildBwcTaskName = { projectName -> 
      return "buildBwc${projectName.replaceAll(/-\w/){ it[1].toUpperCase() }.capitalize()}"
    }

    task buildBwc {}

    Closure createBuildBwcTask = { projectName, projectDir, projectArtifact ->
        Task bwcTask = createRunBwcGradleTask(buildBwcTaskName(projectName)) {
            inputs.file("${project.buildDir}/refspec")
            outputs.files(projectArtifact)
            outputs.cacheIf("BWC distribution caching is disabled on 'master' branch") {
                // Don't bother caching in 'master' since the BWC branches move too quickly to make this cost worthwhile
                project.ext.isCi && System.getenv('GIT_BRANCH')?.endsWith("master") == false
            }
            args ":${projectDir.replace('/', ':')}:assemble"
            if (project.gradle.startParameter.buildCacheEnabled) {
                args "--build-cache"
            }
            doLast {
                if (projectArtifact.exists() == false) {
                    throw new InvalidUserDataException("Building ${bwcVersion} didn't generate expected file ${projectArtifact}")
                }
            }
        }
        buildBwc.dependsOn bwcTask
    }

    Map<String, File> artifactFiles = [:]
    List<String> projectDirs = []
    List<String> projects = ['deb', 'rpm']
    if (bwcVersion.onOrAfter('7.0.0')) {
      projects.addAll(['windows-zip', 'darwin-tar', 'linux-tar'])
    } else {
      projects.add('zip')
    }

    for (String projectName : projects) {
        String baseDir = "distribution"
        String classifier = ""
        String extension = projectName
        if (bwcVersion.onOrAfter('7.0.0') && (projectName.contains('zip') || projectName.contains('tar'))) {
          int index = projectName.indexOf('-')
          classifier = "-${projectName.substring(0, index)}-x86_64"
          extension = projectName.substring(index + 1)
          if (extension.equals('tar')) {
            extension += '.gz'
          }
        }
        if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('deb')) {
          classifier = "-amd64"
        }
        if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('rpm')) {
          classifier = "-x86_64"
        }
        if (bwcVersion.onOrAfter('6.3.0')) {
            baseDir += projectName.endsWith('zip') || projectName.endsWith('tar') ? '/archives' : '/packages'
            // add oss variant first
            projectDirs.add("${baseDir}/oss-${projectName}")
            File ossProjectArtifact = file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT${classifier}.${extension}")
            artifactFiles.put("oss-" + projectName, ossProjectArtifact)
            createBuildBwcTask("oss-${projectName}", "${baseDir}/oss-${projectName}", ossProjectArtifact)
        }
        projectDirs.add("${baseDir}/${projectName}")
        File projectArtifact = file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT${classifier}.${extension}")
        artifactFiles.put(projectName, projectArtifact)
                
        createBuildBwcTask(projectName, "${baseDir}/${projectName}", projectArtifact)
    }

    createRunBwcGradleTask("resolveAllBwcDependencies") {
        args 'resolveAllDependencies'
    }
    Version currentVersion = Version.fromString(version)
    if (currentVersion.getMinor() == 0 && currentVersion.getRevision() == 0) {
        // We only want to resolve dependencies for live versions of master, without cascading this to older versions
        resolveAllDependencies.dependsOn resolveAllBwcDependencies
    }

    for (e in artifactFiles) {
        String projectName = e.key
        String buildBwcTask = buildBwcTaskName(projectName)
        File artifactFile = e.value
        String artifactFileName = artifactFile.name
        String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
        String suffix = artifactFile.toString()[-3..-1]
        int archIndex = artifactFileName.indexOf('x86_64')
        String classifier = ''
        if (archIndex != -1) {
            int osIndex = artifactFileName.lastIndexOf('-', archIndex - 2)
            classifier = "${artifactFileName.substring(osIndex + 1, archIndex - 1)}-x86_64"
        }
        configurations.create(projectName)
        artifacts {
            it.add(projectName, [file: artifactFile, name: artifactName, classifier: classifier, type: suffix, builtBy: buildBwcTask])
        }
    }
    // make sure no dependencies were added to assemble; we want it to be a no-op
    assemble.dependsOn = []
}}

class IndentingOutputStream extends OutputStream {

    public final byte[] indent
    private final OutputStream delegate

    public IndentingOutputStream(OutputStream delegate, Object version) {
        this.delegate = delegate
        indent = " [${version}] ".getBytes(StandardCharsets.UTF_8)
    }

    @Override
    public void write(int b) {
        write([b] as int[], 0, 1)
    }

    public void write(int[] bytes, int offset, int length) {
        for (int i = 0; i < bytes.length; i++) {
            delegate.write(bytes[i])
            if (bytes[i] == '\n') {
                delegate.write(indent)
            }
        }
    }
}