OpenSearch/distribution/bwc/build.gradle

/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.VersionCollection

import java.nio.charset.StandardCharsets

import static org.elasticsearch.gradle.BuildPlugin.getJavaHome

/**
 *  We want to be able to do BWC tests for unreleased versions without relying on and waiting for snapshots.
 *  For this we need to check out and build the unreleased versions.
 *  Since These depend on the current version, we can't name the Gradle projects statically, and don't know what the
 *  unreleased versions are when Gradle projects are set up, so we use "build-unreleased-version-*" as placeholders
 *  and configure them to build various versions here.
 */
bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion -> project("${project.path}:${unreleasedVersion.gradleProjectName}") {
    Version bwcVersion = unreleasedVersion.version
    String bwcBranch = unreleasedVersion.branch
    apply plugin: 'distribution'
    // Not published so no need to assemble
    assemble.enabled = false
    assemble.dependsOn.remove('buildBwcVersion')

    File checkoutDir = file("${buildDir}/bwc/checkout-${bwcBranch}")

    final String remote = System.getProperty("tests.bwc.remote", "elastic")

    final boolean gitFetchLatest
    final String gitFetchLatestProperty = System.getProperty("tests.bwc.git_fetch_latest", "true")
    if ("true".equals(gitFetchLatestProperty)) {
        gitFetchLatest = true
    } else if ("false".equals(gitFetchLatestProperty)) {
        gitFetchLatest = false
    } else {
        throw new GradleException("tests.bwc.git_fetch_latest must be [true] or [false] but was [" + gitFetchLatestProperty + "]")
    }

    task createClone(type: LoggedExec) {
        onlyIf { checkoutDir.exists() == false }
        commandLine = ['git', 'clone', rootDir, checkoutDir]
    }

    task findRemote(type: LoggedExec) {
        dependsOn createClone
        workingDir = checkoutDir
        commandLine = ['git', 'remote', '-v']
        ByteArrayOutputStream output = new ByteArrayOutputStream()
        standardOutput = output
        doLast {
            project.ext.remoteExists = false
            output.toString('UTF-8').eachLine {
                if (it.contains("${remote}\t")) {
                    project.ext.remoteExists = true
                }
            }
        }
    }

    task addRemote(type: LoggedExec) {
        dependsOn findRemote
        onlyIf { project.ext.remoteExists == false }
        workingDir = checkoutDir
        commandLine = ['git', 'remote', 'add', "${remote}", "https://github.com/${remote}/elasticsearch.git"]
    }

    task fetchLatest(type: LoggedExec) {
        onlyIf { project.gradle.startParameter.isOffline() == false && gitFetchLatest }
        dependsOn addRemote
        workingDir = checkoutDir
        commandLine = ['git', 'fetch', '--all']
    }

    String buildMetadataKey = "bwc_refspec_${project.path.substring(1)}"
    task checkoutBwcBranch(type: LoggedExec) {
        String refspec = System.getProperty("tests.bwc.refspec.${bwcBranch}", buildMetadata.get(buildMetadataKey, "${remote}/${bwcBranch}"))
        dependsOn fetchLatest
        workingDir = checkoutDir
        commandLine = ['git', 'checkout', refspec]
        doFirst {
            println "Checking out elasticsearch ${refspec} for branch ${bwcBranch}"
        }
    }

    File buildMetadataFile = project.file("build/${project.name}/build_metadata")
    task writeBuildMetadata(type: LoggedExec) {
        dependsOn checkoutBwcBranch
        workingDir = checkoutDir
        commandLine = ['git', 'rev-parse', 'HEAD']
        ignoreExitValue = true
        ByteArrayOutputStream output = new ByteArrayOutputStream()
        standardOutput = output
        doLast {
            if (execResult.exitValue != 0) {
                output.toString('UTF-8').eachLine { line -> logger.error(line) }
                execResult.assertNormalExitValue()
            }
            project.mkdir(buildMetadataFile.parent)
            String commit = output.toString('UTF-8')
            buildMetadataFile.setText("${buildMetadataKey}=${commit}", 'UTF-8')
            println "Checked out elasticsearch commit ${commit}"
        }
    }

    List<File> artifactFiles = []
    List<String> projectDirs = []
    for (String project : ['zip', 'deb', 'rpm']) {
        String baseDir = "distribution"
        if (bwcVersion.onOrAfter('6.3.0')) {
            baseDir += project == 'zip' ? '/archives' : '/packages'
            // add oss variant first
            projectDirs.add("${baseDir}/oss-${project}")
            artifactFiles.add(file("${checkoutDir}/${baseDir}/oss-${project}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT.${project}"))
        }
        projectDirs.add("${baseDir}/${project}")
        artifactFiles.add(file("${checkoutDir}/${baseDir}/${project}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT.${project}"))
    }

    task buildBwcVersion(type: Exec) {
        dependsOn checkoutBwcBranch, writeBuildMetadata
        workingDir = checkoutDir
        doFirst {
            // Execution time so that the checkouts are available
            List<String> lines = file("${checkoutDir}/.ci/java-versions.properties").readLines()
            environment(
                    'JAVA_HOME',
                    getJavaHome(it, Integer.parseInt(
                            lines
                                    .findAll({ it.startsWith("ES_BUILD_JAVA=java") })
                                    .collect({ it.replace("ES_BUILD_JAVA=java", "").trim() })
                                    .join("!!")
                    ))
            )
            environment(
                    'RUNTIME_JAVA_HOME',
                    getJavaHome(it, Integer.parseInt(
                            lines
                                    .findAll({ it.startsWith("ES_RUNTIME_JAVA=java") })
                                    .collect({ it.replace("ES_RUNTIME_JAVA=java", "").trim() })
                                    .join("!!")
                    ))
            )
        }

        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
            executable 'cmd'
            args '/C', 'call', new File(checkoutDir, 'gradlew').toString()
        } else {
            executable new File(checkoutDir, 'gradlew').toString()
        }
        if (gradle.startParameter.isOffline()) {
            args "--offline"
        }
        for (String dir : projectDirs) {
            args ":${dir.replace('/', ':')}:assemble"
        }
        args "-Dbuild.snapshot=true"
        final LogLevel logLevel = gradle.startParameter.logLevel
        if ([LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG].contains(logLevel)) {
            args "--${logLevel.name().toLowerCase(Locale.ENGLISH)}"
        }
        final String showStacktraceName = gradle.startParameter.showStacktrace.name()
        assert ["INTERNAL_EXCEPTIONS", "ALWAYS", "ALWAYS_FULL"].contains(showStacktraceName)
        if (showStacktraceName.equals("ALWAYS")) {
            args "--stacktrace"
        } else if (showStacktraceName.equals("ALWAYS_FULL")) {
            args "--full-stacktrace"
        }
        standardOutput = new IndentingOutputStream(System.out, bwcVersion)
        errorOutput = new IndentingOutputStream(System.err, bwcVersion)
        doLast {
            List missing = artifactFiles.grep { file ->
                false == file.exists()
            }
            if (false == missing.empty) {
                throw new InvalidUserDataException(
                        "Building ${bwcVersion} didn't generate expected files ${missing}")
            }
        }
    }

    if (gradle.startParameter.taskNames == ["assemble"]) {
        // Gradle needs the `artifacts` declaration, including `builtBy` bellow to make projects dependencies on this
        // project work, but it will also trigger the build of these for the `assemble` task.
        // Since these are only used for testing, we don't want to assemble them if `assemble` is the single command being
        // ran.
        logger.info("Skipping BWC builds since `assemble` is the only task name provided on the command line")
    } else {
        artifacts {
            for (File artifactFile : artifactFiles) {
                String artifactName = artifactFile.name.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
                String suffix = artifactFile.toString()[-3..-1]
                'default' file: artifactFile, name: artifactName, type: suffix, builtBy: buildBwcVersion
            }
        }
    }
}}

class IndentingOutputStream extends OutputStream {

    public final byte[] indent
    private final OutputStream delegate

    public IndentingOutputStream(OutputStream delegate, Object version) {
        this.delegate = delegate
        indent = " [${version}] ".getBytes(StandardCharsets.UTF_8)
    }

    @Override
    public void write(int b) {
        write([b] as int[], 0, 1)
    }

    public void write(int[] bytes, int offset, int length) {
        for (int i = 0; i < bytes.length; i++) {
            delegate.write(bytes[i])
            if (bytes[i] == '\n') {
                delegate.write(indent)
            }
        }
    }
}