OpenSearch/distribution/bwc/build.gradle

/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.BwcVersions
import org.elasticsearch.gradle.info.BuildParams
import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
import org.gradle.util.GradleVersion

import java.nio.charset.StandardCharsets

import static org.elasticsearch.gradle.util.JavaUtil.getJavaHome

/**
 *  We want to be able to do BWC tests for unreleased versions without relying on and waiting for snapshots.
 *  For this we need to check out and build the unreleased versions.
 *  Since These depend on the current version, we can't name the Gradle projects statically, and don't know what the
 *  unreleased versions are when Gradle projects are set up, so we use "build-unreleased-version-*" as placeholders
 *  and configure them to build various versions here.
 */
BuildParams.bwcVersions.forPreviousUnreleased { BwcVersions.UnreleasedVersionInfo unreleasedVersion ->
  project("${unreleasedVersion.gradleProjectPath}") {
    Version bwcVersion = unreleasedVersion.version
    String bwcBranch = unreleasedVersion.branch
    apply plugin: 'distribution'
    // Not published so no need to assemble
    assemble.enabled = false

    File checkoutDir = file("${buildDir}/bwc/checkout-${bwcBranch}")

    final String remote = System.getProperty("bwc.remote", "elastic")

    boolean gitFetchLatest
    final String gitFetchLatestProperty = System.getProperty("tests.bwc.git_fetch_latest", "true")
    if ("true".equals(gitFetchLatestProperty)) {
      gitFetchLatest = true
    } else if ("false".equals(gitFetchLatestProperty)) {
      gitFetchLatest = false
    } else {
      throw new GradleException("tests.bwc.git_fetch_latest must be [true] or [false] but was [" + gitFetchLatestProperty + "]")
    }

    task createClone(type: LoggedExec) {
      onlyIf { checkoutDir.exists() == false }
      commandLine = ['git', 'clone', rootDir, checkoutDir]
    }

    task findRemote(type: LoggedExec) {
      dependsOn createClone
      workingDir = checkoutDir
      commandLine = ['git', 'remote', '-v']
      ByteArrayOutputStream output = new ByteArrayOutputStream()
      standardOutput = output
      doLast {
        project.ext.remoteExists = false
        output.toString('UTF-8').eachLine {
          if (it.contains("${remote}\t")) {
            project.ext.remoteExists = true
          }
        }
      }
    }

    task addRemote(type: LoggedExec) {
      dependsOn findRemote
      onlyIf { project.ext.remoteExists == false }
      workingDir = checkoutDir
      commandLine = ['git', 'remote', 'add', "${remote}", "https://github.com/${remote}/elasticsearch.git"]
    }

    task fetchLatest(type: LoggedExec) {
      onlyIf { project.gradle.startParameter.isOffline() == false && gitFetchLatest }
      dependsOn addRemote
      workingDir = checkoutDir
      commandLine = ['git', 'fetch', '--all']
    }

    Closure execGit = { Action<ExecSpec> action ->
      new ByteArrayOutputStream().withStream { os ->
        ExecResult result = project.exec { spec ->
          workingDir = checkoutDir
          standardOutput os
          action.execute(spec)
        }
        result.assertNormalExitValue()
        return os.toString().trim()
      }
    }
    task checkoutBwcBranch() {
      dependsOn fetchLatest
      doLast {
        def refspec = System.getProperty("bwc.refspec." + bwcBranch) ?: System.getProperty("tests.bwc.refspec." + bwcBranch) ?: "${remote}/${bwcBranch}"
        if (System.getProperty("bwc.checkout.align") != null) {
          /*
              We use a time based approach to make the bwc versions built deterministic and compatible with the current hash.
              Most of the time we want to test against latest, but when running delayed exhaustive tests or wanting
              reproducible builds we want this to be deterministic by using a hash that was the latest when the current
              commit was made.

              This approach doesn't work with merge commits as these can introduce commits in the chronological order
              after the fact e.x. a merge done today can add commits dated with yesterday so the result will no longer be
              deterministic.

              We don't use merge commits, but for additional safety we check that no such commits exist in the time period
              we are interested in.

              Timestamps are at seconds resolution. rev-parse --before and --after are inclusive w.r.t the second
              passed as input. This means the results might not be deterministic in the current second, but this
              should not matter in practice.
          */
          String timeOfCurrent = execGit { spec ->
            spec.commandLine 'git', 'show', '--no-patch', '--no-notes', "--pretty='%cD'"
            spec.workingDir project.rootDir
          }
          logger.lifecycle("Commit date of current: {}", timeOfCurrent)
          String mergeCommits = execGit { spec ->
            spec.commandLine "git", "rev-list", refspec, "--after", timeOfCurrent, "--merges"
          }
          if (mergeCommits.isEmpty() == false) {
            throw new IllegalStateException(
              "Found the following merge commits which prevent determining bwc commits: " + mergeCommits
            )
          }
          refspec = execGit { spec ->
            spec.commandLine "git", "rev-list", refspec, "-n", "1", "--before", timeOfCurrent, "--date-order"
          }
        }

        logger.lifecycle("Performing checkout of ${refspec}...")
        LoggedExec.exec(project) { spec ->
          spec.workingDir = checkoutDir
          spec.commandLine "git", "checkout", refspec
        }
        String checkoutHash = GlobalBuildInfoPlugin.gitInfo(checkoutDir).revision
        logger.lifecycle("Checkout hash for ${project.path} is ${checkoutHash}")
        file("${project.buildDir}/refspec").text = checkoutHash
      }
    }


    Closure createRunBwcGradleTask = { name, extraConfig ->
      return tasks.register("$name", LoggedExec) {
        dependsOn checkoutBwcBranch
        spoolOutput = true
        workingDir = checkoutDir
        doFirst {
          // Execution time so that the checkouts are available
          List<String> lines = file("${checkoutDir}/.ci/java-versions.properties").readLines()
          environment(
            'JAVA_HOME',
            getJavaHome(Integer.parseInt(
              lines
                .findAll({ it.startsWith("ES_BUILD_JAVA=") })
                .collect({ it.replace("ES_BUILD_JAVA=java", "").trim() })
                .collect({ it.replace("ES_BUILD_JAVA=openjdk", "").trim() })
                .join("!!")
            ))
          )
          environment(
            'RUNTIME_JAVA_HOME',
            getJavaHome(Integer.parseInt(
              lines
                .findAll({ it.startsWith("ES_RUNTIME_JAVA=java") })
                .collect({ it.replace("ES_RUNTIME_JAVA=java", "").trim() })
                .join("!!")
            ))
          )
        }

        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
          executable 'cmd'
          args '/C', 'call', new File(checkoutDir, 'gradlew').toString()
        } else {
          executable new File(checkoutDir, 'gradlew').toString()
        }
        if (gradle.startParameter.isOffline()) {
          args "--offline"
        }
        String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url')
        if (buildCacheUrl) {
          args "-Dorg.elasticsearch.build.cache.url=${buildCacheUrl}"
        }

        args "-Dbuild.snapshot=true"
        args "-Dscan.tag.NESTED"
        final LogLevel logLevel = gradle.startParameter.logLevel
        if ([LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG].contains(logLevel)) {
          args "--${logLevel.name().toLowerCase(Locale.ENGLISH)}"
        }
        final String showStacktraceName = gradle.startParameter.showStacktrace.name()
        assert ["INTERNAL_EXCEPTIONS", "ALWAYS", "ALWAYS_FULL"].contains(showStacktraceName)
        if (showStacktraceName.equals("ALWAYS")) {
          args "--stacktrace"
        } else if (showStacktraceName.equals("ALWAYS_FULL")) {
          args "--full-stacktrace"
        }
        if (gradle.getStartParameter().isParallelProjectExecutionEnabled()) {
          args "--parallel"
        }
        standardOutput = new IndentingOutputStream(System.out, bwcVersion)
        errorOutput = new IndentingOutputStream(System.err, bwcVersion)
        configure extraConfig
      }
    }

    Closure buildBwcTaskName = { projectName ->
      return "buildBwc${projectName.replaceAll(/-\w/) { it[1].toUpperCase() }.capitalize()}"
    }

    def buildBwc = tasks.register("buildBwc");

    Closure createBuildBwcTask = { projectName, projectDir, projectArtifact ->
      def bwcTaskName = buildBwcTaskName(projectName)
      createRunBwcGradleTask(bwcTaskName) {
        inputs.file("${project.buildDir}/refspec")
        outputs.files(projectArtifact)
        outputs.cacheIf("BWC distribution caching is disabled on 'master' branch") {
          // Don't bother caching in 'master' since the BWC branches move too quickly to make this cost worthwhile
          BuildParams.ci && System.getenv('GIT_BRANCH')?.endsWith("master") == false
        }
        args ":${projectDir.replace('/', ':')}:assemble"
        if (project.gradle.startParameter.buildCacheEnabled) {
          args "--build-cache"
        }
        doLast {
          if (projectArtifact.exists() == false) {
            throw new InvalidUserDataException("Building ${bwcVersion} didn't generate expected file ${projectArtifact}")
          }
        }
      }
      buildBwc.configure {
        dependsOn(bwcTaskName)
      }
    }

    Map<String, File> artifactFiles = [:]
    List<String> projectDirs = []
    List<String> projects = ['deb', 'rpm']
    if (bwcVersion.onOrAfter('7.0.0')) {
      projects.addAll(['windows-zip', 'darwin-tar', 'linux-tar'])
    } else {
      projects.add('zip')
    }

    for (String projectName : projects) {
      String baseDir = "distribution"
      String classifier = ""
      String extension = projectName
      if (bwcVersion.onOrAfter('7.0.0') && (projectName.contains('zip') || projectName.contains('tar'))) {
        int index = projectName.indexOf('-')
        classifier = "-${projectName.substring(0, index)}-x86_64"
        extension = projectName.substring(index + 1)
        if (extension.equals('tar')) {
          extension += '.gz'
        }
      }
      if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('deb')) {
        classifier = "-amd64"
      }
      if (bwcVersion.onOrAfter('7.0.0') && projectName.contains('rpm')) {
        classifier = "-x86_64"
      }
      if (bwcVersion.onOrAfter('6.3.0')) {
        baseDir += projectName.endsWith('zip') || projectName.endsWith('tar') ? '/archives' : '/packages'
        // add oss variant first
        projectDirs.add("${baseDir}/oss-${projectName}")
        File ossProjectArtifact = file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT${classifier}.${extension}")
        artifactFiles.put("oss-" + projectName, ossProjectArtifact)
        createBuildBwcTask("oss-${projectName}", "${baseDir}/oss-${projectName}", ossProjectArtifact)
      }
      projectDirs.add("${baseDir}/${projectName}")
      File projectArtifact = file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT${classifier}.${extension}")
      artifactFiles.put(projectName, projectArtifact)

      createBuildBwcTask(projectName, "${baseDir}/${projectName}", projectArtifact)
    }

    // Create build tasks for the JDBC driver used for compatibility testing
    String jdbcProjectDir = 'x-pack/plugin/sql/jdbc'
    File jdbcProjectArtifact = file("${checkoutDir}/${jdbcProjectDir}/build/distributions/x-pack-sql-jdbc-${bwcVersion}-SNAPSHOT.jar")
    createBuildBwcTask('jdbc', jdbcProjectDir, jdbcProjectArtifact)

    createRunBwcGradleTask("resolveAllBwcDependencies") {
      args 'resolveAllDependencies'
    }
    Version currentVersion = Version.fromString(version)
    if (currentVersion.getMinor() == 0 && currentVersion.getRevision() == 0) {
      // We only want to resolve dependencies for live versions of master, without cascading this to older versions
      resolveAllDependencies.dependsOn resolveAllBwcDependencies
    }

    for (e in artifactFiles) {
      String projectName = e.key
      String buildBwcTask = buildBwcTaskName(projectName)
      File artifactFile = e.value
      String artifactFileName = artifactFile.name
      String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
      String suffix = artifactFile.toString()[-3..-1]
      int archIndex = artifactFileName.indexOf('x86_64')
      String classifier = ''
      if (archIndex != -1) {
        int osIndex = artifactFileName.lastIndexOf('-', archIndex - 2)
        classifier = "${artifactFileName.substring(osIndex + 1, archIndex - 1)}-x86_64"
      }
      configurations.create(projectName)
      artifacts {
        it.add(projectName, [file: artifactFile, name: artifactName, classifier: classifier, type: suffix, builtBy: buildBwcTask])
      }
    }
    // make sure no dependencies were added to assemble; we want it to be a no-op
    assemble.dependsOn = []
  }
}

class IndentingOutputStream extends OutputStream {

  public final byte[] indent
  private final OutputStream delegate

  public IndentingOutputStream(OutputStream delegate, Object version) {
    this.delegate = delegate
    indent = " [${version}] ".getBytes(StandardCharsets.UTF_8)
  }

  @Override
  public void write(int b) {
    write([b] as int[], 0, 1)
  }

  public void write(int[] bytes, int offset, int length) {
    for (int i = 0; i < bytes.length; i++) {
      delegate.write(bytes[i])
      if (bytes[i] == '\n') {
        delegate.write(indent)
      }
    }
  }
}