OpenSearch/distribution/bwc/build.gradle

/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */



import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.Version

import java.nio.charset.StandardCharsets

import static org.elasticsearch.gradle.BuildPlugin.getJavaHome
/**
 * This is a dummy project which does a local checkout of the previous
 * wire compat version's branch, and builds a snapshot. This allows backcompat
 * tests to test against the next unreleased version, closest to this version,
 * without relying on snapshots.
 */
subprojects {

  Version bwcVersion = bwcVersions.getSnapshotForProject(project.name)
  if (bwcVersion == null) {
    // this project wont do anything
    return
  }

  String bwcBranch
  if (project.name == 'next-minor-snapshot') {
    // this is always a .x series
    bwcBranch = "${bwcVersion.major}.x"
  } else {
    bwcBranch = "${bwcVersion.major}.${bwcVersion.minor}"
  }

  apply plugin: 'distribution'
  // Not published so no need to assemble
  assemble.enabled = false
  assemble.dependsOn.remove('buildBwcVersion')

  File checkoutDir = file("${buildDir}/bwc/checkout-${bwcBranch}")

  final String remote = System.getProperty("tests.bwc.remote", "elastic")

  final boolean gitFetchLatest
  final String gitFetchLatestProperty = System.getProperty("tests.bwc.git_fetch_latest", "true")
  if ("true".equals(gitFetchLatestProperty)) {
    gitFetchLatest = true
  } else if ("false".equals(gitFetchLatestProperty)) {
    gitFetchLatest = false
  } else {
    throw new GradleException("tests.bwc.git_fetch_latest must be [true] or [false] but was [" + gitFetchLatestProperty + "]")
  }

  task createClone(type: LoggedExec) {
    onlyIf { checkoutDir.exists() == false }
    commandLine = ['git', 'clone', rootDir, checkoutDir]
  }

  task findRemote(type: LoggedExec) {
    dependsOn createClone
    workingDir = checkoutDir
    commandLine = ['git', 'remote', '-v']
    ByteArrayOutputStream output = new ByteArrayOutputStream()
    standardOutput = output
    doLast {
      project.ext.remoteExists = false
      output.toString('UTF-8').eachLine {
        if (it.contains("${remote}\t")) {
          project.ext.remoteExists = true
        }
      }
    }
  }

  task addRemote(type: LoggedExec) {
    dependsOn findRemote
    onlyIf { project.ext.remoteExists == false }
    workingDir = checkoutDir
    commandLine = ['git', 'remote', 'add', "${remote}", "https://github.com/${remote}/elasticsearch.git"]
  }

  task fetchLatest(type: LoggedExec) {
    onlyIf { project.gradle.startParameter.isOffline() == false && gitFetchLatest }
    dependsOn addRemote
    workingDir = checkoutDir
    commandLine = ['git', 'fetch', '--all']
  }

  String buildMetadataKey = "bwc_refspec_${project.path.substring(1)}"
  task checkoutBwcBranch(type: LoggedExec) {
    String refspec = System.getProperty("tests.bwc.refspec.${bwcBranch}", buildMetadata.get(buildMetadataKey, "${remote}/${bwcBranch}"))
    dependsOn fetchLatest
    workingDir = checkoutDir
    commandLine = ['git', 'checkout', refspec]
    doFirst {
      println "Checking out elasticsearch ${refspec} for branch ${bwcBranch}"
    }
  }

  File buildMetadataFile = project.file("build/${project.name}/build_metadata")
  task writeBuildMetadata(type: LoggedExec) {
    dependsOn checkoutBwcBranch
    workingDir = checkoutDir
    commandLine = ['git', 'rev-parse', 'HEAD']
    ignoreExitValue = true
    ByteArrayOutputStream output = new ByteArrayOutputStream()
    standardOutput = output
    doLast {
      if (execResult.exitValue != 0) {
        output.toString('UTF-8').eachLine { line -> logger.error(line) }
        execResult.assertNormalExitValue()
      }
      project.mkdir(buildMetadataFile.parent)
      String commit = output.toString('UTF-8')
      buildMetadataFile.setText("${buildMetadataKey}=${commit}", 'UTF-8')
      println "Checked out elasticsearch commit ${commit}"
    }
  }

  List<File> artifactFiles = []
  List<String> projectDirs = []
  for (String project : ['zip', 'deb', 'rpm']) {
    String baseDir = "distribution"
    if (bwcVersion.onOrAfter('6.3.0')) {
      baseDir += project == 'zip' ? '/archives' : '/packages'
      // add oss variant first
      projectDirs.add("${baseDir}/oss-${project}")
      artifactFiles.add(file("${checkoutDir}/${baseDir}/oss-${project}/build/distributions/elasticsearch-oss-${bwcVersion}.${project}"))
    }
    projectDirs.add("${baseDir}/${project}")
    artifactFiles.add(file("${checkoutDir}/${baseDir}/${project}/build/distributions/elasticsearch-${bwcVersion}.${project}"))
  }

  task buildBwcVersion(type: Exec) {
    dependsOn checkoutBwcBranch, writeBuildMetadata
    workingDir = checkoutDir
    doFirst {
      // Execution time so that the checkouts are available  
      List<String> lines = file("$checkoutDir/.ci/java-versions.properties").readLines()
      environment(
              'JAVA_HOME',
              getJavaHome(it, Integer.parseInt(
                      lines
                              .findAll({ it.startsWith("ES_BUILD_JAVA=java") })
                              .collect({ it.replace("ES_BUILD_JAVA=java", "").trim() })
                              .join("!!")
              ))
      )
      environment(
              'RUNTIME_JAVA_HOME',
              getJavaHome(it, Integer.parseInt(
                      lines
                              .findAll({ it.startsWith("ES_RUNTIME_JAVA=java") })
                              .collect({ it.replace("ES_RUNTIME_JAVA=java", "").trim() })
                              .join("!!")
              ))
      )
    }

    if (Os.isFamily(Os.FAMILY_WINDOWS)) {
      executable 'cmd'
      args '/C', 'call', new File(checkoutDir, 'gradlew').toString()
    } else {
      executable new File(checkoutDir, 'gradlew').toString()
    }
    for (String dir : projectDirs) {
      args ":${dir.replace('/', ':')}:assemble"
    }
    args "-Dbuild.snapshot=true"
    final LogLevel logLevel = gradle.startParameter.logLevel
    if ([LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG].contains(logLevel)) {
      args "--${logLevel.name().toLowerCase(Locale.ENGLISH)}"
    }
    final String showStacktraceName = gradle.startParameter.showStacktrace.name()
    assert ["INTERNAL_EXCEPTIONS", "ALWAYS", "ALWAYS_FULL"].contains(showStacktraceName)
    if (showStacktraceName.equals("ALWAYS")) {
      args "--stacktrace"
    } else if (showStacktraceName.equals("ALWAYS_FULL")) {
      args "--full-stacktrace"
    }
    standardOutput = new IndentingOutputStream(System.out)
    errorOutput = new IndentingOutputStream(System.err)
    doLast {
      List missing = artifactFiles.grep { file ->
        false == file.exists()
      }
      if (false == missing.empty) {
        throw new InvalidUserDataException(
                "Building bwc version didn't generate expected files ${missing}")
      }
    }
  }

  if (gradle.startParameter.taskNames == ["assemble"]) {
    // Gradle needs the `artifacts` declaration, including `builtBy` bellow to make projects dependencies on this
    // project work, but it will also trigger the build of these for the `assemble` task.
    // Since these are only used for testing, we don't want to assemble them if `assemble` is the single command being
    // ran.
    logger.info("Skipping BWC builds since `assemble` is the only task name provided on the command line")
  } else {
    artifacts {
      for (File artifactFile : artifactFiles) {
        String artifactName = artifactFile.name.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
        String suffix = artifactFile.toString()[-3..-1]
        'default' file: artifactFile, name: artifactName, type: suffix, builtBy: buildBwcVersion
      }
    }
  }
}

class IndentingOutputStream extends OutputStream {

  public static final byte[] INDENT = "  [bwc] ".getBytes(StandardCharsets.UTF_8)
  private final OutputStream delegate

  public IndentingOutputStream(OutputStream delegate) {
      this.delegate = delegate
  }

  @Override
  public void write(int b) {
    write([b] as int[], 0, 1)
  }

  public void write(int[] bytes, int offset, int length) {
    for (int i = 0; i < bytes.length; i++) {
      delegate.write(bytes[i])
      if (bytes[i] == '\n') {
        delegate.write(INDENT)
      }
    }
  }
}