OpenSearch/plugin/build.gradle

import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.test.NodeInfo

import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardCopyOption

group 'org.elasticsearch.plugin'

apply plugin: 'elasticsearch.esplugin'
esplugin {
  name 'x-pack'
  description 'Elasticsearch Expanded Pack Plugin'
  classname 'org.elasticsearch.xpack.XPackPlugin'
  hasNativeController true
  requiresKeystore true
  licenseFile project(':x-pack-elasticsearch').file('LICENSE.txt')
  noticeFile project(':x-pack-elasticsearch').file('NOTICE.txt')
}
archivesBaseName = 'x-pack' // for api jar

// TODO: fix this! https://github.com/elastic/x-plugins/issues/1066
ext.compactProfile = 'full'

dependencyLicenses {
  mapping from: /netty-.*/, to: 'netty'
  mapping from: /bc.*/, to: 'bouncycastle'
  mapping from: /owasp-java-html-sanitizer.*/, to: 'owasp-java-html-sanitizer'
  mapping from: /transport-netty.*/, to: 'elasticsearch'
  mapping from: /elasticsearch-rest-client.*/, to: 'elasticsearch'
  mapping from: /http.*/, to: 'httpclient' // pulled in by rest client
  mapping from: /commons-.*/, to: 'commons' // pulled in by rest client
  ignoreSha 'elasticsearch-rest-client'
  ignoreSha 'x-pack-client-api-objects'
  ignoreSha 'transport-netty4'
  ignoreSha 'elasticsearch-rest-client-sniffer'
}

licenseHeaders {
  approvedLicenses << 'BCrypt (BSD-like)'
  additionalLicense 'BCRYP', 'BCrypt (BSD-like)', 'Copyright (c) 2006 Damien Miller <djm@mindrot.org>'
}

configurations {
  nativeBundle {
    resolutionStrategy.dependencySubstitution {
      if (findProject(':machine-learning-cpp') != null) {
        substitute module("org.elasticsearch.ml:ml-cpp") with project(":machine-learning-cpp")
      } else {
        substitute module("org.elasticsearch.ml:ml-cpp") with project("${project.path}:ml-cpp-snapshot")
      }
    }
  }
}

dependencies {
  // CLI deps
  compile project(path: ':core:cli', configuration: 'runtime')

  // Request and Response objects
  compile "org.elasticsearch:x-pack-client-api-objects:${version}"

  // security deps
  compile project(path: ':modules:transport-netty4', configuration: 'runtime')
  compile 'com.unboundid:unboundid-ldapsdk:3.2.0'
  compile 'org.bouncycastle:bcprov-jdk15on:1.58'
  compile 'org.bouncycastle:bcpkix-jdk15on:1.58'
  testCompile 'com.google.jimfs:jimfs:1.1'

  // watcher deps
  compile 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:r239'
  compile 'com.google.guava:guava:16.0.1' // needed by watcher for the html sanitizer and security tests for jimfs
  compile 'com.sun.mail:javax.mail:1.5.6'
  // HACK: java 9 removed javax.activation from the default modules, so instead of trying to add modules, which would have
  // to be conditionalized for java 8/9, we pull in the classes directly
  compile 'javax.activation:activation:1.1.1'

  testCompile 'org.subethamail:subethasmtp:3.1.7'
  // needed for subethasmtp, has @GuardedBy annotation
  testCompile 'com.google.code.findbugs:jsr305:3.0.1'

  // monitoring deps
  compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
  compile "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}"

  // ml deps
  compile 'net.sf.supercsv:super-csv:2.4.0'
  nativeBundle "org.elasticsearch.ml:ml-cpp:${project.version}@zip"
  testCompile 'org.ini4j:ini4j:0.5.2'

  // common test deps
  testCompile 'org.elasticsearch:securemock:1.2'
  testCompile "org.elasticsearch:mocksocket:${versions.mocksocket}"
  testCompile 'org.slf4j:slf4j-log4j12:1.6.2'
  testCompile 'org.slf4j:slf4j-api:1.6.2'
  testCompile project(path: ':modules:reindex', configuration: 'runtime')
  testCompile project(path: ':modules:parent-join', configuration: 'runtime')
  testCompile project(path: ':modules:analysis-common', configuration: 'runtime')
}

// make LicenseSigner available for testing signed licenses
sourceSets.test.java {
  srcDir '../license-tools/src/main/java'
}

compileJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked"
compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked"

ext.expansions = [
  'project.version': version,
]

processResources {
  from(sourceSets.main.resources.srcDirs) {
    exclude '**/public.key'
    inputs.properties(expansions)
    MavenFilteringHack.filter(it, expansions)
  }
  boolean snapshot = "true".equals(System.getProperty("build.snapshot", "true"))
  if (snapshot) {
    from 'keys/dev/public.key'
  } else {
    from 'keys/prod/public.key'
  }
}

forbiddenPatterns {
  exclude '**/*.key'
  exclude '**/*.p12'
  exclude '**/*.der'
  exclude '**/*.zip'
}

forbiddenApisMain {
  signaturesURLs += file('forbidden/ldap-signatures.txt').toURI().toURL()
}

task extractNativeLicenses(type: Copy) {
  dependsOn configurations.nativeBundle
  into "${buildDir}"
  from {
    project.zipTree(configurations.nativeBundle.singleFile)
  }
  include 'platform/licenses/**'
  // This is to reduce the risk of credentials used to access the native bundle not
  // having propagated throughout AWS by the time it's downloaded; the time needed
  // to compile the Java is extra time during which the propagation can take place
  shouldRunAfter compileJava
}

// TODO: standardize packaging config for plugins
bundlePlugin {
  dependsOn configurations.nativeBundle
  from('bin/x-pack') {
    into 'bin'
  }
  from('config/x-pack') {
    into 'config'
  }
  from {
    project.zipTree(configurations.nativeBundle.singleFile)
  }
  // We don't ship the individual nativeBundle licenses - instead
  // they get combined into the top level NOTICES file we ship
  exclude 'platform/licenses/**'
}

// add api jar for extension authors to compile against
// note this is just the normal x-pack jar for now, with a different name
project.afterEvaluate {
  task apiJar {
    dependsOn('generatePomFileForApijarPublication', project.jar)
    doFirst {
      Path jarFile = project.jar.outputs.files.singleFile.toPath()
      String apiFileName = jarFile.fileName.toString().replace(project.version, "api-${project.version}")
      Files.copy(jarFile, jarFile.resolveSibling(apiFileName), StandardCopyOption.REPLACE_EXISTING)

      String pomFileName = jarFile.fileName.toString().replace('.jar', '.pom')
      String apiPomFileName = apiFileName.replace('.jar', '.pom')
      Files.copy(jarFile.resolveSibling(pomFileName), jarFile.resolveSibling(apiPomFileName),
                 StandardCopyOption.REPLACE_EXISTING)
    }
  }
  assemble.dependsOn(apiJar)
  project.publishing {
    publications {
      apijar(MavenPublication) {
        from project.components.java
        artifactId = 'x-pack-api'
        pom.withXml { XmlProvider xml ->
          Node root = xml.asNode()
          root.appendNode('name', project.pluginProperties.extension.name)
          root.appendNode('description', project.pluginProperties.extension.description)
        }
      }
    }
  }
  // Add an extra licenses directory to the combined notices
  project.tasks.findByName('generateNotice').dependsOn extractNativeLicenses
  project.tasks.findByName('generateNotice').licensesDir new File("${project.buildDir}/platform/licenses")
}

integTestRunner {
  // TODO: fix this rest test to not depend on a hardcoded port!
  def blacklist = ['getting_started/10_monitor_cluster_health/*']
  boolean snapshot = "true".equals(System.getProperty("build.snapshot", "true"))
  if (!snapshot) {
    // these tests attempt to install basic/internal licenses signed against the dev/public.key
    // Since there is no infrastructure in place (anytime soon) to generate licenses using the production
    // private key, these tests are whitelisted in non-snapshot test runs
    blacklist.addAll(['xpack/15_basic/*', 'license/20_put_license/*'])
  }
  systemProperty 'tests.rest.blacklist', blacklist.join(',')
}

// location of generated keystores and certificates
File keystoreDir = new File(project.buildDir, 'keystore')

// Generate the node's keystore
File nodeKeystore = new File(keystoreDir, 'test-node.jks')
task createNodeKeyStore(type: LoggedExec) {
  doFirst {
    if (nodeKeystore.parentFile.exists() == false) {
      nodeKeystore.parentFile.mkdirs()
    }
    if (nodeKeystore.exists()) {
      delete nodeKeystore
    }
  }
  executable = new File(project.javaHome, 'bin/keytool')
  standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
  args '-genkey',
          '-alias', 'test-node',
          '-keystore', nodeKeystore,
          '-keyalg', 'RSA',
          '-keysize', '2048',
          '-validity', '712',
          '-dname', 'CN=smoke-test-plugins-ssl',
          '-keypass', 'keypass',
          '-storepass', 'keypass'
}

// Add keystores to test classpath: it expects it there
sourceSets.test.resources.srcDir(keystoreDir)
processTestResources.dependsOn(createNodeKeyStore)

integTestCluster {
  dependsOn createNodeKeyStore
  setting 'xpack.ml.enabled', 'true'
  setting 'logger.org.elasticsearch.xpack.ml.datafeed', 'TRACE'
  // Integration tests are supposed to enable/disable exporters before/after each test
  setting 'xpack.monitoring.exporters._local.type', 'local'
  setting 'xpack.monitoring.exporters._local.enabled', 'false'
  setting 'xpack.monitoring.collection.interval', '-1'
  setting 'xpack.security.authc.token.enabled', 'true'
  setting 'xpack.security.transport.ssl.enabled', 'true'
  setting 'xpack.security.transport.ssl.keystore.path', nodeKeystore.name
  setting 'xpack.security.transport.ssl.verification_mode', 'certificate'
  keystoreSetting 'bootstrap.password', 'x-pack-test-password'
  keystoreSetting 'xpack.security.transport.ssl.keystore.secure_password', 'keypass'
  distribution = 'zip' // this is important since we use the reindex module in ML

  setupCommand 'setupTestUser', 'bin/x-pack/users', 'useradd', 'x_pack_rest_user', '-p', 'x-pack-test-password', '-r', 'superuser'

  extraConfigFile nodeKeystore.name, nodeKeystore

  waitCondition = { NodeInfo node, AntBuilder ant ->
      File tmpFile = new File(node.cwd, 'wait.success')

      for (int i = 0; i < 10; i++) {
        // we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned
        HttpURLConnection httpURLConnection = null;
        try {
          httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health?wait_for_nodes=${numNodes}&wait_for_status=yellow").openConnection();
          httpURLConnection.setRequestProperty("Authorization", "Basic " +
                  Base64.getEncoder().encodeToString("x_pack_rest_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)));
          httpURLConnection.setRequestMethod("GET");
          httpURLConnection.connect();
          if (httpURLConnection.getResponseCode() == 200) {
            tmpFile.withWriter StandardCharsets.UTF_8.name(), {
              it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name()))
            }
          }
        } catch (Exception e) {
          if (i == 9) {
            logger.error("final attempt of calling cluster health failed", e)
          } else {
            logger.debug("failed to call cluster health", e)
          }
        } finally {
          if (httpURLConnection != null) {
            httpURLConnection.disconnect();
          }
        }

        // did not start, so wait a bit before trying again
        Thread.sleep(500L);
      }
      return tmpFile.exists()
  }
}

test {
  /*
   * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
   * other if we allow them to set the number of available processors as it's set-once in Netty.
   */
  systemProperty 'es.set.netty.runtime.available.processors', 'false'
}

integTestRunner {
  /*
   * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
   * other if we allow them to set the number of available processors as it's set-once in Netty.
   */
  systemProperty 'es.set.netty.runtime.available.processors', 'false'
}

// TODO: don't publish test artifacts just to run messy tests, fix the tests!
// https://github.com/elastic/x-plugins/issues/724
configurations {
    testArtifacts.extendsFrom testRuntime
}
task testJar(type: Jar) {
    appendix 'test'
    from sourceSets.test.output
}
artifacts {
    // normal es plugins do not publish the jar but we need to since users need it for Transport Clients and extensions
    archives jar
    testArtifacts testJar
}

// classes are missing, e.g. com.ibm.icu.lang.UCharacter
thirdPartyAudit.excludes = [
    // uses internal java api: sun.misc.Unsafe
    'com.google.common.cache.Striped64',
    'com.google.common.cache.Striped64$1',
    'com.google.common.cache.Striped64$Cell',
    'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
    'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',

    // pulled in as external dependency to work on java 9
    'com.sun.activation.registries.LineTokenizer',
    'com.sun.activation.registries.LogSupport',
    'com.sun.activation.registries.MailcapFile',
    'com.sun.activation.registries.MailcapTokenizer',
    'com.sun.activation.registries.MimeTypeEntry',
    'com.sun.activation.registries.MimeTypeFile',
    'javax.activation.MailcapCommandMap',
    'javax.activation.MimetypesFileTypeMap',
]

// pulled in as external dependency to work on java 9
if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
  thirdPartyAudit.excludes += [
    'com.sun.activation.registries.MailcapParseException',
    'javax.activation.ActivationDataFlavor',
    'javax.activation.CommandInfo',
    'javax.activation.CommandMap',
    'javax.activation.CommandObject',
    'javax.activation.DataContentHandler',
    'javax.activation.DataContentHandlerFactory',
    'javax.activation.DataHandler$1',
    'javax.activation.DataHandler',
    'javax.activation.DataHandlerDataSource',
    'javax.activation.DataSource',
    'javax.activation.DataSourceDataContentHandler',
    'javax.activation.FileDataSource',
    'javax.activation.FileTypeMap',
    'javax.activation.MimeType',
    'javax.activation.MimeTypeParameterList',
    'javax.activation.MimeTypeParseException',
    'javax.activation.ObjectDataContentHandler',
    'javax.activation.SecuritySupport$1',
    'javax.activation.SecuritySupport$2',
    'javax.activation.SecuritySupport$3',
    'javax.activation.SecuritySupport$4',
    'javax.activation.SecuritySupport$5',
    'javax.activation.SecuritySupport',
    'javax.activation.URLDataSource',
    'javax.activation.UnsupportedDataTypeException'
  ]
}

run {
  setting 'xpack.ml.enabled', 'true'
  setting 'xpack.graph.enabled', 'true'
  setting 'xpack.security.enabled', 'true'
  setting 'xpack.monitoring.enabled', 'true'
  setting 'xpack.watcher.enabled', 'true'
  keystoreSetting 'bootstrap.password', 'password'
}