OpenSearch/distribution/build.gradle

645 lines
23 KiB
Groovy

/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import org.elasticsearch.gradle.ConcatFilesTask
import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.NoticeTask
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.test.RunTask
import org.apache.tools.ant.filters.FixCrLfFilter
import java.nio.file.Files
import java.nio.file.Path
import java.util.regex.Matcher
import java.util.regex.Pattern
/*****************************************************************************
* Third party dependencies report *
*****************************************************************************/
// Concatenates the dependencies CSV files into a single file
task generateDependenciesReport(type: ConcatFilesTask) {
files = fileTree(dir: project.rootDir, include: '**/dependencies.csv' )
headerLine = "name,version,url,license"
target = new File(System.getProperty('csv')?: "${project.buildDir}/reports/dependencies/es-dependencies.csv")
}
/*****************************************************************************
* Notice file *
*****************************************************************************/
// integ test zip only uses server, so a different notice file is needed there
task buildServerNotice(type: NoticeTask) {
licensesDir new File(project(':server').projectDir, 'licenses')
}
// other distributions include notices from modules as well, which are added below later
task buildDefaultNotice(type: NoticeTask) {
licensesDir new File(project(':server').projectDir, 'licenses')
licensesDir new File(project(':distribution').projectDir, 'licenses')
}
task buildOssNotice(type: NoticeTask) {
licensesDir new File(project(':server').projectDir, 'licenses')
licensesDir new File(project(':distribution').projectDir, 'licenses')
}
task buildDefaultNoJdkNotice(type: NoticeTask) {
licensesDir new File(project(':server').projectDir, 'licenses')
}
task buildOssNoJdkNotice(type: NoticeTask) {
licensesDir new File(project(':server').projectDir, 'licenses')
}
/*****************************************************************************
* Modules *
*****************************************************************************/
String ossOutputs = 'build/outputs/oss'
String defaultOutputs = 'build/outputs/default'
String transportOutputs = 'build/outputs/transport-only'
task processOssOutputs(type: Sync) {
into ossOutputs
}
task processDefaultOutputs(type: Sync) {
into defaultOutputs
from processOssOutputs
}
// Integ tests work over the rest http layer, so we need a transport included with the integ test zip.
// All transport modules are included so that they may be randomized for testing
task processTransportOutputs(type: Sync) {
into transportOutputs
}
// these are dummy tasks that can be used to depend on the relevant sub output dir
task buildOssModules {
dependsOn processOssOutputs
outputs.dir "${ossOutputs}/modules"
}
task buildOssBin {
dependsOn processOssOutputs
outputs.dir "${ossOutputs}/bin"
}
task buildOssConfig {
dependsOn processOssOutputs
outputs.dir "${ossOutputs}/config"
}
task buildDefaultModules {
dependsOn processDefaultOutputs
outputs.dir "${defaultOutputs}/modules"
}
task buildDefaultBin {
dependsOn processDefaultOutputs
outputs.dir "${defaultOutputs}/bin"
}
task buildDefaultConfig {
dependsOn processDefaultOutputs
outputs.dir "${defaultOutputs}/config"
}
task buildTransportModules {
dependsOn processTransportOutputs
outputs.dir "${transportOutputs}/modules"
}
void copyModule(Sync copyTask, Project module) {
copyTask.configure {
dependsOn { module.bundlePlugin }
from({ zipTree(module.bundlePlugin.outputs.files.singleFile) }) {
includeEmptyDirs false
// these are handled separately in the log4j config tasks below
exclude '*/config/log4j2.properties'
exclude 'config/log4j2.properties'
eachFile { details ->
String name = module.plugins.hasPlugin('elasticsearch.esplugin') ? module.esplugin.name : module.es_meta_plugin.name
// Copy all non config/bin files
// Note these might be unde a subdirectory in the case of a meta plugin
if ((details.relativePath.pathString ==~ /([^\/]+\/)?(config|bin)\/.*/) == false) {
details.relativePath = details.relativePath.prepend('modules', name)
} else if ((details.relativePath.pathString ==~ /([^\/]+\/)(config|bin)\/.*/)) {
// this is the meta plugin case, in which we need to remove the intermediate dir
String[] segments = details.relativePath.segments
details.relativePath = new RelativePath(true, segments.takeRight(segments.length - 1))
}
}
}
}
}
// log4j config could be contained in modules, so we must join it together using these tasks
task buildOssLog4jConfig {
dependsOn processOssOutputs
ext.contents = []
ext.log4jFile = file("${ossOutputs}/log4j2.properties")
outputs.file log4jFile
}
task buildDefaultLog4jConfig {
dependsOn processDefaultOutputs
ext.contents = []
ext.log4jFile = file("${defaultOutputs}/log4j2.properties")
outputs.file log4jFile
}
Closure writeLog4jProperties = {
String mainLog4jProperties = file('src/config/log4j2.properties').getText('UTF-8')
it.log4jFile.setText(mainLog4jProperties, 'UTF-8')
for (String moduleLog4jProperties : it.contents.reverse()) {
it.log4jFile.append(moduleLog4jProperties, 'UTF-8')
}
}
buildOssLog4jConfig.doLast(writeLog4jProperties)
buildDefaultLog4jConfig.doLast(writeLog4jProperties)
// copy log4j2.properties from modules that have it
void copyLog4jProperties(Task buildTask, Project module) {
buildTask.doFirst {
FileTree tree = zipTree(module.bundlePlugin.outputs.files.singleFile)
FileTree filtered = tree.matching {
include 'config/log4j2.properties'
include '*/config/log4j2.properties' // could be in a bundled plugin
}
if (filtered.isEmpty() == false) {
buildTask.contents.add('\n\n' + filtered.singleFile.getText('UTF-8'))
}
}
}
ext.restTestExpansions = [
'expected.modules.count': 0,
]
// we create the buildOssModules task above but fill it here so we can do a single
// loop over modules to also setup cross task dependencies and increment our modules counter
project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each { Project module ->
File licenses = new File(module.projectDir, 'licenses')
if (licenses.exists()) {
buildDefaultNotice.licensesDir licenses
buildOssNotice.licensesDir licenses
}
copyModule(processOssOutputs, module)
if (module.name.startsWith('transport-')) {
copyModule(processTransportOutputs, module)
}
copyLog4jProperties(buildOssLog4jConfig, module)
copyLog4jProperties(buildDefaultLog4jConfig, module)
// make sure the module's integration tests run after the integ-test-zip (ie rest tests)
module.afterEvaluate({
module.integTest.mustRunAfter(':distribution:archives:integ-test-zip:integTest')
})
restTestExpansions['expected.modules.count'] += 1
}
// use licenses from each of the bundled xpack plugins
Project xpack = project(':x-pack:plugin')
xpack.subprojects.findAll { it.parent == xpack }.each { Project xpackModule ->
File licenses = new File(xpackModule.projectDir, 'licenses')
if (licenses.exists()) {
buildDefaultNotice.licensesDir licenses
}
copyModule(processDefaultOutputs, xpackModule)
copyLog4jProperties(buildDefaultLog4jConfig, xpackModule)
}
/*****************************************************************************
* JDKs *
*****************************************************************************/
// extract the bundled jdk version, broken into elements as: [feature, interim, update, build]
// Note the "patch" version is not yet handled here, as it has not yet been used by java.
Pattern JDK_VERSION = Pattern.compile("(\\d+)(\\.\\d+\\.\\d+)?\\+(\\d+)@([a-f0-9]{32})?")
Matcher jdkVersionMatcher = JDK_VERSION.matcher(VersionProperties.bundledJdk)
if (jdkVersionMatcher.matches() == false) {
throw new IllegalArgumentException("Malformed jdk version [" + VersionProperties.bundledJdk + "]")
}
String jdkVersion = jdkVersionMatcher.group(1) + (jdkVersionMatcher.group(2) != null ? (jdkVersionMatcher.group(2)) : "")
String jdkMajor = jdkVersionMatcher.group(1)
String jdkBuild = jdkVersionMatcher.group(3)
String hash = jdkVersionMatcher.group(4)
repositories {
// simpler legacy pattern from JDK 9 to JDK 12 that we are advocating to Oracle to bring back
ivy {
name "legacy-jdk"
url "https://download.oracle.com"
metadataSources {
artifact()
}
patternLayout {
artifact "java/GA/jdk${jdkMajor}/${jdkBuild}/GPL/openjdk-[revision]_[module]-x64_bin.[ext]"
}
}
// current pattern since 12.0.1
ivy {
name "jdk"
url "https://download.oracle.com"
metadataSources {
artifact()
}
patternLayout {
artifact "java/GA/jdk${jdkVersion}/${hash}/${jdkBuild}/GPL/openjdk-[revision]_[module]-x64_bin.[ext]"
}
}
}
for (String platform : ['linux', 'darwin', 'windows']) {
String jdkConfigName = "jdk_${platform}"
Configuration jdkConfig = configurations.create(jdkConfigName)
String extension = platform.equals('windows') ? 'zip' : 'tar.gz'
dependencies.add(jdkConfigName, "jdk:${platform.equals('darwin') ? 'osx' : platform}:${jdkVersion}@${extension}")
int rootNdx = platform.equals('darwin') ? 2 : 1
Closure removeRootDir = {
it.eachFile { FileCopyDetails details ->
details.relativePath = new RelativePath(true, details.relativePath.segments[rootNdx..-1] as String[])
}
it.includeEmptyDirs false
}
String extractDir = "${buildDir}/jdks/openjdk-${jdkVersion}_${platform}"
project.task("extract${platform.capitalize()}Jdk", type: Copy) {
doFirst {
project.delete(extractDir)
}
into extractDir
if (extension.equals('zip')) {
from({ zipTree(jdkConfig.singleFile) }, removeRootDir)
} else {
from({ tarTree(resources.gzip(jdkConfig.singleFile)) }, removeRootDir)
}
}
}
// make sure we have a clean task since we aren't a java project, but we have tasks that
// put stuff in the build dir
task clean(type: Delete) {
delete 'build'
}
configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
// TODO: the map needs to be an input of the tasks, so that when it changes, the task will re-run...
/*****************************************************************************
* Properties to expand when copying packaging files *
*****************************************************************************/
project.ext {
/*****************************************************************************
* Common files in all distributions *
*****************************************************************************/
libFiles = { oss ->
copySpec {
// delay by using closures, since they have not yet been configured, so no jar task exists yet
from { project(':server').jar }
from { project(':server').configurations.runtime }
from { project(':libs:plugin-classloader').jar }
from { project(':distribution:tools:java-version-checker').jar }
from { project(':distribution:tools:launchers').jar }
into('tools/plugin-cli') {
from { project(':distribution:tools:plugin-cli').jar }
from { project(':distribution:tools:plugin-cli').configurations.runtime }
}
if (oss == false) {
into('tools/security-cli') {
from { project(':x-pack:plugin:security:cli').jar }
from { project(':x-pack:plugin:security:cli').configurations.compile }
}
}
}
}
modulesFiles = { oss, platform ->
copySpec {
eachFile {
if (it.relativePath.segments[-2] == 'bin') {
// bin files, wherever they are within modules (eg platform specific) should be executable
it.mode = 0755
} else {
it.mode = 0644
}
}
Task buildModules
if (oss) {
buildModules = project(':distribution').buildOssModules
} else {
buildModules = project(':distribution').buildDefaultModules
}
List excludePlatforms = ['linux', 'windows', 'darwin']
if (platform != null) {
excludePlatforms.remove(excludePlatforms.indexOf(platform))
} else {
excludePlatforms = []
}
from(buildModules) {
for (String excludePlatform : excludePlatforms) {
exclude "**/platform/${excludePlatform}-x86_64/**"
}
}
}
}
transportModulesFiles = copySpec {
from project(':distribution').buildTransportModules
}
configFiles = { distributionType, oss, jdk ->
copySpec {
with copySpec {
// main config files, processed with distribution specific substitutions
from '../src/config'
exclude 'log4j2.properties' // this is handled separately below
MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, oss, jdk))
}
if (oss) {
from project(':distribution').buildOssLog4jConfig
from project(':distribution').buildOssConfig
} else {
from project(':distribution').buildDefaultLog4jConfig
from project(':distribution').buildDefaultConfig
}
}
}
binFiles = { distributionType, oss, jdk ->
copySpec {
// non-windows files, for all distributions
with copySpec {
from '../src/bin'
exclude '*.exe'
exclude '*.bat'
eachFile { it.setMode(0755) }
MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, oss, jdk))
}
// windows files, only for zip
if (distributionType == 'zip') {
with copySpec {
from '../src/bin'
include '*.bat'
filter(FixCrLfFilter, eol: FixCrLfFilter.CrLf.newInstance('crlf'))
MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, oss, jdk))
}
with copySpec {
from '../src/bin'
include '*.exe'
}
}
// module provided bin files
with copySpec {
eachFile { it.setMode(0755) }
if (oss) {
from project(':distribution').buildOssBin
} else {
from project(':distribution').buildDefaultBin
}
if (distributionType != 'zip') {
exclude '*.bat'
}
}
}
}
noticeFile = { oss, jdk ->
copySpec {
if (project.name == 'integ-test-zip') {
from buildServerNotice
} else {
if (oss && jdk) {
from buildOssNotice
} else if (oss) {
from buildOssNoJdkNotice
} else if (jdk) {
from buildDefaultNotice
} else {
from buildDefaultNoJdkNotice
}
}
}
}
jdkFiles = { platform ->
copySpec {
from project(':distribution').tasks.getByName("extract${platform.capitalize()}Jdk")
eachFile { FileCopyDetails details ->
if (details.relativePath.segments[-2] == 'bin' || details.relativePath.segments[-1] == 'jspawnhelper') {
details.mode = 0755
}
}
}
}
}
}
task run(type: RunTask) {
distribution = System.getProperty('run.distribution', 'default')
if (distribution == 'default') {
String licenseType = System.getProperty("run.license_type", "basic")
if (licenseType == 'trial') {
setting 'xpack.ml.enabled', 'true'
setting 'xpack.graph.enabled', 'true'
setting 'xpack.watcher.enabled', 'true'
setting 'xpack.license.self_generated.type', 'trial'
setupCommand 'setupTestAdmin',
'bin/elasticsearch-users', 'useradd', 'elastic-admin', '-p', 'elastic-password', '-r', 'superuser'
} else if (licenseType != 'basic') {
throw new IllegalArgumentException("Unsupported self-generated license type: [" + licenseType + "[basic] or [trial].")
}
setting 'xpack.security.enabled', 'true'
setting 'xpack.monitoring.enabled', 'true'
setting 'xpack.sql.enabled', 'true'
setting 'xpack.rollup.enabled', 'true'
keystoreSetting 'bootstrap.password', 'password'
}
}
/**
* Build some variables that are replaced in the packages. This includes both
* scripts like bin/elasticsearch and bin/elasticsearch-plugin that a user might run and also
* scripts like postinst which are run as part of the installation.
*
* <dl>
* <dt>package.name</dt>
* <dd>The name of the project. Its sprinkled throughout the scripts.</dd>
* <dt>package.version</dt>
* <dd>The version of the project. Its mostly used to find the exact jar name.
* </dt>
* <dt>path.conf</dt>
* <dd>The default directory from which to load configuration. This is used in
* the packaging scripts, but in that context it is always
* /etc/elasticsearch. Its also used in bin/elasticsearch-plugin, where it is
* /etc/elasticsearch for the os packages but $ESHOME/config otherwise.</dd>
* <dt>path.env</dt>
* <dd>The env file sourced before bin/elasticsearch to set environment
* variables. Think /etc/defaults/elasticsearch.</dd>
* <dt>heap.min and heap.max</dt>
* <dd>Default min and max heap</dd>
* <dt>scripts.footer</dt>
* <dd>Footer appended to control scripts embedded in the distribution that is
* (almost) entirely there for cosmetic reasons.</dd>
* <dt>stopping.timeout</dt>
* <dd>RPM's init script needs to wait for elasticsearch to stop before
* returning from stop and it needs a maximum time to wait. This is it. One
* day. DEB retries forever.</dd>
* </dl>
*/
subprojects {
ext.expansionsForDistribution = { distributionType, oss, jdk ->
final String defaultHeapSize = "1g"
final String packagingPathData = "path.data: /var/lib/elasticsearch"
final String pathLogs = "/var/log/elasticsearch"
final String packagingPathLogs = "path.logs: ${pathLogs}"
final String packagingLoggc = "${pathLogs}/gc.log"
String licenseText
if (oss) {
licenseText = rootProject.file('licenses/APACHE-LICENSE-2.0.txt').getText('UTF-8')
} else {
licenseText = rootProject.file('licenses/ELASTIC-LICENSE.txt').getText('UTF-8')
}
// license text needs to be indented with a single space
licenseText = ' ' + licenseText.replace('\n', '\n ')
String footer = "# Built for ${project.name}-${project.version} " +
"(${distributionType})"
Map<String, Object> expansions = [
'project.name': project.name,
'project.version': version,
'path.conf': [
'deb': '/etc/elasticsearch',
'rpm': '/etc/elasticsearch',
'def': '"$ES_HOME"/config'
],
'path.data': [
'deb': packagingPathData,
'rpm': packagingPathData,
'def': '#path.data: /path/to/data'
],
'path.env': [
'deb': '/etc/default/elasticsearch',
'rpm': '/etc/sysconfig/elasticsearch',
/* There isn't one of these files for tar or zip but its important to
make an empty string here so the script can properly skip it. */
'def': 'if [ -z "$ES_PATH_CONF" ]; then ES_PATH_CONF="$ES_HOME"/config; done',
],
'source.path.env': [
'deb': 'source /etc/default/elasticsearch',
'rpm': 'source /etc/sysconfig/elasticsearch',
'def': 'if [ -z "$ES_PATH_CONF" ]; then ES_PATH_CONF="$ES_HOME"/config; fi',
],
'path.logs': [
'deb': packagingPathLogs,
'rpm': packagingPathLogs,
'def': '#path.logs: /path/to/logs'
],
'loggc': [
'deb': packagingLoggc,
'rpm': packagingLoggc,
'def': 'logs/gc.log'
],
'heap.min': defaultHeapSize,
'heap.max': defaultHeapSize,
'heap.dump.path': [
'deb': "-XX:HeapDumpPath=/var/lib/elasticsearch",
'rpm': "-XX:HeapDumpPath=/var/lib/elasticsearch",
'def': "-XX:HeapDumpPath=data"
],
'error.file': [
'deb': "-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log",
'rpm': "-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log",
'def': "-XX:ErrorFile=logs/hs_err_pid%p.log"
],
'stopping.timeout': [
'rpm': 86400,
],
'scripts.footer': [
/* Debian needs exit 0 on these scripts so we add it here and preserve
the pretty footer. */
'deb': "exit 0\n${footer}",
'def': footer
],
'es.distribution.flavor': [
'def': oss ? 'oss' : 'default'
],
'es.distribution.type': [
'deb': 'deb',
'rpm': 'rpm',
'tar': 'tar',
'zip': 'zip'
],
'es.bundled_jdk': [
'def': jdk ? 'true' : 'false'
],
'license.name': [
'deb': oss ? 'ASL-2.0' : 'Elastic-License'
],
'license.text': [
'deb': licenseText,
],
]
Map<String, String> result = [:]
expansions = expansions.each { key, value ->
if (value instanceof Map) {
// 'def' is for default but its three characters like 'rpm' and 'deb'
value = value[distributionType] ?: value['def']
if (value == null) {
return
}
}
result[key] = value
}
return result
}
ext.assertLinesInFile = { Path path, List<String> expectedLines ->
final List<String> actualLines = Files.readAllLines(path)
int line = 0
for (final String expectedLine : expectedLines) {
final String actualLine = actualLines.get(line)
if (expectedLine != actualLine) {
throw new GradleException("expected line [${line + 1}] in [${path}] to be [${expectedLine}] but was [${actualLine}]")
}
line++
}
}
}
['archives:windows-zip','archives:oss-windows-zip',
'archives:darwin-tar','archives:oss-darwin-tar',
'archives:linux-tar', 'archives:oss-linux-tar',
'archives:integ-test-zip',
'packages:rpm', 'packages:deb',
'packages:oss-rpm', 'packages:oss-deb',
].forEach { subName ->
Project subproject = project("${project.path}:${subName}")
Configuration configuration = configurations.create(subproject.name)
dependencies {
"${configuration.name}" project(subproject.path)
}
}