/* * SPDX-License-Identifier: Apache-2.0 * * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. * * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. */ /* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright * ownership. Elasticsearch licenses this file to you under * the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ import org.apache.tools.ant.filters.FixCrLfFilter import org.opensearch.gradle.ConcatFilesTask import org.opensearch.gradle.DependenciesInfoTask import org.opensearch.gradle.MavenFilteringHack import org.opensearch.gradle.NoticeTask import org.opensearch.gradle.VersionProperties import org.opensearch.gradle.info.BuildParams import java.nio.file.Files import java.nio.file.Path plugins { id 'base' } /***************************************************************************** * Third party dependencies report * *****************************************************************************/ // Concatenates the dependencies CSV files into a single file tasks.register("generateDependenciesReport", ConcatFilesTask) { dependsOn rootProject.allprojects.collect { it.tasks.withType(DependenciesInfoTask) } files = fileTree(dir: project.rootDir, include: '**/dependencies.csv') headerLine = "name,version,url,license,sourceURL" target = new File(System.getProperty('csv') ?: "${project.buildDir}/reports/dependencies/opensearch-dependencies.csv") // explicitly add our dependency on the JDK String jdkVersion = VersionProperties.versions.get('bundled_jdk') String jdkMajorVersion = jdkVersion.split('[+.]')[0] String sourceUrl = "https://hg.openjdk.java.net/jdk-updates/jdk${jdkMajorVersion}u/archive/jdk-${jdkVersion}.tar.gz" additionalLines << "OpenJDK,${jdkVersion},https://openjdk.java.net/,GPL-2.0-with-classpath-exception,${sourceUrl}".toString() } /***************************************************************************** * Notice file * *****************************************************************************/ // integ test zip only uses server, so a different notice file is needed there def buildServerNoticeTaskProvider = tasks.register("buildServerNotice", NoticeTask) def buildNoticeTaskProvider = tasks.register("buildNotice", NoticeTask) { licensesDir new File(project(':distribution').projectDir, 'licenses') } def buildNoJdkNoticeTaskProvider = tasks.register("buildNoJdkNotice", NoticeTask) // The :server and :libs projects belong to all distributions tasks.withType(NoticeTask).configureEach { licensesDir project(':server').file('licenses') source project(':server').file('src/main/java') project(':libs').subprojects.each { Project lib -> licensesDir lib.file('licenses') source lib.file('src/main/java') } } /***************************************************************************** * Modules * *****************************************************************************/ String distOutputs = 'build/outputs/dist' String systemdOutputs = 'build/outputs/systemd' String transportOutputs = 'build/outputs/transport-only' String externalTestOutputs = 'build/outputs/external-test' def processOutputsTaskProvider = tasks.register("processOutputs", Sync) { into distOutputs } def processSystemdOutputsTaskProvider = tasks.register("processSystemdOutputs", Sync) { into systemdOutputs } def processExternalTestOutputsTaskProvider = tasks.register("processExternalTestOutputs", Sync) { into externalTestOutputs } // Integ tests work over the rest http layer, so we need a transport included with the integ test zip. // All transport modules are included so that they may be randomized for testing def processTransportOutputsTaskProvider = tasks.register("processTransportOutputs", Sync) { into transportOutputs } // these are dummy tasks that can be used to depend on the relevant sub output dir def buildModulesTaskProvider = tasks.register("buildModules") { dependsOn processOutputsTaskProvider outputs.dir "${distOutputs}/modules" } tasks.register("buildBin") { dependsOn "processOutputs" outputs.dir "${distOutputs}/bin" } tasks.register("buildConfig") { dependsOn "processOutputs" outputs.dir "${distOutputs}/config" } def buildSystemdModuleTaskProvider = tasks.register("buildSystemdModule") { dependsOn "processSystemdOutputs" outputs.dir "${systemdOutputs}/modules" } def buildTransportModulesTaskProvider = tasks.register("buildTransportModules") { dependsOn processTransportOutputsTaskProvider outputs.dir "${transportOutputs}/modules" } def buildExternalTestModulesTaskProvider = tasks.register("buildExternalTestModules") { dependsOn "processExternalTestOutputs" outputs.dir "${externalTestOutputs}/modules" } Configuration moduleZip(Project module) { Dependency dep = project.dependencies.project(path: module.path, configuration: 'zip') Configuration config = project.configurations.detachedConfiguration(dep) return config } void copyModule(TaskProvider copyTask, Project module) { copyTask.configure { Configuration moduleConfig = moduleZip(module) dependsOn moduleConfig from({ zipTree(moduleConfig.singleFile) }) { includeEmptyDirs false // these are handled separately in the log4j config tasks below exclude '*/config/log4j2.properties' exclude 'config/log4j2.properties' eachFile { details -> String name = module.plugins.hasPlugin('opensearch.opensearchplugin') ? module.opensearchplugin.name : module.opensearch_meta_plugin.name // Copy all non config/bin files // Note these might be unde a subdirectory in the case of a meta plugin if ((details.relativePath.pathString ==~ /([^\/]+\/)?(config|bin)\/.*/) == false) { details.relativePath = details.relativePath.prepend('modules', name) } else if ((details.relativePath.pathString ==~ /([^\/]+\/)(config|bin)\/.*/)) { // this is the meta plugin case, in which we need to remove the intermediate dir String[] segments = details.relativePath.segments details.relativePath = new RelativePath(true, segments.takeRight(segments.length - 1)) } } } } } // log4j config could be contained in modules, so we must join it together using these tasks def buildLog4jConfigTaskProvider = tasks.register("buildLog4jConfig") { dependsOn "processOutputs" ext.contents = [] ext.log4jFile = file("${distOutputs}/log4j2.properties") outputs.file log4jFile } Closure writeLog4jProperties = { String mainLog4jProperties = file('src/config/log4j2.properties').getText('UTF-8') it.log4jFile.setText(mainLog4jProperties, 'UTF-8') for (String moduleLog4jProperties : it.contents.reverse()) { it.log4jFile.append(moduleLog4jProperties, 'UTF-8') } } buildLog4jConfigTaskProvider.configure { doLast(writeLog4jProperties) } // copy log4j2.properties from modules that have it void copyLog4jProperties(TaskProvider buildTask, Project module) { buildTask.configure { Configuration moduleConfig = moduleZip(module) dependsOn moduleConfig doFirst { FileTree tree = zipTree(moduleConfig.singleFile) FileTree filtered = tree.matching { include 'config/log4j2.properties' include '*/config/log4j2.properties' // could be in a bundled plugin } if (filtered.isEmpty() == false) { contents.add('\n\n' + filtered.singleFile.getText('UTF-8')) } } } } ext.restTestExpansions = [ 'expected.modules.count': 0, ] // we create the buildModules task above but fill it here so we can do a single // loop over modules to also setup cross task dependencies and increment our modules counter project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each { Project module -> if (module.name == 'systemd') { // the systemd module is only included in the package distributions return } File licenses = new File(module.projectDir, 'licenses') if (licenses.exists()) { buildNotice.configure { licensesDir licenses source module.file('src/main/java') } } copyModule(processOutputsTaskProvider, module) if (module.name.startsWith('transport-')) { copyModule(processTransportOutputsTaskProvider, module) } copyLog4jProperties(buildLog4jConfigTaskProvider, module) restTestExpansions['expected.modules.count'] += 1 } // copy all sandbox modules if the distribution is a snapshot if (VersionProperties.isOpenSearchSnapshot()) { Properties sysProps = System.getProperties(); // setting this property to false will exclude the sandbox modules from the distribution final String enableSandbox = sysProps.getProperty("sandbox.enabled", "true"); if(sysProps != null && enableSandbox == "true") { tasks.withType(NoticeTask).configureEach { project(':sandbox:libs').subprojects.each { Project lib -> licensesDir lib.file('licenses') source lib.file('src/main/java') } } project.rootProject.subprojects.findAll { it.parent.path == ':sandbox:modules' }.each { Project module -> File licenses = new File(module.projectDir, 'licenses') if (licenses.exists()) { buildNotice.configure { licensesDir licenses source module.file('src/main/java') } } copyModule(processOutputsTaskProvider, module) copyLog4jProperties(buildLog4jConfigTaskProvider, module) restTestExpansions['expected.modules.count'] += 1 } } } copyModule(processSystemdOutputsTaskProvider, project(':modules:systemd')) project(':test:external-modules').subprojects.each { Project testModule -> copyModule(processExternalTestOutputsTaskProvider, testModule) } configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) { apply plugin: 'opensearch.jdk-download' apply plugin: 'opensearch.repositories' // Setup all required JDKs project.jdks { ['darwin', 'linux', 'windows'].each { platform -> (platform == 'linux' ? ['x64', 'aarch64'] : ['x64']).each { architecture -> "bundled_${platform}_${architecture}" { it.platform = platform it.version = VersionProperties.getBundledJdk(platform) it.vendor = VersionProperties.bundledJdkVendor it.architecture = architecture } } } } // TODO: the map needs to be an input of the tasks, so that when it changes, the task will re-run... /***************************************************************************** * Properties to expand when copying packaging files * *****************************************************************************/ configurations { ['libs', 'libsPluginCli', 'libsKeystoreCli', 'libsUpgradeCli'].each { create(it) { canBeConsumed = false canBeResolved = true attributes { attribute(Category.CATEGORY_ATTRIBUTE, objects.named(Category, Category.LIBRARY)) attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage, Usage.JAVA_RUNTIME)) attribute(Bundling.BUNDLING_ATTRIBUTE, objects.named(Bundling, Bundling.EXTERNAL)) } } } } dependencies { libs project(':server') libs project(':libs:opensearch-plugin-classloader') libs project(':distribution:tools:java-version-checker') libs project(':distribution:tools:launchers') libsPluginCli project(':distribution:tools:plugin-cli') libsKeystoreCli project(path: ':distribution:tools:keystore-cli') libsUpgradeCli project(path: ':distribution:tools:upgrade-cli') } project.ext { /***************************************************************************** * Common files in all distributions * *****************************************************************************/ libFiles = { copySpec { // delay by using closures, since they have not yet been configured, so no jar task exists yet from(configurations.libs) into('tools/plugin-cli') { from(configurations.libsPluginCli) } into('tools/keystore-cli') { from(configurations.libsKeystoreCli) } into('tools/upgrade-cli') { from(configurations.libsUpgradeCli) } } } modulesFiles = { platform -> copySpec { eachFile { if (it.relativePath.segments[-2] == 'bin' || (platform == 'darwin-x64' && it.relativePath.segments[-2] == 'MacOS')) { // bin files, wherever they are within modules (eg platform specific) should be executable // and MacOS is an alternative to bin on macOS it.mode = 0755 } else { it.mode = 0644 } } def buildModules = buildModulesTaskProvider List excludePlatforms = ['darwin-x64', 'freebsd-x64', 'linux-x64', 'linux-arm64', 'windows-x64'] if (platform != null) { excludePlatforms.remove(excludePlatforms.indexOf(platform)) } else { excludePlatforms = [] } from(buildModules) { for (String excludePlatform : excludePlatforms) { exclude "**/platform/${excludePlatform}/**" } } if (BuildParams.isSnapshotBuild()) { from(buildExternalTestModulesTaskProvider) } if (project.path.startsWith(':distribution:packages')) { from(buildSystemdModuleTaskProvider) } } } transportModulesFiles = copySpec { from buildTransportModulesTaskProvider } configFiles = { distributionType, jdk -> copySpec { with copySpec { // main config files, processed with distribution specific substitutions from '../src/config' exclude 'log4j2.properties' // this is handled separately below MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, jdk)) } from project(':distribution').buildLog4jConfig from project(':distribution').buildConfig } } binFiles = { distributionType, jdk -> copySpec { // non-windows files, for all distributions with copySpec { from '../src/bin' exclude '*.exe' exclude '*.bat' eachFile { it.setMode(0755) } MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, jdk)) } // windows files, only for zip if (distributionType == 'zip') { with copySpec { from '../src/bin' include '*.bat' filter(FixCrLfFilter, eol: FixCrLfFilter.CrLf.newInstance('crlf')) MavenFilteringHack.filter(it, expansionsForDistribution(distributionType, jdk)) } with copySpec { from '../src/bin' include '*.exe' } } // module provided bin files with copySpec { eachFile { it.setMode(0755) } from project(':distribution').buildBin if (distributionType != 'zip') { exclude '*.bat' } } } } noticeFile = { jdk -> copySpec { if (project.name == 'integ-test-zip') { from buildServerNoticeTaskProvider } else { if (jdk) { from buildNoticeTaskProvider } else { from buildNoJdkNoticeTaskProvider } } } } jdkFiles = { Project project, String platform, String architecture -> return copySpec { /* * Jdk uses aarch64 from ARM. Translating from arm64 to aarch64 which Jdk understands. */ if ("arm64".equals(architecture)) { architecture = "aarch64" } from project.jdks."bundled_${platform}_${architecture}" exclude "demo/**" /* * The Contents/MacOS directory interferes with notarization, and is unused by our distribution, so we exclude * it from the build. */ if ("darwin".equals(platform)) { exclude "Contents/MacOS" } eachFile { FileCopyDetails details -> if (details.relativePath.segments[-2] == 'bin' || details.relativePath.segments[-1] == 'jspawnhelper') { details.mode = 0755 } if (details.name == 'src.zip') { details.exclude() } } } } } } /** * Build some variables that are replaced in the packages. This includes both * scripts like bin/opensearch and bin/opensearch-plugin that a user might run and also * scripts like postinst which are run as part of the installation. * *
*
package.name
*
The name of the project. Its sprinkled throughout the scripts.
*
package.version
*
The version of the project. Its mostly used to find the exact jar name. * *
path.conf
*
The default directory from which to load configuration. This is used in * the packaging scripts, but in that context it is always * /etc/opensearch. Its also used in bin/opensearch-plugin, where it is * /etc/opensearch for the os packages but $OPENSEARCH_HOME/config otherwise.
*
path.env
*
The env file sourced before bin/opensearch to set environment * variables. Think /etc/defaults/opensearch.
*
heap.min and heap.max
*
Default min and max heap
*
scripts.footer
*
Footer appended to control scripts embedded in the distribution that is * (almost) entirely there for cosmetic reasons.
*
stopping.timeout
*
RPM's init script needs to wait for opensearch to stop before * returning from stop and it needs a maximum time to wait. This is it. One * day. DEB retries forever.
*
*/ subprojects { ext.expansionsForDistribution = { distributionType, jdk -> final String defaultHeapSize = "1g" final String packagingPathData = "path.data: /var/lib/opensearch" final String pathLogs = "/var/log/opensearch" final String packagingPathLogs = "path.logs: ${pathLogs}" final String packagingLoggc = "${pathLogs}/gc.log" String licenseText = rootProject.file('licenses/APACHE-LICENSE-2.0.txt').getText('UTF-8') // license text needs to be indented with a single space licenseText = ' ' + licenseText.replace('\n', '\n ') String footer = "# Built for ${project.name}-${project.version} " + "(${distributionType})" Map expansions = [ 'project.name': project.name, 'project.version': version, 'path.conf': [ 'deb': '/etc/opensearch', 'rpm': '/etc/opensearch', 'def': '"$OPENSEARCH_HOME"/config' ], 'path.data': [ 'deb': packagingPathData, 'rpm': packagingPathData, 'def': '#path.data: /path/to/data' ], 'path.env': [ 'deb': '/etc/default/opensearch', 'rpm': '/etc/sysconfig/opensearch', /* There isn't one of these files for tar or zip but its important to make an empty string here so the script can properly skip it. */ 'def': 'if [ -z "$OPENSEARCH_PATH_CONF" ]; then OPENSEARCH_PATH_CONF="$OPENSEARCH_HOME"/config; done', ], 'source.path.env': [ 'deb': 'source /etc/default/opensearch', 'rpm': 'source /etc/sysconfig/opensearch', 'def': 'if [ -z "$OPENSEARCH_PATH_CONF" ]; then OPENSEARCH_PATH_CONF="$OPENSEARCH_HOME"/config; fi', ], 'path.logs': [ 'deb': packagingPathLogs, 'rpm': packagingPathLogs, 'def': '#path.logs: /path/to/logs' ], 'loggc': [ 'deb': packagingLoggc, 'rpm': packagingLoggc, 'def': 'logs/gc.log' ], 'heap.min': defaultHeapSize, 'heap.max': defaultHeapSize, 'heap.dump.path': [ 'deb': "-XX:HeapDumpPath=/var/lib/opensearch", 'rpm': "-XX:HeapDumpPath=/var/lib/opensearch", 'def': "-XX:HeapDumpPath=data" ], 'error.file': [ 'deb': "-XX:ErrorFile=/var/log/opensearch/hs_err_pid%p.log", 'rpm': "-XX:ErrorFile=/var/log/opensearch/hs_err_pid%p.log", 'def': "-XX:ErrorFile=logs/hs_err_pid%p.log" ], 'stopping.timeout': [ 'rpm': 86400, ], 'scripts.footer': [ /* Debian needs exit 0 on these scripts so we add it here and preserve the pretty footer. */ 'deb': "exit 0\n${footer}", 'def': footer ], 'opensearch.distribution.type': [ 'deb': 'deb', 'rpm': 'rpm', 'tar': 'tar', 'zip': 'zip' ], 'opensearch.bundled_jdk': [ 'def': jdk ? 'true' : 'false' ], 'license.name': [ 'deb': 'ASL-2.0' ], 'license.text': [ 'deb': licenseText, ], ] Map result = [:] expansions = expansions.each { key, value -> if (value instanceof Map) { // 'def' is for default but its three characters like 'rpm' and 'deb' value = value[distributionType] ?: value['def'] if (value == null) { return } } result[key] = value } return result } ext.assertLinesInFile = { Path path, List expectedLines -> final List actualLines = Files.readAllLines(path) int line = 0 for (final String expectedLine : expectedLines) { final String actualLine = actualLines.get(line) if (expectedLine != actualLine) { throw new GradleException("expected line [${line + 1}] in [${path}] to be [${expectedLine}] but was [${actualLine}]") } line++ } } } ['archives:darwin-tar', 'archives:integ-test-zip', 'archives:linux-arm64-tar', 'archives:linux-tar', 'archives:windows-zip', 'packages:arm64-rpm', 'packages:arm64-deb', 'packages:rpm', 'packages:deb' ].forEach { subName -> Project subproject = project("${project.path}:${subName}") Configuration configuration = configurations.create(subproject.name) dependencies { "${configuration.name}" project(path: subproject.path, configuration:'default') } }