/* * SPDX-License-Identifier: Apache-2.0 * * The OpenSearch Contributors require contributions made to * this file be licensed under the Apache-2.0 license or a * compatible open source license. * * Modifications Copyright OpenSearch Contributors. See * GitHub history for details. */ /* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright * ownership. Elasticsearch licenses this file to you under * the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ import com.avast.gradle.dockercompose.tasks.ComposePull import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin import org.apache.tools.ant.taskdefs.condition.Os import org.opensearch.gradle.BuildPlugin import org.opensearch.gradle.Version import org.opensearch.gradle.VersionProperties import org.opensearch.gradle.info.BuildParams import org.opensearch.gradle.plugin.PluginBuildPlugin import org.gradle.plugins.ide.eclipse.model.AccessRule import org.gradle.plugins.ide.eclipse.model.EclipseJdt import org.gradle.plugins.ide.eclipse.model.SourceFolder import org.gradle.util.DistributionLocator import org.gradle.util.GradleVersion import static org.opensearch.gradle.util.GradleUtils.maybeConfigure plugins { id 'lifecycle-base' id 'opensearch.docker-support' id 'opensearch.global-build-info' id "com.diffplug.spotless" version "6.6.1" apply false id "org.gradle.test-retry" version "1.4.0" apply false id "test-report-aggregation" id 'jacoco-report-aggregation' } apply from: 'gradle/build-complete.gradle' apply from: 'gradle/runtime-jdk-provision.gradle' apply from: 'gradle/ide.gradle' apply from: 'gradle/forbidden-dependencies.gradle' apply from: 'gradle/formatting.gradle' apply from: 'gradle/local-distribution.gradle' apply from: 'gradle/fips.gradle' apply from: 'gradle/run.gradle' apply from: 'gradle/missing-javadoc.gradle' apply from: 'gradle/code-coverage.gradle' // common maven publishing configuration allprojects { group = 'org.opensearch' version = VersionProperties.getOpenSearch() description = "OpenSearch subproject ${project.path}" } configure(allprojects - project(':distribution:archives:integ-test-zip')) { project.pluginManager.withPlugin('nebula.maven-base-publish') { if (project.pluginManager.hasPlugin('opensearch.build') == false) { throw new GradleException("Project ${path} publishes a pom but doesn't apply the build plugin.") } } } subprojects { // Default to the apache license project.ext.licenseName = 'The Apache Software License, Version 2.0' project.ext.licenseUrl = 'http://www.apache.org/licenses/LICENSE-2.0.txt' // we only use maven publish to add tasks for pom generation plugins.withType(MavenPublishPlugin).whenPluginAdded { publishing { publications { // add license information to generated poms all { pom.withXml { XmlProvider xml -> Node node = xml.asNode() node.appendNode('inceptionYear', '2021') Node license = node.appendNode('licenses').appendNode('license') license.appendNode('name', project.licenseName) license.appendNode('url', project.licenseUrl) license.appendNode('distribution', 'repo') Node developer = node.appendNode('developers').appendNode('developer') developer.appendNode('name', 'OpenSearch') developer.appendNode('url', 'https://github.com/opensearch-project/OpenSearch') } } } repositories { maven { name = 'test' url = "${rootProject.buildDir}/local-test-repo" } maven { name = 'Snapshots' url = 'https://aws.oss.sonatype.org/content/repositories/snapshots' credentials { username "$System.env.SONATYPE_USERNAME" password "$System.env.SONATYPE_PASSWORD" } } } } } plugins.withType(BuildPlugin).whenPluginAdded { project.licenseFile = project.rootProject.file('licenses/APACHE-LICENSE-2.0.txt') project.noticeFile = project.rootProject.file('NOTICE.txt') } } tasks.register("updateCIBwcVersions") { doLast { File yml = file(".ci/bwcVersions") yml.text = "" yml << "BWC_VERSION:\n" BuildParams.bwcVersions.indexCompatible.each { yml << " - \"$it\"\n" } } } // build metadata from previous build, contains eg hashes for bwc builds String buildMetadataValue = System.getenv('BUILD_METADATA') if (buildMetadataValue == null) { buildMetadataValue = '' } Map buildMetadataMap = buildMetadataValue.tokenize(';').collectEntries { def (String key, String value) = it.split('=') return [key, value] } // injecting groovy property variables into all projects allprojects { project.ext { // for ide hacks... isEclipse = System.getProperty("eclipse.launcher") != null || // Detects gradle launched from Eclipse's IDE System.getProperty("eclipse.application") != null || // Detects gradle launched from the Eclipse compiler server gradle.startParameter.taskNames.contains('eclipse') || // Detects gradle launched from the command line to do eclipse stuff gradle.startParameter.taskNames.contains('cleanEclipse') buildMetadata = buildMetadataMap } } tasks.register("verifyVersions") { doLast { if (gradle.startParameter.isOffline()) { throw new GradleException("Must run in online mode to verify versions") } // Read the list from maven central. // Fetch the metadata and parse the xml into Version instances because it's more straight forward here // rather than bwcVersion ( VersionCollection ). new URL('https://repo1.maven.org/maven2/org/opensearch/opensearch/maven-metadata.xml').openStream().withStream { s -> BuildParams.bwcVersions.compareToAuthoritative( new XmlParser().parse(s) .versioning.versions.version .collect { it.text() }.findAll { it ==~ /\d+\.\d+\.\d+/ } .collect { Version.fromString(it) } ) } String ciYml = file(".ci/bwcVersions").text BuildParams.bwcVersions.indexCompatible.each { if (ciYml.contains("\"$it\"\n") == false) { throw new Exception(".ci/bwcVersions is outdated, run `./gradlew updateCIBwcVersions` and check in the results"); } } } } /* * When adding backcompat behavior that spans major versions, temporarily * disabling the backcompat tests is necessary. This flag controls * the enabled state of every bwc task. It should be set back to true * after the backport of the backcompat code is complete. */ boolean bwc_tests_enabled = true /* place an issue link here when committing bwc changes */ final String bwc_tests_disabled_issue = "" if (bwc_tests_enabled == false) { if (bwc_tests_disabled_issue.isEmpty()) { throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false") } println "========================= WARNING =========================" println " Backwards compatibility tests are disabled!" println "See ${bwc_tests_disabled_issue}" println "===========================================================" } if (project.gradle.startParameter.taskNames.find { it.startsWith("checkPart") } != null) { // Disable BWC tests for checkPart* tasks as it's expected that this will run un it's own check bwc_tests_enabled = false } subprojects { ext.bwc_tests_enabled = bwc_tests_enabled } tasks.register("verifyBwcTestsEnabled") { doLast { if (bwc_tests_enabled == false) { throw new GradleException('Bwc tests are disabled. They must be re-enabled after completing backcompat behavior backporting.') } } } tasks.register("branchConsistency") { description 'Ensures this branch is internally consistent. For example, that versions constants match released versions.' group 'Verification' dependsOn ":verifyVersions", ":verifyBwcTestsEnabled" } allprojects { // configure compiler options tasks.withType(JavaCompile).configureEach { JavaCompile compile -> // See please https://bugs.openjdk.java.net/browse/JDK-8209058 if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_11) { compile.options.compilerArgs << '-Werror' } compile.options.compilerArgs << '-Xlint:auxiliaryclass' compile.options.compilerArgs << '-Xlint:cast' compile.options.compilerArgs << '-Xlint:classfile' compile.options.compilerArgs << '-Xlint:dep-ann' compile.options.compilerArgs << '-Xlint:divzero' compile.options.compilerArgs << '-Xlint:empty' compile.options.compilerArgs << '-Xlint:exports' compile.options.compilerArgs << '-Xlint:fallthrough' compile.options.compilerArgs << '-Xlint:finally' compile.options.compilerArgs << '-Xlint:module' compile.options.compilerArgs << '-Xlint:opens' compile.options.compilerArgs << '-Xlint:overloads' compile.options.compilerArgs << '-Xlint:overrides' compile.options.compilerArgs << '-Xlint:-processing' compile.options.compilerArgs << '-Xlint:rawtypes' compile.options.compilerArgs << '-Xlint:removal' compile.options.compilerArgs << '-Xlint:requires-automatic' compile.options.compilerArgs << '-Xlint:requires-transitive-automatic' compile.options.compilerArgs << '-Xlint:static' compile.options.compilerArgs << '-Xlint:unchecked' compile.options.compilerArgs << '-Xlint:varargs' compile.options.compilerArgs << '-Xlint:preview' // TODO: disabled warnings: path, serial, options, deprecation, try // -path because gradle will send in paths that don't always exist. // -missing because we have tons of missing @returns and @param. // -serial because we don't use java serialization. compile.options.compilerArgs << '-Xdoclint:accessibility' compile.options.compilerArgs << '-Xdoclint:html' compile.options.compilerArgs << '-Xdoclint:reference' compile.options.compilerArgs << '-Xdoclint:syntax' } // ignore missing javadocs tasks.withType(Javadoc).configureEach { Javadoc javadoc -> // the -quiet here is because of a bug in gradle, in that adding a string option // by itself is not added to the options. By adding quiet, both this option and // the "value" -quiet is added, separated by a space. This is ok since the javadoc // command already adds -quiet, so we are just duplicating it // see https://discuss.gradle.org/t/add-custom-javadoc-option-that-does-not-take-an-argument/5959 javadoc.options.encoding = 'UTF8' javadoc.options.addStringOption('Xdoclint:all,-missing', '-quiet') javadoc.options.tags = ["opensearch.internal", "opensearch.api", "opensearch.experimental"] } // support for reproducible builds tasks.withType(AbstractArchiveTask).configureEach { // ignore file timestamps // be consistent in archive file order preserveFileTimestamps = false reproducibleFileOrder = true } project.afterEvaluate { // Handle javadoc dependencies across projects. Order matters: the linksOffline for // org.opensearch:opensearch must be the last one or all the links for the // other packages (e.g org.opensearch.client) will point to server rather than // their own artifacts. if (project.plugins.hasPlugin(BuildPlugin) || project.plugins.hasPlugin(PluginBuildPlugin)) { String artifactsHost = VersionProperties.getOpenSearch().endsWith("-SNAPSHOT") ? "https://artifacts.opensearch.org/snapshots/" : "https://artifacts.opensearch.org/releases/" Closure sortClosure = { a, b -> b.group <=> a.group } Closure depJavadocClosure = { shadowed, dep -> if ((dep instanceof ProjectDependency) == false) { return } Project upstreamProject = dep.dependencyProject if (upstreamProject == null) { return } if (shadowed) { /* * Include the source of shadowed upstream projects so we don't * have to publish their javadoc. */ project.evaluationDependsOn(upstreamProject.path) project.javadoc.source += upstreamProject.javadoc.source /* * Instead we need the upstream project's javadoc classpath so * we don't barf on the classes that it references. */ project.javadoc.classpath += upstreamProject.javadoc.classpath } else { // Link to non-shadowed dependant projects project.javadoc.dependsOn "${upstreamProject.path}:javadoc" String externalLinkName = upstreamProject.archivesBaseName String artifactPath = dep.group.replaceAll('\\.', '/') + '/' + externalLinkName.replaceAll('\\.', '/') + '/' + dep.version project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${upstreamProject.buildDir}/docs/javadoc/" } } boolean hasShadow = project.plugins.hasPlugin(ShadowPlugin) project.configurations.implementation.dependencies .findAll() .toSorted(sortClosure) .each({ c -> depJavadocClosure(hasShadow, c) }) project.configurations.compileOnly.dependencies .findAll() .toSorted(sortClosure) .each({ c -> depJavadocClosure(false, c) }) if (hasShadow) { // include any dependencies for shadow JAR projects that are *not* bundled in the shadow JAR project.configurations.shadow.dependencies .findAll() .toSorted(sortClosure) .each({ c -> depJavadocClosure(false, c) }) } } } } // Ensure similar tasks in dependent projects run first. The projectsEvaluated here is // important because, while dependencies.all will pickup future dependencies, // it is not necessarily true that the task exists in both projects at the time // the dependency is added. gradle.projectsEvaluated { allprojects { if (project.path == ':test:framework') { // :test:framework:test cannot run before and after :server:test return } if (tasks.findByPath('test') != null && tasks.findByPath('integTest') != null) { integTest.mustRunAfter test } project.tasks.withType(Test) { task -> if (task != null) { if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_17) { task.jvmArgs += ["-Djava.security.manager=allow"] } } } configurations.matching { it.canBeResolved }.all { Configuration configuration -> dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep -> Project upstreamProject = dep.dependencyProject if (upstreamProject != null) { if (project.path == upstreamProject.path) { // TODO: distribution integ tests depend on themselves (!), fix that return } for (String taskName : ['test', 'integTest']) { Task task = project.tasks.findByName(taskName) Task upstreamTask = upstreamProject.tasks.findByName(taskName) if (task != null && upstreamTask != null) { task.shouldRunAfter(upstreamTask) } } } } } } dependencies { subprojects.findAll { it.pluginManager.hasPlugin('java') }.forEach { testReportAggregation it } subprojects.findAll { it.pluginManager.hasPlugin('jacoco') }.forEach { jacocoAggregation it } } } // test retry configuration subprojects { apply plugin: "org.gradle.test-retry" tasks.withType(Test).configureEach { retry { failOnPassedAfterRetry = false maxRetries = 3 maxFailures = 10 } } } // eclipse configuration allprojects { apply plugin: 'eclipse' // Name all the non-root projects after their path so that paths get grouped together when imported into eclipse. if (path != ':') { eclipse.project.name = path if (Os.isFamily(Os.FAMILY_WINDOWS)) { eclipse.project.name = eclipse.project.name.replace(':', '_') } } plugins.withType(JavaBasePlugin) { eclipse.classpath.defaultOutputDir = file('build-eclipse') eclipse.classpath.file.whenMerged { classpath -> // give each source folder a unique corresponding output folder int i = 0; classpath.entries.findAll { it instanceof SourceFolder }.each { folder -> i++; folder.output = "build-eclipse/" + i } } } /* * Allow accessing com/sun/net/httpserver in projects that have * configured forbidden apis to allow it. */ plugins.withType(ForbiddenApisPlugin) { eclipse.classpath.file.whenMerged { classpath -> if (false == forbiddenApisTest.bundledSignatures.contains('jdk-non-portable')) { classpath.entries .findAll { it.kind == "con" && it.toString().contains("org.eclipse.jdt.launching.JRE_CONTAINER") } .each { it.accessRules.add(new AccessRule("accessible", "com/sun/net/httpserver/*")) } } } } File licenseHeaderFile licenseHeaderFile = new File(project.rootDir, 'buildSrc/src/main/resources/license-headers/license-header.txt') String lineSeparator = Os.isFamily(Os.FAMILY_WINDOWS) ? '\\\\r\\\\n' : '\\\\n' String licenseHeader = licenseHeaderFile.getText('UTF-8').replace(System.lineSeparator(), lineSeparator) tasks.register('copyEclipseSettings', Copy) { mustRunAfter 'wipeEclipseSettings' // TODO: "package this up" for external builds from new File(project.rootDir, 'buildSrc/src/main/resources/eclipse.settings') into '.settings' filter { it.replaceAll('@@LICENSE_HEADER_TEXT@@', licenseHeader) } } // otherwise .settings is not nuked entirely tasks.register('wipeEclipseSettings', Delete) { delete '.settings' } tasks.named('cleanEclipse') { dependsOn 'wipeEclipseSettings' } // otherwise the eclipse merging is *super confusing* tasks.named('eclipse') { dependsOn 'cleanEclipse', 'copyEclipseSettings' } afterEvaluate { tasks.findByName("eclipseJdt")?.configure { dependsOn 'copyEclipseSettings' } } } wrapper { distributionType = 'ALL' doLast { final DistributionLocator locator = new DistributionLocator() final GradleVersion version = GradleVersion.version(wrapper.gradleVersion) final URI distributionUri = locator.getDistributionFor(version, wrapper.distributionType.name().toLowerCase(Locale.ENGLISH)) final URI sha256Uri = new URI(distributionUri.toString() + ".sha256") final String sha256Sum = new String(sha256Uri.toURL().bytes) wrapper.getPropertiesFile() << "distributionSha256Sum=${sha256Sum}\n" println "Added checksum to wrapper properties" // Update build-tools to reflect the Gradle upgrade // TODO: we can remove this once we have tests to make sure older versions work. project(':build-tools').file('src/main/resources/minimumGradleVersion').text = gradleVersion println "Updated minimum Gradle Version" } } gradle.projectsEvaluated { subprojects { /* * Remove assemble/dependenciesInfo on all qa projects because we don't * need to publish artifacts for them. */ if (project.name.equals('qa') || project.path.contains(':qa:')) { maybeConfigure(project.tasks, 'assemble') { it.enabled = false } maybeConfigure(project.tasks, 'dependenciesInfo') { it.enabled = false } } } // Having the same group and name for distinct projects causes Gradle to consider them equal when resolving // dependencies leading to hard to debug failures. Run a check across all project to prevent this from happening. // see: https://github.com/gradle/gradle/issues/847 Map coordsToProject = [:] project.allprojects.forEach { p -> String coords = "${p.group}:${p.name}" if (false == coordsToProject.putIfAbsent(coords, p)) { throw new GradleException( "Detected that two projects: ${p.path} and ${coordsToProject[coords].path} " + "have the same name and group: ${coords}. " + "This doesn't currently work correctly in Gradle, see: " + "https://github.com/gradle/gradle/issues/847" ) } } } allprojects { tasks.register('resolveAllDependencies', org.opensearch.gradle.ResolveAllDependencies) { configs = project.configurations if (project.path.contains("fixture")) { dependsOn tasks.withType(ComposePull) } } // helper task to print direct dependencies of a single task project.tasks.addRule("Pattern: Dependencies") { String taskName -> if (taskName.endsWith("Dependencies") == false) { return } if (project.tasks.findByName(taskName) != null) { return } String realTaskName = taskName.substring(0, taskName.length() - "Dependencies".length()) Task realTask = project.tasks.findByName(realTaskName) if (realTask == null) { return } project.tasks.register(taskName) { doLast { println("${realTask.path} dependencies:") for (Task dep : realTask.getTaskDependencies().getDependencies(realTask)) { println(" - ${dep.path}") } } } } def checkPart1 = tasks.register('checkPart1') def checkPart2 = tasks.register('checkPart2') plugins.withId('lifecycle-base') { checkPart1.configure { dependsOn 'check' } } } subprojects { project.ext.disableTasks = { String... tasknames -> for (String taskname : tasknames) { project.tasks.named(taskname).configure { onlyIf { false } } } } } reporting { reports { testAggregateTestReport(AggregateTestReport) { testType = TestSuiteType.UNIT_TEST } } } tasks.named(JavaBasePlugin.CHECK_TASK_NAME) { dependsOn tasks.named('testAggregateTestReport', TestReport) }