diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 51237a6e0e8..0cd869ddf6a 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -116,8 +116,7 @@ class BuildPlugin implements Plugin { "Gradle ${minimumGradleVersion}+ is required to use elasticsearch.build plugin" ) } - project.pluginManager.apply('java') - configureConfigurations(project) + project.pluginManager.apply('elasticsearch.java') configureJars(project) configureJarManifest(project) @@ -128,12 +127,9 @@ class BuildPlugin implements Plugin { configureRepositories(project) project.extensions.getByType(ExtraPropertiesExtension).set('versions', VersionProperties.versions) - configureInputNormalization(project) - configureCompile(project) configureJavadoc(project) configureSourcesJar(project) configurePomGeneration(project) - configureTestTasks(project) configurePrecommit(project) configureDependenciesInfo(project) configureFips140(project) @@ -373,84 +369,6 @@ class BuildPlugin implements Plugin { scmNode.appendNode('url', BuildParams.gitOrigin) } - /** - * Apply runtime classpath input normalization so that changes in JAR manifests don't break build cacheability - */ - static void configureInputNormalization(Project project) { - project.normalization.runtimeClasspath.ignore('META-INF/MANIFEST.MF') - } - - /** Adds compiler settings to the project */ - static void configureCompile(Project project) { - ExtraPropertiesExtension ext = project.extensions.getByType(ExtraPropertiesExtension) - if (BuildParams.compilerJavaVersion < JavaVersion.VERSION_1_10) { - ext.set('compactProfile', 'compact3') - } else { - ext.set('compactProfile', 'full') - } - ext.set('compactProfile', 'full') - - project.extensions.getByType(JavaPluginExtension).sourceCompatibility = BuildParams.minimumRuntimeVersion - project.extensions.getByType(JavaPluginExtension).targetCompatibility = BuildParams.minimumRuntimeVersion - - project.afterEvaluate { - project.tasks.withType(JavaCompile).configureEach({ JavaCompile compileTask -> - final JavaVersion targetCompatibilityVersion = JavaVersion.toVersion(compileTask.targetCompatibility) - // we only fork if the Gradle JDK is not the same as the compiler JDK - if (BuildParams.compilerJavaHome.canonicalPath == Jvm.current().javaHome.canonicalPath) { - compileTask.options.fork = false - } else { - compileTask.options.fork = true - compileTask.options.forkOptions.javaHome = BuildParams.compilerJavaHome - } - if (targetCompatibilityVersion == JavaVersion.VERSION_1_8) { - // compile with compact 3 profile by default - // NOTE: this is just a compile time check: does not replace testing with a compact3 JRE - if (ext.get('compactProfile') != 'full') { - compileTask.options.compilerArgs << '-profile' << ext.get('compactProfile').toString() - } - } - /* - * -path because gradle will send in paths that don't always exist. - * -missing because we have tons of missing @returns and @param. - * -serial because we don't use java serialization. - */ - // don't even think about passing args with -J-xxx, oracle will ask you to submit a bug report :) - // fail on all javac warnings - compileTask.options.compilerArgs << '-Werror' << '-Xlint:all,-path,-serial,-options,-deprecation,-try' << '-Xdoclint:all' << '-Xdoclint:-missing' - - // either disable annotation processor completely (default) or allow to enable them if an annotation processor is explicitly defined - if (compileTask.options.compilerArgs.contains("-processor") == false) { - compileTask.options.compilerArgs << '-proc:none' - } - - compileTask.options.encoding = 'UTF-8' - compileTask.options.incremental = true - - // TODO: use native Gradle support for --release when available (cf. https://github.com/gradle/gradle/issues/2510) - compileTask.options.compilerArgs << '--release' << targetCompatibilityVersion.majorVersion - } as Action) - // also apply release flag to groovy, which is used in build-tools - project.tasks.withType(GroovyCompile).configureEach({ GroovyCompile compileTask -> - // we only fork if the Gradle JDK is not the same as the compiler JDK - if (BuildParams.compilerJavaHome.canonicalPath == Jvm.current().javaHome.canonicalPath) { - compileTask.options.fork = false - } else { - compileTask.options.fork = true - compileTask.options.forkOptions.javaHome = BuildParams.compilerJavaHome - compileTask.options.compilerArgs << '--release' << JavaVersion.toVersion(compileTask.targetCompatibility).majorVersion - } - } as Action) - } - - project.pluginManager.withPlugin('com.github.johnrengelman.shadow') { - // Ensure that when we are compiling against the "original" JAR that we also include any "shadow" dependencies on the compile classpath - project.configurations.getByName(ShadowBasePlugin.CONFIGURATION_NAME).dependencies.all { Dependency dependency -> - project.configurations.getByName(JavaPlugin.API_ELEMENTS_CONFIGURATION_NAME).dependencies.add(dependency) - } - } - } - static void configureJavadoc(Project project) { // remove compiled classes from the Javadoc classpath: http://mail.openjdk.java.net/pipermail/javadoc-dev/2018-January/000400.html final List classes = new ArrayList<>() @@ -583,148 +501,6 @@ class BuildPlugin implements Plugin { } } - static void configureTestTasks(Project project) { - // Default test task should run only unit tests - maybeConfigure(project.tasks, 'test', Test) { Test task -> - task.include '**/*Tests.class' - } - - // none of this stuff is applicable to the `:buildSrc` project tests - if (project.path != ':build-tools') { - File heapdumpDir = new File(project.buildDir, 'heapdump') - - project.tasks.withType(Test).configureEach { Test test -> - File testOutputDir = new File(test.reports.junitXml.getDestination(), "output") - - ErrorReportingTestListener listener = new ErrorReportingTestListener(test.testLogging, testOutputDir) - test.extensions.add(ErrorReportingTestListener, 'errorReportingTestListener', listener) - test.addTestOutputListener(listener) - test.addTestListener(listener) - - /* - * We use lazy-evaluated strings in order to configure system properties whose value will not be known until - * execution time (e.g. cluster port numbers). Adding these via the normal DSL doesn't work as these get treated - * as task inputs and therefore Gradle attempts to snapshot them before/after task execution. This fails due - * to the GStrings containing references to non-serializable objects. - * - * We bypass this by instead passing this system properties vi a CommandLineArgumentProvider. This has the added - * side-effect that these properties are NOT treated as inputs, therefore they don't influence things like the - * build cache key or up to date checking. - */ - SystemPropertyCommandLineArgumentProvider nonInputProperties = new SystemPropertyCommandLineArgumentProvider() - - test.doFirst { - project.mkdir(testOutputDir) - project.mkdir(heapdumpDir) - project.mkdir(test.workingDir) - project.mkdir(test.workingDir.toPath().resolve('temp')) - } - //TODO remove once jvm.options are added to test system properties - if (BuildParams.runtimeJavaVersion == JavaVersion.VERSION_1_8) { - test.systemProperty ('java.locale.providers','SPI,JRE') - } else if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_1_9) { - test.systemProperty ('java.locale.providers','SPI,COMPAT') - test.jvmArgs '--illegal-access=warn' - } - if (inFipsJvm()) { - project.dependencies.add('testRuntimeOnly', "org.bouncycastle:bc-fips:1.0.1") - project.dependencies.add('testRuntimeOnly', "org.bouncycastle:bctls-fips:1.0.9") - } - test.jvmArgumentProviders.add(nonInputProperties) - test.extensions.add('nonInputProperties', nonInputProperties) - - test.workingDir = project.file("${project.buildDir}/testrun/${test.name}") - test.maxParallelForks = System.getProperty('tests.jvms', BuildParams.defaultParallel.toString()) as Integer - - test.exclude '**/*$*.class' - - test.jvmArgs "-Xmx${System.getProperty('tests.heap.size', '512m')}", - "-Xms${System.getProperty('tests.heap.size', '512m')}", - '-XX:+HeapDumpOnOutOfMemoryError' - - test.jvmArgumentProviders.add({ ["-XX:HeapDumpPath=$heapdumpDir"] } as CommandLineArgumentProvider) - - if (System.getProperty('tests.jvm.argline')) { - test.jvmArgs System.getProperty('tests.jvm.argline').split(" ") - } - - if (Boolean.parseBoolean(System.getProperty('tests.asserts', 'true'))) { - test.jvmArgs '-ea', '-esa' - } - - test.systemProperties 'java.awt.headless': 'true', - 'tests.gradle': 'true', - 'tests.artifact': project.name, - 'tests.task': test.path, - 'tests.security.manager': 'true', - 'jna.nosys': 'true' - - // ignore changing test seed when build is passed -Dignore.tests.seed for cacheability experimentation - if (System.getProperty('ignore.tests.seed') != null) { - nonInputProperties.systemProperty('tests.seed', BuildParams.testSeed) - } else { - test.systemProperty('tests.seed', BuildParams.testSeed) - } - - // don't track these as inputs since they contain absolute paths and break cache relocatability - nonInputProperties.systemProperty('gradle.dist.lib', new File(project.class.location.toURI()).parent) - nonInputProperties.systemProperty('gradle.worker.jar', "${project.gradle.getGradleUserHomeDir()}/caches/${project.gradle.gradleVersion}/workerMain/gradle-worker.jar") - nonInputProperties.systemProperty('gradle.user.home', project.gradle.getGradleUserHomeDir()) - // we use 'temp' relative to CWD since this is per JVM and tests are forbidden from writing to CWD - nonInputProperties.systemProperty('java.io.tmpdir', test.workingDir.toPath().resolve('temp')) - - nonInputProperties.systemProperty('compiler.java', BuildParams.compilerJavaVersion.majorVersion) - nonInputProperties.systemProperty('runtime.java', BuildParams.runtimeJavaVersion.majorVersion) - - // TODO: remove setting logging level via system property - test.systemProperty 'tests.logger.level', 'WARN' - System.getProperties().each { key, value -> - if ((key.toString().startsWith('tests.') || key.toString().startsWith('es.'))) { - test.systemProperty key.toString(), value - } - } - - // TODO: remove this once ctx isn't added to update script params in 7.0 - test.systemProperty 'es.scripting.update.ctx_in_params', 'false' - - // TODO: remove this property in 8.0 - test.systemProperty 'es.search.rewrite_sort', 'true' - - // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0 - test.systemProperty 'es.transport.cname_in_publish_address', 'true' - - // Set netty system properties to the properties we configure in jvm.options - test.systemProperty('io.netty.noUnsafe', 'true') - test.systemProperty('io.netty.noKeySetOptimization', 'true') - test.systemProperty('io.netty.recycler.maxCapacityPerThread', '0') - - test.testLogging { TestLoggingContainer logging -> - logging.showExceptions = true - logging.showCauses = true - logging.exceptionFormat = 'full' - } - - if (OS.current().equals(OS.WINDOWS) && System.getProperty('tests.timeoutSuite') == null) { - // override the suite timeout to 30 mins for windows, because it has the most inefficient filesystem known to man - test.systemProperty 'tests.timeoutSuite', '1800000!' - } - - /* - * If this project builds a shadow JAR than any unit tests should test against that artifact instead of - * compiled class output and dependency jars. This better emulates the runtime environment of consumers. - */ - project.pluginManager.withPlugin('com.github.johnrengelman.shadow') { - // Remove output class files and any other dependencies from the test classpath, since the shadow JAR includes these - test.classpath -= project.extensions.getByType(SourceSetContainer).getByName(SourceSet.MAIN_SOURCE_SET_NAME).runtimeClasspath - // Add any "shadow" dependencies. These are dependencies that are *not* bundled into the shadow JAR - test.classpath += project.configurations.getByName(ShadowBasePlugin.CONFIGURATION_NAME) - // Add the shadow JAR artifact itself - test.classpath += project.files(project.tasks.named('shadowJar')) - } - } - } - } - private static configurePrecommit(Project project) { TaskProvider precommit = PrecommitTasks.create(project, true) project.tasks.named(LifecycleBasePlugin.CHECK_TASK_NAME).configure { it.dependsOn(precommit) } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy index e9891dbad49..4aac7eba1a8 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy @@ -22,6 +22,7 @@ package org.elasticsearch.gradle.test import groovy.transform.CompileStatic import org.elasticsearch.gradle.BuildPlugin +import org.elasticsearch.gradle.ElasticsearchJavaPlugin import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin @@ -62,10 +63,10 @@ class StandaloneRestTestPlugin implements Plugin { project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask) BuildPlugin.configureRepositories(project) - BuildPlugin.configureTestTasks(project) - BuildPlugin.configureInputNormalization(project) + ElasticsearchJavaPlugin.configureTestTasks(project) + ElasticsearchJavaPlugin.configureInputNormalization(project) BuildPlugin.configureFips140(project) - BuildPlugin.configureCompile(project) + ElasticsearchJavaPlugin.configureCompile(project) project.extensions.getByType(JavaPluginExtension).sourceCompatibility = BuildParams.minimumRuntimeVersion project.extensions.getByType(JavaPluginExtension).targetCompatibility = BuildParams.minimumRuntimeVersion diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy index 3ba4ca7e1c4..d8daabd9ffb 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy @@ -21,6 +21,7 @@ package org.elasticsearch.gradle.test import groovy.transform.CompileStatic import org.elasticsearch.gradle.BuildPlugin +import org.elasticsearch.gradle.ElasticsearchJavaPlugin import org.gradle.api.Plugin import org.gradle.api.Project import org.gradle.api.plugins.JavaBasePlugin @@ -44,7 +45,7 @@ class StandaloneTestPlugin implements Plugin { t.mustRunAfter(project.tasks.getByName('precommit')) } - BuildPlugin.configureCompile(project) + ElasticsearchJavaPlugin.configureCompile(project) project.tasks.named('check').configure { it.dependsOn(project.tasks.named('test')) } } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java new file mode 100644 index 00000000000..c61a8d09294 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchJavaPlugin.java @@ -0,0 +1,380 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle; + +import com.github.jengelman.gradle.plugins.shadow.ShadowBasePlugin; +import org.elasticsearch.gradle.info.BuildParams; +import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin; +import org.elasticsearch.gradle.test.ErrorReportingTestListener; +import org.elasticsearch.gradle.util.Util; +import org.gradle.api.GradleException; +import org.gradle.api.JavaVersion; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.ModuleDependency; +import org.gradle.api.artifacts.ProjectDependency; +import org.gradle.api.artifacts.ResolutionStrategy; +import org.gradle.api.file.FileCollection; +import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.plugins.JavaPluginExtension; +import org.gradle.api.tasks.SourceSet; +import org.gradle.api.tasks.SourceSetContainer; +import org.gradle.api.tasks.compile.CompileOptions; +import org.gradle.api.tasks.compile.GroovyCompile; +import org.gradle.api.tasks.compile.JavaCompile; +import org.gradle.api.tasks.testing.Test; +import org.gradle.internal.jvm.Jvm; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; + +import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure; + +/** + * A wrapper around Gradle's Java plugin that applies our common configuration. + */ +public class ElasticsearchJavaPlugin implements Plugin { + @Override + public void apply(Project project) { + // make sure the global build info plugin is applied to the root project + project.getRootProject().getPluginManager().apply(GlobalBuildInfoPlugin.class); + + project.getPluginManager().apply(JavaPlugin.class); + configureConfigurations(project); + configureCompile(project); + configureInputNormalization(project); + configureTestTasks(project); + } + + /** + * Makes dependencies non-transitive. + * + * Gradle allows setting all dependencies as non-transitive very easily. + * Sadly this mechanism does not translate into maven pom generation. In order + * to effectively make the pom act as if it has no transitive dependencies, + * we must exclude each transitive dependency of each direct dependency. + * + * Determining the transitive deps of a dependency which has been resolved as + * non-transitive is difficult because the process of resolving removes the + * transitive deps. To sidestep this issue, we create a configuration per + * direct dependency version. This specially named and unique configuration + * will contain all of the transitive dependencies of this particular + * dependency. We can then use this configuration during pom generation + * to iterate the transitive dependencies and add excludes. + */ + public static void configureConfigurations(Project project) { + // we want to test compileOnly deps! + Configuration compileOnlyConfig = project.getConfigurations().getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME); + Configuration testCompileConfig = project.getConfigurations().getByName(JavaPlugin.TEST_COMPILE_CONFIGURATION_NAME); + testCompileConfig.extendsFrom(compileOnlyConfig); + + // we are not shipping these jars, we act like dumb consumers of these things + if (project.getPath().startsWith(":test:fixtures") || project.getPath().equals(":build-tools")) { + return; + } + // fail on any conflicting dependency versions + project.getConfigurations().all(configuration -> { + if (configuration.getName().endsWith("Fixture")) { + // just a self contained test-fixture configuration, likely transitive and hellacious + return; + } + configuration.resolutionStrategy(ResolutionStrategy::failOnVersionConflict); + }); + + // force all dependencies added directly to compile/testCompile to be non-transitive, except for ES itself + Consumer disableTransitiveDeps = configName -> { + Configuration config = project.getConfigurations().getByName(configName); + config.getDependencies().all(dep -> { + if (dep instanceof ModuleDependency + && dep instanceof ProjectDependency == false + && dep.getGroup().startsWith("org.elasticsearch") == false) { + ((ModuleDependency) dep).setTransitive(false); + } + }); + }; + disableTransitiveDeps.accept(JavaPlugin.COMPILE_CONFIGURATION_NAME); + disableTransitiveDeps.accept(JavaPlugin.TEST_COMPILE_CONFIGURATION_NAME); + disableTransitiveDeps.accept(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME); + disableTransitiveDeps.accept(JavaPlugin.RUNTIME_ONLY_CONFIGURATION_NAME); + } + + /** Adds compiler settings to the project */ + public static void configureCompile(Project project) { + project.getExtensions().getExtraProperties().set("compactProfile", "full"); + + JavaPluginExtension java = project.getExtensions().getByType(JavaPluginExtension.class); + java.setSourceCompatibility(BuildParams.getMinimumRuntimeVersion()); + java.setTargetCompatibility(BuildParams.getMinimumRuntimeVersion()); + + Function canonicalPath = file -> { + try { + return file.getCanonicalPath(); + } catch (IOException e) { + throw new GradleException("Failed to get canonical path for " + file, e); + } + }; + // common options to both java and groovy + Consumer configureFork = compileOptions -> { + // we only fork if the Gradle JDK is not the same as the compiler JDK + String compilerJavaHome = canonicalPath.apply(BuildParams.getCompilerJavaHome()); + String currentJavaHome = canonicalPath.apply(Jvm.current().getJavaHome()); + if (compilerJavaHome.equals(currentJavaHome)) { + compileOptions.setFork(false); + } else { + compileOptions.setFork(true); + compileOptions.getForkOptions().setJavaHome(BuildParams.getCompilerJavaHome()); + } + }; + + project.afterEvaluate(p -> { + project.getTasks().withType(JavaCompile.class).configureEach(compileTask -> { + CompileOptions compileOptions = compileTask.getOptions(); + + configureFork.accept(compileOptions); + /* + * -path because gradle will send in paths that don't always exist. + * -missing because we have tons of missing @returns and @param. + * -serial because we don't use java serialization. + */ + // don't even think about passing args with -J-xxx, oracle will ask you to submit a bug report :) + // fail on all javac warnings + List compilerArgs = compileOptions.getCompilerArgs(); + compilerArgs.add("-Werror"); + compilerArgs.add("-Xlint:all,-path,-serial,-options,-deprecation,-try"); + compilerArgs.add("-Xdoclint:all"); + compilerArgs.add("-Xdoclint:-missing"); + + // either disable annotation processor completely (default) or allow to enable them if an annotation processor is explicitly + // defined + if (compilerArgs.contains("-processor") == false) { + compilerArgs.add("-proc:none"); + } + + compileOptions.setEncoding("UTF-8"); + compileOptions.setIncremental(true); + + // TODO: use native Gradle support for --release when available (cf. https://github.com/gradle/gradle/issues/2510) + final JavaVersion targetCompatibilityVersion = JavaVersion.toVersion(compileTask.getTargetCompatibility()); + compilerArgs.add("--release"); + compilerArgs.add(targetCompatibilityVersion.getMajorVersion()); + + }); + // also apply release flag to groovy, which is used in build-tools + project.getTasks().withType(GroovyCompile.class).configureEach(compileTask -> { + configureFork.accept(compileTask.getOptions()); + + // TODO: this probably shouldn't apply to groovy at all? + // TODO: use native Gradle support for --release when available (cf. https://github.com/gradle/gradle/issues/2510) + final JavaVersion targetCompatibilityVersion = JavaVersion.toVersion(compileTask.getTargetCompatibility()); + final List compilerArgs = compileTask.getOptions().getCompilerArgs(); + compilerArgs.add("--release"); + compilerArgs.add(targetCompatibilityVersion.getMajorVersion()); + }); + }); + + project.getPluginManager().withPlugin("com.github.johnrengelman.shadow", plugin -> { + // Ensure that when we are compiling against the "original" JAR that we also include any "shadow" dependencies on the compile + // classpath + Configuration shadowConfig = project.getConfigurations().getByName(ShadowBasePlugin.getCONFIGURATION_NAME()); + Configuration apiConfig = project.getConfigurations().getByName(JavaPlugin.API_ELEMENTS_CONFIGURATION_NAME); + shadowConfig.getDependencies().all(dependency -> apiConfig.getDependencies().add(dependency)); + }); + } + + /** + * Apply runtime classpath input normalization so that changes in JAR manifests don't break build cacheability + */ + public static void configureInputNormalization(Project project) { + project.getNormalization().getRuntimeClasspath().ignore("META-INF/MANIFEST.MF"); + } + + public static void configureTestTasks(Project project) { + // Default test task should run only unit tests + maybeConfigure(project.getTasks(), "test", Test.class, task -> task.include("**/*Tests.class")); + + // none of this stuff is applicable to the `:buildSrc` project tests + if (project.getPath().equals(":build-tools")) { + return; + } + + File heapdumpDir = new File(project.getBuildDir(), "heapdump"); + + project.getTasks().withType(Test.class).configureEach(test -> { + File testOutputDir = new File(test.getReports().getJunitXml().getDestination(), "output"); + + ErrorReportingTestListener listener = new ErrorReportingTestListener(test.getTestLogging(), testOutputDir); + test.getExtensions().add("errorReportingTestListener", listener); + test.addTestOutputListener(listener); + test.addTestListener(listener); + + /* + * We use lazy-evaluated strings in order to configure system properties whose value will not be known until + * execution time (e.g. cluster port numbers). Adding these via the normal DSL doesn't work as these get treated + * as task inputs and therefore Gradle attempts to snapshot them before/after task execution. This fails due + * to the GStrings containing references to non-serializable objects. + * + * We bypass this by instead passing this system properties vi a CommandLineArgumentProvider. This has the added + * side-effect that these properties are NOT treated as inputs, therefore they don't influence things like the + * build cache key or up to date checking. + */ + SystemPropertyCommandLineArgumentProvider nonInputProperties = new SystemPropertyCommandLineArgumentProvider(); + + test.doFirst(t -> { + project.mkdir(testOutputDir); + project.mkdir(heapdumpDir); + project.mkdir(test.getWorkingDir()); + project.mkdir(test.getWorkingDir().toPath().resolve("temp")); + + // TODO remove once jvm.options are added to test system properties + + //TODO remove once jvm.options are added to test system properties + if (BuildParams.getRuntimeJavaVersion() == JavaVersion.VERSION_1_8) { + test.systemProperty("java.locale.providers", "SPI,JRE"); + } else { + test.systemProperty("java.locale.providers", "SPI,COMPAT"); + } + }); + + if (BuildParams.isInFipsJvm()) { + project.getDependencies().add("testRuntimeOnly", "org.bouncycastle:bc-fips:1.0.1"); + project.getDependencies().add("testRuntimeOnly", "org.bouncycastle:bctls-fips:1.0.9"); + } + test.getJvmArgumentProviders().add(nonInputProperties); + test.getExtensions().add("nonInputProperties", nonInputProperties); + + test.setWorkingDir(project.file(project.getBuildDir() + "/testrun/" + test.getName())); + test.setMaxParallelForks(Integer.parseInt(System.getProperty("tests.jvms", BuildParams.getDefaultParallel().toString()))); + + test.exclude("**/*$*.class"); + + test.jvmArgs( + "-Xmx" + System.getProperty("tests.heap.size", "512m"), + "-Xms" + System.getProperty("tests.heap.size", "512m"), + "--illegal-access=warn", + "-XX:+HeapDumpOnOutOfMemoryError" + ); + + test.getJvmArgumentProviders().add(() -> List.of("-XX:HeapDumpPath=$heapdumpDir")); + + String argline = System.getProperty("tests.jvm.argline"); + if (argline != null) { + test.jvmArgs((Object[]) argline.split(" ")); + } + + if (Util.getBooleanProperty("tests.asserts", true)) { + test.jvmArgs("-ea", "-esa"); + } + + Map sysprops = Map.of( + "java.awt.headless", + "true", + "tests.gradle", + "true", + "tests.artifact", + project.getName(), + "tests.task", + test.getPath(), + "tests.security.manager", + "true", + "jna.nosys", + "true" + ); + test.systemProperties(sysprops); + + // ignore changing test seed when build is passed -Dignore.tests.seed for cacheability experimentation + if (System.getProperty("ignore.tests.seed") != null) { + nonInputProperties.systemProperty("tests.seed", BuildParams.getTestSeed()); + } else { + test.systemProperty("tests.seed", BuildParams.getTestSeed()); + } + + // don't track these as inputs since they contain absolute paths and break cache relocatability + File gradleHome = project.getGradle().getGradleUserHomeDir(); + String gradleVersion = project.getGradle().getGradleVersion(); + nonInputProperties.systemProperty("gradle.dist.lib", new File(project.getGradle().getGradleHomeDir(), "lib")); + nonInputProperties.systemProperty( + "gradle.worker.jar", + gradleHome + "/caches/" + gradleVersion + "/workerMain/gradle-worker.jar" + ); + nonInputProperties.systemProperty("gradle.user.home", gradleHome); + // we use 'temp' relative to CWD since this is per JVM and tests are forbidden from writing to CWD + nonInputProperties.systemProperty("java.io.tmpdir", test.getWorkingDir().toPath().resolve("temp")); + + nonInputProperties.systemProperty("compiler.java", BuildParams.getCompilerJavaVersion().getMajorVersion()); + nonInputProperties.systemProperty("runtime.java", BuildParams.getRuntimeJavaVersion().getMajorVersion()); + + // TODO: remove setting logging level via system property + test.systemProperty("tests.logger.level", "WARN"); + System.getProperties().entrySet().forEach(entry -> { + if ((entry.getKey().toString().startsWith("tests.") || entry.getKey().toString().startsWith("es."))) { + test.systemProperty(entry.getKey().toString(), entry.getValue()); + } + }); + + // TODO: remove this once ctx isn't added to update script params in 7.0 + test.systemProperty("es.scripting.update.ctx_in_params", "false"); + + // TODO: remove this property in 8.0 + test.systemProperty("es.search.rewrite_sort", "true"); + + // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0 + test.systemProperty("es.transport.cname_in_publish_address", "true"); + + // Set netty system properties to the properties we configure in jvm.options + test.systemProperty("io.netty.noUnsafe", "true"); + test.systemProperty("io.netty.noKeySetOptimization", "true"); + test.systemProperty("io.netty.recycler.maxCapacityPerThread", "0"); + + test.testLogging(logging -> { + logging.setShowExceptions(true); + logging.setShowCauses(true); + logging.setExceptionFormat("full"); + }); + + if (OS.current().equals(OS.WINDOWS) && System.getProperty("tests.timeoutSuite") == null) { + // override the suite timeout to 30 mins for windows, because it has the most inefficient filesystem known to man + test.systemProperty("tests.timeoutSuite", "1800000!"); + } + + /* + * If this project builds a shadow JAR than any unit tests should test against that artifact instead of + * compiled class output and dependency jars. This better emulates the runtime environment of consumers. + */ + project.getPluginManager().withPlugin("com.github.johnrengelman.shadow", p -> { + // Remove output class files and any other dependencies from the test classpath, since the shadow JAR includes these + FileCollection mainRuntime = project.getExtensions() + .getByType(SourceSetContainer.class) + .getByName(SourceSet.MAIN_SOURCE_SET_NAME) + .getRuntimeClasspath(); + // Add any "shadow" dependencies. These are dependencies that are *not* bundled into the shadow JAR + Configuration shadowConfig = project.getConfigurations().getByName(ShadowBasePlugin.getCONFIGURATION_NAME()); + // Add the shadow JAR artifact itself + FileCollection shadowJar = project.files(project.getTasks().named("shadowJar")); + + test.setClasspath(test.getClasspath().minus(mainRuntime).plus(shadowConfig).plus(shadowJar)); + }); + }); + } +} diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.java.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.java.properties new file mode 100644 index 00000000000..61bd1c03e7a --- /dev/null +++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.java.properties @@ -0,0 +1 @@ +implementation-class=org.elasticsearch.gradle.ElasticsearchJavaPlugin