Simplify usage of Gradle Shadow plugin (#48478) (#48597)

This commit simplifies and standardizes our usage of the Gradle Shadow
plugin to conform more to plugin conventions. The custom "bundle" plugin
has been removed as it's not necessary and performs the same function
as the Shadow plugin's default behavior with existing configurations.

Additionally, this removes unnecessary creation of a "nodeps" artifact,
which is unnecessary because by default project dependencies will in
fact use the non-shadowed JAR unless explicitly depending on the
"shadow" configuration.

Finally, we've cleaned up the logic used for unit testing, so we are
now correctly testing against the shadow JAR when the plugin is applied.
This better represents a real-world scenario for consumers and provides
better test coverage for incorrectly declared dependencies.

(cherry picked from commit 3698131109c7e78bdd3a3340707e1c7b4740d310)
This commit is contained in:
Mark Vieira 2019-10-28 12:11:55 -07:00 committed by GitHub
parent 605500df7e
commit e5c6440a4f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 85 additions and 209 deletions

View File

@ -294,16 +294,17 @@ allprojects {
project.configurations.compile.dependencies
.findAll()
.toSorted(sortClosure)
.each({ c -> depJavadocClosure(false, c) })
.each({ c -> depJavadocClosure(hasShadow, c) })
project.configurations.compileOnly.dependencies
.findAll()
.toSorted(sortClosure)
.each({ c -> depJavadocClosure(false, c) })
if (hasShadow) {
project.configurations.bundle.dependencies
// include any dependencies for shadow JAR projects that are *not* bundled in the shadow JAR
project.configurations.shadow.dependencies
.findAll()
.toSorted(sortClosure)
.each({ c -> depJavadocClosure(true, c) })
.each({ c -> depJavadocClosure(false, c) })
}
}
}
@ -425,24 +426,6 @@ allprojects {
tasks.named('eclipse') { dependsOn 'cleanEclipse', 'copyEclipseSettings' }
}
allprojects {
/*
* IntelliJ and Eclipse don't know about the shadow plugin so when we're
* in "IntelliJ mode" or "Eclipse mode" switch "bundle" dependencies into
* regular "compile" dependencies. This isn't needed for the project
* itself because the IDE configuration is done by SourceSets but it is
* *is* needed for projects that depends on the project doing the shadowing.
* Without this they won't properly depend on the shadowed project.
*/
if (isEclipse || isIdea) {
project.plugins.withType(ShadowPlugin).whenPluginAdded {
project.afterEvaluate {
project.configurations.compile.extendsFrom project.configurations.bundle
}
}
}
}
// we need to add the same --debug-jvm option as
// the real RunTask has, so we can pass it through
class Run extends DefaultTask {

View File

@ -114,7 +114,7 @@ dependencies {
compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
compile 'org.apache.rat:apache-rat:0.11'
compile "org.elasticsearch:jna:4.5.1"
compile 'com.github.jengelman.gradle.plugins:shadow:4.0.3'
compile 'com.github.jengelman.gradle.plugins:shadow:5.1.0'
compile 'de.thetaphi:forbiddenapis:2.7'
compile 'com.avast.gradle:gradle-docker-compose-plugin:0.8.12'
testCompile "junit:junit:${props.getProperty('junit')}"

View File

@ -19,6 +19,9 @@
package org.elasticsearch.gradle
import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
import com.github.jengelman.gradle.plugins.shadow.ShadowBasePlugin
import com.github.jengelman.gradle.plugins.shadow.ShadowExtension
import com.github.jengelman.gradle.plugins.shadow.ShadowJavaPlugin
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
import groovy.transform.CompileDynamic
import groovy.transform.CompileStatic
@ -86,6 +89,7 @@ import java.nio.file.Files
import java.util.regex.Matcher
import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
/**
* Encapsulates build configuration for elasticsearch projects.
*/
@ -132,7 +136,6 @@ class BuildPlugin implements Plugin<Project> {
configureRepositories(project)
project.extensions.getByType(ExtraPropertiesExtension).set('versions', VersionProperties.versions)
configureInputNormalization(project)
configureSourceSets(project)
configureCompile(project)
configureJavadoc(project)
configureSourcesJar(project)
@ -392,11 +395,6 @@ class BuildPlugin implements Plugin<Project> {
project.configurations.getByName(JavaPlugin.COMPILE_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
project.configurations.getByName(JavaPlugin.TEST_COMPILE_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME).dependencies.all(disableTransitiveDeps)
project.plugins.withType(ShadowPlugin).whenPluginAdded {
Configuration bundle = project.configurations.create('bundle')
bundle.dependencies.all(disableTransitiveDeps)
}
}
/** Adds repositories used by ES dependencies */
@ -533,58 +531,45 @@ class BuildPlugin implements Plugin<Project> {
/**Configuration generation of maven poms. */
static void configurePomGeneration(Project project) {
// Only works with `enableFeaturePreview('STABLE_PUBLISHING')`
// https://github.com/gradle/gradle/issues/5696#issuecomment-396965185
project.tasks.withType(GenerateMavenPom.class).configureEach({ GenerateMavenPom generatePOMTask ->
// The GenerateMavenPom task is aggressive about setting the destination, instead of fighting it,
// just make a copy.
ExtraPropertiesExtension ext = generatePOMTask.extensions.getByType(ExtraPropertiesExtension)
ext.set('pomFileName', null)
generatePOMTask.doLast {
project.copy { CopySpec spec ->
spec.from generatePOMTask.destination
spec.into "${project.buildDir}/distributions"
spec.rename {
ext.has('pomFileName') && ext.get('pomFileName') == null ?
"${project.convention.getPlugin(BasePluginConvention).archivesBaseName}-${project.version}.pom" :
ext.get('pomFileName')
}
}
}
} as Action<GenerateMavenPom>)
// build poms with assemble (if the assemble task exists)
maybeConfigure(project.tasks, 'assemble') { assemble ->
if (assemble.enabled) {
assemble.dependsOn(project.tasks.withType(GenerateMavenPom))
}
}
project.plugins.withType(MavenPublishPlugin).whenPluginAdded {
TaskProvider generatePomTask = project.tasks.register("generatePom") { Task task ->
task.dependsOn 'generatePomFileForNebulaPublication'
}
maybeConfigure(project.tasks, LifecycleBasePlugin.ASSEMBLE_TASK_NAME) { assemble ->
assemble.dependsOn(generatePomTask)
}
project.tasks.withType(GenerateMavenPom).configureEach({ GenerateMavenPom pomTask ->
pomTask.destination = "${project.buildDir}/distributions/${project.convention.getPlugin(BasePluginConvention).archivesBaseName}-${project.version}.pom"
} as Action<GenerateMavenPom>)
PublishingExtension publishing = project.extensions.getByType(PublishingExtension)
publishing.publications.all { MavenPublication publication -> // we only deal with maven
project.extensions.getByType(PublishingExtension).publications.all { MavenPublication publication -> // we only deal with maven
// add exclusions to the pom directly, for each of the transitive deps of this project's deps
publication.pom.withXml(fixupDependencies(project))
}
project.plugins.withType(ShadowPlugin).whenPluginAdded {
MavenPublication publication = publishing.publications.maybeCreate('nebula', MavenPublication)
publication.with {
artifacts = [ project.tasks.getByName('shadowJar') ]
}
}
}
}
/**
* Add dependencies that we are going to bundle to the compile classpath.
*/
static void configureSourceSets(Project project) {
project.plugins.withType(ShadowPlugin).whenPluginAdded {
['main', 'test'].each {name ->
SourceSet sourceSet = project.extensions.getByType(SourceSetContainer).findByName(name)
if (sourceSet != null) {
sourceSet.compileClasspath += project.configurations.getByName('bundle')
project.pluginManager.withPlugin('com.github.johnrengelman.shadow') {
MavenPublication publication = publishing.publications.maybeCreate('shadow', MavenPublication)
ShadowExtension shadow = project.extensions.getByType(ShadowExtension)
shadow.component(publication)
// Workaround for https://github.com/johnrengelman/shadow/issues/334
// Here we manually add any project dependencies in the "shadow" configuration to our generated POM
publication.pom.withXml { xml ->
Node dependenciesNode = (xml.asNode().get('dependencies') as NodeList).get(0) as Node
project.configurations.getByName(ShadowBasePlugin.CONFIGURATION_NAME).allDependencies.each { dependency ->
if (dependency instanceof ProjectDependency) {
def dependencyNode = dependenciesNode.appendNode('dependency')
dependencyNode.appendNode('groupId', dependency.group)
dependencyNode.appendNode('artifactId', dependency.getDependencyProject().convention.getPlugin(BasePluginConvention).archivesBaseName)
dependencyNode.appendNode('version', dependency.version)
dependencyNode.appendNode('scope', 'runtime')
}
}
}
generatePomTask.configure({ Task t -> t.dependsOn = ['generatePomFileForShadowPublication'] } as Action<Task>)
}
}
}
@ -665,6 +650,11 @@ class BuildPlugin implements Plugin<Project> {
}
} as Action<GroovyCompile>)
}
project.pluginManager.withPlugin('com.github.johnrengelman.shadow') {
// Ensure that when we are compiling against the "original" JAR that we also include any "shadow" dependencies on the compile classpath
project.configurations.getByName(JavaPlugin.API_ELEMENTS_CONFIGURATION_NAME).extendsFrom(project.configurations.getByName(ShadowBasePlugin.CONFIGURATION_NAME))
}
}
static void configureJavadoc(Project project) {
@ -765,34 +755,27 @@ class BuildPlugin implements Plugin<Project> {
}
}
}
project.plugins.withType(ShadowPlugin).whenPluginAdded {
/*
* When we use the shadow plugin we entirely replace the
* normal jar with the shadow jar so we no longer want to run
* the jar task.
*/
project.tasks.named(JavaPlugin.JAR_TASK_NAME).configure { it.enabled = false }
project.tasks.named('shadowJar').configure { ShadowJar shadowJar ->
project.pluginManager.withPlugin('com.github.johnrengelman.shadow') {
project.tasks.getByName(ShadowJavaPlugin.SHADOW_JAR_TASK_NAME).configure { ShadowJar shadowJar ->
/*
* Replace the default "shadow" classifier with null
* Replace the default "-all" classifier with null
* which will leave the classifier off of the file name.
*/
shadowJar.classifier = null
shadowJar.archiveClassifier.set((String) null)
/*
* Not all cases need service files merged but it is
* better to be safe
*/
shadowJar.mergeServiceFiles()
/*
* Bundle dependencies of the "bundled" configuration.
*/
shadowJar.configurations = [project.configurations.getByName('bundle')]
}
// Add "original" classifier to the non-shadowed JAR to distinguish it from the shadow JAR
project.tasks.getByName(JavaPlugin.JAR_TASK_NAME).configure { Jar jar ->
jar.archiveClassifier.set('original')
}
// Make sure we assemble the shadow jar
project.tasks.named(BasePlugin.ASSEMBLE_TASK_NAME).configure {
it.dependsOn project.tasks.named('shadowJar')
project.tasks.named(BasePlugin.ASSEMBLE_TASK_NAME).configure { Task task ->
task.dependsOn 'shadowJar'
}
project.artifacts.add('apiElements', project.tasks.getByName('shadowJar'))
}
}
@ -926,12 +909,17 @@ class BuildPlugin implements Plugin<Project> {
test.systemProperty 'tests.timeoutSuite', '1800000!'
}
project.plugins.withType(ShadowPlugin).whenPluginAdded {
// Test against a shadow jar if we made one
test.classpath -= project.tasks.getByName('compileJava').outputs.files
test.classpath += project.tasks.getByName('shadowJar').outputs.files
test.dependsOn project.tasks.getByName('shadowJar')
/*
* If this project builds a shadow JAR than any unit tests should test against that artifact instead of
* compiled class output and dependency jars. This better emulates the runtime environment of consumers.
*/
project.pluginManager.withPlugin('com.github.johnrengelman.shadow') {
// Remove output class files and any other dependencies from the test classpath, since the shadow JAR includes these
test.classpath -= project.extensions.getByType(SourceSetContainer).getByName(SourceSet.MAIN_SOURCE_SET_NAME).runtimeClasspath
// Add any "shadow" dependencies. These are dependencies that are *not* bundled into the shadow JAR
test.classpath += project.configurations.getByName(ShadowBasePlugin.CONFIGURATION_NAME)
// Add the shadow JAR artifact itself
test.classpath += project.files(project.tasks.named('shadowJar'))
}
}
}
@ -943,33 +931,20 @@ class BuildPlugin implements Plugin<Project> {
project.tasks.named(JavaPlugin.TEST_TASK_NAME).configure { it.mustRunAfter(precommit) }
// only require dependency licenses for non-elasticsearch deps
project.tasks.withType(DependencyLicensesTask).named('dependencyLicenses').configure {
it.dependencies = project.configurations.getByName(JavaPlugin.RUNTIME_CONFIGURATION_NAME).fileCollection { Dependency dependency ->
it.dependencies = project.configurations.getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME).fileCollection { Dependency dependency ->
dependency.group.startsWith('org.elasticsearch') == false
} - project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME)
}
project.plugins.withType(ShadowPlugin).whenPluginAdded {
project.tasks.withType(DependencyLicensesTask).named('dependencyLicenses').configure {
it.dependencies += project.configurations.getByName('bundle').fileCollection { Dependency dependency ->
dependency.group.startsWith('org.elasticsearch') == false
}
}
}
}
private static configureDependenciesInfo(Project project) {
TaskProvider<DependenciesInfoTask> deps = project.tasks.register("dependenciesInfo", DependenciesInfoTask, { DependenciesInfoTask task ->
task.runtimeConfiguration = project.configurations.getByName(JavaPlugin.RUNTIME_CONFIGURATION_NAME)
project.tasks.register("dependenciesInfo", DependenciesInfoTask, { DependenciesInfoTask task ->
task.runtimeConfiguration = project.configurations.getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME)
task.compileOnlyConfiguration = project.configurations.getByName(JavaPlugin.COMPILE_ONLY_CONFIGURATION_NAME)
task.getConventionMapping().map('mappings') {
(project.tasks.getByName('dependencyLicenses') as DependencyLicensesTask).mappings
}
} as Action<DependenciesInfoTask>)
project.plugins.withType(ShadowPlugin).whenPluginAdded {
deps.configure { task ->
task.runtimeConfiguration = project.configurations.create('infoDeps')
task.runtimeConfiguration.extendsFrom(project.configurations.getByName(JavaPlugin.RUNTIME_CONFIGURATION_NAME), project.configurations.getByName('bundle'))
}
}
}
private static class TestFailureReportingPlugin implements Plugin<Project> {

View File

@ -18,13 +18,12 @@
*/
package org.elasticsearch.gradle.precommit
import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.tool.ClasspathUtils
import org.gradle.api.JavaVersion
import org.gradle.api.Project
import org.gradle.api.artifacts.Configuration
import org.gradle.api.plugins.JavaBasePlugin
@ -121,13 +120,10 @@ class PrecommitTasks {
}
}
private static TaskProvider configureJarHell(Project project, Configuration jarHelConfig) {
private static TaskProvider configureJarHell(Project project, Configuration jarHellConfig) {
return project.tasks.register('jarHell', JarHellTask) { task ->
task.classpath = project.sourceSets.test.runtimeClasspath + jarHelConfig;
if (project.plugins.hasPlugin(ShadowPlugin)) {
task.classpath += project.configurations.bundle
}
task.dependsOn(jarHelConfig);
task.classpath = project.sourceSets.test.runtimeClasspath + jarHellConfig
task.dependsOn(jarHellConfig)
}
}

View File

@ -23,7 +23,6 @@ apply plugin: 'elasticsearch.build'
apply plugin: 'elasticsearch.rest-test'
apply plugin: 'nebula.maven-base-publish'
apply plugin: 'nebula.maven-scm'
apply plugin: 'com.github.johnrengelman.shadow'
group = 'org.elasticsearch.client'
archivesBaseName = 'elasticsearch-rest-high-level-client'
@ -49,10 +48,6 @@ idea {
}
dependencies {
/*
* Everything in the "shadow" configuration is *not* copied into the
* shadowJar.
*/
compile project(':server')
compile project(':client:rest')
compile project(':modules:mapper-extras')
@ -78,7 +73,7 @@ dependencies {
//we need to copy the yaml spec so we can check naming (see RestHighlevelClientTests#testApiNamingConventions)
processTestResources {
dependsOn jar // so that configurations resolve
dependsOn configurations.restSpec // so that configurations resolve
from({ zipTree(configurations.restSpec.singleFile) }) {
include 'rest-api-spec/api/**'
}

View File

@ -53,10 +53,7 @@ dependencies {
testCompile project(path: ':modules:parent-join', configuration: 'runtime')
testCompile project(path: ':modules:analysis-common', configuration: 'runtime')
bin(project(path: xpackModule('sql:sql-cli'))) {
// sql-cli bundles all of its dependencies into a single executable jar
transitive = false
}
bin(project(path: xpackModule('sql:sql-cli'), configuration: 'shadow'))
}
/* Bundle the sql-cli into the binary files. It should end up

View File

@ -40,41 +40,6 @@ dependencyLicenses {
ignoreSha 'elasticsearch'
}
test {
// don't use the shaded jar for tests
classpath += project.tasks.compileJava.outputs.files
classpath -= project.tasks.shadowJar.outputs.files
}
shadowJar {
relocate 'com.fasterxml', 'org.elasticsearch.fasterxml'
// set the shaded configuration back to runtime instead of bundle because
// we need tests to use the non-shaded deps to allow editing/testing in intellij
configurations = [project.configurations.runtime]
}
// We need a no-depenencies jar though for qa testing so it doesn't conflict with cli
configurations {
nodeps
}
task nodepsJar(type: Jar) {
appendix 'nodeps'
from sourceSets.main.output
}
artifacts {
nodeps nodepsJar
}
publishing {
publications {
nebula {
artifactId = archivesBaseName
pom.withXml {
// Nebula is mistakenly including all dependencies that are already shadowed into the shadow jar
asNode().remove(asNode().dependencies)
}
}
}
}

View File

@ -7,13 +7,13 @@ dependencies {
compile project(":test:framework")
// JDBC testing dependencies
compile project(path: xpackModule('sql:jdbc'), configuration: 'nodeps')
compile project(path: xpackModule('sql:jdbc'))
compile project(path: xpackModule('sql:sql-action'))
compile "net.sourceforge.csvjdbc:csvjdbc:${csvjdbcVersion}"
// CLI testing dependencies
compile project(path: xpackModule('sql:sql-cli'), configuration: 'nodeps')
compile project(path: xpackModule('sql:sql-cli'))
// H2GIS testing dependencies
compile ("org.orbisgis:h2gis:${h2gisVersion}") {
@ -82,7 +82,7 @@ subprojects {
exclude group: "com.fasterxml.jackson.core"
}
testRuntime project(path: xpackModule('sql:jdbc'), configuration: 'nodeps')
testRuntime project(path: xpackModule('sql:jdbc'))
testRuntime xpackProject('plugin:sql:sql-client')
// TODO check if needed
@ -91,7 +91,7 @@ subprojects {
}
// CLI testing dependencies
testRuntime project(path: xpackModule('sql:sql-cli'), configuration: 'nodeps')
testRuntime project(path: xpackModule('sql:sql-cli'))
testRuntime (xpackProject('plugin:sql:sql-action')) {
transitive = false
}

View File

@ -6,6 +6,7 @@
*/
apply plugin: 'elasticsearch.build'
apply plugin: 'com.github.johnrengelman.shadow'
/* We don't use the 'application' plugin because it builds a zip and tgz which
* we don't want. */
@ -46,39 +47,12 @@ dependencyLicenses {
ignoreSha 'sql-client'
}
/*
* Bundle all dependencies into the main jar and mark it as executable it
* can be easily shipped around and used.
*/
jar {
dependsOn configurations.runtimeClasspath
from({
configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) }
}) {
// We don't need the META-INF from the things we bundle. For now.
exclude 'META-INF/*'
}
shadowJar {
manifest {
attributes 'Main-Class': 'org.elasticsearch.xpack.sql.cli.Cli'
}
}
/*
* Build a jar that doesn't include the dependencies bundled that we can
* include with QA tests along side Elasticsearch without breaking
* jarhell.
*/
configurations {
nodeps
}
task nodepsJar(type: Jar) {
appendix 'nodeps'
from sourceSets.main.output
}
artifacts {
nodeps nodepsJar
}
forbiddenApisMain {
//sql does not depend on server, so only jdk signatures should be checked
replaceSignatureFiles 'jdk-signatures'
@ -87,13 +61,13 @@ forbiddenApisMain {
task runcli {
description = 'Run the CLI and connect to elasticsearch running on 9200'
dependsOn 'assemble'
dependsOn shadowJar
doLast {
List command = [new File(project.runtimeJavaHome, 'bin/java').absolutePath]
if ('true'.equals(System.getProperty('debug', 'false'))) {
command += '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000'
}
command += ['-jar', jar.archivePath.absolutePath]
command += ['-jar', shadowJar.archivePath.absolutePath]
logger.info("running the cli with: ${command}")
new ProcessBuilder(command)
@ -104,12 +78,3 @@ task runcli {
.waitFor()
}
}
// Use the jar for testing so we can get the proper version information
test {
classpath -= compileJava.outputs.files
classpath -= configurations.compile
classpath -= configurations.runtime
classpath += jar.outputs.files
dependsOn jar
}