diff --git a/build.gradle b/build.gradle index 015db80d325..6517d0292ba 100644 --- a/build.gradle +++ b/build.gradle @@ -543,7 +543,7 @@ subprojects { project -> } } -/* Remove assemble on all qa projects because we don't need to publish +/* Remove assemble/dependenciesInfo on all qa projects because we don't need to publish * artifacts for them. */ gradle.projectsEvaluated { subprojects { @@ -553,6 +553,11 @@ gradle.projectsEvaluated { project.tasks.remove(assemble) project.build.dependsOn.remove('assemble') } + Task dependenciesInfo = project.tasks.findByName('dependenciesInfo') + if (dependenciesInfo) { + project.tasks.remove(dependenciesInfo) + project.precommit.dependsOn.remove('dependenciesInfo') + } } } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 85fe712fd8d..eb3cd1dc8c6 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -762,6 +762,10 @@ class BuildPlugin implements Plugin { private static configureDependenciesInfo(Project project) { Task deps = project.tasks.create("dependenciesInfo", DependenciesInfoTask.class) - deps.dependencies = project.configurations.compile.allDependencies + deps.runtimeConfiguration = project.configurations.runtime + deps.compileOnlyConfiguration = project.configurations.compileOnly + project.afterEvaluate { + deps.mappings = project.dependencyLicenses.mappings + } } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy index b42e6cc8e3c..e62fe4db954 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy @@ -19,14 +19,19 @@ package org.elasticsearch.gradle +import org.elasticsearch.gradle.precommit.DependencyLicensesTask import org.gradle.api.DefaultTask +import org.gradle.api.artifacts.Configuration import org.gradle.api.artifacts.Dependency +import org.gradle.api.artifacts.DependencyResolutionListener import org.gradle.api.artifacts.DependencySet import org.gradle.api.tasks.Input import org.gradle.api.tasks.InputDirectory import org.gradle.api.tasks.OutputFile import org.gradle.api.tasks.TaskAction +import java.util.regex.Matcher +import java.util.regex.Pattern /** * A task to gather information about the dependencies and export them into a csv file. @@ -44,7 +49,14 @@ public class DependenciesInfoTask extends DefaultTask { /** Dependencies to gather information from. */ @Input - public DependencySet dependencies + public Configuration runtimeConfiguration + + /** We subtract compile-only dependencies. */ + @Input + public Configuration compileOnlyConfiguration + + @Input + public LinkedHashMap mappings /** Directory to read license files */ @InputDirectory @@ -59,15 +71,34 @@ public class DependenciesInfoTask extends DefaultTask { @TaskAction public void generateDependenciesInfo() { + + final DependencySet runtimeDependencies = runtimeConfiguration.getAllDependencies() + // we have to resolve the transitive dependencies and create a group:artifactId:version map + final Set compileOnlyArtifacts = + compileOnlyConfiguration + .getResolvedConfiguration() + .resolvedArtifacts + .collect { it -> "${it.moduleVersion.id.group}:${it.moduleVersion.id.name}:${it.moduleVersion.id.version}" } + final StringBuilder output = new StringBuilder() - for (Dependency dependency : dependencies) { - // Only external dependencies are checked - if (dependency.group != null && dependency.group.contains("elasticsearch") == false) { - final String url = createURL(dependency.group, dependency.name, dependency.version) - final String licenseType = getLicenseType(dependency.group, dependency.name) - output.append("${dependency.group}:${dependency.name},${dependency.version},${url},${licenseType}\n") + for (final Dependency dependency : runtimeDependencies) { + // we do not need compile-only dependencies here + if (compileOnlyArtifacts.contains("${dependency.group}:${dependency.name}:${dependency.version}")) { + continue } + // only external dependencies are checked + if (dependency.group != null && dependency.group.contains("org.elasticsearch")) { + continue + } + + final String url = createURL(dependency.group, dependency.name, dependency.version) + final String dependencyName = DependencyLicensesTask.getDependencyName(mappings, dependency.name) + logger.info("mapped dependency ${dependency.group}:${dependency.name} to ${dependencyName} for license info") + + final String licenseType = getLicenseType(dependency.group, dependencyName) + output.append("${dependency.group}:${dependency.name},${dependency.version},${url},${licenseType}\n") + } outputFile.setText(output.toString(), 'UTF-8') } @@ -173,7 +204,7 @@ are met: derived from this software without specific prior written permission\\.| (3\\.)? Neither the name of .+ nor the names of its contributors may be used to endorse or promote products derived from - this software without specific prior written permission\\.) + this software without specific prior written permission\\.) THIS SOFTWARE IS PROVIDED BY .+ (``|''|")AS IS(''|") AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy index df30326a59d..04fb023e205 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy @@ -109,6 +109,10 @@ public class DependencyLicensesTask extends DefaultTask { mappings.put(from, to) } + public LinkedHashMap getMappings() { + return new LinkedHashMap<>(mappings) + } + /** * Add a rule which will skip SHA checking for the given dependency name. This should be used for * locally build dependencies, which cause the sha to change constantly. @@ -129,10 +133,6 @@ public class DependencyLicensesTask extends DefaultTask { throw new GradleException("Licences dir ${licensesDir} does not exist, but there are dependencies") } - - // order is the same for keys and values iteration since we use a linked hashmap - List mapped = new ArrayList<>(mappings.values()) - Pattern mappingsPattern = Pattern.compile('(' + mappings.keySet().join(')|(') + ')') Map licenses = new HashMap<>() Map notices = new HashMap<>() Set shaFiles = new HashSet() @@ -162,16 +162,10 @@ public class DependencyLicensesTask extends DefaultTask { checkSha(dependency, jarName, shaFiles) } - logger.info("Checking license/notice for " + depName) - Matcher match = mappingsPattern.matcher(depName) - if (match.matches()) { - int i = 0 - while (i < match.groupCount() && match.group(i + 1) == null) ++i; - logger.info("Mapped dependency name ${depName} to ${mapped.get(i)} for license check") - depName = mapped.get(i) - } - checkFile(depName, jarName, licenses, 'LICENSE') - checkFile(depName, jarName, notices, 'NOTICE') + final String dependencyName = getDependencyName(mappings, depName) + logger.info("mapped dependency name ${depName} to ${dependencyName} for license/notice check") + checkFile(dependencyName, jarName, licenses, 'LICENSE') + checkFile(dependencyName, jarName, notices, 'NOTICE') } licenses.each { license, count -> @@ -189,6 +183,19 @@ public class DependencyLicensesTask extends DefaultTask { } } + public static String getDependencyName(final LinkedHashMap mappings, final String dependencyName) { + // order is the same for keys and values iteration since we use a linked hashmap + List mapped = new ArrayList<>(mappings.values()) + Pattern mappingsPattern = Pattern.compile('(' + mappings.keySet().join(')|(') + ')') + Matcher match = mappingsPattern.matcher(dependencyName) + if (match.matches()) { + int i = 0 + while (i < match.groupCount() && match.group(i + 1) == null) ++i; + return mapped.get(i) + } + return dependencyName + } + private File getShaFile(String jarName) { return new File(licensesDir, jarName + SHA_EXTENSION) } diff --git a/client/client-benchmark-noop-api-plugin/build.gradle b/client/client-benchmark-noop-api-plugin/build.gradle index bee41034c3c..cc84207d90d 100644 --- a/client/client-benchmark-noop-api-plugin/build.gradle +++ b/client/client-benchmark-noop-api-plugin/build.gradle @@ -31,6 +31,9 @@ esplugin { tasks.remove(assemble) build.dependsOn.remove('assemble') +dependencyLicenses.enabled = false +dependenciesInfo.enabled = false + compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked" // no unit tests diff --git a/distribution/build.gradle b/distribution/build.gradle index fa62513a540..7f08c244f45 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -32,6 +32,11 @@ Collection distributions = project('archives').subprojects + project('packages') // Concatenates the dependencies CSV files into a single file task generateDependenciesReport(type: ConcatFilesTask) { + project.rootProject.allprojects { + afterEvaluate { + if (it.tasks.findByName("dependenciesInfo")) dependsOn it.tasks.dependenciesInfo + } + } files = fileTree(dir: project.rootDir, include: '**/dependencies.csv' ) headerLine = "name,version,url,license" target = new File(System.getProperty('csv')?: "${project.buildDir}/dependencies/es-dependencies.csv") diff --git a/test/fixtures/example-fixture/build.gradle b/test/fixtures/example-fixture/build.gradle index 225a2cf9deb..ce562e89abb 100644 --- a/test/fixtures/example-fixture/build.gradle +++ b/test/fixtures/example-fixture/build.gradle @@ -22,3 +22,5 @@ test.enabled = false // Not published so no need to assemble tasks.remove(assemble) build.dependsOn.remove('assemble') + +dependenciesInfo.enabled = false diff --git a/x-pack/qa/build.gradle b/x-pack/qa/build.gradle index 1570b218592..24b6618b7d8 100644 --- a/x-pack/qa/build.gradle +++ b/x-pack/qa/build.gradle @@ -28,5 +28,9 @@ gradle.projectsEvaluated { project.tasks.remove(assemble) project.build.dependsOn.remove('assemble') } + Task dependenciesInfo = project.tasks.findByName('dependenciesInfo') + if (dependenciesInfo) { + project.precommit.dependsOn.remove('dependenciesInfo') + } } } diff --git a/x-pack/qa/sql/build.gradle b/x-pack/qa/sql/build.gradle index a3c147bbc04..0bea3a9364b 100644 --- a/x-pack/qa/sql/build.gradle +++ b/x-pack/qa/sql/build.gradle @@ -22,6 +22,7 @@ dependencies { test.enabled = false dependencyLicenses.enabled = false +dependenciesInfo.enabled = false // the main files are actually test files, so use the appropriate forbidden api sigs forbiddenApisMain { diff --git a/x-pack/test/feature-aware/build.gradle b/x-pack/test/feature-aware/build.gradle index 217ed25a2d4..11b0e67183c 100644 --- a/x-pack/test/feature-aware/build.gradle +++ b/x-pack/test/feature-aware/build.gradle @@ -10,6 +10,7 @@ dependencies { forbiddenApisMain.enabled = true dependencyLicenses.enabled = false +dependenciesInfo.enabled = false jarHell.enabled = false