honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15168 from rjernst/precommit_cleanup 

Cleanup precommit task gradle code
This commit is contained in:
Ryan Ernst 2015-12-01 23:50:53 -08:00
parent 9053c9a002 d68c6673a2
commit 0867d5da28
12 changed files with 269 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
View File

@ -62,7 +62,7 @@ class BuildPlugin implements Plugin<Project> {
configureCompile(project)
configureTest(project)
PrecommitTasks.configure(project)
configurePrecommit(project)
}
/** Performs checks on the build environment and prints information about the build environment. */
@ -416,4 +416,11 @@ class BuildPlugin implements Plugin<Project> {
}
return test
}
private static configurePrecommit(Project project) {
Task precommit = PrecommitTasks.create(project, true)
project.check.dependsOn(precommit)
project.test.mustRunAfter(precommit)
project.dependencyLicenses.dependencies = project.configurations.runtime - project.configurations.provided
}
}

101
buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy
View File

@ -18,63 +18,100 @@
*/
package org.elasticsearch.gradle.precommit
import org.gradle.api.DefaultTask
import org.gradle.api.GradleException
import org.gradle.api.InvalidUserDataException
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.*
import org.gradle.api.file.FileCollection
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputDirectory
import org.gradle.api.tasks.InputFiles
import org.gradle.api.tasks.StopActionException
import org.gradle.api.tasks.TaskAction
import org.gradle.api.tasks.VerificationTask
import java.nio.file.Files
import java.security.MessageDigest
import java.util.regex.Matcher
import java.util.regex.Pattern
class DependencyLicensesTask extends DefaultTask {
static final String SHA_EXTENSION = '.sha1'
static Task configure(Project project, Closure closure) {
DependencyLicensesTask task = project.tasks.create(type: DependencyLicensesTask, name: 'dependencyLicenses')
UpdateShasTask update = project.tasks.create(type: UpdateShasTask, name: 'updateShas')
update.parentTask = task
task.configure(closure)
project.check.dependsOn(task)
return task
}
/**
* A task to check licenses for dependencies.
*
* There are two parts to the check:
* <ul>
* <li>LICENSE and NOTICE files</li>
* <li>SHA checksums for each dependency jar</li>
* </ul>
*
* The directory to find the license and sha files in defaults to the dir @{code licenses}
* in the project directory for this task. You can override this directory:
* <pre>
* dependencyLicenses {
* licensesDir = project.file('mybetterlicensedir')
* }
* </pre>
*
* The jar files to check default to the dependencies from the default configuration. You
* can override this, for example, to only check compile dependencies:
* <pre>
* dependencyLicenses {
* dependencies = project.configurations.compile
* }
* </pre>
*
* Every jar must have a {@code .sha1} file in the licenses dir. These can be managed
* automatically using the {@code updateShas} helper task that is created along
* with this task. It will add {@code .sha1} files for new jars that are in dependencies
* and remove old {@code .sha1} files that are no longer needed.
*
* Every jar must also have a LICENSE and NOTICE file. However, multiple jars can share
* LICENSE and NOTICE files by mapping a pattern to the same name.
* <pre>
* dependencyLicenses {
* mapping from: &#47;lucene-.*&#47;, to: 'lucene'
* }
* </pre>
*/
public class DependencyLicensesTask extends DefaultTask {
private static final String SHA_EXTENSION = '.sha1'
// TODO: we should be able to default this to eg compile deps, but we need to move the licenses
// check from distribution to core (ie this should only be run on java projects)
/** A collection of jar files that should be checked. */
@InputFiles
FileCollection dependencies
public FileCollection dependencies
/** The directory to find the license and sha files in. */
@InputDirectory
File licensesDir = new File(project.projectDir, 'licenses')
public File licensesDir = new File(project.projectDir, 'licenses')
LinkedHashMap<String, String> mappings = new LinkedHashMap<>()
/** A map of patterns to prefix, used to find the LICENSE and NOTICE file. */
private LinkedHashMap<String, String> mappings = new LinkedHashMap<>()
/**
* Add a mapping from a regex pattern for the jar name, to a prefix to find
* the LICENSE and NOTICE file for that jar.
*/
@Input
void mapping(Map<String, String> props) {
String from = props.get('from')
public void mapping(Map<String, String> props) {
String from = props.remove('from')
if (from == null) {
throw new InvalidUserDataException('Missing "from" setting for license name mapping')
}
String to = props.get('to')
String to = props.remove('to')
if (to == null) {
throw new InvalidUserDataException('Missing "to" setting for license name mapping')
}
if (props.isEmpty() == false) {
throw new InvalidUserDataException("Unknown properties for mapping on dependencyLicenses: ${props.keySet()}")
}
mappings.put(from, to)
}
@TaskAction
void checkDependencies() {
// TODO: empty license dir (or error when dir exists and no deps)
public void checkDependencies() {
if (licensesDir.exists() == false && dependencies.isEmpty() == false) {
throw new GradleException("Licences dir ${licensesDir} does not exist, but there are dependencies")
}
if (licensesDir.exists() && dependencies.isEmpty()) {
throw new GradleException("Licenses dir ${licensesDir} exists, but there are no dependencies")
}
// order is the same for keys and values iteration since we use a linked hashmap
List<String> mapped = new ArrayList<>(mappings.values())
@ -127,7 +164,7 @@ class DependencyLicensesTask extends DefaultTask {
}
}
void checkSha(File jar, String jarName, Set<File> shaFiles) {
private void checkSha(File jar, String jarName, Set<File> shaFiles) {
File shaFile = new File(licensesDir, jarName + SHA_EXTENSION)
if (shaFile.exists() == false) {
throw new GradleException("Missing SHA for ${jarName}. Run 'gradle updateSHAs' to create")
@ -143,7 +180,7 @@ class DependencyLicensesTask extends DefaultTask {
shaFiles.remove(shaFile)
}
void checkFile(String name, String jarName, Map<String, Integer> counters, String type) {
private void checkFile(String name, String jarName, Map<String, Integer> counters, String type) {
String fileName = "${name}-${type}"
Integer count = counters.get(fileName)
if (count == null) {
@ -158,10 +195,12 @@ class DependencyLicensesTask extends DefaultTask {
counters.put(fileName, count + 1)
}
static class UpdateShasTask extends DefaultTask {
DependencyLicensesTask parentTask
/** A helper task to update the sha files in the license dir. */
public static class UpdateShasTask extends DefaultTask {
private DependencyLicensesTask parentTask
@TaskAction
void updateShas() {
public void updateShas() {
Set<File> shaFiles = new HashSet<File>()
parentTask.licensesDir.eachFile {
String name = it.getName()

41
buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ForbiddenPatternsTask.groovy
View File

@ -19,10 +19,11 @@
package org.elasticsearch.gradle.precommit
import org.gradle.api.DefaultTask
import org.gradle.api.GradleException
import org.gradle.api.InvalidUserDataException
import org.gradle.api.file.FileCollection
import org.gradle.api.tasks.InputFiles
import org.gradle.api.tasks.OutputFile
import org.gradle.api.tasks.OutputFiles
import org.gradle.api.tasks.SourceSet
import org.gradle.api.tasks.TaskAction
import org.gradle.api.tasks.util.PatternFilterable
@ -33,14 +34,19 @@ import java.util.regex.Pattern
/**
* Checks for patterns in source files for the project which are forbidden.
*/
class ForbiddenPatternsTask extends DefaultTask {
Map<String,String> patterns = new LinkedHashMap<>()
PatternFilterable filesFilter = new PatternSet()
public class ForbiddenPatternsTask extends DefaultTask {
/** The rules: a map from the rule name, to a rule regex pattern. */
private Map<String,String> patterns = new LinkedHashMap<>()
/** A pattern set of which files should be checked. */
private PatternFilterable filesFilter = new PatternSet()
@OutputFile
File outputMarker = new File(project.buildDir, "markers/forbiddenPatterns")
ForbiddenPatternsTask() {
public ForbiddenPatternsTask() {
description = 'Checks source files for invalid patterns like nocommits or tabs'
// we always include all source files, and exclude what should not be checked
filesFilter.include('**')
// exclude known binary extensions
@ -52,23 +58,28 @@ class ForbiddenPatternsTask extends DefaultTask {
filesFilter.exclude('**/*.crt')
filesFilter.exclude('**/*.png')
// TODO: add compile and test compile outputs as this tasks outputs, so we don't rerun when source files haven't changed
// add mandatory rules
patterns.put('nocommit', /nocommit/)
patterns.put('tab', /\t/)
}
/** Adds a file glob pattern to be excluded */
void exclude(String... excludes) {
public void exclude(String... excludes) {
this.filesFilter.exclude(excludes)
}
/** Adds pattern to forbid */
/** Adds a pattern to forbid. T */
void rule(Map<String,String> props) {
String name = props.get('name')
String name = props.remove('name')
if (name == null) {
throw new IllegalArgumentException('Missing [name] for invalid pattern rule')
throw new InvalidUserDataException('Missing [name] for invalid pattern rule')
}
String pattern = props.get('pattern')
String pattern = props.remove('pattern')
if (pattern == null) {
throw new IllegalArgumentException('Missing [pattern] for invalid pattern rule')
throw new InvalidUserDataException('Missing [pattern] for invalid pattern rule')
}
if (props.isEmpty() == false) {
throw new InvalidUserDataException("Unknown arguments for ForbiddenPatterns rule mapping: ${props.keySet()}")
}
// TODO: fail if pattern contains a newline, it won't work (currently)
patterns.put(name, pattern)
@ -89,14 +100,14 @@ class ForbiddenPatternsTask extends DefaultTask {
Pattern allPatterns = Pattern.compile('(' + patterns.values().join(')|(') + ')')
List<String> failures = new ArrayList<>()
for (File f : files()) {
f.eachLine('UTF-8') { line, lineNumber ->
f.eachLine('UTF-8') { String line, int lineNumber ->
if (allPatterns.matcher(line).find()) {
addErrorMessages(failures, f, (String)line, (int)lineNumber)
addErrorMessages(failures, f, line, lineNumber)
}
}
}
if (failures.isEmpty() == false) {
throw new IllegalArgumentException('Found invalid patterns:\n' + failures.join('\n'))
throw new GradleException('Found invalid patterns:\n' + failures.join('\n'))
}
outputMarker.setText('done', 'UTF-8')
}

62
buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/JarHellTask.groovy Normal file
View File

@ -0,0 +1,62 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.precommit
import org.elasticsearch.gradle.LoggedExec
import org.gradle.api.file.FileCollection
import org.gradle.api.tasks.InputFile
import org.gradle.api.tasks.OutputFile
/**
* Runs CheckJarHell on a classpath.
*/
public class JarHellTask extends LoggedExec {
/**
* We use a simple "marker" file that we touch when the task succeeds
* as the task output. This is compared against the modified time of the
* inputs (ie the jars/class files).
*/
@OutputFile
public File successMarker = new File(project.buildDir, 'markers/jarHell')
/** The classpath to run jarhell check on, defaults to the test runtime classpath */
@InputFile
public FileCollection classpath = project.sourceSets.test.runtimeClasspath
public JarHellTask() {
project.afterEvaluate {
dependsOn(classpath)
description = "Runs CheckJarHell on ${classpath}"
executable = new File(project.javaHome, 'bin/java')
doFirst({
/* JarHell doesn't like getting directories that don't exist but
gradle isn't especially careful about that. So we have to do it
filter it ourselves. */
FileCollection taskClasspath = classpath.filter { it.exists() }
args('-cp', taskClasspath.asPath, 'org.elasticsearch.bootstrap.JarHell')
})
doLast({
successMarker.parentFile.mkdirs()
successMarker.setText("", 'UTF-8')
})
}
}
}

110
buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
View File

@ -18,16 +18,10 @@
*/
package org.elasticsearch.gradle.precommit
import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApisExtension
import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
import org.gradle.api.GradleException
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.file.FileCollection
import org.gradle.api.plugins.JavaBasePlugin
import org.gradle.api.tasks.Exec
import org.gradle.api.tasks.TaskContainer
/**
* Validation tasks which should be run before committing. These run before tests.
@ -35,36 +29,34 @@ import org.gradle.api.tasks.TaskContainer
class PrecommitTasks {
/** Adds a precommit task, which depends on non-test verification tasks. */
static void configure(Project project) {
List precommitTasks = [
configureForbiddenApis(project),
configureForbiddenPatterns(project.tasks),
configureJarHell(project)]
public static Task create(Project project, boolean includeDependencyLicenses) {
Map precommitOptions = [
name: 'precommit',
group: JavaBasePlugin.VERIFICATION_GROUP,
description: 'Runs all non-test checks.',
dependsOn: precommitTasks
]
Task precommit = project.tasks.create(precommitOptions)
project.check.dependsOn(precommit)
List<Task> precommitTasks = [
configureForbiddenApis(project),
project.tasks.create('forbiddenPatterns', ForbiddenPatternsTask.class),
project.tasks.create('jarHell', JarHellTask.class)]
// delay ordering relative to test tasks, since they may not be setup yet
project.afterEvaluate {
Task test = project.tasks.findByName('test')
if (test != null) {
test.mustRunAfter(precommit)
}
Task integTest = project.tasks.findByName('integTest')
if (integTest != null) {
integTest.mustRunAfter(precommit)
}
// tasks with just tests don't need dependency licenses, so this flag makes adding
// the task optional
if (includeDependencyLicenses) {
DependencyLicensesTask dependencyLicenses = project.tasks.create('dependencyLicenses', DependencyLicensesTask.class)
precommitTasks.add(dependencyLicenses)
// we also create the updateShas helper task that is associated with dependencyLicenses
UpdateShasTask updateShas = project.tasks.create('updateShas', UpdateShasTask.class)
updateShas.parentTask = dependencyLicenses
}
Map<String, Object> precommitOptions = [
name: 'precommit',
group: JavaBasePlugin.VERIFICATION_GROUP,
description: 'Runs all non-test checks.',
dependsOn: precommitTasks
]
return project.tasks.create(precommitOptions)
}
static Task configureForbiddenApis(Project project) {
project.pluginManager.apply('de.thetaphi.forbiddenapis')
private static Task configureForbiddenApis(Project project) {
project.pluginManager.apply(ForbiddenApisPlugin.class)
project.forbiddenApis {
internalRuntimeForbidden = true
failOnUnsupportedJava = false
@ -75,7 +67,7 @@ class PrecommitTasks {
Task mainForbidden = project.tasks.findByName('forbiddenApisMain')
if (mainForbidden != null) {
mainForbidden.configure {
bundledSignatures += ['jdk-system-out']
bundledSignatures += 'jdk-system-out'
signaturesURLs += [
getClass().getResource('/forbidden/core-signatures.txt'),
getClass().getResource('/forbidden/third-party-signatures.txt')]
@ -84,63 +76,11 @@ class PrecommitTasks {
Task testForbidden = project.tasks.findByName('forbiddenApisTest')
if (testForbidden != null) {
testForbidden.configure {
signaturesURLs += [getClass().getResource('/forbidden/test-signatures.txt')]
signaturesURLs += getClass().getResource('/forbidden/test-signatures.txt')
}
}
Task forbiddenApis = project.tasks.findByName('forbiddenApis')
forbiddenApis.group = "" // clear group, so this does not show up under verification tasks
return forbiddenApis
}
static Task configureForbiddenPatterns(TaskContainer tasks) {
Map options = [
name: 'forbiddenPatterns',
type: ForbiddenPatternsTask,
description: 'Checks source files for invalid patterns like nocommits or tabs',
]
return tasks.create(options) {
rule name: 'nocommit', pattern: /nocommit/
rule name: 'tab', pattern: /\t/
}
}
/**
* Adds a task to run jar hell before on the test classpath.
*
* We use a simple "marker" file that we touch when the task succeeds
* as the task output. This is compared against the modified time of the
* inputs (ie the jars/class files).
*/
static Task configureJarHell(Project project) {
File successMarker = new File(project.buildDir, 'markers/jarHell')
Exec task = project.tasks.create(name: 'jarHell', type: Exec)
FileCollection testClasspath = project.sourceSets.test.runtimeClasspath
task.dependsOn(testClasspath)
task.inputs.files(testClasspath)
task.outputs.file(successMarker)
task.executable = new File(project.javaHome, 'bin/java')
task.doFirst({
/* JarHell doesn't like getting directories that don't exist but
gradle isn't especially careful about that. So we have to do it
filter it ourselves. */
def taskClasspath = testClasspath.filter { it.exists() }
task.args('-cp', taskClasspath.asPath, 'org.elasticsearch.bootstrap.JarHell')
})
if (task.logger.isInfoEnabled() == false) {
task.standardOutput = new ByteArrayOutputStream()
task.errorOutput = task.standardOutput
task.ignoreExitValue = true
task.doLast({
if (execResult.exitValue != 0) {
logger.error(standardOutput.toString())
throw new GradleException("JarHell failed")
}
})
}
task.doLast({
successMarker.parentFile.mkdirs()
successMarker.setText("", 'UTF-8')
})
return task
}
}

65
buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/UpdateShasTask.groovy Normal file
View File

@ -0,0 +1,65 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.precommit
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.TaskAction
import java.nio.file.Files
import java.security.MessageDigest
/**
* A task to update shas used by {@code DependencyLicensesCheck}
*/
public class UpdateShasTask extends DefaultTask {
/** The parent dependency licenses task to use configuration from */
public DependencyLicensesTask parentTask
public UpdateShasTask() {
description = 'Updates the sha files for the dependencyLicenses check'
}
@TaskAction
public void updateShas() {
Set<File> shaFiles = new HashSet<File>()
parentTask.licensesDir.eachFile {
String name = it.getName()
if (name.endsWith(SHA_EXTENSION)) {
shaFiles.add(it)
}
}
for (File dependency : parentTask.dependencies) {
String jarName = dependency.getName()
File shaFile = new File(parentTask.licensesDir, jarName + SHA_EXTENSION)
if (shaFile.exists() == false) {
logger.lifecycle("Adding sha for ${jarName}")
String sha = MessageDigest.getInstance("SHA-1").digest(dependency.getBytes()).encodeHex().toString()
shaFile.setText(sha, 'UTF-8')
} else {
shaFiles.remove(shaFile)
}
}
shaFiles.each { shaFile ->
logger.lifecycle("Removing unused sha ${shaFile.getName()}")
Files.delete(shaFile.toPath())
}
}
}

1
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
View File

@ -58,6 +58,7 @@ class RestIntegTestTask extends RandomizedTestingTask {
integTest.testClassesDir = test.testClassesDir
integTest.mustRunAfter(test)
}
integTest.mustRunAfter(project.precommit)
project.check.dependsOn(integTest)
RestSpecHack.configureDependencies(project)
project.afterEvaluate {

3
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestBasePlugin.groovy
View File

@ -56,6 +56,7 @@ class StandaloneTestBasePlugin implements Plugin<Project> {
plusConfigurations = [project.configurations.testRuntime]
}
}
PrecommitTasks.configure(project)
PrecommitTasks.create(project, false)
project.check.dependsOn(project.precommit)
}
}

1
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneTestPlugin.groovy
View File

@ -45,6 +45,7 @@ class StandaloneTestPlugin implements Plugin<Project> {
classpath = project.sourceSets.test.runtimeClasspath
testClassesDir project.sourceSets.test.output.classesDir
}
test.mustRunAfter(project.precommit)
project.check.dependsOn(test)
}
}

3
core/build.gradle
View File

@ -117,6 +117,9 @@ forbiddenPatterns {
exclude '**/org/elasticsearch/cluster/routing/shard_routes.txt'
}
// dependency license are currently checked in distribution
dependencyLicenses.enabled = false
if (isEclipse == false || project.path == ":core-tests") {
task integTest(type: RandomizedTestingTask,
group: JavaBasePlugin.VERIFICATION_GROUP,

8
distribution/build.gradle
View File

@ -20,6 +20,7 @@
import org.apache.tools.ant.filters.FixCrLfFilter
import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.precommit.DependencyLicensesTask
import org.elasticsearch.gradle.precommit.UpdateShasTask
import org.elasticsearch.gradle.test.RunTask
import org.elasticsearch.gradle.EmptyDirTask
import org.elasticsearch.gradle.MavenFilteringHack
@ -293,13 +294,16 @@ configure(subprojects.findAll { it.name == 'deb' || it.name == 'rpm' }) {
// TODO: dependency checks should really be when building the jar itself, which would remove the need
// for this hackery and instead we can do this inside the BuildPlugin
task check(group: 'Verification', description: 'Runs all checks.') {} // dummy task!
DependencyLicensesTask.configure(project) {
task dependencyLicenses(type: DependencyLicensesTask) {
dependsOn = [dependencyFiles]
dependencies = dependencyFiles
mapping from: /lucene-.*/, to: 'lucene'
mapping from: /jackson-.*/, to: 'jackson'
}
task check(group: 'Verification', description: 'Runs all checks.', dependsOn: dependencyLicenses) {} // dummy task!
task updateShas(type: UpdateShasTask) {
parentTask = dependencyLicenses
}
RunTask.configure(project)

5
plugins/build.gradle
View File

@ -29,9 +29,4 @@ subprojects {
// for local ES plugins, the name of the plugin is the same as the directory
name project.name
}
Task dependencyLicensesTask = DependencyLicensesTask.configure(project) {
dependencies = project.configurations.runtime - project.configurations.provided
}
project.precommit.dependsOn(dependencyLicensesTask)
}