honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

converting ForbiddenPatternsTask to .java (#36194) 

* converting ForbiddenPatternsTask to java impl & unit tests
This commit is contained in:
Christian Schneider 2018-12-11 06:15:44 -06:00 committed by Alpar Torok
parent 5f7524bb89
commit 00eadd93eb
7 changed files with 291 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
buildSrc/build.gradle
View File

@ -217,7 +217,7 @@ if (project != rootProject) {
forbiddenPatterns {
exclude '**/*.wav'
// the file that actually defines nocommit
exclude '**/ForbiddenPatternsTask.groovy'
exclude '**/ForbiddenPatternsTask.java'
}
namingConventions {

130
buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ForbiddenPatternsTask.groovy
View File

@ -1,130 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.precommit
import org.gradle.api.DefaultTask
import org.gradle.api.GradleException
import org.gradle.api.InvalidUserDataException
import org.gradle.api.file.FileCollection
import org.gradle.api.tasks.InputFiles
import org.gradle.api.tasks.OutputFile
import org.gradle.api.tasks.SourceSet
import org.gradle.api.tasks.TaskAction
import org.gradle.api.tasks.util.PatternFilterable
import org.gradle.api.tasks.util.PatternSet
import java.util.regex.Pattern
/**
* Checks for patterns in source files for the project which are forbidden.
*/
public class ForbiddenPatternsTask extends DefaultTask {
/** The rules: a map from the rule name, to a rule regex pattern. */
private Map<String,String> patterns = new LinkedHashMap<>()
/** A pattern set of which files should be checked. */
private PatternFilterable filesFilter = new PatternSet()
@OutputFile
File outputMarker = new File(project.buildDir, "markers/forbiddenPatterns")
public ForbiddenPatternsTask() {
description = 'Checks source files for invalid patterns like nocommits or tabs'
// we always include all source files, and exclude what should not be checked
filesFilter.include('**')
// exclude known binary extensions
filesFilter.exclude('**/*.gz')
filesFilter.exclude('**/*.ico')
filesFilter.exclude('**/*.jar')
filesFilter.exclude('**/*.zip')
filesFilter.exclude('**/*.jks')
filesFilter.exclude('**/*.crt')
filesFilter.exclude('**/*.png')
// add mandatory rules
patterns.put('nocommit', /nocommit|NOCOMMIT/)
patterns.put('nocommit should be all lowercase or all uppercase',
/((?i)nocommit)(?<!(nocommit|NOCOMMIT))/)
patterns.put('tab', /\t/)
inputs.property("excludes", filesFilter.excludes)
inputs.property("rules", patterns)
}
/** Adds a file glob pattern to be excluded */
public void exclude(String... excludes) {
filesFilter.exclude(excludes)
}
/** Adds a pattern to forbid. T */
void rule(Map<String,String> props) {
String name = props.remove('name')
if (name == null) {
throw new InvalidUserDataException('Missing [name] for invalid pattern rule')
}
String pattern = props.remove('pattern')
if (pattern == null) {
throw new InvalidUserDataException('Missing [pattern] for invalid pattern rule')
}
if (props.isEmpty() == false) {
throw new InvalidUserDataException("Unknown arguments for ForbiddenPatterns rule mapping: ${props.keySet()}")
}
// TODO: fail if pattern contains a newline, it won't work (currently)
patterns.put(name, pattern)
}
/** Returns the files this task will check */
@InputFiles
FileCollection files() {
List<FileCollection> collections = new ArrayList<>()
for (SourceSet sourceSet : project.sourceSets) {
collections.add(sourceSet.allSource.matching(filesFilter))
}
return project.files(collections.toArray())
}
@TaskAction
void checkInvalidPatterns() {
Pattern allPatterns = Pattern.compile('(' + patterns.values().join(')|(') + ')')
List<String> failures = new ArrayList<>()
for (File f : files()) {
f.eachLine('UTF-8') { String line, int lineNumber ->
if (allPatterns.matcher(line).find()) {
addErrorMessages(failures, f, line, lineNumber)
}
}
}
if (failures.isEmpty() == false) {
throw new GradleException('Found invalid patterns:\n' + failures.join('\n'))
}
outputMarker.setText('done', 'UTF-8')
}
// iterate through patterns to find the right ones for nice error messages
void addErrorMessages(List<String> failures, File f, String line, int lineNumber) {
String path = project.getRootProject().projectDir.toURI().relativize(f.toURI()).toString()
for (Map.Entry<String,String> pattern : patterns.entrySet()) {
if (Pattern.compile(pattern.value).matcher(line).find()) {
failures.add('- ' + pattern.key + ' on line ' + lineNumber + ' of ' + path)
}
}
}
}

161
buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ForbiddenPatternsTask.java Normal file
View File

@ -0,0 +1,161 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.precommit;
import org.gradle.api.DefaultTask;
import org.gradle.api.GradleException;
import org.gradle.api.InvalidUserDataException;
import org.gradle.api.file.FileCollection;
import org.gradle.api.file.FileTree;
import org.gradle.api.plugins.JavaPluginConvention;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.InputFiles;
import org.gradle.api.tasks.OutputFile;
import org.gradle.api.tasks.SkipWhenEmpty;
import org.gradle.api.tasks.TaskAction;
import org.gradle.api.tasks.util.PatternFilterable;
import org.gradle.api.tasks.util.PatternSet;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
/**
* Checks for patterns in source files for the project which are forbidden.
*/
public class ForbiddenPatternsTask extends DefaultTask {
/*
* A pattern set of which files should be checked.
*/
private final PatternFilterable filesFilter = new PatternSet()
// we always include all source files, and exclude what should not be checked
.include("**")
// exclude known binary extensions
.exclude("**/*.gz")
.exclude("**/*.ico")
.exclude("**/*.jar")
.exclude("**/*.zip")
.exclude("**/*.jks")
.exclude("**/*.crt")
.exclude("**/*.png");
/*
* The rules: a map from the rule name, to a rule regex pattern.
*/
private final Map<String, String> patterns = new HashMap<>();
public ForbiddenPatternsTask() {
setDescription("Checks source files for invalid patterns like nocommits or tabs");
getInputs().property("excludes", filesFilter.getExcludes());
getInputs().property("rules", patterns);
// add mandatory rules
patterns.put("nocommit", "nocommit|NOCOMMIT");
patterns.put("nocommit should be all lowercase or all uppercase", "((?i)nocommit)(?<!(nocommit|NOCOMMIT))");
patterns.put("tab", "\t");
}
@InputFiles
@SkipWhenEmpty
public FileCollection files() {
return getProject().getConvention().getPlugin(JavaPluginConvention.class).getSourceSets()
.stream()
.map(sourceSet -> sourceSet.getAllSource().matching(filesFilter))
.reduce(FileTree::plus)
.orElse(getProject().files().getAsFileTree());
}
@TaskAction
public void checkInvalidPatterns() throws IOException {
Pattern allPatterns = Pattern.compile("(" + String.join(")|(", getPatterns().values()) + ")");
List<String> failures = new ArrayList<>();
for (File f : files()) {
List<String> lines;
try(Stream<String> stream = Files.lines(f.toPath(), StandardCharsets.UTF_8)) {
lines = stream.collect(Collectors.toList());
} catch (UncheckedIOException e) {
throw new IllegalArgumentException("Failed to read " + f + " as UTF_8", e);
}
List<Integer> invalidLines = IntStream.range(0, lines.size())
.filter(i -> allPatterns.matcher(lines.get(i)).find())
.boxed()
.collect(Collectors.toList());
String path = getProject().getRootProject().getProjectDir().toURI().relativize(f.toURI()).toString();
failures = invalidLines.stream()
.map(l -> new AbstractMap.SimpleEntry<>(l+1, lines.get(l)))
.flatMap(kv -> patterns.entrySet().stream()
.filter(p -> Pattern.compile(p.getValue()).matcher(kv.getValue()).find())
.map(p -> "- " + p.getKey() + " on line " + kv.getKey() + " of " + path)
)
.collect(Collectors.toList());
}
if (failures.isEmpty() == false) {
throw new GradleException("Found invalid patterns:\n" + String.join("\n", failures));
}
File outputMarker = getOutputMarker();
outputMarker.getParentFile().mkdirs();
Files.write(outputMarker.toPath(), "done".getBytes(StandardCharsets.UTF_8));
}
@OutputFile
public File getOutputMarker() {
return new File(getProject().getBuildDir(), "markers/" + getName());
}
@Input
public Map<String, String> getPatterns() {
return Collections.unmodifiableMap(patterns);
}
public void exclude(String... excludes) {
filesFilter.exclude(excludes);
}
public void rule(Map<String,String> props) {
String name = props.remove("name");
if (name == null) {
throw new InvalidUserDataException("Missing [name] for invalid pattern rule");
}
String pattern = props.remove("pattern");
if (pattern == null) {
throw new InvalidUserDataException("Missing [pattern] for invalid pattern rule");
}
if (props.isEmpty() == false) {
throw new InvalidUserDataException("Unknown arguments for ForbiddenPatterns rule mapping: "
+ props.keySet().toString());
}
// TODO: fail if pattern contains a newline, it won't work (currently)
patterns.put(name, pattern);
}
}

111
buildSrc/src/test/java/org/elasticsearch/gradle/precommit/ForbiddenPatternsTaskTests.java Normal file
View File

@ -0,0 +1,111 @@
package org.elasticsearch.gradle.precommit;
import org.elasticsearch.gradle.test.GradleUnitTestCase;
import org.gradle.api.GradleException;
import org.gradle.api.Project;
import org.gradle.api.plugins.JavaPlugin;
import org.gradle.testfixtures.ProjectBuilder;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
public class ForbiddenPatternsTaskTests extends GradleUnitTestCase {
public void testCheckInvalidPatternsWhenNoSourceFilesExist() throws Exception {
Project project = createProject();
ForbiddenPatternsTask task = createTask(project);
checkAndAssertTaskSuccessful(task);
}
public void testCheckInvalidPatternsWhenSourceFilesExistNoViolation() throws Exception {
Project project = createProject();
ForbiddenPatternsTask task = createTask(project);
writeSourceFile(project, "src/main/java/Foo.java", "public void bar() {}");
checkAndAssertTaskSuccessful(task);
}
public void testCheckInvalidPatternsWhenSourceFilesExistHavingTab() throws Exception {
Project project = createProject();
ForbiddenPatternsTask task = createTask(project);
writeSourceFile(project, "src/main/java/Bar.java", "\tpublic void bar() {}");
checkAndAssertTaskThrowsException(task);
}
public void testCheckInvalidPatternsWithCustomRule() throws Exception {
Map<String, String> rule = new HashMap<>();
rule.put("name", "TODO comments are not allowed");
rule.put("pattern", "\\/\\/.*(?i)TODO");
Project project = createProject();
ForbiddenPatternsTask task = createTask(project);
task.rule(rule);
writeSourceFile(project, "src/main/java/Moot.java", "GOOD LINE", "//todo", "// some stuff, toDo");
checkAndAssertTaskThrowsException(task);
}
public void testCheckInvalidPatternsWhenExcludingFiles() throws Exception {
Project project = createProject();
ForbiddenPatternsTask task = createTask(project);
task.exclude("**/*.java");
writeSourceFile(project, "src/main/java/FooBarMoot.java", "\t");
checkAndAssertTaskSuccessful(task);
}
private Project createProject() {
Project project = ProjectBuilder.builder().build();
project.getPlugins().apply(JavaPlugin.class);
return project;
}
private ForbiddenPatternsTask createTask(Project project) {
return project.getTasks().create("forbiddenPatterns", ForbiddenPatternsTask.class);
}
private ForbiddenPatternsTask createTask(Project project, String taskName) {
return project.getTasks().create(taskName, ForbiddenPatternsTask.class);
}
private void writeSourceFile(Project project, String name, String... lines) throws IOException {
File file = new File(project.getProjectDir(), name);
file.getParentFile().mkdirs();
file.createNewFile();
if (lines.length != 0)
Files.write(file.toPath(), Arrays.asList(lines), StandardCharsets.UTF_8);
}
private void checkAndAssertTaskSuccessful(ForbiddenPatternsTask task) throws IOException {
task.checkInvalidPatterns();
assertTaskSuccessful(task.getProject(), task.getName());
}
private void checkAndAssertTaskThrowsException(ForbiddenPatternsTask task) throws IOException {
try {
task.checkInvalidPatterns();
fail("GradleException was expected to be thrown in this case!");
} catch (GradleException e) {
assertTrue(e.getMessage().startsWith("Found invalid patterns"));
}
}
private void assertTaskSuccessful(Project project, String fileName) throws IOException {
File outputMarker = new File(project.getBuildDir(), "markers/" + fileName);
assertTrue(outputMarker.exists());
Optional<String> result = Files.readAllLines(outputMarker.toPath(), StandardCharsets.UTF_8).stream().findFirst();
assertTrue(result.isPresent());
assertEquals("done", result.get());
}
}

3
server/build.gradle
View File

@ -156,6 +156,9 @@ compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-tr
forbiddenPatterns {
exclude '**/*.json'
exclude '**/*.jmx'
exclude '**/*.dic'
exclude '**/*.binary'
exclude '**/*.st'
}
task generateModulesList {

7
x-pack/qa/full-cluster-restart/build.gradle
View File

@ -86,6 +86,9 @@ licenseHeaders {
approvedLicenses << 'Apache'
}
forbiddenPatterns {
exclude '**/system_key'
}
/**
* Subdirectories of this project are test rolling upgrades with various
* configuration options based on their name.
@ -115,6 +118,10 @@ subprojects {
approvedLicenses << 'Apache'
}
forbiddenPatterns {
exclude '**/system_key'
}
String outputDir = "${buildDir}/generated-resources/${project.name}"
// This is a top level task which we will add dependencies to below.

8
x-pack/qa/rolling-upgrade/build.gradle
View File

@ -72,6 +72,10 @@ Project mainProject = project
compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
forbiddenPatterns {
exclude '**/system_key'
}
/**
* Subdirectories of this project are test rolling upgrades with various
* configuration options based on their name.
@ -97,6 +101,10 @@ subprojects {
}
}
forbiddenPatterns {
exclude '**/system_key'
}
String outputDir = "${buildDir}/generated-resources/${project.name}"
// This is a top level task which we will add dependencies to below.