From b988d557d75d92f4378e98804261f6733f04378a Mon Sep 17 00:00:00 2001 From: Dawid Weiss Date: Tue, 8 Sep 2020 10:08:51 +0200 Subject: [PATCH] =?UTF-8?q?LUCENE-9506:=20Gradle:=20split=20validateSource?= =?UTF-8?q?Patterns=20into=20per-project=20an=E2=80=A6=20(#1830)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * LUCENE-9506: Gradle: split validateSourcePatterns into per-project and root-specific tasks (allow parallelism) Co-authored-by: Uwe Schindler --- gradle/validation/precommit.gradle | 5 +- gradle/validation/validate-log-calls.gradle | 5 +- .../validate-source-patterns.gradle | 155 +++++++++++++----- 3 files changed, 122 insertions(+), 43 deletions(-) diff --git a/gradle/validation/precommit.gradle b/gradle/validation/precommit.gradle index aad8c6f071a..e4de4be7620 100644 --- a/gradle/validation/precommit.gradle +++ b/gradle/validation/precommit.gradle @@ -26,7 +26,6 @@ configure(rootProject) { dependsOn ":verifyLocks" dependsOn ":versionsPropsAreSorted" dependsOn ":checkWorkingCopyClean" - dependsOn ":validateSourcePatterns" // Solr validation tasks. dependsOn ":solr:validateConfigFileSanity" @@ -40,8 +39,8 @@ configure(rootProject) { "javadoc", "rat", "ecjLint", - "checkMissingDocs", - "validateLogCalls" + "validateLogCalls", + "validateSourcePatterns", ]} } } diff --git a/gradle/validation/validate-log-calls.gradle b/gradle/validation/validate-log-calls.gradle index b548c2116ce..741f4e76fe1 100644 --- a/gradle/validation/validate-log-calls.gradle +++ b/gradle/validation/validate-log-calls.gradle @@ -45,7 +45,10 @@ configure(subprojects.findAll { it.path.startsWith(':solr') }) { sourceFiles = files(sourceSets*.java) } - } + + // Add log validation to per-project checks as well. + check.dependsOn validateLogCalls + } } class ValidateLogCallsTask extends DefaultTask { diff --git a/gradle/validation/validate-source-patterns.gradle b/gradle/validation/validate-source-patterns.gradle index 5b027d8d6bb..7363308e364 100644 --- a/gradle/validation/validate-source-patterns.gradle +++ b/gradle/validation/validate-source-patterns.gradle @@ -15,9 +15,13 @@ * limitations under the License. */ -import org.apache.rat.Defaults; -import org.apache.rat.document.impl.FileDocument; -import org.apache.rat.api.MetaData; +import org.apache.rat.Defaults +import org.apache.rat.document.impl.FileDocument +import org.apache.rat.api.MetaData + +import javax.inject.Inject; +import org.gradle.internal.logging.progress.ProgressLoggerFactory +import org.gradle.internal.logging.progress.ProgressLogger buildscript { repositories { @@ -29,50 +33,119 @@ buildscript { } } -configure(rootProject) { - task("validateSourcePatterns", type: ValidateSourcePatternsTask) { task -> +def extensions = [ + 'adoc', + 'bat', + 'cmd', + 'css', + 'g4', + 'gradle', + 'groovy', + 'html', + 'java', + 'jflex', + 'jj', + 'js', + 'json', + 'md', + 'mdtext', + 'pl', + 'policy', + 'properties', + 'py', + 'sh', + 'template', + 'vm', + 'xml', + 'xsl', +] + +// Create source validation task local for each project's files. +subprojects { + task validateSourcePatterns(type: ValidateSourcePatternsTask) { task -> group = 'Verification' description = 'Validate Source Patterns' // This task has no proper outputs. setupDummyOutputs(task) - sourceFiles = project.fileTree(project.rootDir) { - [ - 'java', 'jflex', 'py', 'pl', 'g4', 'jj', 'html', 'js', - 'css', 'xml', 'xsl', 'vm', 'sh', 'cmd', 'bat', 'policy', - 'properties', 'mdtext', 'groovy', 'gradle', - 'template', 'adoc', 'json', - ].each{ - include "lucene/**/*.${it}" - include "solr/**/*.${it}" - include "dev-tools/**/*.${it}" - include "gradle/**/*.${it}" - include "*.${it}" + sourceFiles = fileTree(projectDir) { + extensions.each{ + include "**/*.${it}" } - // TODO: For now we don't scan txt / md files, so we - // check licenses in top-level folders separately: - include '*.txt' - include '*/*.txt' - include '*.md' - include '*/*.md' - // excludes: - exclude '**/build/**' - exclude '**/dist/**' - exclude 'dev-tools/missing-doclet/src/**/*.java' // <-- TODO: remove once we allow "var" on master - exclude 'lucene/benchmark/work/**' - exclude 'lucene/benchmark/temp/**' - exclude '**/CheckLoggingConfiguration.java' - exclude 'solr/core/src/test/org/apache/hadoop/**' - exclude '**/validate-source-patterns.gradle' // ourselves :-) + + // Don't go into child projects (scanned separately). + childProjects.keySet().each{ + exclude "${it}/**" + } + + // default excludes. + exclude 'build/**' } } + + // Add source validation to per-project checks as well. + check.dependsOn validateSourcePatterns +} + +configure(project(':lucene:benchmark')) { + project.tasks.withType(ValidateSourcePatternsTask) { + sourceFiles.exclude 'temp/**' + sourceFiles.exclude 'work/**' + } +} + +configure(project(':solr:core')) { + project.tasks.withType(ValidateSourcePatternsTask) { + sourceFiles.exclude 'src/**/CheckLoggingConfiguration.java' + sourceFiles.exclude 'src/test/org/apache/hadoop/**' + } +} + +configure(rootProject) { + task validateSourcePatterns(type: ValidateSourcePatternsTask) { task -> + group = 'Verification' + description = 'Validate Source Patterns' + + // This task has no proper outputs. + setupDummyOutputs(task) + + sourceFiles = fileTree(projectDir) { + extensions.each{ + include "**/*.${it}" + } + + // We do not scan for *.txt by default (broken files in subprojects), + // but in root we can do this). + include '**/*.txt' + + // Don't go into child projects (scanned separately). + childProjects.keySet().each{ + exclude "${it}/**" + } + + // default excludes. + exclude '**/build/**' + exclude 'dev-tools/missing-doclet/src/**/*.java' // <-- TODO: remove once we allow "var" on master + + // ourselves :-) + exclude 'gradle/validation/validate-source-patterns.gradle' + } + } + + check.dependsOn validateSourcePatterns } class ValidateSourcePatternsTask extends DefaultTask { + private ProgressLoggerFactory progressLoggerFactory @InputFiles - ConfigurableFileTree sourceFiles + FileTree sourceFiles + + @Inject + ValidateSourcePatternsTask(ProgressLoggerFactory progressLoggerFactory) { + this.progressLoggerFactory = progressLoggerFactory + } @TaskAction public void check() { @@ -99,12 +172,10 @@ class ValidateSourcePatternsTask extends DefaultTask { (~$/\n\s*var\s+/$) : 'var is not allowed in until we stop development on the 8x code line' ] - def baseDirLen = sourceFiles.dir.toString().length() + 1; - def found = 0; def violations = new TreeSet(); def reportViolation = { f, name -> - logger.error('{}: {}', name, f.toString().substring(baseDirLen).replace(File.separatorChar, (char)'/')); + logger.error('{}: {}', name, f); violations.add(name); found++; } @@ -181,8 +252,13 @@ class ValidateSourcePatternsTask extends DefaultTask { } } + ProgressLogger progress = progressLoggerFactory.newOperation(this.class) + progress.start(this.name, this.name) + sourceFiles.each{ f -> + progress.progress("Scanning ${f.name}") logger.debug('Scanning source file: {}', f); + def text = f.getText('UTF-8'); invalidPatterns.each{ pattern,name -> if (pattern.matcher(text).find()) { @@ -221,7 +297,7 @@ class ValidateSourcePatternsTask extends DefaultTask { checkMockitoAssume(f, text); } - if (f.path.substring(baseDirLen).contains("solr/") + if (project.path.startsWith(':solr:') && f.name.equals("SolrTestCase.java") == false && f.name.equals("TestXmlQParser.java") == false) { if (extendsLuceneTestCasePattern.matcher(text).find()) { @@ -234,13 +310,14 @@ class ValidateSourcePatternsTask extends DefaultTask { } } } - if (f.name.endsWith('.xml') || f.name.endsWith('.xml.template')) { + if (f.name.endsWith('.xml')) { checkLicenseHeaderPrecedes(f, '', xmlTagPattern, xmlCommentPattern, text, ratDocument); } if (f.name.endsWith('.adoc')) { checkForUnescapedSymbolSubstitutions(f, text); } - }; + } + progress.completed() if (found) { throw new GradleException(String.format(Locale.ENGLISH, 'Found %d violations in source files (%s).',