LUCENE-9506: Gradle: split validateSourcePatterns into per-project an… (#1830)

* LUCENE-9506: Gradle: split validateSourcePatterns into per-project and root-specific tasks (allow parallelism)

Co-authored-by: Uwe Schindler <uschindler@apache.org>

gradle/validation/precommit.gradle

@@ -26,7 +26,6 @@ configure(rootProject) {
     dependsOn ":verifyLocks"
     dependsOn ":versionsPropsAreSorted"
     dependsOn ":checkWorkingCopyClean"
-    dependsOn ":validateSourcePatterns"
 
     // Solr validation tasks.
     dependsOn ":solr:validateConfigFileSanity"
@@ -40,8 +39,8 @@ configure(rootProject) {
           "javadoc",
           "rat",
           "ecjLint",
-          "checkMissingDocs",
+          "validateLogCalls",
-          "validateLogCalls"
+          "validateSourcePatterns",
       ]}
     }
   }

gradle/validation/validate-log-calls.gradle

@@ -45,7 +45,10 @@ configure(subprojects.findAll { it.path.startsWith(':solr') }) {
 
       sourceFiles = files(sourceSets*.java)
     }
-  }
+ 
+    // Add log validation to per-project checks as well.
+    check.dependsOn validateLogCalls
+ }
 }
 
 class ValidateLogCallsTask extends DefaultTask {

gradle/validation/validate-source-patterns.gradle

@@ -15,9 +15,13 @@
  * limitations under the License.
  */
 
-import org.apache.rat.Defaults;
+import org.apache.rat.Defaults
-import org.apache.rat.document.impl.FileDocument;
+import org.apache.rat.document.impl.FileDocument
-import org.apache.rat.api.MetaData;
+import org.apache.rat.api.MetaData
+
+import javax.inject.Inject;
+import org.gradle.internal.logging.progress.ProgressLoggerFactory
+import org.gradle.internal.logging.progress.ProgressLogger
 
 buildscript {
   repositories {
@@ -29,50 +33,119 @@ buildscript {
   }
 }
 
-configure(rootProject) {
+def extensions = [
-  task("validateSourcePatterns", type: ValidateSourcePatternsTask) { task ->
+    'adoc',
+    'bat',
+    'cmd',
+    'css',
+    'g4',
+    'gradle',
+    'groovy',
+    'html',
+    'java',
+    'jflex',
+    'jj',
+    'js',
+    'json',
+    'md',
+    'mdtext',
+    'pl',
+    'policy',
+    'properties',
+    'py',
+    'sh',
+    'template',
+    'vm',
+    'xml',
+    'xsl',
+]
+
+// Create source validation task local for each project's files.
+subprojects {
+  task validateSourcePatterns(type: ValidateSourcePatternsTask) { task ->
     group = 'Verification'
     description = 'Validate Source Patterns'
 
     // This task has no proper outputs.
     setupDummyOutputs(task)
 
-    sourceFiles = project.fileTree(project.rootDir) {
+    sourceFiles = fileTree(projectDir) {
-      [
+      extensions.each{
-        'java', 'jflex', 'py', 'pl', 'g4', 'jj', 'html', 'js',
+        include "**/*.${it}"
-        'css', 'xml', 'xsl', 'vm', 'sh', 'cmd', 'bat', 'policy',
-        'properties', 'mdtext', 'groovy', 'gradle',
-        'template', 'adoc', 'json',
-      ].each{
-        include "lucene/**/*.${it}"
-        include "solr/**/*.${it}"
-        include "dev-tools/**/*.${it}"
-        include "gradle/**/*.${it}"
-        include "*.${it}"
       }
-      // TODO: For now we don't scan txt / md files, so we
+      
-      // check licenses in top-level folders separately:
+      // Don't go into child projects (scanned separately).
-      include '*.txt'
+      childProjects.keySet().each{
-      include '*/*.txt'
+        exclude "${it}/**"
-      include '*.md'
+      }
-      include '*/*.md'
+
-      // excludes:
+      // default excludes.
-      exclude '**/build/**'
+      exclude 'build/**'
-      exclude '**/dist/**'
-      exclude 'dev-tools/missing-doclet/src/**/*.java' // <-- TODO: remove once we allow "var" on master
-      exclude 'lucene/benchmark/work/**'
-      exclude 'lucene/benchmark/temp/**'
-      exclude '**/CheckLoggingConfiguration.java'
-      exclude 'solr/core/src/test/org/apache/hadoop/**'
-      exclude '**/validate-source-patterns.gradle' // ourselves :-)
     }
   }
+
+  // Add source validation to per-project checks as well.
+  check.dependsOn validateSourcePatterns
+}
+
+configure(project(':lucene:benchmark')) {
+  project.tasks.withType(ValidateSourcePatternsTask) {
+    sourceFiles.exclude 'temp/**'
+    sourceFiles.exclude 'work/**'
+  }
+}
+
+configure(project(':solr:core')) {
+  project.tasks.withType(ValidateSourcePatternsTask) {
+    sourceFiles.exclude 'src/**/CheckLoggingConfiguration.java'
+    sourceFiles.exclude 'src/test/org/apache/hadoop/**'
+  }
+}
+
+configure(rootProject) {
+  task validateSourcePatterns(type: ValidateSourcePatternsTask) { task ->
+    group = 'Verification'
+    description = 'Validate Source Patterns'
+
+    // This task has no proper outputs.
+    setupDummyOutputs(task)
+
+    sourceFiles = fileTree(projectDir) {
+      extensions.each{
+        include "**/*.${it}"
+      }
+
+      // We do not scan for *.txt by default (broken files in subprojects),
+      // but in root we can do this).
+      include '**/*.txt'
+
+      // Don't go into child projects (scanned separately).
+      childProjects.keySet().each{
+        exclude "${it}/**"
+      }
+
+      // default excludes.
+      exclude '**/build/**'
+      exclude 'dev-tools/missing-doclet/src/**/*.java' // <-- TODO: remove once we allow "var" on master
+
+      // ourselves :-)
+      exclude 'gradle/validation/validate-source-patterns.gradle'
+    }
+  }
+
+  check.dependsOn validateSourcePatterns
 }
 
 class ValidateSourcePatternsTask extends DefaultTask {
+  private ProgressLoggerFactory progressLoggerFactory
 
   @InputFiles
-  ConfigurableFileTree sourceFiles
+  FileTree sourceFiles
+
+  @Inject
+  ValidateSourcePatternsTask(ProgressLoggerFactory progressLoggerFactory) {
+    this.progressLoggerFactory = progressLoggerFactory
+  }
   
   @TaskAction
   public void check() {
@@ -99,12 +172,10 @@ class ValidateSourcePatternsTask extends DefaultTask {
       (~$/\n\s*var\s+/$) : 'var is not allowed in until we stop development on the 8x code line'
     ]
 
-    def baseDirLen = sourceFiles.dir.toString().length() + 1;
-
     def found = 0;
     def violations = new TreeSet();
     def reportViolation = { f, name ->
-      logger.error('{}: {}', name, f.toString().substring(baseDirLen).replace(File.separatorChar, (char)'/'));
+      logger.error('{}: {}', name, f);
       violations.add(name);
       found++;
     }
@@ -181,8 +252,13 @@ class ValidateSourcePatternsTask extends DefaultTask {
       }
     }
 
+    ProgressLogger progress = progressLoggerFactory.newOperation(this.class)
+    progress.start(this.name, this.name)
+
     sourceFiles.each{ f ->
+      progress.progress("Scanning ${f.name}")
       logger.debug('Scanning source file: {}', f);
+
       def text = f.getText('UTF-8');
       invalidPatterns.each{ pattern,name ->
         if (pattern.matcher(text).find()) {
@@ -221,7 +297,7 @@ class ValidateSourcePatternsTask extends DefaultTask {
           checkMockitoAssume(f, text);
         }
 
-        if (f.path.substring(baseDirLen).contains("solr/")
+        if (project.path.startsWith(':solr:')
             && f.name.equals("SolrTestCase.java") == false
             && f.name.equals("TestXmlQParser.java") == false) {
           if (extendsLuceneTestCasePattern.matcher(text).find()) {
@@ -234,13 +310,14 @@ class ValidateSourcePatternsTask extends DefaultTask {
           }
         }
       }
-      if (f.name.endsWith('.xml') || f.name.endsWith('.xml.template')) {
+      if (f.name.endsWith('.xml')) {
         checkLicenseHeaderPrecedes(f, '<tag>', xmlTagPattern, xmlCommentPattern, text, ratDocument);
       }
       if (f.name.endsWith('.adoc')) {
         checkForUnescapedSymbolSubstitutions(f, text);
       }
-    };
+    }
+    progress.completed()
 
     if (found) {
       throw new GradleException(String.format(Locale.ENGLISH, 'Found %d violations in source files (%s).',