LUCENE-9465: 'beast' task from within gradle (#1757)

build.gradle

@@ -132,6 +132,7 @@ apply from: file('gradle/testing/per-project-summary.gradle')
 apply from: file('gradle/testing/slowest-tests-at-end.gradle')
 apply from: file('gradle/testing/failed-tests-at-end.gradle')
 apply from: file('gradle/testing/profiling.gradle')
+apply from: file('gradle/testing/beasting.gradle')
 apply from: file('gradle/help.gradle')
 
 // Ant-compatibility layer. ALL OF THESE SHOULD BE GONE at some point. They are

gradle/testing/beasting.gradle

@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This adds 'beast' task which clones tests a given number of times (preferably
+// constrained with a filtering pattern passed via '--tests').
+
+// TODO: subtasks are not run in parallel (sigh, gradle removed this capability for intra-project tasks).
+// TODO: maybe it would be better to take a deeper approach and just feed the task
+//       runner duplicated suite names (much like https://github.com/gradle/test-retry-gradle-plugin)
+// TODO: this is a somewhat related issue: https://github.com/gradle/test-retry-gradle-plugin/issues/29
+
+def beastingMode = gradle.startParameter.taskNames.contains("beast");
+
+if (beastingMode) {
+  if (rootProject.rootSeedUserProvided) {
+    logger.warn("Root randomization seed is externally provided, all duplicated runs will use the same starting seed.")
+  }
+
+  allprojects {
+    plugins.withType(JavaPlugin) {
+      task beast(type: BeastTask) {
+        description "Run a test suite (or a set of tests) many times over (duplicate 'test' task)."
+        group "Verification"
+      }
+
+      def dups = Integer.parseInt(propertyOrDefault("tests.dups", "0"))
+      if (dups <= 0) {
+        throw new GradleException("Specify -Ptests.dups=[count] for beast task.")
+      }
+
+      // generate N test tasks and attach them to the beasting task for this project;
+      // the test filter will be applied by the beast task once it is received from
+      // command line.
+      def subtasks = (1..dups).collect { value ->
+        return tasks.create(name: "test_${value}", type: Test, {
+          failFast = true
+          doFirst {
+            // If there is a global root seed, use it (all duplicated tasks will run
+            // from the same starting seed). Otherwise pick a sequential derivative.
+            if (!rootProject.rootSeedUserProvided) {
+              systemProperty("tests.seed",
+                  String.format("%08X", new Random(rootProject.rootSeedLong + value).nextLong()))
+            }
+          }
+        })
+      }
+
+      beast.dependsOn subtasks
+    }
+  }
+}
+
+/**
+ * We have to declare a dummy task here to be able to reuse the same syntax for 'test' task
+ * filter option.
+ */
+class BeastTask extends DefaultTask {
+  @Option(option = "tests", description = "Sets test class or method name to be included, '*' is supported.")
+  public void setTestNamePatterns(List<String> patterns) {
+    taskDependencies.getDependencies(this).each { subtask ->
+      subtask.filter.setCommandLineIncludePatterns(patterns)
+    }
+  }
+
+  @TaskAction
+  void run() {
+  }
+}

gradle/testing/defaults-tests.gradle

@@ -50,7 +50,7 @@ allprojects {
       }
     }
 
-    test {
+    tasks.withType(Test) {
       ext {
         testOutputsDir = file("${reports.junitXml.destination}/outputs")
       }

gradle/testing/fail-on-no-tests.gradle

@@ -19,7 +19,7 @@
 
 configure(allprojects) {
   plugins.withType(JavaPlugin) {
-    test {
+    tasks.withType(Test) {
       filter {
         failOnNoMatchingTests = false
       }

gradle/testing/randomization.gradle

@@ -37,6 +37,7 @@ buildscript {
 configure(rootProject) {
   ext {
     rootSeed = propertyOrDefault('tests.seed', String.format("%08X", new Random().nextLong()))
+    rootSeedUserProvided = (propertyOrDefault('tests.seed', null) != null)
     rootSeedLong = SeedUtils.parseSeedChain(rootSeed)[0]
     projectSeedLong = rootSeedLong ^ project.path.hashCode()
   }
@@ -62,7 +63,7 @@ allprojects {
       testOptions = [
           // seed, repetition and amplification.
           [propName: 'tests.seed', value: "random", description: "Sets the master randomization seed."],
-          [propName: 'tests.iters', value: null, description: "Duplicate (re-run) each test N times."],
+          [propName: 'tests.iters', value: null, description: "Duplicate (re-run) each test case N times."],
           [propName: 'tests.multiplier', value: 1, description: "Value multiplier for randomized tests."],
           [propName: 'tests.maxfailures', value: null, description: "Skip tests after a given number of failures."],
           [propName: 'tests.timeoutSuite', value: null, description: "Timeout (in millis) for an entire suite."],
@@ -157,7 +158,7 @@ allprojects {
       }
 
       // Append resolved test properties to the test task.
-      test {
+      tasks.withType(Test) { task ->
         // TODO: we could remove opts with "buildOnly: true" (?)
         systemProperties testOptionsResolved
 
@@ -237,9 +238,10 @@ if (vmName =~ /(?i)(hotspot|openjdk|jrockit)/ &&
   logger.debug("Enabling HashMap assertions.")
   allprojects {
     plugins.withType(JavaPlugin) {
-      test {
+      tasks.withType(Test) { task ->  
         jvmArgs("-da:java.util.HashMap")
       }
     }
   }
 }
+

help/tests.txt

@@ -101,6 +101,21 @@ cleanTest task:
 
 gradlew -p lucene/core cleanTest test -Ptests.seed=deadbeef
 
+The 'tests.iters' option should be sufficient for individual test cases
+and is *much* faster than trying to duplicate re-runs of the entire
+test suites. When it is absolutely needed to re-run an entire suite (because
+of randomization in the static initialization, for example), you can do it
+by running the 'beast' task with 'tests.dups' option:
+
+gradlew -p lucene/core beast -Ptests.dups=10 --tests TestPerFieldDocValuesFormat
+
+Note the filter (--tests) used to narrow down test reiterations to a particular
+class. You can use any filter, including no filter at all, but it rarely makes
+sense (will take ages). By default the test tasks generated by the 'beast' mode
+use a random starting seed for randomization. If you pass an explicit seed, this
+won't be the case (all tasks will use exactly the same starting seed):
+
+gradlew -p lucene/core beast -Ptests.dups=10 --tests TestPerFieldDocValuesFormat -Dtests.seed=deadbeef
 
 Verbose mode and debugging
 --------------------------