LUCENE-9188: Add jacoco code coverage support to gradle (#119)

Co-authored-by: Dawid Weiss <dawid.weiss@carrotsearch.com>
Co-authored-by: Uwe Schindler <uschindler@apache.org>

build.gradle

@@ -164,6 +164,7 @@ apply from: file('gradle/testing/slowest-tests-at-end.gradle')
 apply from: file('gradle/testing/failed-tests-at-end.gradle')
 apply from: file('gradle/testing/profiling.gradle')
 apply from: file('gradle/testing/beasting.gradle')
+apply from: file('gradle/testing/coverage.gradle')
 apply from: file('gradle/help.gradle')
 
 apply from: file('gradle/documentation/documentation.gradle')

gradle/testing/coverage.gradle

@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This adds jacoco code coverage to tests.
+
+// Run with jacoco if either 'coverage' is passed as a task on input or
+// tests.coverage property is true.
+def withCoverage = gradle.startParameter.taskNames.contains("coverage") ||
+    Boolean.parseBoolean(propertyOrDefault("tests.coverage", "false"))
+
+if (withCoverage) {
+  allprojects {
+    plugins.withType(JavaPlugin) {
+      // Apply jacoco once we know the project has a Java plugin too.
+      project.plugins.apply("jacoco")
+
+      // Synthetic task to enable test coverage (and reports).
+      task coverage() {
+        dependsOn jacocoTestReport
+      }
+
+      tasks.withType(Test) { Task testTask ->
+        // Configure jacoco destination file to be within the test
+        // task's working directory - this is related to security
+        // manager permissions (access to this file from within test jvm).
+        jacoco {
+          destinationFile = file("${testTask.workingDir}/jacoco.exec")
+        }
+
+        // Test reports, if any, must be preceded by test execution.
+        jacocoTestReport.dependsOn testTask
+      }
+
+      configure(jacocoTestReport) {
+        doLast {
+          logger.lifecycle("Code coverage report at: ${reports.html.destination}.\n")
+        }
+      }
+    }
+  }
+}

gradle/testing/randomization/policies/replicator-tests.policy

@@ -58,6 +58,7 @@ grant {
   permission java.lang.RuntimePermission "fileSystemProvider";
   // needed to test unmap hack on platforms that support it
   permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
+  permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
 };
 
 // Permissions to support ant build
@@ -65,9 +66,16 @@ grant {
   permission java.io.FilePermission "${user.home}${/}.ivy2${/}cache${/}-", "read";
   permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,write,delete";
   permission java.io.FilePermission "${clover.db.dir}${/}-", "read,write,delete";
-  permission java.io.FilePermission "${junit4.childvm.cwd}${/}jacoco.db", "write";
+};
+
+// Permissions for jacoco code coverage
+grant {
+  // permission to write the per-jvm code coverage output
+  permission java.io.FilePermission "${user.dir}${/}jacoco.exec", "write";
   // needed by jacoco to dump coverage on shutdown
   permission java.lang.RuntimePermission "shutdownHooks";
+  // needed by jacoco to instrument classes
+  permission java.lang.RuntimePermission "defineClass";
 };
 
 // Grant all permissions to Gradle test runner classes.

gradle/testing/randomization/policies/tests.policy

@@ -104,9 +104,16 @@ grant {
   permission java.io.FilePermission "${user.home}${/}.ivy2${/}cache${/}-", "read";
   permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,write,delete";
   permission java.io.FilePermission "${clover.db.dir}${/}-", "read,write,delete";
-  permission java.io.FilePermission "${junit4.childvm.cwd}${/}jacoco.db", "write";
+};
+
+// Permissions for jacoco code coverage
+grant {
+  // permission to write the per-jvm code coverage output
+  permission java.io.FilePermission "${user.dir}${/}jacoco.exec", "write";
   // needed by jacoco to dump coverage on shutdown
   permission java.lang.RuntimePermission "shutdownHooks";
+  // needed by jacoco to instrument classes
+  permission java.lang.RuntimePermission "defineClass";
 };
 
 // Grant all permissions to Gradle test runner classes.

help/tests.txt

@@ -117,6 +117,7 @@ won't be the case (all tasks will use exactly the same starting seed):
 
 gradlew -p lucene/core beast -Ptests.dups=10 --tests TestPerFieldDocValuesFormat -Dtests.seed=deadbeef
 
+
 Verbose mode and debugging
 --------------------------
 
@@ -157,6 +158,18 @@ to increase the top-N count:
 gradlew -p lucene/core test -Ptests.profile=true -Ptests.profile.count=100
 
 
+Generating Coverage Reports
+------------------------------
+
+Running the "coverage" task (or setting the property "tests.coverage" to true)
+will run the tests with instrumentation to record code coverage.
+
+Example:
+
+gradlew -p lucene/core coverage
+open lucene/core/build/reports/jacoco/test/html/index.html
+
+
 External data sets
 ------------------
 

lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimplePrimaryNode.java

@@ -203,7 +203,7 @@ class SimplePrimaryNode extends PrimaryNode {
           c.out.writeByte(SimpleReplicaNode.CMD_PRE_COPY_MERGE);
           c.out.writeVLong(primaryGen);
           c.out.writeVInt(tcpPort);
-          SimpleServer.writeFilesMetaData(c.out, files);
+          TestSimpleServer.writeFilesMetaData(c.out, files);
           c.flush();
           c.s.shutdownOutput();
           message("warm connection " + c.s);
@@ -390,7 +390,7 @@ class SimplePrimaryNode extends PrimaryNode {
     out.writeBytes(state.infosBytes, 0, state.infosBytes.length);
     out.writeVLong(state.gen);
     out.writeVLong(state.version);
-    SimpleServer.writeFilesMetaData(out, state.files);
+    TestSimpleServer.writeFilesMetaData(out, state.files);
 
     out.writeVInt(state.completedMergeFiles.size());
     for (String fileName : state.completedMergeFiles) {
@@ -813,7 +813,7 @@ class SimplePrimaryNode extends PrimaryNode {
               c.out.writeByte(SimpleReplicaNode.CMD_PRE_COPY_MERGE);
               c.out.writeVLong(primaryGen);
               c.out.writeVInt(tcpPort);
-              SimpleServer.writeFilesMetaData(c.out, preCopy.files);
+              TestSimpleServer.writeFilesMetaData(c.out, preCopy.files);
               c.flush();
               c.s.shutdownOutput();
               message("successfully started warming");

lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimpleReplicaNode.java

@@ -133,7 +133,7 @@ class SimpleReplicaNode extends ReplicaNode {
         // No incoming CopyState: ask primary for latest one now
         c.out.writeByte((byte) 1);
         c.flush();
-        copyState = SimpleServer.readCopyState(c.in);
+        copyState = TestSimpleServer.readCopyState(c.in);
         files = copyState.files;
       } else {
         c.out.writeByte((byte) 0);
@@ -331,7 +331,7 @@ class SimpleReplicaNode extends ReplicaNode {
 
           long newPrimaryGen = in.readVLong();
           curPrimaryTCPPort = in.readVInt();
-          Map<String, FileMetaData> files = SimpleServer.readFilesMetaData(in);
+          Map<String, FileMetaData> files = TestSimpleServer.readFilesMetaData(in);
           message("done reading files to copy files=" + files.keySet());
           AtomicBoolean finished = new AtomicBoolean();
           launchPreCopyMerge(finished, newPrimaryGen, files);

lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestNRTReplication.java

@@ -39,6 +39,7 @@ import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
 import org.apache.lucene.util.SuppressForbidden;
+import org.apache.lucene.util.TestRuleIgnoreTestSuites;
 import org.apache.lucene.util.TestUtil;
 
 // MockRandom's .sd file has no index header/footer:
@@ -93,17 +94,19 @@ public class TestNRTReplication extends LuceneTestCase {
       cmd.add("-Dtests.nrtreplication.forcePrimaryVersion=" + forcePrimaryVersion);
     }
 
+    // Mark as running nested.
+    cmd.add("-D" + TestRuleIgnoreTestSuites.PROPERTY_RUN_NESTED + "=true");
+
     // Mixin our own counter because this is called from a fresh thread which means the seed
     // otherwise isn't changing each time we spawn a
     // new node:
     long seed = random().nextLong() * nodeStartCounter.incrementAndGet();
-
     cmd.add("-Dtests.seed=" + SeedUtils.formatSeed(seed));
     cmd.add("-ea");
     cmd.add("-cp");
     cmd.add(System.getProperty("java.class.path"));
     cmd.add("org.junit.runner.JUnitCore");
-    cmd.add(getClass().getName().replace(getClass().getSimpleName(), "SimpleServer"));
+    cmd.add(TestSimpleServer.class.getName());
 
     message("child process command: " + cmd);
     ProcessBuilder pb = new ProcessBuilder(cmd);

lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestSimpleServer.java

@@ -46,8 +46,11 @@ import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
 import org.apache.lucene.util.SuppressForbidden;
+import org.apache.lucene.util.TestRuleIgnoreTestSuites;
 import org.apache.lucene.util.TestUtil;
+import org.junit.Assume;
 import org.junit.AssumptionViolatedException;
+import org.junit.BeforeClass;
 
 /**
  * Child process with silly naive TCP socket server to handle between-node commands, launched for
@@ -56,7 +59,7 @@ import org.junit.AssumptionViolatedException;
 @SuppressCodecs({"MockRandom", "Direct", "SimpleText"})
 @SuppressSysoutChecks(bugUrl = "Stuff gets printed, important stuff for debugging a failure")
 @SuppressForbidden(reason = "We need Unsafe to actually crush :-)")
-public class SimpleServer extends LuceneTestCase {
+public class TestSimpleServer extends LuceneTestCase {
 
   static final Set<Thread> clientThreads = Collections.synchronizedSet(new HashSet<>());
   static final AtomicBoolean stop = new AtomicBoolean();
@@ -222,6 +225,11 @@ public class SimpleServer extends LuceneTestCase {
     return new CopyState(files, version, gen, infosBytes, completedMergeFiles, primaryGen, null);
   }
 
+  @BeforeClass
+  public static void ensureNested() {
+    Assume.assumeTrue(TestRuleIgnoreTestSuites.isRunningNested());
+  }
+
   @SuppressWarnings("try")
   public void test() throws Exception {
     String nodeId = System.getProperty("tests.nrtreplication.nodeid");

lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestStressNRTReplication.java

@@ -51,6 +51,7 @@ import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
 import org.apache.lucene.util.SuppressForbidden;
+import org.apache.lucene.util.TestRuleIgnoreTestSuites;
 import org.apache.lucene.util.TestUtil;
 import org.apache.lucene.util.ThreadInterruptedException;
 
@@ -607,17 +608,19 @@ public class TestStressNRTReplication extends LuceneTestCase {
     long myPrimaryGen = primaryGen;
     cmd.add("-Dtests.nrtreplication.primaryGen=" + myPrimaryGen);
 
+    // Mark as running nested.
+    cmd.add("-D" + TestRuleIgnoreTestSuites.PROPERTY_RUN_NESTED + "=true");
+
     // Mixin our own counter because this is called from a fresh thread which means the seed
     // otherwise isn't changing each time we spawn a
     // new node:
     long seed = random().nextLong() * nodeStartCounter.incrementAndGet();
-
     cmd.add("-Dtests.seed=" + SeedUtils.formatSeed(seed));
     cmd.add("-ea");
     cmd.add("-cp");
     cmd.add(System.getProperty("java.class.path"));
     cmd.add("org.junit.runner.JUnitCore");
-    cmd.add(getClass().getName().replace(getClass().getSimpleName(), "SimpleServer"));
+    cmd.add(TestSimpleServer.class.getName());
 
     Writer childLog;