From 06907a2c1272f9004b3b892c3656df53a616462b Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Sun, 2 May 2021 10:24:06 -0400 Subject: [PATCH] LUCENE-9188: Add jacoco code coverage support to gradle (#119) Co-authored-by: Dawid Weiss Co-authored-by: Uwe Schindler --- build.gradle | 1 + gradle/testing/coverage.gradle | 55 +++++++++++++++++++ .../policies/replicator-tests.policy | 10 +++- .../randomization/policies/tests.policy | 9 ++- help/tests.txt | 13 +++++ .../replicator/nrt/SimplePrimaryNode.java | 6 +- .../replicator/nrt/SimpleReplicaNode.java | 4 +- .../replicator/nrt/TestNRTReplication.java | 7 ++- ...impleServer.java => TestSimpleServer.java} | 10 +++- .../nrt/TestStressNRTReplication.java | 7 ++- 10 files changed, 110 insertions(+), 12 deletions(-) create mode 100644 gradle/testing/coverage.gradle rename lucene/replicator/src/test/org/apache/lucene/replicator/nrt/{SimpleServer.java => TestSimpleServer.java} (98%) diff --git a/build.gradle b/build.gradle index c6cfc8de454..f64eff2b0d8 100644 --- a/build.gradle +++ b/build.gradle @@ -164,6 +164,7 @@ apply from: file('gradle/testing/slowest-tests-at-end.gradle') apply from: file('gradle/testing/failed-tests-at-end.gradle') apply from: file('gradle/testing/profiling.gradle') apply from: file('gradle/testing/beasting.gradle') +apply from: file('gradle/testing/coverage.gradle') apply from: file('gradle/help.gradle') apply from: file('gradle/documentation/documentation.gradle') diff --git a/gradle/testing/coverage.gradle b/gradle/testing/coverage.gradle new file mode 100644 index 00000000000..38b23c156f5 --- /dev/null +++ b/gradle/testing/coverage.gradle @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This adds jacoco code coverage to tests. + +// Run with jacoco if either 'coverage' is passed as a task on input or +// tests.coverage property is true. +def withCoverage = gradle.startParameter.taskNames.contains("coverage") || + Boolean.parseBoolean(propertyOrDefault("tests.coverage", "false")) + +if (withCoverage) { + allprojects { + plugins.withType(JavaPlugin) { + // Apply jacoco once we know the project has a Java plugin too. + project.plugins.apply("jacoco") + + // Synthetic task to enable test coverage (and reports). + task coverage() { + dependsOn jacocoTestReport + } + + tasks.withType(Test) { Task testTask -> + // Configure jacoco destination file to be within the test + // task's working directory - this is related to security + // manager permissions (access to this file from within test jvm). + jacoco { + destinationFile = file("${testTask.workingDir}/jacoco.exec") + } + + // Test reports, if any, must be preceded by test execution. + jacocoTestReport.dependsOn testTask + } + + configure(jacocoTestReport) { + doLast { + logger.lifecycle("Code coverage report at: ${reports.html.destination}.\n") + } + } + } + } +} diff --git a/gradle/testing/randomization/policies/replicator-tests.policy b/gradle/testing/randomization/policies/replicator-tests.policy index 7fa4a0f2857..0d9659ee83e 100644 --- a/gradle/testing/randomization/policies/replicator-tests.policy +++ b/gradle/testing/randomization/policies/replicator-tests.policy @@ -58,6 +58,7 @@ grant { permission java.lang.RuntimePermission "fileSystemProvider"; // needed to test unmap hack on platforms that support it permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; }; // Permissions to support ant build @@ -65,9 +66,16 @@ grant { permission java.io.FilePermission "${user.home}${/}.ivy2${/}cache${/}-", "read"; permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,write,delete"; permission java.io.FilePermission "${clover.db.dir}${/}-", "read,write,delete"; - permission java.io.FilePermission "${junit4.childvm.cwd}${/}jacoco.db", "write"; +}; + +// Permissions for jacoco code coverage +grant { + // permission to write the per-jvm code coverage output + permission java.io.FilePermission "${user.dir}${/}jacoco.exec", "write"; // needed by jacoco to dump coverage on shutdown permission java.lang.RuntimePermission "shutdownHooks"; + // needed by jacoco to instrument classes + permission java.lang.RuntimePermission "defineClass"; }; // Grant all permissions to Gradle test runner classes. diff --git a/gradle/testing/randomization/policies/tests.policy b/gradle/testing/randomization/policies/tests.policy index 469892c90cc..71dc7991fdb 100644 --- a/gradle/testing/randomization/policies/tests.policy +++ b/gradle/testing/randomization/policies/tests.policy @@ -104,9 +104,16 @@ grant { permission java.io.FilePermission "${user.home}${/}.ivy2${/}cache${/}-", "read"; permission java.io.FilePermission "${junit4.tempDir}${/}*", "read,write,delete"; permission java.io.FilePermission "${clover.db.dir}${/}-", "read,write,delete"; - permission java.io.FilePermission "${junit4.childvm.cwd}${/}jacoco.db", "write"; +}; + +// Permissions for jacoco code coverage +grant { + // permission to write the per-jvm code coverage output + permission java.io.FilePermission "${user.dir}${/}jacoco.exec", "write"; // needed by jacoco to dump coverage on shutdown permission java.lang.RuntimePermission "shutdownHooks"; + // needed by jacoco to instrument classes + permission java.lang.RuntimePermission "defineClass"; }; // Grant all permissions to Gradle test runner classes. diff --git a/help/tests.txt b/help/tests.txt index b4a14449c77..aae3da293ad 100644 --- a/help/tests.txt +++ b/help/tests.txt @@ -117,6 +117,7 @@ won't be the case (all tasks will use exactly the same starting seed): gradlew -p lucene/core beast -Ptests.dups=10 --tests TestPerFieldDocValuesFormat -Dtests.seed=deadbeef + Verbose mode and debugging -------------------------- @@ -157,6 +158,18 @@ to increase the top-N count: gradlew -p lucene/core test -Ptests.profile=true -Ptests.profile.count=100 +Generating Coverage Reports +------------------------------ + +Running the "coverage" task (or setting the property "tests.coverage" to true) +will run the tests with instrumentation to record code coverage. + +Example: + +gradlew -p lucene/core coverage +open lucene/core/build/reports/jacoco/test/html/index.html + + External data sets ------------------ diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimplePrimaryNode.java b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimplePrimaryNode.java index 53dda82e70f..8a7492fde9f 100644 --- a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimplePrimaryNode.java +++ b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimplePrimaryNode.java @@ -203,7 +203,7 @@ class SimplePrimaryNode extends PrimaryNode { c.out.writeByte(SimpleReplicaNode.CMD_PRE_COPY_MERGE); c.out.writeVLong(primaryGen); c.out.writeVInt(tcpPort); - SimpleServer.writeFilesMetaData(c.out, files); + TestSimpleServer.writeFilesMetaData(c.out, files); c.flush(); c.s.shutdownOutput(); message("warm connection " + c.s); @@ -390,7 +390,7 @@ class SimplePrimaryNode extends PrimaryNode { out.writeBytes(state.infosBytes, 0, state.infosBytes.length); out.writeVLong(state.gen); out.writeVLong(state.version); - SimpleServer.writeFilesMetaData(out, state.files); + TestSimpleServer.writeFilesMetaData(out, state.files); out.writeVInt(state.completedMergeFiles.size()); for (String fileName : state.completedMergeFiles) { @@ -813,7 +813,7 @@ class SimplePrimaryNode extends PrimaryNode { c.out.writeByte(SimpleReplicaNode.CMD_PRE_COPY_MERGE); c.out.writeVLong(primaryGen); c.out.writeVInt(tcpPort); - SimpleServer.writeFilesMetaData(c.out, preCopy.files); + TestSimpleServer.writeFilesMetaData(c.out, preCopy.files); c.flush(); c.s.shutdownOutput(); message("successfully started warming"); diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimpleReplicaNode.java b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimpleReplicaNode.java index cb1281ac15b..d4b7b5ee1cd 100644 --- a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimpleReplicaNode.java +++ b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimpleReplicaNode.java @@ -133,7 +133,7 @@ class SimpleReplicaNode extends ReplicaNode { // No incoming CopyState: ask primary for latest one now c.out.writeByte((byte) 1); c.flush(); - copyState = SimpleServer.readCopyState(c.in); + copyState = TestSimpleServer.readCopyState(c.in); files = copyState.files; } else { c.out.writeByte((byte) 0); @@ -331,7 +331,7 @@ class SimpleReplicaNode extends ReplicaNode { long newPrimaryGen = in.readVLong(); curPrimaryTCPPort = in.readVInt(); - Map files = SimpleServer.readFilesMetaData(in); + Map files = TestSimpleServer.readFilesMetaData(in); message("done reading files to copy files=" + files.keySet()); AtomicBoolean finished = new AtomicBoolean(); launchPreCopyMerge(finished, newPrimaryGen, files); diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestNRTReplication.java b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestNRTReplication.java index d02bd410592..d3632f2ce6a 100644 --- a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestNRTReplication.java +++ b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestNRTReplication.java @@ -39,6 +39,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks; import org.apache.lucene.util.SuppressForbidden; +import org.apache.lucene.util.TestRuleIgnoreTestSuites; import org.apache.lucene.util.TestUtil; // MockRandom's .sd file has no index header/footer: @@ -93,17 +94,19 @@ public class TestNRTReplication extends LuceneTestCase { cmd.add("-Dtests.nrtreplication.forcePrimaryVersion=" + forcePrimaryVersion); } + // Mark as running nested. + cmd.add("-D" + TestRuleIgnoreTestSuites.PROPERTY_RUN_NESTED + "=true"); + // Mixin our own counter because this is called from a fresh thread which means the seed // otherwise isn't changing each time we spawn a // new node: long seed = random().nextLong() * nodeStartCounter.incrementAndGet(); - cmd.add("-Dtests.seed=" + SeedUtils.formatSeed(seed)); cmd.add("-ea"); cmd.add("-cp"); cmd.add(System.getProperty("java.class.path")); cmd.add("org.junit.runner.JUnitCore"); - cmd.add(getClass().getName().replace(getClass().getSimpleName(), "SimpleServer")); + cmd.add(TestSimpleServer.class.getName()); message("child process command: " + cmd); ProcessBuilder pb = new ProcessBuilder(cmd); diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimpleServer.java b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestSimpleServer.java similarity index 98% rename from lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimpleServer.java rename to lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestSimpleServer.java index 0b8784026e9..c776c29851e 100644 --- a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimpleServer.java +++ b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestSimpleServer.java @@ -46,8 +46,11 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks; import org.apache.lucene.util.SuppressForbidden; +import org.apache.lucene.util.TestRuleIgnoreTestSuites; import org.apache.lucene.util.TestUtil; +import org.junit.Assume; import org.junit.AssumptionViolatedException; +import org.junit.BeforeClass; /** * Child process with silly naive TCP socket server to handle between-node commands, launched for @@ -56,7 +59,7 @@ import org.junit.AssumptionViolatedException; @SuppressCodecs({"MockRandom", "Direct", "SimpleText"}) @SuppressSysoutChecks(bugUrl = "Stuff gets printed, important stuff for debugging a failure") @SuppressForbidden(reason = "We need Unsafe to actually crush :-)") -public class SimpleServer extends LuceneTestCase { +public class TestSimpleServer extends LuceneTestCase { static final Set clientThreads = Collections.synchronizedSet(new HashSet<>()); static final AtomicBoolean stop = new AtomicBoolean(); @@ -222,6 +225,11 @@ public class SimpleServer extends LuceneTestCase { return new CopyState(files, version, gen, infosBytes, completedMergeFiles, primaryGen, null); } + @BeforeClass + public static void ensureNested() { + Assume.assumeTrue(TestRuleIgnoreTestSuites.isRunningNested()); + } + @SuppressWarnings("try") public void test() throws Exception { String nodeId = System.getProperty("tests.nrtreplication.nodeid"); diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestStressNRTReplication.java b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestStressNRTReplication.java index b7b6b1c8574..197ba2f6dfa 100644 --- a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestStressNRTReplication.java +++ b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/TestStressNRTReplication.java @@ -51,6 +51,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks; import org.apache.lucene.util.SuppressForbidden; +import org.apache.lucene.util.TestRuleIgnoreTestSuites; import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.ThreadInterruptedException; @@ -607,17 +608,19 @@ public class TestStressNRTReplication extends LuceneTestCase { long myPrimaryGen = primaryGen; cmd.add("-Dtests.nrtreplication.primaryGen=" + myPrimaryGen); + // Mark as running nested. + cmd.add("-D" + TestRuleIgnoreTestSuites.PROPERTY_RUN_NESTED + "=true"); + // Mixin our own counter because this is called from a fresh thread which means the seed // otherwise isn't changing each time we spawn a // new node: long seed = random().nextLong() * nodeStartCounter.incrementAndGet(); - cmd.add("-Dtests.seed=" + SeedUtils.formatSeed(seed)); cmd.add("-ea"); cmd.add("-cp"); cmd.add(System.getProperty("java.class.path")); cmd.add("org.junit.runner.JUnitCore"); - cmd.add(getClass().getName().replace(getClass().getSimpleName(), "SimpleServer")); + cmd.add(TestSimpleServer.class.getName()); Writer childLog;