From 9b049b0dbd8a80fe34cb8f7665175f3468d6c4c6 Mon Sep 17 00:00:00 2001
From: Greg Bowyer <gbowyer@apache.org>
Date: Tue, 4 Sep 2012 23:54:13 +0000
Subject: [PATCH] LUCENE-4332: Integrate PiTest mutation coverage

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1380938 13f79535-47bb-0310-9956-ffa450edef68
---
 build.xml                          |  9 +++
 lucene/build.xml                   |  8 +++
 lucene/common-build.xml            | 97 +++++++++++++++++++++++++++++-
 lucene/tools/build.xml             |  1 +
 lucene/tools/junit4/tests.policy   | 10 ++-
 solr/build.xml                     | 19 ++++++
 solr/example/build.xml             |  3 +
 solr/example/example-DIH/build.xml |  3 +
 8 files changed, 147 insertions(+), 3 deletions(-)

diff --git a/build.xml b/build.xml
index 5bcd84ee6b5..46e33ab9d48 100644
--- a/build.xml
+++ b/build.xml
@@ -33,6 +33,15 @@
     </sequential>
   </target>
 
+  <target name="pitest" description="Test both Lucene and Solr">
+    <sequential>
+      <subant target="pitest" inheritall="false" failonerror="false">
+        <fileset dir="lucene" includes="build.xml" />
+        <fileset dir="solr" includes="build.xml" />
+      </subant>
+    </sequential>
+  </target>
+
   <target name="javadocs" description="Generate Lucene and Solr javadocs">
     <sequential>
       <subant target="javadocs" inheritall="false" failonerror="true">
diff --git a/lucene/build.xml b/lucene/build.xml
index 7f394e3eefb..ceef56e587d 100644
--- a/lucene/build.xml
+++ b/lucene/build.xml
@@ -55,6 +55,10 @@
           description="Runs all unit tests (core, modules and back-compat)"
   />
 
+  <target name="pitest" depends="pitest-modules"
+          description="Runs pitests (core, modules and back-compat)"
+  />
+
   <path id="backwards.test.compile.classpath">
     <path refid="junit-path"/>
     <path refid="ant-path"/>
@@ -478,6 +482,10 @@
     <build-changes changes.src.dir="${changes.src.dir}" changes.target.dir="${changes.target.dir}" />
   </target>
 
+  <target name="pitest-modules" depends="compile-test">
+    <modules-crawl target="pitest" failonerror="false"/>
+  </target>
+
   <!--
    Committer helpers
    -->
diff --git a/lucene/common-build.xml b/lucene/common-build.xml
index 002788468b6..13193122975 100644
--- a/lucene/common-build.xml
+++ b/lucene/common-build.xml
@@ -106,7 +106,7 @@
     <isset property="run.clover"/>
   </condition>
   <property name="tests.clover.args" value=""/>
-  
+
   <property name="tests.tempDir" value="${build.dir}/test"/>
 
   <property name="tests.cachefile" location="${common.dir}/tools/junit4/cached-timehints.txt" />
@@ -218,6 +218,15 @@
   <property name="clover.db.dir" location="${common.dir}/build/clover/db"/>
   <property name="clover.report.dir" location="${common.dir}/build/clover/reports"/>
 
+  <property name="pitest.report.dir" location="${common.dir}/build/pitest/${name}/reports"/>
+  <property name="pitest.distance" value="0" />
+  <property name="pitest.threads" value="2" />
+  <property name="pitest.testCases" value="org.apache.*" />
+  <property name="pitest.maxMutations" value="0" />
+  <property name="pitest.timeoutFactor" value="1.25" />
+  <property name="pitest.timeoutConst" value="3000" />
+  <property name="pitest.targetClasses" value="org.apache.*" />
+
   <!-- a reasonable default exclusion set, can be overridden for special cases -->
   <property name="rat.excludes" value="**/TODO,**/*.txt"/>
 
@@ -1102,6 +1111,16 @@ ${tests-output}/junit4-*.suites     - per-JVM executed suites
     <path id="clover.classpath"/>
   </target>
 
+  <target name="pitest" if="run.pitest" depends="compile-test,install-junit4-taskdef,clover,validate"
+      description="Run Unit tests using pitest mutation testing. To use, specify -Drun.pitest=true on the command line.">
+    <echo>Code coverage with pitest enabled.</echo>
+    <ivy:cachepath
+        organisation="org.pitest" module="pitest-ant"
+        inline="true"
+        pathid="pitest.framework.classpath" />
+    <pitest-macro />
+  </target>
+
   <target name="generate-test-reports" description="Generates test reports">
     <mkdir dir="${junit.reports}"/>
     <junitreport todir="${junit.output.dir}">
@@ -1705,4 +1724,80 @@ ${tests-output}/junit4-*.suites     - per-JVM executed suites
     </sequential>
   </macrodef>
 
+  <macrodef name="pitest-macro" description="Executes junit tests.">
+    <attribute name="pitest.report.dir" default="${pitest.report.dir}"/>
+    <attribute name="pitest.framework.classpath" default="pitest.framework.classpath"/>
+    <attribute name="pitest.distance" default="${pitest.distance}" />
+    <attribute name="pitest.sysprops" default="${pitest.sysprops}" />
+    <attribute name="pitest.threads" default="${pitest.threads}" />
+    <attribute name="pitest.testCases" default="${pitest.testCases}" />
+    <attribute name="pitest.maxMutations" default="${pitest.maxMutations}" />
+    <attribute name="pitest.timeoutFactor" default="${pitest.timeoutFactor}" />
+    <attribute name="pitest.timeoutConst" default="${pitest.timeoutConst}" />
+    <attribute name="pitest.targetClasses" default="${pitest.targetClasses}" />
+
+    <attribute name="junit.classpath" default="junit.classpath"/>
+
+    <attribute name="src.dir" default="${src.dir}"/>
+    <attribute name="build.dir" default="${build.dir}"/>
+
+    <sequential>
+
+        <echo>
+PiTest mutation coverage can take a *long* time on even large hardware.
+(EC2 32core sandy bridge takes at least 12 hours to run PiTest for the lucene test cases)
+
+The following arguments can be provided to ant to alter its behaviour and target specific tests::
+
+-Dpitest.report.dir (@{pitest.report.dir}) - Change where PiTest writes output reports
+
+-Dpitest.distance (@{pitest.distance}) - How far away from the test class should be mutated
+   0 being immeditate callees only
+
+-Dpitest.threads (@{pitest.threads}) - How many threads to use in PiTest 
+   (note this is independent of junit threads)
+
+-Dpitest.testCases (@{pitest.testCases}) - Glob of testcases to run
+
+-Dpitest.maxMutations (@{pitest.maxMutations}) - Maximum number of mutations per class under test
+    0 being unlimited
+
+-Dpitest.timeoutFactor (@{pitest.timeoutFactor}) - Tunable factor used to determine
+    if a test is potentially been mutated to be an infinate loop or O(n!) (or similar)
+
+-Dpitest.timeoutConst (@{pitest.timeoutConst}) - Base constant used for working out timeouts
+
+-Dpitest.targetClasses (@{pitest.targetClasses}) - Classes to consider for mutation
+        </echo>
+
+        <taskdef name="pitest" classname="org.pitest.ant.PitestTask"
+            classpathref="pitest.framework.classpath" />
+
+        <path id="pitest.classpath">
+            <path refid="junit.classpath"/>
+            <path refid="pitest.framework.classpath"/>
+            <pathelement path="${java.class.path}"/>
+        </path>
+
+        <junit4:pickseed property="pitest.seed" />
+
+        <property name="pitest.sysprops" value="-Dlucene.version=${dev.version},-Dtest.seed=${pitest.seed},-Djava.security.manager,-Djava.security.policy=${common.dir}/tools/junit4/tests.policy,-Djava.io.tmpdir=${tests.tempDir},-Dtests.sandbox.dir=${build.dir}" />
+
+        <pitest
+            classPath="pitest.classpath"
+            targetClasses="@{pitest.targetClasses}"
+            targetTests="@{pitest.testCases}"
+            reportDir="@{pitest.report.dir}"
+            sourceDir="@{src.dir}"
+            threads="@{pitest.threads}"
+            maxMutationsPerClass="@{pitest.maxMutations}"
+            timeoutFactor="@{pitest.timeoutFactor}"
+            timeoutConst="@{pitest.timeoutConst}"
+            verbose="false"
+            dependencyDistance="@{pitest.distance}"
+            mutableCodePaths="@{build.dir}/classes/java"
+            jvmArgs="-ea,@{pitest.sysprops}" />
+    </sequential>
+  </macrodef>
+
 </project>
diff --git a/lucene/tools/build.xml b/lucene/tools/build.xml
index 106c2da6f19..60f9c517fa3 100644
--- a/lucene/tools/build.xml
+++ b/lucene/tools/build.xml
@@ -45,4 +45,5 @@
   </target>
 
   <target name="javadocs"/> <!-- to make common-build.xml happy -->
+  <target name="pitest"/> <!-- to make common-build.xml happy -->
 </project>
diff --git a/lucene/tools/junit4/tests.policy b/lucene/tools/junit4/tests.policy
index c0a69c02281..5b0ec0a7a2e 100644
--- a/lucene/tools/junit4/tests.policy
+++ b/lucene/tools/junit4/tests.policy
@@ -45,11 +45,17 @@ grant {
   permission java.util.PropertyPermission "*", "read,write";
   permission java.lang.reflect.ReflectPermission "*";
   permission java.lang.RuntimePermission "*";
-  
+
+  // Needed for some things in DNS caching in the JVM
+  permission java.security.SecurityPermission "*", "read,write";
+
+  // These two *have* to be spelled out a seperate
+  permission java.lang.management.ManagementPermission "control";
+  permission java.lang.management.ManagementPermission "monitor";
+
   // Solr needs those:
   permission java.net.NetPermission "*";
   permission java.util.logging.LoggingPermission "control";
-  permission java.lang.management.ManagementPermission "monitor";
   permission javax.management.MBeanPermission "*", "*";
   permission javax.management.MBeanServerPermission "*";
   permission javax.management.MBeanTrustPermission "*";
diff --git a/solr/build.xml b/solr/build.xml
index 6fab138d640..87cf22bc491 100644
--- a/solr/build.xml
+++ b/solr/build.xml
@@ -133,6 +133,8 @@
           depends="test-core, test-contrib"/>
   <target name="test-core" description="Runs the core and solrj unit tests."
           depends="test-solr-core, test-solrj"/>
+  <target name="pitest" description="Validate, then run core, solrj, and contrib unit tests."
+          depends="pitest-core, pitest-contrib"/>
   <target name="compile-test" description="Compile unit tests."
           depends="compile-solr-test-framework, compile-test-solr-core, compile-test-solrj, compile-test-contrib"/>
   <target name="javadocs" description="Calls javadocs-all, javadocs-solrj, and javadocs-test-framework"
@@ -162,6 +164,23 @@
   <target name="test-contrib" description="Run contrib unit tests.">
     <contrib-crawl target="test" failonerror="true"/>
   </target>
+
+  <!-- Pitest targets -->
+  <target name="pitest-core" description="PiTest solr core">
+    <ant dir="core" target="pitest" inheritAll="false">
+      <propertyset refid="uptodate.and.compiled.properties"/>
+    </ant>
+  </target>
+
+  <target name="pitest-solrj" description="PiTest java client">
+    <ant dir="solrj" target="pitest" inheritAll="false">
+      <propertyset refid="uptodate.and.compiled.properties"/>
+    </ant>
+  </target>
+
+  <target name="pitest-contrib" description="Run contrib PiTests.">
+    <contrib-crawl target="pitest" failonerror="false"/>
+  </target>
   
   <!-- test-framework targets -->
   <target name="javadocs-test-framework">
diff --git a/solr/example/build.xml b/solr/example/build.xml
index fdfbe95b829..91ef7b4a3c4 100644
--- a/solr/example/build.xml
+++ b/solr/example/build.xml
@@ -33,6 +33,9 @@
   <target name="compile-core"/>
   <target name="compile-test"/>
 
+  <!-- nothing to cover -->
+  <target name="pitest"/>
+
   <target name="resolve" depends="ivy-availability-check,ivy-fail,ivy-configure">
     <sequential>
     <!-- jetty libs in lib/ -->
diff --git a/solr/example/example-DIH/build.xml b/solr/example/example-DIH/build.xml
index b48b1496123..fb82edf96f5 100644
--- a/solr/example/example-DIH/build.xml
+++ b/solr/example/example-DIH/build.xml
@@ -35,4 +35,7 @@
   <target name="compile-core"/>
   <target name="compile-test"/>
 
+  <!-- nothing to cover -->
+  <target name="pitest"/>
+
 </project>