Use parallel build to speed up building and running tests

Enable parallel building and parallel test-execution
Provide a configuration to still build/test serially on CI
via CI_BUILD=TRUE gradle --no-parallel --max-workers=1 clean build
Adjust Jenkins DSL to set CI_BUILD to avoid problems in CI
Adjust TestAllFiles to not fail when parallel builds are enabled
Reduce memory settings to reduce requirements on build-environments
Add gradle plugin to list task-dependencies

Thanks to Andreas Reichel for the PR

Closes #275

git-svn-id: https://svn.apache.org/repos/asf/poi/trunk@1894812 13f79535-47bb-0310-9956-ffa450edef68

build.gradle

@@ -37,6 +37,7 @@ buildscript {
 
 plugins {
     id 'base'
+    id "com.dorongold.task-tree" version "2.1.0"
     id('org.nosphere.apache.rat') version '0.7.0'
     id 'distribution'
 }
@@ -51,6 +52,8 @@ if (project.hasProperty('enableSonar')) {
     apply plugin: 'org.sonarqube'
 }
 
+boolean isCIBuild = false;
+
 // For help converting an Ant build to a Gradle build, see
 // https://docs.gradle.org/current/userguide/ant.html
 
@@ -145,6 +148,7 @@ subprojects {
         options.encoding = 'UTF-8'
         options.compilerArgs << '-Xlint:unchecked'
         options.deprecation = true
+        options.incremental = true
 
         onlyIf {
             (name != "compileJava9" && name != "compileTest9") || JavaVersion.current() != JavaVersion.VERSION_1_8
@@ -258,7 +262,8 @@ subprojects {
 
         // set heap size for the test JVM(s)
         minHeapSize = "128m"
-        maxHeapSize = "1512m"
+        maxHeapSize = "1G"
+
 
         // Specifying the local via system properties did not work, so we set them this way
         jvmArgs << [
@@ -271,11 +276,38 @@ subprojects {
             '-Djavax.xml.stream.XMLInputFactory=com.sun.xml.internal.stream.XMLInputFactoryImpl',
             "-Dversion.id=${project.version}",
             '-ea',
-            '-Djunit.jupiter.execution.parallel.config.strategy=fixed',
-            '-Djunit.jupiter.execution.parallel.config.fixed.parallelism=2'
             // -Xjit:verbose={compileStart|compileEnd},vlog=build/jit.log${no.jit.sherlock}   ... if ${isIBMVM}
         ]
 
+        // detect if running on Jenkins/CI
+        isCIBuild |= Boolean.valueOf(System.getenv("CI_BUILD"));
+
+        if (isCIBuild) {
+        	jvmArgs += [
+        	    // Strictly serial
+        	    // '-Djunit.jupiter.execution.parallel.enabled=false',
+
+        	    // OR parallel on 2 threads
+				'-Djunit.jupiter.execution.parallel.config.strategy=fixed',
+				'-Djunit.jupiter.execution.parallel.config.fixed.parallelism=2'
+			]
+			maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
+		} else {
+			jvmArgs += [
+			    '-Djunit.jupiter.execution.parallel.enabled=true',
+			    '-Djunit.jupiter.execution.parallel.config.strategy=dynamic',
+
+                // this setting breaks the test builds, do not use it!
+				//'-Djunit.jupiter.execution.parallel.mode.default=concurrent'
+			]
+
+			 // Explicitly defining the maxParallelForks was always slower than not setting it
+			 // So we leave this to Gradle itself, which seems to be very smart
+			 // maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
+			 // maxParallelForks = Math.max( Runtime.runtime.availableProcessors() - 1, 1 )
+
+        }
+
         // show standard out and standard error of the test JVM(s) on the console
         //testLogging.showStandardStreams = true
 

gradle.properties

@@ -1,3 +1,15 @@
 # Specifies the JVM arguments used for the daemon process.
 # The setting is particularly useful for tweaking memory settings.
-org.gradle.jvmargs=-Xmx4096m
+# Less than 2G definitely slows things down. -XX:+HeapDumpOnOutOfMemoryError
+org.gradle.jvmargs=-Xmx2G -XX:MaxPermSize=512m -Dfile.encoding=UTF-8
+
+# Activating will be much faster, but break the build of 'poi-ooxml-lite'
+# @todo: look into poi-ooxml-lite task generateModuleInfo and enforce running whatever is needed before
+org.gradle.caching=false
+
+# Modularise your project and enable parallel build
+org.gradle.parallel=true
+
+# Enable configure on demand.
+org.gradle.configureondemand=true
+

jenkins/create_jobs.groovy

@@ -262,6 +262,7 @@ poijobs.each { poijob ->
         label(slaves)
         environmentVariables {
             env('LANG', 'en_US.UTF-8')
+            env('CI_BUILD', 'TRUE')
             if(jdkKey == '1.10') {
                 // when using JDK 9/10 for running Ant, we need to provide more modules for the forbidden-api-checks task
                 // on JDK 11 and newer there is no such module any more, so do not add it here

poi-integration/src/test/java/org/apache/poi/stress/TestAllFiles.java

@@ -36,6 +36,8 @@ import java.util.stream.Stream;
 import org.apache.poi.hssf.record.crypto.Biff8EncryptionKey;
 import org.apache.tools.ant.DirectoryScanner;
 import org.junit.jupiter.api.function.Executable;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
@@ -66,7 +68,7 @@ import org.opentest4j.AssertionFailedError;
  *  that we do not remove expected sanity checks.
  */
 // also need to set JVM parameter: -Djunit.jupiter.execution.parallel.enabled=true
-//@Execution(ExecutionMode.CONCURRENT)
+@Execution(ExecutionMode.CONCURRENT)
 public class TestAllFiles {
     private static final String DEFAULT_TEST_DATA_PATH = "test-data";
     public static final File ROOT_DIR = new File(System.getProperty("POI.testdata.path", DEFAULT_TEST_DATA_PATH));
@@ -117,6 +119,11 @@ public class TestAllFiles {
 
         final List<Arguments> result = new ArrayList<>(100);
         for (String file : scanner.getIncludedFiles()) {
+			// avoid running on files leftover from previous failed runs
+			if(file.endsWith("-saved.xls") || file.endsWith("TestHPSFWritingFunctionality.doc")) {
+				continue;
+			}
+
             for (FileHandlerKnown handler : sm.getHandler(file)) {
                 ExcInfo info1 = sm.getExcInfo(file, testName, handler);
                 if (info1 == null || info1.isValid(testName, handler.name())) {

poi/build.gradle

@@ -183,6 +183,7 @@ javadocJar {
 }
 
 sourcesJar {
+    dependsOn generateVersionJava
     metaInf {
         from("$projectDir/../legal/LICENSE")
         from("$projectDir/../legal/NOTICE")

settings.gradle

@@ -1,5 +1,4 @@
 rootProject.name = 'poi'
 
 include 'poi', 'poi-ooxml-full', 'poi-ooxml-lite-agent', 'poi-scratchpad',
-        'poi-ooxml', 'poi-excelant', 'poi-examples', 'poi-integration',
-        'poi-ooxml-lite'
+        'poi-ooxml', 'poi-excelant', 'poi-examples', 'poi-integration' , 'poi-ooxml-lite'