From 53217e2b1e9d6871b2babfd74fb52129f6cf7461 Mon Sep 17 00:00:00 2001 From: Dominik Stadler Date: Sun, 7 Nov 2021 14:59:48 +0000 Subject: [PATCH] Use parallel build to speed up building and running tests Enable parallel building and parallel test-execution Provide a configuration to still build/test serially on CI via CI_BUILD=TRUE gradle --no-parallel --max-workers=1 clean build Adjust Jenkins DSL to set CI_BUILD to avoid problems in CI Adjust TestAllFiles to not fail when parallel builds are enabled Reduce memory settings to reduce requirements on build-environments Add gradle plugin to list task-dependencies Thanks to Andreas Reichel for the PR Closes #275 git-svn-id: https://svn.apache.org/repos/asf/poi/trunk@1894812 13f79535-47bb-0310-9956-ffa450edef68 --- build.gradle | 38 +++++++++++++++++-- gradle.properties | 14 ++++++- jenkins/create_jobs.groovy | 1 + .../org/apache/poi/stress/TestAllFiles.java | 9 ++++- poi/build.gradle | 1 + settings.gradle | 3 +- 6 files changed, 59 insertions(+), 7 deletions(-) diff --git a/build.gradle b/build.gradle index 68a503520e..21b7b78ced 100644 --- a/build.gradle +++ b/build.gradle @@ -37,6 +37,7 @@ buildscript { plugins { id 'base' + id "com.dorongold.task-tree" version "2.1.0" id('org.nosphere.apache.rat') version '0.7.0' id 'distribution' } @@ -51,6 +52,8 @@ if (project.hasProperty('enableSonar')) { apply plugin: 'org.sonarqube' } +boolean isCIBuild = false; + // For help converting an Ant build to a Gradle build, see // https://docs.gradle.org/current/userguide/ant.html @@ -145,6 +148,7 @@ subprojects { options.encoding = 'UTF-8' options.compilerArgs << '-Xlint:unchecked' options.deprecation = true + options.incremental = true onlyIf { (name != "compileJava9" && name != "compileTest9") || JavaVersion.current() != JavaVersion.VERSION_1_8 @@ -258,7 +262,8 @@ subprojects { // set heap size for the test JVM(s) minHeapSize = "128m" - maxHeapSize = "1512m" + maxHeapSize = "1G" + // Specifying the local via system properties did not work, so we set them this way jvmArgs << [ @@ -271,11 +276,38 @@ subprojects { '-Djavax.xml.stream.XMLInputFactory=com.sun.xml.internal.stream.XMLInputFactoryImpl', "-Dversion.id=${project.version}", '-ea', - '-Djunit.jupiter.execution.parallel.config.strategy=fixed', - '-Djunit.jupiter.execution.parallel.config.fixed.parallelism=2' // -Xjit:verbose={compileStart|compileEnd},vlog=build/jit.log${no.jit.sherlock} ... if ${isIBMVM} ] + // detect if running on Jenkins/CI + isCIBuild |= Boolean.valueOf(System.getenv("CI_BUILD")); + + if (isCIBuild) { + jvmArgs += [ + // Strictly serial + // '-Djunit.jupiter.execution.parallel.enabled=false', + + // OR parallel on 2 threads + '-Djunit.jupiter.execution.parallel.config.strategy=fixed', + '-Djunit.jupiter.execution.parallel.config.fixed.parallelism=2' + ] + maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1 + } else { + jvmArgs += [ + '-Djunit.jupiter.execution.parallel.enabled=true', + '-Djunit.jupiter.execution.parallel.config.strategy=dynamic', + + // this setting breaks the test builds, do not use it! + //'-Djunit.jupiter.execution.parallel.mode.default=concurrent' + ] + + // Explicitly defining the maxParallelForks was always slower than not setting it + // So we leave this to Gradle itself, which seems to be very smart + // maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1 + // maxParallelForks = Math.max( Runtime.runtime.availableProcessors() - 1, 1 ) + + } + // show standard out and standard error of the test JVM(s) on the console //testLogging.showStandardStreams = true diff --git a/gradle.properties b/gradle.properties index 4f4e5ef86a..b7042fcd77 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,15 @@ # Specifies the JVM arguments used for the daemon process. # The setting is particularly useful for tweaking memory settings. -org.gradle.jvmargs=-Xmx4096m \ No newline at end of file +# Less than 2G definitely slows things down. -XX:+HeapDumpOnOutOfMemoryError +org.gradle.jvmargs=-Xmx2G -XX:MaxPermSize=512m -Dfile.encoding=UTF-8 + +# Activating will be much faster, but break the build of 'poi-ooxml-lite' +# @todo: look into poi-ooxml-lite task generateModuleInfo and enforce running whatever is needed before +org.gradle.caching=false + +# Modularise your project and enable parallel build +org.gradle.parallel=true + +# Enable configure on demand. +org.gradle.configureondemand=true + diff --git a/jenkins/create_jobs.groovy b/jenkins/create_jobs.groovy index c8397fa1cb..761e1af34a 100644 --- a/jenkins/create_jobs.groovy +++ b/jenkins/create_jobs.groovy @@ -262,6 +262,7 @@ poijobs.each { poijob -> label(slaves) environmentVariables { env('LANG', 'en_US.UTF-8') + env('CI_BUILD', 'TRUE') if(jdkKey == '1.10') { // when using JDK 9/10 for running Ant, we need to provide more modules for the forbidden-api-checks task // on JDK 11 and newer there is no such module any more, so do not add it here diff --git a/poi-integration/src/test/java/org/apache/poi/stress/TestAllFiles.java b/poi-integration/src/test/java/org/apache/poi/stress/TestAllFiles.java index 30338f0f8b..524ec0a2f2 100644 --- a/poi-integration/src/test/java/org/apache/poi/stress/TestAllFiles.java +++ b/poi-integration/src/test/java/org/apache/poi/stress/TestAllFiles.java @@ -36,6 +36,8 @@ import java.util.stream.Stream; import org.apache.poi.hssf.record.crypto.Biff8EncryptionKey; import org.apache.tools.ant.DirectoryScanner; import org.junit.jupiter.api.function.Executable; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -66,7 +68,7 @@ import org.opentest4j.AssertionFailedError; * that we do not remove expected sanity checks. */ // also need to set JVM parameter: -Djunit.jupiter.execution.parallel.enabled=true -//@Execution(ExecutionMode.CONCURRENT) +@Execution(ExecutionMode.CONCURRENT) public class TestAllFiles { private static final String DEFAULT_TEST_DATA_PATH = "test-data"; public static final File ROOT_DIR = new File(System.getProperty("POI.testdata.path", DEFAULT_TEST_DATA_PATH)); @@ -117,6 +119,11 @@ public class TestAllFiles { final List result = new ArrayList<>(100); for (String file : scanner.getIncludedFiles()) { + // avoid running on files leftover from previous failed runs + if(file.endsWith("-saved.xls") || file.endsWith("TestHPSFWritingFunctionality.doc")) { + continue; + } + for (FileHandlerKnown handler : sm.getHandler(file)) { ExcInfo info1 = sm.getExcInfo(file, testName, handler); if (info1 == null || info1.isValid(testName, handler.name())) { diff --git a/poi/build.gradle b/poi/build.gradle index 21b4627d3e..fbf2ba4725 100644 --- a/poi/build.gradle +++ b/poi/build.gradle @@ -183,6 +183,7 @@ javadocJar { } sourcesJar { + dependsOn generateVersionJava metaInf { from("$projectDir/../legal/LICENSE") from("$projectDir/../legal/NOTICE") diff --git a/settings.gradle b/settings.gradle index 7d799fd606..2d4eef563f 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,5 +1,4 @@ rootProject.name = 'poi' include 'poi', 'poi-ooxml-full', 'poi-ooxml-lite-agent', 'poi-scratchpad', - 'poi-ooxml', 'poi-excelant', 'poi-examples', 'poi-integration', - 'poi-ooxml-lite' \ No newline at end of file + 'poi-ooxml', 'poi-excelant', 'poi-examples', 'poi-integration' , 'poi-ooxml-lite' \ No newline at end of file