diff --git a/.gitignore b/.gitignore index 023c82798d6..264d7caf87d 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,6 @@ __pycache__ build/ .gradle/ .idea/ + +# Ignore the generated local settings file. +gradle.properties diff --git a/build.gradle b/build.gradle index 2d1c56113f5..b5d39d3ea12 100644 --- a/build.gradle +++ b/build.gradle @@ -16,6 +16,8 @@ allprojects { // if the build file is incorrectly written and evaluates something // eagerly). +apply from: file('gradle/generate-defaults.gradle') + // CI systems. apply from: file('gradle/ci/buildscan.gradle') apply from: file('gradle/ci/travis.gradle') diff --git a/gradle.properties b/gradle.properties deleted file mode 100644 index a7a04d65c7b..00000000000 --- a/gradle.properties +++ /dev/null @@ -1,5 +0,0 @@ -systemProp.file.encoding=UTF-8 - -org.gradle.jvmargs=-Xmx1g -org.gradle.parallel=true -org.gradle.priority=low diff --git a/gradle/defaults.gradle b/gradle/defaults.gradle index 474826eca9d..1b534c41ffb 100644 --- a/gradle/defaults.gradle +++ b/gradle/defaults.gradle @@ -15,4 +15,19 @@ allprojects { // Artifacts will have names after full gradle project path // so :solr:core will have solr-core.jar, etc. project.archivesBaseName = project.path.replaceAll("^:", "").replace(':', '-') + + ext { + // Utility method to support passing overrides via -P or -D. + propertyOrDefault = { propName, defValue -> + def result + if (project.hasProperty(propName)) { + result = project.getProperty(propName) + } else if (System.properties.containsKey(propName)) { + result = System.properties.get(propName) + } else { + result = defValue + } + return result + } + } } diff --git a/gradle/generate-defaults.gradle b/gradle/generate-defaults.gradle new file mode 100644 index 00000000000..72f0dafd89f --- /dev/null +++ b/gradle/generate-defaults.gradle @@ -0,0 +1,73 @@ + +// This script tries to guess sensible defaults for gradle parallelism +// and local machine's resources and save them under 'gradle.properties'. + +def hasDefaults = rootProject.file("gradle.properties").exists() + +// If we don't have the defaults yet, create them and re-run the build +// recursively with the same parameters as originally passed. +// +// Sadly, the recursive build doesn't seem to pick up the parallelism +// tweaks from gradle.properties file. + +if (!hasDefaults) { + configure(rootProject) { + task setupLocalDefaultsOnce(type: GradleBuild) { + // Approximate a common-sense default for running gradle with parallel + // workers: half the count of available cpus but not more than 12. + def cpus = Runtime.runtime.availableProcessors() + def maxWorkers = (int) Math.max(1d, Math.min(cpus * 0.5d, 12)) + def testsJvms = (int) Math.max(1d, Math.min(cpus * 0.5d, 4)) + + // Reuse the same set of parameters for the recursive invocation and apply + // some of these eagerly. + def startParams = gradle.startParameter.newInstance() + startParams.setParallelProjectExecutionEnabled(true) + startParams.setMaxWorkerCount(maxWorkers) + startParameter(startParams) + + // Write the defaults for this machine. + rootProject.file("gradle.properties").write( + [ + "# These settings have been generated automatically on the first run.", + "# See gradlew :helpLocalSettings for more information.", + "systemProp.file.encoding=UTF-8", + "org.gradle.daemon=true", + "org.gradle.jvmargs=-Xmx1g", + "org.gradle.parallel=true", + "org.gradle.priority=normal", + "", + "# Maximum number of parallel gradle workers.", + "org.gradle.workers.max=${maxWorkers}", + "", + "# Maximum number of test JVMs forked per test task.", + "tests.jvms=${testsJvms}" + ].join("\n"), "UTF-8") + + doFirst { + logger.log(LogLevel.WARN, "\nIMPORTANT. This is the first time you ran the build. " + + "I wrote some sane defaults (for this machine) to 'gradle.properties', " + + "they will be picked up on consecutive gradle invocations (not this one).\n\n" + + "Run gradlew :helpLocalSettings for more information.") + } + } + } + + // Disable any tasks in this build, they were forked recursively. + gradle.taskGraph.whenReady { graph -> + graph.allTasks.each { task -> + if (task != rootProject.setupLocalDefaultsOnce) { + task.enabled = false + } + } + } + + // Make all tasks depend on local setup to make sure it'll run though. + allprojects { + tasks.all { task -> + if (task != rootProject.setupLocalDefaultsOnce) { + task.dependsOn rootProject.setupLocalDefaultsOnce + } + } + } +} \ No newline at end of file diff --git a/gradle/help.gradle b/gradle/help.gradle index 0ee82ac18fa..e5f8014dd0d 100644 --- a/gradle/help.gradle +++ b/gradle/help.gradle @@ -6,6 +6,7 @@ configure(rootProject) { ["Ant", "help/ant.txt", "Ant-gradle migration help."], ["Tests", "help/tests.txt", "Tests, filtering, beasting, etc."], ["ForbiddenApis", "help/forbiddenApis.txt", "How to add/apply rules for forbidden APIs."], + ["LocalSettings", "help/localSettings.txt", "Local settings, overrides and build performance tweaks."] ] helpFiles.each { section, path, sectionInfo -> @@ -24,7 +25,8 @@ configure(rootProject) { println "This is an experimental Lucene/Solr gradle build. See some" println "guidelines, ant-equivalent commands etc. under help/*; or type:" helpFiles.each { section, path, sectionInfo -> - println " gradlew :help${section} # ${sectionInfo}" + println String.format(Locale.ROOT, + " gradlew :help%-14s # %s", section, sectionInfo) } } } diff --git a/gradle/testing/defaults-tests.gradle b/gradle/testing/defaults-tests.gradle index 668cb84f02d..d685328b763 100644 --- a/gradle/testing/defaults-tests.gradle +++ b/gradle/testing/defaults-tests.gradle @@ -13,8 +13,7 @@ allprojects { useJUnit() - // Set up default parallel execution limits. - maxParallelForks = (int) Math.max(1, Math.min(Runtime.runtime.availableProcessors() / 2.0, 3.0)) + maxParallelForks = propertyOrDefault("tests.jvms", (int) Math.max(1, Math.min(Runtime.runtime.availableProcessors() / 2.0, 4.0))) minHeapSize = "256m" maxHeapSize = "512m" diff --git a/gradle/testing/randomization.gradle b/gradle/testing/randomization.gradle index 8024264cabe..6330777b1ed 100644 --- a/gradle/testing/randomization.gradle +++ b/gradle/testing/randomization.gradle @@ -1,23 +1,6 @@ // Configure test randomization seeds and derived test properties. -allprojects { - ext { - // Support passing overrides via -P or -D. - propertyOrDefault = { propName, defValue -> - def result - if (project.hasProperty(propName)) { - result = project.getProperty(propName) - } else if (System.properties.containsKey(propName)) { - result = System.properties.get(propName) - } else { - result = defValue - } - return result - } - } -} - // Pick the "root" seed from which everything else is derived. configure(rootProject) { ext { diff --git a/gradle/testing/slowest-tests-at-end.gradle b/gradle/testing/slowest-tests-at-end.gradle index 2e339173dc6..b3e6d204965 100644 --- a/gradle/testing/slowest-tests-at-end.gradle +++ b/gradle/testing/slowest-tests-at-end.gradle @@ -8,7 +8,7 @@ allprojects { def duration = (result.getEndTime() - result.getStartTime()) allTests << [ - name : "${desc.className.replaceAll('.+\\.', "")}.${desc.name} (${project.name})", + name : "${desc.className.replaceAll('.+\\.', "")}.${desc.name} (${project.path})", duration: duration ] } diff --git a/help/localSettings.txt b/help/localSettings.txt new file mode 100644 index 00000000000..c67c895b095 --- /dev/null +++ b/help/localSettings.txt @@ -0,0 +1,46 @@ +Local developer settings +======================== + +The first invocation of any task in Lucene/Solr gradle build will generate +and save a project-local 'gradle.properties' file. This file contains +the defaults you may (but don't have to) tweak for your particular hardware +(or taste). + +This is an overview of some of these settings. + +Parallelism +----------- + +Gradle build can run tasks in parallel but by default it consumes all CPU cores which +is too optimistic a default for Lucene/Solr tests. You can disable the parallelism +entirely or assign it a 'low' priority with these properties: + +org.gradle.parallel=[true, false] +org.gradle.priority=[normal, low] + +The default level of parallelism is computed based on the number of cores on +your machine (on the first run of gradle build). By default these are fairly conservative +settings (half the number of cores for workers, for example): + +org.gradle.workers.max=[X] +tests.jvms=[N <= X] + +The number of test JVMs can be lower than the number of workers: this just means +that two projects can run tests in parallel to saturate all the workers. The I/O and memory +bandwidth limits will kick in quickly so even if you have a very beefy machine bumping +it too high may not help. + +You can always override these settings locally using command line as well: +gradlew -Ptests.jvms=N --max-workers=X + +Gradle Daemon +------------- + +The gradle daemon is a background process that keeps an evaluated copy of the project +structure, some caches, etc. It speeds up repeated builds quite a bit but if you don't +like the idea of having a (sizeable) background process running in the background, +disable it. + +org.gradle.daemon=[true, false] +org.gradle.jvmargs=... +