Generate gradle.properties from gradlew (#12131)

* SOLR-16641 - Generate gradle.properties from gradlew (#1320)
* Adapt for Lucene
* Remove localSettings from smoker; thanks @colvinco
* Print properties at end for debugging
* Add CHANGES.txt entry

---------

Co-authored-by: Colvin Cowie <colvin.cowie.dev@gmail.com>
Co-authored-by: Colvin Cowie <51863265+colvinco@users.noreply.github.com>

.github/workflows/distribution.yml

@@ -37,8 +37,5 @@ jobs:
     - name: Prepare caches
       uses: ./.github/actions/gradle-caches
 
-    - name: Initialize gradle settings (${{ matrix.os }})
-      run: ./gradlew localSettings
-
     - name: Run all distribution tests including GUI tests (${{ matrix.os }})
       run: ./gradlew -p lucene/distribution.tests test

.github/workflows/gradle-precommit.yml

@@ -39,9 +39,6 @@ jobs:
     - name: Prepare caches
       uses: ./.github/actions/gradle-caches
 
-    - name: Initialize gradle settings
-      run: ./gradlew localSettings --max-workers 2
-
     - name: Run gradle check (without tests)
       run: ./gradlew check -x test -Ptask.times=true --max-workers 2
 
@@ -72,11 +69,8 @@ jobs:
     - name: Prepare caches
       uses: ./.github/actions/gradle-caches
 
-    - name: Initialize gradle settings
-      run: ./gradlew localSettings --max-workers 2
+    - name: Run gradle tests
+      run: ./gradlew test "-Ptask.times=true" --max-workers 2
 
     - name: Echo settings
       run: cat gradle.properties
-
-    - name: Run gradle tests
-      run: ./gradlew test "-Ptask.times=true" --max-workers 2

.github/workflows/hunspell.yml

@@ -27,8 +27,5 @@ jobs:
     - name: Prepare caches
       uses: ./.github/actions/gradle-caches
 
-    - name: Initialize gradle settings
-      run: ./gradlew localSettings
-
     - name: Run regular and regression tests
       run: ./gradlew -p lucene/analysis/common check testRegressions

buildSrc/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java

@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.gradle;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.Map;
+
+/**
+ * Standalone class that generates a populated gradle.properties from a template.
+ *
+ * <p>Has no dependencies outside of standard java libraries
+ */
+public class GradlePropertiesGenerator {
+  public static void main(String[] args) {
+    if (args.length != 2) {
+      System.err.println("Usage: java GradlePropertiesGenerator.java <source> <destination>");
+      System.exit(2);
+    }
+
+    try {
+      new GradlePropertiesGenerator().run(Paths.get(args[0]), Paths.get(args[1]));
+    } catch (Exception e) {
+      System.err.println("ERROR: " + e.getMessage());
+      System.exit(3);
+    }
+  }
+
+  public void run(Path source, Path destination) throws IOException {
+    if (!Files.exists(source)) {
+      throw new IOException("template file not found: " + source);
+    }
+    if (Files.exists(destination)) {
+      System.out.println(destination + " already exists, skipping generation.");
+      return;
+    }
+
+    // Approximate a common-sense default for running gradle/tests with parallel
+    // workers: half the count of available cpus but not more than 12.
+    var cpus = Runtime.getRuntime().availableProcessors();
+    var maxWorkers = (int) Math.max(1d, Math.min(cpus * 0.5d, 12));
+    var testsJvms = (int) Math.max(1d, Math.min(cpus * 0.5d, 12));
+
+    var replacements = Map.of("@MAX_WORKERS@", maxWorkers, "@TEST_JVMS@", testsJvms);
+
+    System.out.println("Generating gradle.properties");
+    String fileContent = Files.readString(source, StandardCharsets.UTF_8);
+    for (var entry : replacements.entrySet()) {
+      fileContent = fileContent.replace(entry.getKey(), String.valueOf(entry.getValue()));
+    }
+    Files.writeString(
+            destination, fileContent, StandardCharsets.UTF_8, StandardOpenOption.CREATE_NEW);
+  }
+}

dev-tools/scripts/releaseWizard.yaml

@@ -342,8 +342,6 @@ groups:
       - !Command
         cmd: git pull --ff-only
         stdout: true
-      - !Command
-        cmd: "{{ gradle_cmd }} localSettings"
       - !Command
         cmd: "{{ gradle_cmd }} clean check -x test"
   - !Todo

dev-tools/scripts/smokeTestRelease.py

@@ -609,9 +609,6 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
         print('      %s' % line.strip())
       raise RuntimeError('source release has WARs...')
 
-    print('    initialize local settings for Gradle...')
-    java.run_java17('./gradlew --no-daemon localSettings', '%s/localsettings.log' % unpackPath)
-
     validateCmd = './gradlew --no-daemon check -p lucene/documentation'
     print('    run "%s"' % validateCmd)
     java.run_java17(validateCmd, '%s/validate.log' % unpackPath)

gradle/generation/local-settings.gradle

@@ -41,87 +41,12 @@ configure(rootProject) {
         throw new GradleException(
             "Certain gradle tasks and plugins require access to jdk.compiler" +
                 " internals, your gradle.properties might have just been generated or could be" +
-                " out of sync (see help/localSettings.txt)")
-      }
-    }
-  }
-
-  task localSettings() {
-    doFirst {
-      // If we don't have the defaults yet, create them.
-      if (hasDefaults) {
-        logger.lifecycle("Local settings already exist, skipping generation.")
-      } else {
-        // Approximate a common-sense default for running gradle/tests with parallel
-        // workers: half the count of available cpus but not more than 12.
-        def cpus = Runtime.runtime.availableProcessors()
-        def maxWorkers = (int) Math.max(1d, Math.min(cpus * 0.5d, 12))
-        def testsJvms = (int) Math.max(1d, Math.min(cpus * 0.5d, 12))
-
-        // Write the defaults for this machine.
-        rootProject.file("gradle.properties").write("""
-# These settings have been generated automatically on the first run.
-# See gradlew :helpLocalSettings for more information.
-systemProp.file.encoding=UTF-8
-
-# Set up gradle JVM defaults.
-#
-# We also open up internal compiler modules for spotless/ google java format.
-org.gradle.jvmargs=-Xmx1g -XX:TieredStopAtLevel=1 -XX:+UseParallelGC -XX:ActiveProcessorCount=1 \\
- --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \\
- --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \\
- --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \\
- --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \\
- --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
-
-# Run at normal priority, in parallel
-org.gradle.parallel=true
-org.gradle.priority=normal
-
-# This setting enables local task output caches. This will speed up
-# your local builds in most cases but will also consume disk space in your
-# gradle home. See LUCENE-10195 and/or SOLR-15603 for details.
-# org.gradle.caching=true
-
-# Silence gradle warnings. We'll deal with them when we upgrade the wrapper.
-org.gradle.warning.mode=none
-
-# You may disable the background daemon if it consumes too much memory.
-org.gradle.daemon=true
-# timeout after 15 mins of inactivity.
-org.gradle.daemon.idletimeout=900000
-
-# Maximum number of parallel gradle workers.
-org.gradle.workers.max=${maxWorkers}
-
-# Maximum number of test JVMs forked per test task.
-tests.jvms=${testsJvms}
-
-# Enable auto JVM provisioning.
-org.gradle.java.installations.auto-download=true
-
-# Set these to enable automatic JVM location discovery.
-org.gradle.java.installations.fromEnv=JAVA17_HOME,JAVA19_HOME,JAVA20_HOME,JAVA21_HOME,RUNTIME_JAVA_HOME
-#org.gradle.java.installations.paths=(custom paths)
-
-""", "UTF-8")
-
-        logger.log(LogLevel.WARN, "\nIMPORTANT. This is the first time you ran the build. " +
-            "I wrote some sane defaults (for this machine) to 'gradle.properties', " +
-            "they will be picked up on consecutive gradle invocations (not this one).\n\n" +
-            "Run gradlew :helpLocalSettings for more information.")
+                " out of sync (see gradle/template.gradle.properties)")
       }
     }
   }
 }
 
-if (!hasDefaults) {
-  // Make all tasks depend on local setup to make sure it'll run.
-  allprojects {
-    tasks.all { task ->
-      if (task != rootProject.localSettings) {
-        task.dependsOn rootProject.localSettings
-      }
-    }
-  }
+task localSettings() {
+  // This is just a placeholder until all references to the localSettings task are removed
 }

gradle/template.gradle.properties

@@ -0,0 +1,106 @@
+#############################
+#  Local developer settings #
+#############################
+#
+# The first invocation of any task in Lucene gradlew will generate and save this project-local 'gradle.properties' file.
+# This file contains the defaults you may (but don't have to) tweak for your particular hardware (or taste). Note there
+# are certain settings in that file that may be _required_ at runtime for certain plugins (an example is the spotless/
+# google java format plugin, which requires adding custom exports to JVM modules). gradlew only generates this file
+# if it's not already present (it never overwrites the defaults) -- occasionally you may have to manually delete (or move)
+# this file and regenerate from scratch.
+#
+# This is an overview of some of these settings.
+#
+###############
+# Parallelism #
+###############
+#
+# Gradle build can run tasks in parallel but by default it consumes all CPU cores which
+# is too optimistic a default for Lucene tests. You can disable the parallelism
+# entirely or assign it a 'low' priority with these properties:
+#
+# org.gradle.parallel=[true, false]
+# org.gradle.priority=[normal, low]
+#
+# The default level of parallelism is computed based on the number of cores on
+# your machine (on the first run of gradle build). By default these are fairly conservative
+# settings (half the number of cores for workers, for example):
+#
+# org.gradle.workers.max=[X]
+# tests.jvms=[N <= X]
+#
+# The number of test JVMs can be lower than the number of workers: this just means
+# that two projects can run tests in parallel to saturate all the workers. The I/O and memory
+# bandwidth limits will kick in quickly so even if you have a very beefy machine bumping
+# it too high may not help.
+#
+# You can always override these settings locally using command line as well:
+# gradlew -Ptests.jvms=N --max-workers=X
+#
+#############
+# Test JVMS #
+#############
+#
+# Test JVMs have their own set of arguments which can be customized. These are configured
+# separately from the gradle workers, for example:
+#
+# tests.jvms=3
+# tests.heapsize=512m
+# tests.minheapsize=512m
+# tests.jvmargs=-XX:+UseParallelGC -XX:TieredStopAtLevel=1 -XX:ActiveProcessorCount=1
+#
+#################
+# Gradle Daemon #
+#################
+#
+# The gradle daemon is a background process that keeps an evaluated copy of the project
+# structure, some caches, etc. It speeds up repeated builds quite a bit but if you don't
+# like the idea of having a (sizeable) background process running in the background,
+# disable it.
+#
+# org.gradle.daemon=[true, false]
+# org.gradle.jvmargs=...
+#############################################################################################
+
+# UTF-8 as standard file encoding
+systemProp.file.encoding=UTF-8
+
+# Set up gradle JVM defaults.
+#
+# We also open up internal compiler modules for spotless/ google java format.
+org.gradle.jvmargs=-Xmx1g -XX:TieredStopAtLevel=1 -XX:+UseParallelGC -XX:ActiveProcessorCount=1 \
+ --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+ --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
+ --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
+ --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+ --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+
+# Run at normal priority, in parallel
+org.gradle.parallel=true
+org.gradle.priority=normal
+
+# This setting enables local task output caches. This will speed up
+# your local builds in most cases but will also consume disk space in your
+# gradle home. See LUCENE-10195 for details.
+# org.gradle.caching=true
+
+# Silence gradle warnings. We'll deal with them when we upgrade the wrapper.
+org.gradle.warning.mode=none
+
+# You may disable the background daemon if it consumes too much memory.
+org.gradle.daemon=true
+# timeout after 15 mins of inactivity.
+org.gradle.daemon.idletimeout=900000
+
+# Maximum number of parallel gradle workers.
+org.gradle.workers.max=@MAX_WORKERS@
+
+# Maximum number of test JVMs forked per test task.
+tests.jvms=@TEST_JVMS@
+
+# Enable auto JVM provisioning.
+org.gradle.java.installations.auto-download=true
+
+# Set these to enable automatic JVM location discovery.
+org.gradle.java.installations.fromEnv=JAVA17_HOME,JAVA19_HOME,JAVA20_HOME,JAVA21_HOME,RUNTIME_JAVA_HOME
+#org.gradle.java.installations.paths=(custom paths)

gradle/validation/gradlew-scripts-tweaked.gradle

@@ -32,8 +32,8 @@ configure(rootProject) {
     doFirst {
       scripts.each { file ->
         def content = new String(file.readBytes(), StandardCharsets.US_ASCII)
-        if (content.indexOf("Don't fork a daemon mode on initial run that generates local defaults") < 0) {
-          throw new GradleException("Launch script ${file} does not have a manual daemon tweak (see LUCENE-9232).")
+        if (content.indexOf("GradlePropertiesGenerator") < 0) {
+          throw new GradleException("Launch script ${file} does not have a tweak to generate gradle.properties.")
         }
       }
     }

gradlew

@@ -132,11 +132,16 @@ fi
 
 CLASSPATH=$GRADLE_WRAPPER_JAR
 
-# Don't fork a daemon mode on initial run that generates local defaults.
-GRADLE_DAEMON_CTRL=
+# START OF LUCENE CUSTOMIZATION
+# Generate gradle.properties if they don't exist
 if [ ! -e "$APP_HOME/gradle.properties" ]; then
-    GRADLE_DAEMON_CTRL=--no-daemon
+    "$JAVACMD" $JAVA_OPTS --source 11 "$APP_HOME/buildSrc/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java" "$APP_HOME/gradle/template.gradle.properties" "$APP_HOME/gradle.properties"
+    GENERATOR_STATUS=$?
+    if [ "$GENERATOR_STATUS" -ne 0 ]; then
+        exit $GENERATOR_STATUS
+    fi
 fi
+# END OF LUCENE CUSTOMIZATION
 
 # Increase the maximum file descriptors if we can.
 if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
@@ -213,7 +218,7 @@ APP_ARGS=$(save "$@")
 export GIT_CONFIG_NOSYSTEM=1
 
 # Collect all arguments for the java command, following the shell quoting and substitution rules
-eval set -- $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain $GRADLE_DAEMON_CTRL "$APP_ARGS"
+eval set -- $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
 
 # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
 if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then

gradlew.bat

@@ -83,15 +83,22 @@ IF %ERRORLEVEL% NEQ 0 goto fail
 @rem Setup the command line
 set CLASSPATH=%GRADLE_WRAPPER_JAR%
 
-@rem Don't fork a daemon mode on initial run that generates local defaults.
-SET GRADLE_DAEMON_CTRL=
-IF NOT EXIST "%DIRNAME%\gradle.properties" SET GRADLE_DAEMON_CTRL=--no-daemon
+@rem START OF LUCENE CUSTOMIZATION
+@rem Generate gradle.properties if they don't exist
+IF NOT EXIST "%APP_HOME%\gradle.properties" (
+  @rem local expansion is needed to check ERRORLEVEL inside control blocks.
+  setlocal enableDelayedExpansion
+  "%JAVA_EXE%" %JAVA_OPTS% --source 11 "%APP_HOME%/buildSrc/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java" "%APP_HOME%\gradle\template.gradle.properties" "%APP_HOME%\gradle.properties"
+  IF %ERRORLEVEL% NEQ 0 goto fail
+  endlocal
+)
+@rem END OF LUCENE CUSTOMIZATION
 
 @rem Prevent jgit from forking/searching git.exe
 SET GIT_CONFIG_NOSYSTEM=1
 
 @rem Execute Gradle
-"%JAVA_EXE%" %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %GRADLE_DAEMON_CTRL% %*
+"%JAVA_EXE%" %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
 
 :end
 @rem End local scope for the variables with windows NT shell

help/localSettings.txt

@@ -2,61 +2,7 @@ Local developer settings
 ========================
 
 The first invocation of any task in Lucene's gradle build will generate
-and save a project-local 'gradle.properties' file. This file contains
-the defaults you may (but don't have to) tweak for your particular hardware
-(or taste). Note there are certain settings in that file that may
-be _required_ at runtime for certain plugins (an example is the spotless/
-google java format plugin, which requires adding custom exports to JVM modules). Gradle
-build only generates this file if it's not already present (it never overwrites
-the defaults) -- occasionally you may have to manually delete (or move) this
-file and regenerate from scratch.
-
-This is an overview of some settings present in gradle.properties.
-
-Parallelism
------------
-
-Gradle build can run tasks in parallel but by default it consumes all CPU cores which
-is too optimistic a default for Lucene tests. You can disable the parallelism
-entirely or assign it a 'low' priority with these properties:
-
-org.gradle.parallel=[true, false]
-org.gradle.priority=[normal, low]
-
-The default level of parallelism is computed based on the number of cores on
-your machine (on the first run of gradle build). By default these are fairly conservative
-settings (half the number of cores for workers, for example):
-
-org.gradle.workers.max=[X]
-tests.jvms=[N <= X]
-
-The number of test JVMs can be lower than the number of workers: this just means
-that two projects can run tests in parallel to saturate all the workers. The I/O and memory
-bandwidth limits will kick in quickly so even if you have a very beefy machine bumping
-it too high may not help.
-
-You can always override these settings locally using command line as well:
-gradlew -Ptests.jvms=N --max-workers=X
-
-Test JVMS
----------
-
-Test JVMs have their own set of arguments which can be customized. These are configured
-separately from the gradle workers, for example:
-
-tests.jvms=3
-tests.heapsize=512m
-tests.minheapsize=512m
-tests.jvmargs=-XX:+UseParallelGC -XX:TieredStopAtLevel=1 -XX:ActiveProcessorCount=1
-
-Gradle Daemon
--------------
-
-The gradle daemon is a background process that keeps an evaluated copy of the project
-structure, some caches, etc. It speeds up repeated builds quite a bit but if you don't
-like the idea of having a (sizeable) background process running in the background,
-disable it.
-
-org.gradle.daemon=[true, false]
-org.gradle.jvmargs=...
+and save a project-local 'gradle.properties' file from a template,
+with some default settings which you can modify.
 
+See your 'gradle.properties' file for more details of those settings.

lucene/CHANGES.txt

@@ -133,6 +133,11 @@ Bug Fixes
 ---------------------
 (No changes)
 
+Build
+---------------------
+
+* GITHUB#12131: Generate gradle.properties from gradlew, if absent (Colvin Cowie, Uwe Schindler)
+
 Other
 ---------------------