LUCENE-9266 remove gradle wrapper jar from source

ASF Release Policy states that we cannot have binary JAR files checked
in to our source releases, a few other projects have solved this by
modifying their generated gradlew scripts to download a copy of the
wrapper jar.

We now have a version and checksum file in ./gradle/wrapper directory
used for verifying the wrapper jar, and will take advantage of single
source java execution to verify and download.

The gradle wrapper jar will continue to be available in the git
repository, but will be excluded from src tarball generation. This
should not change workflows for any users, since we expect the gradlew
script to get the jar when it is missing.

Co-authored-by: Dawid Weiss <dweiss@apache.org>

.github/workflows/gradle-precommit.yml

@@ -12,6 +12,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+    # Setup
     - uses: actions/checkout@v2
     - name: Set up JDK 11
       uses: actions/setup-java@v1
@@ -21,3 +22,4 @@ jobs:
       run: chmod +x gradlew
     - name: Build with Gradle
       run: ./gradlew precommit
+    - uses: gradle/wrapper-validation-action@v1

.github/workflows/gradle-wrapper-validation.yml

@@ -1,11 +0,0 @@
-name: "Validate Gradle Wrapper"
-on: [push, pull_request]
-
-jobs:
-  validation:
-    name: "Validation"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: gradle/wrapper-validation-action@v1
-

buildSrc/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java

@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.gradle;
+
+import java.io.IOException;
+import java.net.URL;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.EnumSet;
+import java.util.Locale;
+
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+import static java.nio.file.StandardOpenOption.APPEND;
+
+/**
+ * Standalone class that can be used to download a gradle-wrapper.jar
+ * <p>
+ * Has no dependencies outside of standard java libraries
+ */
+public class WrapperDownloader {
+  public static void main(String[] args) {
+    if (args.length != 1) {
+      System.err.println("Usage: java WrapperDownloader.java <destination>");
+      System.exit(1);
+    }
+
+    try {
+      new WrapperDownloader().run(Paths.get(args[0]));
+    } catch (Exception e) {
+      System.err.println("ERROR: " + e.getMessage());
+      System.exit(1);
+    }
+  }
+
+  public void run(Path destination) throws IOException, NoSuchAlgorithmException {
+    Path checksumPath = destination.resolveSibling(destination.getFileName().toString() + ".sha256");
+    if (!Files.exists(checksumPath)) {
+      throw new IOException("Checksum file not found: " + checksumPath);
+    }
+    String expectedChecksum = Files.readString(checksumPath, StandardCharsets.UTF_8).trim();
+
+    Path versionPath = destination.resolveSibling(destination.getFileName().toString() + ".version");
+    if (!Files.exists(versionPath)) {
+      throw new IOException("Wrapper version file not found: " + versionPath);
+    }
+    String wrapperVersion = Files.readString(versionPath, StandardCharsets.UTF_8).trim();
+
+    MessageDigest digest = MessageDigest.getInstance("SHA-256");
+
+    if (Files.exists(destination)) {
+      if (checksum(digest, destination).equalsIgnoreCase(expectedChecksum)) {
+        // File exists, checksum matches, good to go!
+        return;
+      } else {
+        System.err.println("Checksum mismatch, will attempt to re-download gradle-wrapper.jar");
+        System.out.println(destination);
+        Files.delete(destination);
+      }
+    }
+
+    URL url = new URL("https://github.com/gradle/gradle/raw/v" + wrapperVersion + "/gradle/wrapper/gradle-wrapper.jar");
+    System.err.println("Downloading gradle-wrapper.jar from " + url);
+
+    // As of v6.0.1 the wrapper is approximately 60K
+    // Can increase this if gradle wrapper ever goes beyond 500K, but keep a safety check
+    final int maxSize = 512 * 1024;
+
+    // Zero-copy save the jar to a temp file
+    Path temp = Files.createTempFile(destination.getParent(), ".gradle-wrapper", ".tmp");
+    try {
+      try (ReadableByteChannel in = Channels.newChannel(url.openStream());
+           FileChannel out = FileChannel.open(temp, EnumSet.of(APPEND))) {
+        out.transferFrom(in, 0, maxSize);
+      } catch (IOException e) {
+        throw new IOException("Could not download gradle-wrapper.jar (" + e.getMessage() + ").");
+      }
+
+      String checksum = checksum(digest, temp);
+      if (!checksum.equalsIgnoreCase(expectedChecksum)) {
+        throw new IOException(String.format(Locale.ROOT,
+                "Checksum mismatch on downloaded gradle-wrapper.jar (was: %s, expected: %s).",
+                checksum,
+                expectedChecksum));
+      }
+
+      Files.move(temp, destination, REPLACE_EXISTING);
+      temp = null;
+    } finally {
+      if (temp != null) {
+        Files.deleteIfExists(temp);
+      }
+    }
+  }
+
+  private String checksum(MessageDigest messageDigest, Path path) throws IOException {
+    try {
+      char[] hex = "0123456789abcdef".toCharArray();
+      byte[] digest = messageDigest.digest(Files.readAllBytes(path));
+      StringBuilder sb = new StringBuilder();
+      for (byte b : digest) {
+        sb.append(hex[(b >> 4) & 0xf]).append(hex[b & 0xf]);
+      }
+      return sb.toString();
+    } catch (IOException e) {
+      throw new IOException("Could not compute digest of file: " + path + " (" + e.getMessage() + ")");
+    }
+  }
+}

gradle/wrapper/gradle-wrapper.jar.sha256

@@ -0,0 +1 @@
+28b330c20a9a73881dfe9702df78d4d78bf72368e8906c70080ab6932462fe9e

gradle/wrapper/gradle-wrapper.jar.version

@@ -0,0 +1 @@
+6.0.1

gradlew

@@ -80,8 +80,6 @@ case "`uname`" in
     ;;
 esac
 
-CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
-
 # Determine the Java command to use to start the JVM.
 if [ -n "$JAVA_HOME" ] ; then
     if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
@@ -104,6 +102,14 @@ Please set the JAVA_HOME variable in your environment to match the
 location of your Java installation."
 fi
 
+# LUCENE-9266: verify and download the gradle wrapper jar if we don't have one.
+GRADLE_WRAPPER_JAR=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+if ! $JAVACMD buildSrc/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java $GRADLE_WRAPPER_JAR ; then
+    exit $?
+fi
+
+CLASSPATH=$GRADLE_WRAPPER_JAR
+
 # Don't fork a daemon mode on initial run that generates local defaults.
 GRADLE_DAEMON_CTRL=
 if [ ! -e "$APP_HOME/gradle.properties" ]; then

gradlew.bat

@@ -77,9 +77,14 @@ if "x%~1" == "x" goto execute
 set CMD_LINE_ARGS=%*
 
 :execute
-@rem Setup the command line
 
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+@rem LUCENE-9266: verify and download the gradle wrapper jar if we don't have one.
+set GRADLE_WRAPPER_JAR=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+"%JAVA_EXE%" buildSrc/src/main/java/org/apache/lucene/gradle/WrapperDownloader.java "%GRADLE_WRAPPER_JAR%"
+IF %ERRORLEVEL% NEQ 0 goto fail
+
+@rem Setup the command line
+set CLASSPATH=%GRADLE_WRAPPER_JAR%
 
 @rem Don't fork a daemon mode on initial run that generates local defaults.
 SET GRADLE_DAEMON_CTRL=

solr/build.xml

@@ -459,7 +459,7 @@
                   fullpath="${fullnamever}/solr/LUCENE_CHANGES.txt" />
       <tarfileset dir="${src.export.dir}"
                   prefix="${fullnamever}"
-                  excludes="solr/example/**/*.sh solr/example/**/bin/ solr/scripts/**"/>
+                  excludes="solr/example/**/*.sh solr/example/**/bin/ solr/scripts/** gradle/wrapper/gradle-wrapper.jar"/>
       <tarfileset dir="${src.export.dir}"
                   prefix="${fullnamever}"
                   filemode="755"