Detangle JdkJarHellCheck from build tool building (#58601) (#58755)

* Detangle JdkJarHellCheck from build tool building

- allows building the tool with same runtime as es
- allows building build tools with newer runtime version and keep ThirdPartyAuditTask
running with minimum runtime to ensure we check against correct jre
- add jdkjarhell test jar setup into fixture

buildSrc/build.gradle

@@ -206,6 +206,8 @@ if (project != rootProject) {
     distribution project(':distribution:archives:linux-tar')
     distribution project(':distribution:archives:oss-linux-tar')
     distribution project(':distribution:archives:oss-linux-aarch64-tar')
+
+    integTestRuntimeOnly(project(":libs:elasticsearch-core"))
   }
 
   // for external projects we want to remove the marker file indicating we are running the Elasticsearch project

buildSrc/src/integTest/java/org/elasticsearch/gradle/BuildPluginIT.java

@@ -37,6 +37,8 @@ import java.util.List;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
+import static org.elasticsearch.gradle.test.TestClasspathUtils.setupJarJdkClasspath;
+
 public class BuildPluginIT extends GradleIntegrationTestCase {
 
     @Rule
@@ -49,6 +51,7 @@ public class BuildPluginIT extends GradleIntegrationTestCase {
     }
 
     public void testCheckTask() {
+        setupJarJdkClasspath(getProjectDir("elasticsearch.build"));
         BuildResult result = getGradleRunner("elasticsearch.build").withArguments("check", "assemble", "-s").build();
         assertTaskSuccessful(result, ":check");
     }

buildSrc/src/integTest/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTaskIT.java

@@ -23,12 +23,16 @@ import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
 import org.gradle.testkit.runner.BuildResult;
 import org.junit.Before;
 
+import static org.elasticsearch.gradle.test.TestClasspathUtils.setupJarJdkClasspath;
+
 public class ThirdPartyAuditTaskIT extends GradleIntegrationTestCase {
 
     @Before
     public void setUp() throws Exception {
         // Build the sample jars
         getGradleRunner("thirdPartyAudit").withArguments(":sample_jars:build", "-s").build();
+        // propagate jdkjarhell jar
+        setupJarJdkClasspath(getProjectDir("thirdPartyAudit"));
     }
 
     public void testElasticsearchIgnored() {

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditPrecommitPlugin.java

@@ -24,17 +24,30 @@ import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin;
 import org.elasticsearch.gradle.info.BuildParams;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
+import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.tasks.TaskProvider;
 
 import java.nio.file.Path;
 
 public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin {
+
+    public static final String JDK_JAR_HELL_CONFIG_NAME = "jdkJarHell";
+    public static final String LIBS_ELASTICSEARCH_CORE_PROJECT_PATH = ":libs:elasticsearch-core";
+
     @Override
     public TaskProvider<? extends Task> createTask(Project project) {
         project.getPlugins().apply(CompileOnlyResolvePlugin.class);
         project.getConfigurations().create("forbiddenApisCliJar");
         project.getDependencies().add("forbiddenApisCliJar", "de.thetaphi:forbiddenapis:2.7");
 
+        Configuration jdkJarHellConfig = project.getConfigurations().create(JDK_JAR_HELL_CONFIG_NAME);
+        if (BuildParams.isInternal() && project.getPath().equals(":libs:elasticsearch-core") == false) {
+            // External plugins will depend on this already via transitive dependencies.
+            // Internal projects are not all plugins, so make sure the check is available
+            // we are not doing this for this project itself to avoid jar hell with itself
+            project.getDependencies().add(JDK_JAR_HELL_CONFIG_NAME, project.project(LIBS_ELASTICSEARCH_CORE_PROJECT_PATH));
+        }
+
         TaskProvider<ExportElasticsearchBuildResourcesTask> resourcesTask = project.getTasks()
             .register("thirdPartyAuditResources", ExportElasticsearchBuildResourcesTask.class);
         Path resourcesDir = project.getBuildDir().toPath().resolve("third-party-audit-config");
@@ -49,6 +62,7 @@ public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin {
             t.getTargetCompatibility().set(project.provider(BuildParams::getRuntimeJavaVersion));
             t.setSignatureFile(resourcesDir.resolve("forbidden/third-party-audit.txt").toFile());
         });
+        project.getTasks().withType(ThirdPartyAuditTask.class).configureEach(t -> t.setJdkJarHellClasspath(jdkJarHellConfig));
         return audit;
     }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java

@@ -20,19 +20,19 @@ package org.elasticsearch.gradle.precommit;
 
 import de.thetaphi.forbiddenapis.cli.CliMain;
 import org.apache.commons.io.output.NullOutputStream;
-import org.elasticsearch.gradle.JdkJarHellCheck;
 import org.elasticsearch.gradle.OS;
 import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin;
 import org.gradle.api.DefaultTask;
-import org.gradle.api.GradleException;
 import org.gradle.api.JavaVersion;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.provider.Property;
 import org.gradle.api.specs.Spec;
 import org.gradle.api.tasks.CacheableTask;
 import org.gradle.api.tasks.Classpath;
+import org.gradle.api.tasks.CompileClasspath;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputFile;
 import org.gradle.api.tasks.InputFiles;
@@ -48,8 +48,6 @@ import org.gradle.process.ExecResult;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
-import java.net.URISyntaxException;
-import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.util.Arrays;
@@ -77,6 +75,7 @@ public class ThirdPartyAuditTask extends DefaultTask {
         CliMain.EXIT_VIOLATION,
         CliMain.EXIT_UNSUPPORTED_JDK
     );
+    private static final String JDK_JAR_HELL_MAIN_CLASS = "org.elasticsearch.bootstrap.JdkJarHellCheck";
 
     private Set<String> missingClassExcludes = new TreeSet<>();
 
@@ -88,6 +87,8 @@ public class ThirdPartyAuditTask extends DefaultTask {
 
     private String javaHome;
 
+    private FileCollection jdkJarHellClasspath;
+
     private final Property<JavaVersion> targetCompatibility = getProject().getObjects().property(JavaVersion.class);
 
     @Input
@@ -131,6 +132,17 @@ public class ThirdPartyAuditTask extends DefaultTask {
         return new File(getProject().getBuildDir(), "markers/" + getName());
     }
 
+    // We use compile classpath normalization here because class implementation changes are irrelevant for the purposes of jdk jar hell.
+    // We only care about the runtime classpath ABI here.
+    @CompileClasspath
+    public FileCollection getJdkJarHellClasspath() {
+        return jdkJarHellClasspath.filter(File::exists);
+    }
+
+    public void setJdkJarHellClasspath(FileCollection jdkJarHellClasspath) {
+        this.jdkJarHellClasspath = jdkJarHellClasspath;
+    }
+
     public void ignoreMissingClasses(String... classesOrPackages) {
         if (classesOrPackages.length == 0) {
             missingClassExcludes = null;
@@ -357,20 +369,13 @@ public class ThirdPartyAuditTask extends DefaultTask {
     private Set<String> runJdkJarHellCheck() throws IOException {
         ByteArrayOutputStream standardOut = new ByteArrayOutputStream();
         ExecResult execResult = getProject().javaexec(spec -> {
-            URL location = JdkJarHellCheck.class.getProtectionDomain().getCodeSource().getLocation();
+            spec.classpath(
-            if (location.getProtocol().equals("file") == false) {
+                jdkJarHellClasspath,
-                throw new GradleException("Unexpected location for JdkJarHellCheck class: " + location);
+                getRuntimeConfiguration(),
-            }
+                getProject().getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)
-            try {
+            );
-                spec.classpath(
+
-                    location.toURI().getPath(),
+            spec.setMain(JDK_JAR_HELL_MAIN_CLASS);
-                    getRuntimeConfiguration(),
-                    getProject().getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)
-                );
-            } catch (URISyntaxException e) {
-                throw new AssertionError(e);
-            }
-            spec.setMain(JdkJarHellCheck.class.getName());
             spec.args(getJarExpandDir());
             spec.setIgnoreExitValue(true);
             if (javaHome != null) {

buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/TestClasspathUtils.java

@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+
+import static org.junit.Assert.fail;
+
+public class TestClasspathUtils {
+
+    public static void setupJarJdkClasspath(File projectRoot) {
+        try {
+            URL originLocation = TestClasspathUtils.class.getClassLoader()
+                .loadClass("org.elasticsearch.bootstrap.JdkJarHellCheck")
+                .getProtectionDomain()
+                .getCodeSource()
+                .getLocation();
+            File targetFile = new File(
+                projectRoot,
+                "sample_jars/build/testrepo/org/elasticsearch/elasticsearch-core/current/elasticsearch-core-current.jar"
+            );
+            targetFile.getParentFile().mkdirs();
+            Path originalPath = Paths.get(originLocation.toURI());
+            Files.copy(originalPath, targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+        } catch (ClassNotFoundException | URISyntaxException | IOException e) {
+            e.printStackTrace();
+            fail("Cannot setup jdk jar hell classpath");
+        }
+    }
+
+}

buildSrc/src/testKit/elasticsearch.build/build.gradle

@@ -10,9 +10,38 @@ dependencies {
   api "junit:junit:${versions.junit}"
   // missing classes in thirdparty audit
   api 'org.hamcrest:hamcrest-core:1.3'
+  jdkJarHell 'org.elasticsearch:elasticsearch-core:current'
 }
 
 repositories {
+  /**
+   * Local test repo contains dummy jars with different group names and versions.
+   *   - broken-log4j creates a log4j logger but has no pom, so the class will be missing
+   *   - dummy-io has a class that creates a new java.io.File ( something which third-party-audit-absurd.txt forbids )
+   *   - version 0.0.2 has the same class and one extra file just to make the jar different
+   *   - used for propagating jar containing jdkjarhell jar from integ test runtime classpath
+   */
+  maven {
+    name = "local-test"
+    url = file("sample_jars/build/testrepo")
+    metadataSources {
+      artifact()
+    }
+  }
+  jcenter()
+}
+
+repositories {
+  /**
+   * Local test repo contains jdkjarhell jar from integ test runtime classpath propagated at runtime
+   */
+  maven {
+    name = "local-test"
+    url = file("sample_jars/build/testrepo")
+    metadataSources {
+      artifact()
+    }
+  }
   jcenter()
 }
 

buildSrc/src/testKit/thirdPartyAudit/build.gradle

@@ -5,7 +5,7 @@ import org.elasticsearch.gradle.precommit.ThirdPartyAuditTask
 plugins {
   id 'java'
   // bring in build-tools onto the classpath
-  id 'elasticsearch.global-build-info' apply false
+  id 'elasticsearch.global-build-info'
 }
 
 plugins.apply(ThirdPartyAuditPrecommitPlugin)
@@ -16,6 +16,7 @@ repositories {
    *   - broken-log4j creates a log4j logger but has no pom, so the class will be missing
    *   - dummy-io has a class that creates a new java.io.File ( something which third-party-audit-absurd.txt forbids )
    *   - version 0.0.2 has the same class and one extra file just to make the jar different
+   *   - used for propagating jar containing jdkjarhell jar from integ test runtime classpath
    */
   maven {
     name = "local-test"
@@ -29,6 +30,7 @@ repositories {
 
 dependencies {
   forbiddenApisCliJar 'de.thetaphi:forbiddenapis:2.7'
+  jdkJarHell 'org.elasticsearch:elasticsearch-core:current'
   compileOnly "org.${project.properties.compileOnlyGroup}:${project.properties.compileOnlyVersion}"
   implementation "org.${project.properties.compileGroup}:${project.properties.compileVersion}"
 }

libs/core/src/main/java/org/elasticsearch/bootstrap/JdkJarHellCheck.java

@@ -7,7 +7,7 @@
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *    http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
@@ -16,7 +16,9 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.gradle;
+package org.elasticsearch.bootstrap;
+
+import org.elasticsearch.common.SuppressForbidden;
 
 import java.io.IOException;
 import java.nio.file.FileVisitResult;
@@ -57,6 +59,7 @@ public class JdkJarHellCheck {
         return Collections.unmodifiableSet(detected);
     }
 
+    @SuppressForbidden(reason = "command line tool")
     public static void main(String[] argv) throws IOException {
         JdkJarHellCheck checker = new JdkJarHellCheck();
         for (String location : argv) {