honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Detangle JdkJarHellCheck from build tool building (#58601) (#58755) 

* Detangle JdkJarHellCheck from build tool building

- allows building the tool with same runtime as es
- allows building build tools with newer runtime version and keep ThirdPartyAuditTask
running with minimum runtime to ensure we check against correct jre
- add jdkjarhell test jar setup into fixture
This commit is contained in:
Rene Groeschke 2020-07-01 10:41:01 +02:00 committed by GitHub
parent 417f7062c5
commit 2e278eaf3c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 138 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
buildSrc/build.gradle
View File

@ -206,6 +206,8 @@ if (project != rootProject) {
distribution project(':distribution:archives:linux-tar')
distribution project(':distribution:archives:oss-linux-tar')
distribution project(':distribution:archives:oss-linux-aarch64-tar')
integTestRuntimeOnly(project(":libs:elasticsearch-core"))
}
// for external projects we want to remove the marker file indicating we are running the Elasticsearch project

3
buildSrc/src/integTest/java/org/elasticsearch/gradle/BuildPluginIT.java
View File

@ -37,6 +37,8 @@ import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import static org.elasticsearch.gradle.test.TestClasspathUtils.setupJarJdkClasspath;
public class BuildPluginIT extends GradleIntegrationTestCase {
@Rule
@ -49,6 +51,7 @@ public class BuildPluginIT extends GradleIntegrationTestCase {
}
public void testCheckTask() {
setupJarJdkClasspath(getProjectDir("elasticsearch.build"));
BuildResult result = getGradleRunner("elasticsearch.build").withArguments("check", "assemble", "-s").build();
assertTaskSuccessful(result, ":check");
}

4
buildSrc/src/integTest/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTaskIT.java
View File

@ -23,12 +23,16 @@ import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
import org.gradle.testkit.runner.BuildResult;
import org.junit.Before;
import static org.elasticsearch.gradle.test.TestClasspathUtils.setupJarJdkClasspath;
public class ThirdPartyAuditTaskIT extends GradleIntegrationTestCase {
@Before
public void setUp() throws Exception {
// Build the sample jars
getGradleRunner("thirdPartyAudit").withArguments(":sample_jars:build", "-s").build();
// propagate jdkjarhell jar
setupJarJdkClasspath(getProjectDir("thirdPartyAudit"));
}
public void testElasticsearchIgnored() {

14
buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditPrecommitPlugin.java
View File

@ -24,17 +24,30 @@ import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin;
import org.elasticsearch.gradle.info.BuildParams;
import org.gradle.api.Project;
import org.gradle.api.Task;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.tasks.TaskProvider;
import java.nio.file.Path;
public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin {
public static final String JDK_JAR_HELL_CONFIG_NAME = "jdkJarHell";
public static final String LIBS_ELASTICSEARCH_CORE_PROJECT_PATH = ":libs:elasticsearch-core";
@Override
public TaskProvider<? extends Task> createTask(Project project) {
project.getPlugins().apply(CompileOnlyResolvePlugin.class);
project.getConfigurations().create("forbiddenApisCliJar");
project.getDependencies().add("forbiddenApisCliJar", "de.thetaphi:forbiddenapis:2.7");
Configuration jdkJarHellConfig = project.getConfigurations().create(JDK_JAR_HELL_CONFIG_NAME);
if (BuildParams.isInternal() && project.getPath().equals(":libs:elasticsearch-core") == false) {
// External plugins will depend on this already via transitive dependencies.
// Internal projects are not all plugins, so make sure the check is available
// we are not doing this for this project itself to avoid jar hell with itself
project.getDependencies().add(JDK_JAR_HELL_CONFIG_NAME, project.project(LIBS_ELASTICSEARCH_CORE_PROJECT_PATH));
}
TaskProvider<ExportElasticsearchBuildResourcesTask> resourcesTask = project.getTasks()
.register("thirdPartyAuditResources", ExportElasticsearchBuildResourcesTask.class);
Path resourcesDir = project.getBuildDir().toPath().resolve("third-party-audit-config");
@ -49,6 +62,7 @@ public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin {
t.getTargetCompatibility().set(project.provider(BuildParams::getRuntimeJavaVersion));
t.setSignatureFile(resourcesDir.resolve("forbidden/third-party-audit.txt").toFile());
});
project.getTasks().withType(ThirdPartyAuditTask.class).configureEach(t -> t.setJdkJarHellClasspath(jdkJarHellConfig));
return audit;
}
}

41
buildSrc/src/main/java/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.java
View File

@ -20,19 +20,19 @@ package org.elasticsearch.gradle.precommit;
import de.thetaphi.forbiddenapis.cli.CliMain;
import org.apache.commons.io.output.NullOutputStream;
import org.elasticsearch.gradle.JdkJarHellCheck;
import org.elasticsearch.gradle.OS;
import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin;
import org.gradle.api.DefaultTask;
import org.gradle.api.GradleException;
import org.gradle.api.JavaVersion;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.artifacts.Dependency;
import org.gradle.api.file.FileCollection;
import org.gradle.api.file.FileTree;
import org.gradle.api.provider.Property;
import org.gradle.api.specs.Spec;
import org.gradle.api.tasks.CacheableTask;
import org.gradle.api.tasks.Classpath;
import org.gradle.api.tasks.CompileClasspath;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.InputFile;
import org.gradle.api.tasks.InputFiles;
@ -48,8 +48,6 @@ import org.gradle.process.ExecResult;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.Arrays;
@ -77,6 +75,7 @@ public class ThirdPartyAuditTask extends DefaultTask {
CliMain.EXIT_VIOLATION,
CliMain.EXIT_UNSUPPORTED_JDK
);
private static final String JDK_JAR_HELL_MAIN_CLASS = "org.elasticsearch.bootstrap.JdkJarHellCheck";
private Set<String> missingClassExcludes = new TreeSet<>();
@ -88,6 +87,8 @@ public class ThirdPartyAuditTask extends DefaultTask {
private String javaHome;
private FileCollection jdkJarHellClasspath;
private final Property<JavaVersion> targetCompatibility = getProject().getObjects().property(JavaVersion.class);
@Input
@ -131,6 +132,17 @@ public class ThirdPartyAuditTask extends DefaultTask {
return new File(getProject().getBuildDir(), "markers/" + getName());
}
// We use compile classpath normalization here because class implementation changes are irrelevant for the purposes of jdk jar hell.
// We only care about the runtime classpath ABI here.
@CompileClasspath
public FileCollection getJdkJarHellClasspath() {
return jdkJarHellClasspath.filter(File::exists);
}
public void setJdkJarHellClasspath(FileCollection jdkJarHellClasspath) {
this.jdkJarHellClasspath = jdkJarHellClasspath;
}
public void ignoreMissingClasses(String... classesOrPackages) {
if (classesOrPackages.length == 0) {
missingClassExcludes = null;
@ -357,20 +369,13 @@ public class ThirdPartyAuditTask extends DefaultTask {
private Set<String> runJdkJarHellCheck() throws IOException {
ByteArrayOutputStream standardOut = new ByteArrayOutputStream();
ExecResult execResult = getProject().javaexec(spec -> {
URL location = JdkJarHellCheck.class.getProtectionDomain().getCodeSource().getLocation();
if (location.getProtocol().equals("file") == false) {
throw new GradleException("Unexpected location for JdkJarHellCheck class: " + location);
}
try {
spec.classpath(
location.toURI().getPath(),
getRuntimeConfiguration(),
getProject().getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)
);
} catch (URISyntaxException e) {
throw new AssertionError(e);
}
spec.setMain(JdkJarHellCheck.class.getName());
spec.classpath(
jdkJarHellClasspath,
getRuntimeConfiguration(),
getProject().getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)
);
spec.setMain(JDK_JAR_HELL_MAIN_CLASS);
spec.args(getJarExpandDir());
spec.setIgnoreExitValue(true);
if (javaHome != null) {

55
buildSrc/src/testFixtures/java/org/elasticsearch/gradle/test/TestClasspathUtils.java Normal file
View File

@ -0,0 +1,55 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.test;
import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import static org.junit.Assert.fail;
public class TestClasspathUtils {
public static void setupJarJdkClasspath(File projectRoot) {
try {
URL originLocation = TestClasspathUtils.class.getClassLoader()
.loadClass("org.elasticsearch.bootstrap.JdkJarHellCheck")
.getProtectionDomain()
.getCodeSource()
.getLocation();
File targetFile = new File(
projectRoot,
"sample_jars/build/testrepo/org/elasticsearch/elasticsearch-core/current/elasticsearch-core-current.jar"
);
targetFile.getParentFile().mkdirs();
Path originalPath = Paths.get(originLocation.toURI());
Files.copy(originalPath, targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
} catch (ClassNotFoundException | URISyntaxException | IOException e) {
e.printStackTrace();
fail("Cannot setup jdk jar hell classpath");
}
}
}

29
buildSrc/src/testKit/elasticsearch.build/build.gradle
View File

@ -10,9 +10,38 @@ dependencies {
api "junit:junit:${versions.junit}"
// missing classes in thirdparty audit
api 'org.hamcrest:hamcrest-core:1.3'
jdkJarHell 'org.elasticsearch:elasticsearch-core:current'
}
repositories {
/**
* Local test repo contains dummy jars with different group names and versions.
* - broken-log4j creates a log4j logger but has no pom, so the class will be missing
* - dummy-io has a class that creates a new java.io.File ( something which third-party-audit-absurd.txt forbids )
* - version 0.0.2 has the same class and one extra file just to make the jar different
* - used for propagating jar containing jdkjarhell jar from integ test runtime classpath
*/
maven {
name = "local-test"
url = file("sample_jars/build/testrepo")
metadataSources {
artifact()
}
}
jcenter()
}
repositories {
/**
* Local test repo contains jdkjarhell jar from integ test runtime classpath propagated at runtime
*/
maven {
name = "local-test"
url = file("sample_jars/build/testrepo")
metadataSources {
artifact()
}
}
jcenter()
}

4
buildSrc/src/testKit/thirdPartyAudit/build.gradle
View File

@ -5,7 +5,7 @@ import org.elasticsearch.gradle.precommit.ThirdPartyAuditTask
plugins {
id 'java'
// bring in build-tools onto the classpath
id 'elasticsearch.global-build-info' apply false
id 'elasticsearch.global-build-info'
}
plugins.apply(ThirdPartyAuditPrecommitPlugin)
@ -16,6 +16,7 @@ repositories {
* - broken-log4j creates a log4j logger but has no pom, so the class will be missing
* - dummy-io has a class that creates a new java.io.File ( something which third-party-audit-absurd.txt forbids )
* - version 0.0.2 has the same class and one extra file just to make the jar different
* - used for propagating jar containing jdkjarhell jar from integ test runtime classpath
*/
maven {
name = "local-test"
@ -29,6 +30,7 @@ repositories {
dependencies {
forbiddenApisCliJar 'de.thetaphi:forbiddenapis:2.7'
jdkJarHell 'org.elasticsearch:elasticsearch-core:current'
compileOnly "org.${project.properties.compileOnlyGroup}:${project.properties.compileOnlyVersion}"
implementation "org.${project.properties.compileGroup}:${project.properties.compileVersion}"
}

7
buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/JdkJarHellCheck.java → libs/core/src/main/java/org/elasticsearch/bootstrap/JdkJarHellCheck.java
View File

@ -7,7 +7,7 @@
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
@ -16,7 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle;
package org.elasticsearch.bootstrap;
import org.elasticsearch.common.SuppressForbidden;
import java.io.IOException;
import java.nio.file.FileVisitResult;
@ -57,6 +59,7 @@ public class JdkJarHellCheck {
return Collections.unmodifiableSet(detected);
}
@SuppressForbidden(reason = "command line tool")
public static void main(String[] argv) throws IOException {
JdkJarHellCheck checker = new JdkJarHellCheck();
for (String location : argv) {