From 0ebc0fa9ce30bc6cd4bdeca7e95942b345a6c587 Mon Sep 17 00:00:00 2001 From: Olivier Lamy Date: Thu, 23 May 2019 14:12:44 +1000 Subject: [PATCH] Issue #3659 jmh tests are creating to many tmp files and never cleaned (#3650) * cleanup tmp files created by jmh tests Signed-off-by: olivier lamy --- Jmh_Jenkinsfile | 14 +++----- .../jetty/http/jmh/MultiPartBenchmark.java | 32 +++++++++++++------ .../jetty/server/jmh/ForwardBenchmark.java | 27 ++++------------ .../thread/strategy/jmh/EWYKBenchmark.java | 29 ++++++++++------- .../util/thread/strategy/jmh/TestServer.java | 3 +- 5 files changed, 52 insertions(+), 53 deletions(-) diff --git a/Jmh_Jenkinsfile b/Jmh_Jenkinsfile index 9c0a1d2fafe..a771e7594ac 100644 --- a/Jmh_Jenkinsfile +++ b/Jmh_Jenkinsfile @@ -4,20 +4,15 @@ def branch = params.get("JETTY_BRANCH" ,"jetty-9.4.x") def owner = params.get("REPO_OWNER", "eclipse") def jdk = params.get("JDK", "jdk8") def jmhJarPath = params.get("jmhJarPath","jetty-jmh/target/benchmarks.jar") +currentBuild.description = "Build branch $branch with jdk $jdk" node("linux") { // System Dependent Locations - def mvntool = tool name: 'maven3.5', type: 'hudson.tasks.Maven$MavenInstallation' - def jdktool = tool name: "$jdk", type: 'hudson.model.JDK' def mvnName = 'maven3.5' def localRepo = "${env.JENKINS_HOME}/${env.EXECUTOR_NUMBER}" def settingsName = 'oss-settings.xml' def mavenOpts = '-Xms1g -Xmx4g -Djava.awt.headless=true' - // Environment - List mvnEnv = ["PATH+MVN=${mvntool}/bin", "PATH+JDK=${jdktool}/bin", "JAVA_HOME=${jdktool}/", "MAVEN_HOME=${mvntool}"] - mvnEnv.add("MAVEN_OPTS=$mavenOpts") - stage("Checkout") { git url: "https://github.com/$owner/jetty.project.git", branch: "$branch" @@ -41,15 +36,14 @@ node("linux") { } // jmh run - -stage("jmh-run") { - node( 'jmh-build-node' ) { +node( 'jmh-build-node' ) { + stage("jmh-run") { timeout( time: 180, unit: 'MINUTES' ) { withEnv( ["JAVA_HOME=${tool "$jdk"}"] ) { unstash name: 'perf-tests' sh "rm -rf jmh_results" sh "mkdir jmh_results" - sh "${env.JAVA_HOME}/bin/java -jar $jmhJarPath -rff jmh_results/jmh_result.json -rf json" + sh "${env.JAVA_HOME}/bin/java -jar $jmhJarPath -rff jmh_results/jmh_result.json -rf json -foe true" jmhReport 'jmh_results/jmh_result.json' } } diff --git a/jetty-jmh/src/main/java/org/eclipse/jetty/http/jmh/MultiPartBenchmark.java b/jetty-jmh/src/main/java/org/eclipse/jetty/http/jmh/MultiPartBenchmark.java index 754d3cc6801..f2d62f8eb8b 100644 --- a/jetty-jmh/src/main/java/org/eclipse/jetty/http/jmh/MultiPartBenchmark.java +++ b/jetty-jmh/src/main/java/org/eclipse/jetty/http/jmh/MultiPartBenchmark.java @@ -26,7 +26,7 @@ import static org.hamcrest.Matchers.notNullValue; import org.eclipse.jetty.http.MultiPartFormInputStream; import org.eclipse.jetty.toolchain.test.Hex; -import org.eclipse.jetty.toolchain.test.IO; +import org.eclipse.jetty.util.IO; import org.eclipse.jetty.util.QuotedStringTokenizer; import org.eclipse.jetty.util.StringUtil; import org.hamcrest.Matchers; @@ -57,6 +57,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import java.security.DigestOutputStream; import java.security.MessageDigest; @@ -177,7 +178,7 @@ public class MultiPartBenchmark public long testLargeGenerated() throws Exception { Path multipartRawFile = _file.toPath(); - Path outputDir = new File("/tmp").toPath(); + Path outputDir = Files.createTempDirectory( "jetty_multipart_benchmark"); MultipartConfigElement config = newMultipartConfigElement(outputDir); @@ -188,11 +189,14 @@ public class MultiPartBenchmark case "HTTP": { MultiPartFormInputStream parser = new MultiPartFormInputStream(in, _contentType, config, outputDir.toFile()); + parser.setDeleteOnExit(true); if (parser.getParts().size() != _numSections) throw new IllegalStateException("Incorrect Parsing"); for (Part p : parser.getParts()) { count += p.getSize(); + if(p instanceof MultiPartFormInputStream.MultiPart) ((MultiPartFormInputStream.MultiPart)p).cleanUp(); + else p.delete(); } } break; @@ -200,11 +204,14 @@ public class MultiPartBenchmark case "UTIL": { org.eclipse.jetty.util.MultiPartInputStreamParser parser = new org.eclipse.jetty.util.MultiPartInputStreamParser(in, _contentType, config, outputDir.toFile()); + parser.setDeleteOnExit(true); if (parser.getParts().size() != _numSections) throw new IllegalStateException("Incorrect Parsing"); for (Part p : parser.getParts()) { count += p.getSize(); + if(p instanceof org.eclipse.jetty.util.MultiPartInputStreamParser.MultiPart) ((org.eclipse.jetty.util.MultiPartInputStreamParser.MultiPart)p).cleanUp(); + else p.delete(); } } break; @@ -213,7 +220,7 @@ public class MultiPartBenchmark throw new IllegalStateException("Unknown parserType Parameter"); } } - + IO.delete(outputDir.toFile()); return count; } @@ -236,18 +243,16 @@ public class MultiPartBenchmark { for (String multiPart : data) { - //Path multipartRawFile = MavenTestingUtils.getTestResourcePathFile("multipart/" + multiPart + ".raw"); String expectationPath = "multipart/" + multiPart + ".expected.txt"; - //Path expectationPath = MavenTestingUtils.getTestResourcePathFile("multipart/" + multiPart + ".expected.txt"); File expectationFile = File.createTempFile( expectationPath, ".tmp" ); - try(InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(expectationPath); - OutputStream os = Files.newOutputStream( expectationFile.toPath() )) { - IO.copy( inputStream, os ); + try(InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(expectationPath); // + OutputStream os = Files.newOutputStream( expectationFile.toPath())) { + IO.copy(inputStream, os); } - Path outputDir = Files.createTempDirectory( "expected_output_jmh_jetty" );// new File("/tmp").toPath(); + Path outputDir = Files.createTempDirectory("expected_output_jmh_jetty"); MultipartExpectations multipartExpectations = new MultipartExpectations(expectationFile.toPath()); MultipartConfigElement config = newMultipartConfigElement(outputDir); @@ -259,18 +264,24 @@ public class MultiPartBenchmark case "HTTP": { MultiPartFormInputStream parser = new MultiPartFormInputStream(in, multipartExpectations.contentType, config, outputDir.toFile()); + parser.setDeleteOnExit(true); for (Part p : parser.getParts()) { count += p.getSize(); + if(p instanceof MultiPartFormInputStream.MultiPart) ((MultiPartFormInputStream.MultiPart)p).cleanUp(); + else p.delete(); } } break; case "UTIL": { org.eclipse.jetty.util.MultiPartInputStreamParser parser = new org.eclipse.jetty.util.MultiPartInputStreamParser(in, multipartExpectations.contentType, config, outputDir.toFile()); + parser.setDeleteOnExit(true); for (Part p : parser.getParts()) { count += p.getSize(); + if(p instanceof org.eclipse.jetty.util.MultiPartInputStreamParser.MultiPart) ((org.eclipse.jetty.util.MultiPartInputStreamParser.MultiPart)p).cleanUp(); + else p.delete(); } } break; @@ -278,7 +289,8 @@ public class MultiPartBenchmark throw new IllegalStateException("Unknown parserType Parameter"); } } - + Files.deleteIfExists(expectationFile.toPath()); + IO.delete(outputDir.toFile()); } return count; } diff --git a/jetty-jmh/src/main/java/org/eclipse/jetty/server/jmh/ForwardBenchmark.java b/jetty-jmh/src/main/java/org/eclipse/jetty/server/jmh/ForwardBenchmark.java index 20660748b67..2f77f105103 100644 --- a/jetty-jmh/src/main/java/org/eclipse/jetty/server/jmh/ForwardBenchmark.java +++ b/jetty-jmh/src/main/java/org/eclipse/jetty/server/jmh/ForwardBenchmark.java @@ -18,28 +18,9 @@ package org.eclipse.jetty.server.jmh; -import java.io.IOException; -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; -import java.util.Arrays; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - import org.eclipse.jetty.http.HttpField; import org.eclipse.jetty.http.HttpFields; import org.eclipse.jetty.http.HttpHeader; -import org.eclipse.jetty.server.ForwardedRequestCustomizer; -import org.eclipse.jetty.server.HttpConfiguration; -import org.eclipse.jetty.server.HttpConnectionFactory; -import org.eclipse.jetty.server.LocalConnector; -import org.eclipse.jetty.server.Request; -import org.eclipse.jetty.server.Server; -import org.eclipse.jetty.server.handler.AbstractHandler; import org.eclipse.jetty.util.ArrayTrie; import org.eclipse.jetty.util.Trie; import org.openjdk.jmh.annotations.Benchmark; @@ -56,8 +37,12 @@ import org.openjdk.jmh.runner.RunnerException; import org.openjdk.jmh.runner.options.Options; import org.openjdk.jmh.runner.options.OptionsBuilder; -import static java.lang.invoke.MethodHandles.dropArguments; -import static java.lang.invoke.MethodHandles.foldArguments; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.util.Arrays; +import java.util.concurrent.TimeUnit; + import static java.lang.invoke.MethodType.methodType; diff --git a/jetty-jmh/src/main/java/org/eclipse/jetty/util/thread/strategy/jmh/EWYKBenchmark.java b/jetty-jmh/src/main/java/org/eclipse/jetty/util/thread/strategy/jmh/EWYKBenchmark.java index 7c81298cc83..2292281ede2 100644 --- a/jetty-jmh/src/main/java/org/eclipse/jetty/util/thread/strategy/jmh/EWYKBenchmark.java +++ b/jetty-jmh/src/main/java/org/eclipse/jetty/util/thread/strategy/jmh/EWYKBenchmark.java @@ -19,11 +19,16 @@ package org.eclipse.jetty.util.thread.strategy.jmh; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import org.eclipse.jetty.util.IO; import org.eclipse.jetty.util.component.LifeCycle; import org.eclipse.jetty.util.thread.ExecutionStrategy; import org.eclipse.jetty.util.thread.Invocable; @@ -51,7 +56,7 @@ public class EWYKBenchmark { static TestServer server; static ReservedThreadExecutor reserved; - static File directory; + static Path directory; @Param({"PC","PEC","EWYK"}) public static String strategyName; @@ -66,21 +71,15 @@ public class EWYKBenchmark public static void setupServer() throws Exception { // Make a test directory - directory = File.createTempFile("ewyk","dir"); - if (directory.exists()) - directory.delete(); - directory.mkdir(); - directory.deleteOnExit(); - + directory = Files.createTempDirectory("ewyk"); + // Make some test files for (int i=0;i<75;i++) { - File file =new File(directory,i+".txt"); - file.createNewFile(); - file.deleteOnExit(); + File.createTempFile( "ewyk_benchmark", i + ".txt" , directory.toFile()); } - server=new TestServer(directory); + server=new TestServer(directory.toFile()); server.start(); reserved = new ReservedThreadExecutor(server,20); reserved.start(); @@ -89,6 +88,14 @@ public class EWYKBenchmark @TearDown(Level.Trial) public static void stopServer() throws Exception { + try + { + IO.delete(directory.toFile()); + } + catch ( Exception e ) + { + System.out.println("cannot delete directory:"+directory); + } reserved.stop(); server.stop(); } diff --git a/jetty-jmh/src/main/java/org/eclipse/jetty/util/thread/strategy/jmh/TestServer.java b/jetty-jmh/src/main/java/org/eclipse/jetty/util/thread/strategy/jmh/TestServer.java index ab99582cad6..3f05463f2ab 100644 --- a/jetty-jmh/src/main/java/org/eclipse/jetty/util/thread/strategy/jmh/TestServer.java +++ b/jetty-jmh/src/main/java/org/eclipse/jetty/util/thread/strategy/jmh/TestServer.java @@ -19,6 +19,7 @@ package org.eclipse.jetty.util.thread.strategy.jmh; import java.io.File; +import java.nio.file.Files; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -44,7 +45,7 @@ public class TestServer implements Executor, TryExecutor TestServer() { - this(new File("/tmp/")); + this(new File(System.getProperty("java.io.tmpdir"))); }