From 8a8324cb18b30438de26dfd8e8463860af89bec0 Mon Sep 17 00:00:00 2001 From: Lachlan Roberts Date: Thu, 29 Mar 2018 16:29:48 +1100 Subject: [PATCH] Completed benchmark for UTIL vs HTTP multipart parser. Fixed some tests in MultiPartCaptureTest. Signed-off-by: Lachlan Roberts --- jetty-http/pom.xml | 56 +++- .../jetty/http/MultiPartFormInputStream.java | 8 +- .../jetty/http/MultiPartCaptureTest.java | 12 +- .../http/MultiPartFormInputStreamTest.java | 33 ++ .../jetty/http/MultiPartParserTest.java | 46 +++ .../jetty/http/jmh/MultiPartBenchmark.java | 297 ++++++++++++++++++ ...s-charset-form-android-chrome.expected.txt | 2 +- ...-charset-form-android-firefox.expected.txt | 2 +- ...ture-sjis-charset-form-chrome.expected.txt | 2 +- ...ure-sjis-charset-form-firefox.expected.txt | 2 +- ...-sjis-charset-form-ios-safari.expected.txt | 2 +- ...ture-sjis-charset-form-safari.expected.txt | 2 +- 12 files changed, 449 insertions(+), 15 deletions(-) create mode 100644 jetty-http/src/test/java/org/eclipse/jetty/http/jmh/MultiPartBenchmark.java diff --git a/jetty-http/pom.xml b/jetty-http/pom.xml index 6c174f5e67d..e371ebe3e98 100644 --- a/jetty-http/pom.xml +++ b/jetty-http/pom.xml @@ -33,6 +33,18 @@ jetty-test-helper test + + org.openjdk.jmh + jmh-core + ${jmh.version} + test + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + test + @@ -69,7 +81,49 @@ org.eclipse.jetty.http.* - + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + ${jmhjar.name} + true + + + org.openjdk.jmh:jmh-core + + + + + org.openjdk.jmh.Main + + + + + org.openjdk.jmh:jmh-core + + ** + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + diff --git a/jetty-http/src/main/java/org/eclipse/jetty/http/MultiPartFormInputStream.java b/jetty-http/src/main/java/org/eclipse/jetty/http/MultiPartFormInputStream.java index 49eb302ca03..c2d1d14fe29 100644 --- a/jetty-http/src/main/java/org/eclipse/jetty/http/MultiPartFormInputStream.java +++ b/jetty-http/src/main/java/org/eclipse/jetty/http/MultiPartFormInputStream.java @@ -610,8 +610,6 @@ public class MultiPartFormInputStream } - //TODO implement proper toString - //TODO more debugging class Handler implements MultiPartParser.Handler { @@ -768,6 +766,12 @@ public class MultiPartFormInputStream contentType = null; headers = new MultiMap<>(); } + + @Override + public String toString() { + return("contentDisposition: "+contentDisposition+" contentType:"+contentType); + } + } public void setDeleteOnExit(boolean deleteOnExit) diff --git a/jetty-http/src/test/java/org/eclipse/jetty/http/MultiPartCaptureTest.java b/jetty-http/src/test/java/org/eclipse/jetty/http/MultiPartCaptureTest.java index 4ea4da84283..347614f4573 100644 --- a/jetty-http/src/test/java/org/eclipse/jetty/http/MultiPartCaptureTest.java +++ b/jetty-http/src/test/java/org/eclipse/jetty/http/MultiPartCaptureTest.java @@ -110,14 +110,14 @@ public class MultiPartCaptureTest // ret.add(new String[]{"browser-capture-sjis-form-safari"}); // contains html encoded character and unspecified charset defaults to utf-8 // form submitted as shift-jis (with HTML5 specific hidden _charset_ field) - // ret.add(new String[]{"browser-capture-sjis-charset-form-android-chrome"}); // contains html encoded character - // ret.add(new String[]{"browser-capture-sjis-charset-form-android-firefox"}); // contains html encoded character - // ret.add(new String[]{"browser-capture-sjis-charset-form-chrome"}); // contains html encoded character + ret.add(new String[]{"browser-capture-sjis-charset-form-android-chrome"}); // contains html encoded character + ret.add(new String[]{"browser-capture-sjis-charset-form-android-firefox"}); // contains html encoded character + ret.add(new String[]{"browser-capture-sjis-charset-form-chrome"}); // contains html encoded character ret.add(new String[]{"browser-capture-sjis-charset-form-edge"}); - // ret.add(new String[]{"browser-capture-sjis-charset-form-firefox"}); // contains html encoded character - // ret.add(new String[]{"browser-capture-sjis-charset-form-ios-safari"}); // contains html encoded character + ret.add(new String[]{"browser-capture-sjis-charset-form-firefox"}); // contains html encoded character + ret.add(new String[]{"browser-capture-sjis-charset-form-ios-safari"}); // contains html encoded character ret.add(new String[]{"browser-capture-sjis-charset-form-msie"}); - // ret.add(new String[]{"browser-capture-sjis-charset-form-safari"}); // contains html encoded character + ret.add(new String[]{"browser-capture-sjis-charset-form-safari"}); // contains html encoded character // form submitted with simple file upload ret.add(new String[]{"browser-capture-form-fileupload-android-chrome"}); diff --git a/jetty-http/src/test/java/org/eclipse/jetty/http/MultiPartFormInputStreamTest.java b/jetty-http/src/test/java/org/eclipse/jetty/http/MultiPartFormInputStreamTest.java index c7b558108cb..f48b4d4a309 100644 --- a/jetty-http/src/test/java/org/eclipse/jetty/http/MultiPartFormInputStreamTest.java +++ b/jetty-http/src/test/java/org/eclipse/jetty/http/MultiPartFormInputStreamTest.java @@ -1062,8 +1062,41 @@ public class MultiPartFormInputStreamTest } + @Test + public void testGeneratedForm() + throws Exception + { + String contentType = "multipart/form-data, boundary=WebKitFormBoundary7MA4YWf7OaKlSxkTrZu0gW"; + String body = "Content-Type: multipart/form-data; boundary=WebKitFormBoundary7MA4YWf7OaKlSxkTrZu0gW\r\n" + + "\r\n" + + "--WebKitFormBoundary7MA4YWf7OaKlSxkTrZu0gW\r\n" + + "Content-Disposition: form-data; name=\"part1\"\r\n" + + "\n" + + "wNfﾐxVam﾿t\r\n" + + "--WebKitFormBoundary7MA4YWf7OaKlSxkTrZu0gW\n" + + "Content-Disposition: form-data; name=\"part2\"\r\n" + + "\r\n" + + "&ﾳﾺ￙ￖￃO\r\n" + + "--WebKitFormBoundary7MA4YWf7OaKlSxkTrZu0gW--"; + MultipartConfigElement config = new MultipartConfigElement(_dirname, 1024, 3072, 50); + MultiPartFormInputStream mpis = new MultiPartFormInputStream(new ByteArrayInputStream(body.getBytes()), + contentType, + config, + _tmpDir); + mpis.setDeleteOnExit(true); + + Collection parts = mpis.getParts(); + assertThat(parts, notNullValue()); + assertThat(parts.size(), is(2)); + + Part part1 = mpis.getPart("part1"); + assertThat(part1, notNullValue()); + Part part2 = mpis.getPart("part2"); + assertThat(part2, notNullValue()); + } + private String createMultipartRequestString(String filename) { diff --git a/jetty-http/src/test/java/org/eclipse/jetty/http/MultiPartParserTest.java b/jetty-http/src/test/java/org/eclipse/jetty/http/MultiPartParserTest.java index 07b38af84b0..bc1c77b8fad 100644 --- a/jetty-http/src/test/java/org/eclipse/jetty/http/MultiPartParserTest.java +++ b/jetty-http/src/test/java/org/eclipse/jetty/http/MultiPartParserTest.java @@ -620,6 +620,52 @@ public class MultiPartParserTest } + @Test + public void testGeneratedForm() + { + TestHandler handler = new TestHandler() + { + @Override + public boolean messageComplete() + { + return true; + } + + @Override + public boolean content(ByteBuffer buffer, boolean last) + { + super.content(buffer,last); + return false; + } + + @Override + public boolean headerComplete() + { + return false; + } + }; + + MultiPartParser parser = new MultiPartParser(handler,"WebKitFormBoundary7MA4YWf7OaKlSxkTrZu0gW"); + ByteBuffer data = BufferUtil.toBuffer("" + + "Content-Type: multipart/form-data; boundary=WebKitFormBoundary7MA4YWf7OaKlSxkTrZu0gW\r\n" + + "\r\n" + + "--WebKitFormBoundary7MA4YWf7OaKlSxkTrZu0gW\r\n" + + "Content-Disposition: form-data; name=\"part1\"\r\n" + + "\n" + + "wNfﾐxVam﾿t\r\n" + + "--WebKitFormBoundary7MA4YWf7OaKlSxkTrZu0gW\n" + + "Content-Disposition: form-data; name=\"part2\"\r\n" + + "\r\n" + + "&ﾳﾺ￙ￖￃO\r\n" + + "--WebKitFormBoundary7MA4YWf7OaKlSxkTrZu0gW--"); + + parser.parse(data,true); + assertThat(parser.getState(), is(State.END)); + assertThat(handler.fields.size(), is(2)); + + } + + static class TestHandler implements MultiPartParser.Handler { List fields = new ArrayList<>(); diff --git a/jetty-http/src/test/java/org/eclipse/jetty/http/jmh/MultiPartBenchmark.java b/jetty-http/src/test/java/org/eclipse/jetty/http/jmh/MultiPartBenchmark.java new file mode 100644 index 00000000000..49924222a0c --- /dev/null +++ b/jetty-http/src/test/java/org/eclipse/jetty/http/jmh/MultiPartBenchmark.java @@ -0,0 +1,297 @@ +// +// ======================================================================== +// Copyright (c) 1995-2018 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.http.jmh; + +import java.io.File; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import javax.servlet.MultipartConfigElement; +import javax.servlet.http.Part; + +import org.eclipse.jetty.http.MultiPartFormInputStream; +import org.eclipse.jetty.http.MultiPartCaptureTest.MultipartExpectations; +import org.eclipse.jetty.toolchain.test.MavenTestingUtils; +import org.eclipse.jetty.toolchain.test.TestingDir; +import org.eclipse.jetty.util.BufferUtil; +import org.junit.Rule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.profile.CompilerProfiler; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@State(Scope.Benchmark) +public class MultiPartBenchmark +{ + + public static final int MAX_FILE_SIZE = Integer.MAX_VALUE; + public static final int MAX_REQUEST_SIZE = Integer.MAX_VALUE; + public static final int FILE_SIZE_THRESHOLD = 50; + + public int count = 0; + static String _contentType; + static File _file; + static int _numSections; + static int _numBytesPerSection; + + + public static List data = new ArrayList<>(); + static + { + // Capture of raw request body contents from various browsers + + // simple form - 2 fields + data.add("browser-capture-form1-android-chrome"); + data.add("browser-capture-form1-android-firefox"); + data.add("browser-capture-form1-chrome"); + data.add("browser-capture-form1-edge"); + data.add("browser-capture-form1-firefox"); + data.add("browser-capture-form1-ios-safari"); + data.add("browser-capture-form1-msie"); + data.add("browser-capture-form1-osx-safari"); + + // form submitted as shift-jis + data.add("browser-capture-sjis-form-edge"); + data.add("browser-capture-sjis-form-msie"); + + // form submitted as shift-jis (with HTML5 specific hidden _charset_ field) + data.add("browser-capture-sjis-charset-form-edge"); + data.add("browser-capture-sjis-charset-form-msie"); + + // form submitted with simple file upload + data.add("browser-capture-form-fileupload-android-chrome"); + data.add("browser-capture-form-fileupload-android-firefox"); + data.add("browser-capture-form-fileupload-chrome"); + data.add("browser-capture-form-fileupload-edge"); + data.add("browser-capture-form-fileupload-firefox"); + data.add("browser-capture-form-fileupload-ios-safari"); + data.add("browser-capture-form-fileupload-msie"); + data.add("browser-capture-form-fileupload-safari"); + + // form submitted with 2 files (1 binary, 1 text) and 2 text fields + data.add("browser-capture-form-fileupload-alt-chrome"); + data.add("browser-capture-form-fileupload-alt-edge"); + data.add("browser-capture-form-fileupload-alt-firefox"); + data.add("browser-capture-form-fileupload-alt-msie"); + data.add("browser-capture-form-fileupload-alt-safari"); + } + + + @Param({"UTIL","HTTP"}) + public static String parserType; + + @Setup(Level.Trial) + public static void setupTrial() throws Exception + { + _file = File.createTempFile("test01",null); + _file.deleteOnExit(); + + _numSections = 1; + _numBytesPerSection = 1024*1024*10; + + _contentType = "multipart/form-data, boundary=WebKitFormBoundary7MA4YWf7OaKlSxkTrZu0gW"; + String initialBoundary = "--WebKitFormBoundary7MA4YWf7OaKlSxkTrZu0gW\r\n"; + String boundary = "\r\n--WebKitFormBoundary7MA4YWf7OaKlSxkTrZu0gW\r\n"; + String closingBoundary = "\r\n--WebKitFormBoundary7MA4YWf7OaKlSxkTrZu0gW--\r\n"; + String headerStart = "Content-Disposition: form-data; name=\""; + + + for(int i=0; i<_numSections; i++) { + //boundary and headers + if(i==0) + Files.write(_file.toPath(), initialBoundary.getBytes(), StandardOpenOption.APPEND); + else + Files.write(_file.toPath(), boundary.getBytes(), StandardOpenOption.APPEND); + + Files.write(_file.toPath(), headerStart.getBytes(), StandardOpenOption.APPEND); + Files.write(_file.toPath(), new String("part"+(i+1)).getBytes(), StandardOpenOption.APPEND); + Files.write(_file.toPath(), new String("\"\r\n\r\n").getBytes(), StandardOpenOption.APPEND); + + //append random data + byte[] data = new byte[_numBytesPerSection]; + new Random().nextBytes(data); + Files.write(_file.toPath(), data, StandardOpenOption.APPEND); + } + + //closing boundary + Files.write(_file.toPath(), closingBoundary.getBytes(), StandardOpenOption.APPEND); + + /* + // print out file to verify that it contains valid contents (just for testing) + InputStream in = Files.newInputStream(_file.toPath()); + System.out.println(); + while(in.available()>0) { + byte b[] = new byte[100]; + int read = in.read(b,0,100); + for(int i=0; i