From a2011abec0bd7e31ca8d339fe33e210fd1d2c762 Mon Sep 17 00:00:00 2001 From: Simone Bordet Date: Thu, 11 Jul 2019 11:46:06 +0200 Subject: [PATCH 01/33] Fixes #3829 - Avoid sending empty trailer frames for http/2 responses. Fixed the logic to send response trailers. Signed-off-by: Simone Bordet --- .../client/http/ResponseTrailerTest.java | 127 ++++++++++++++++++ .../test/resources/jetty-logging.properties | 1 + .../http2/server/HttpTransportOverHTTP2.java | 72 ++++++---- 3 files changed, 177 insertions(+), 23 deletions(-) create mode 100644 jetty-http2/http2-http-client-transport/src/test/java/org/eclipse/jetty/http2/client/http/ResponseTrailerTest.java diff --git a/jetty-http2/http2-http-client-transport/src/test/java/org/eclipse/jetty/http2/client/http/ResponseTrailerTest.java b/jetty-http2/http2-http-client-transport/src/test/java/org/eclipse/jetty/http2/client/http/ResponseTrailerTest.java new file mode 100644 index 00000000000..11727ee7425 --- /dev/null +++ b/jetty-http2/http2-http-client-transport/src/test/java/org/eclipse/jetty/http2/client/http/ResponseTrailerTest.java @@ -0,0 +1,127 @@ +// +// ======================================================================== +// Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.http2.client.http; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.eclipse.jetty.http.HttpFields; +import org.eclipse.jetty.http.HttpMethod; +import org.eclipse.jetty.http.HttpURI; +import org.eclipse.jetty.http.HttpVersion; +import org.eclipse.jetty.http.MetaData; +import org.eclipse.jetty.http2.api.Session; +import org.eclipse.jetty.http2.api.Stream; +import org.eclipse.jetty.http2.client.HTTP2Client; +import org.eclipse.jetty.http2.frames.DataFrame; +import org.eclipse.jetty.http2.frames.HeadersFrame; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Response; +import org.eclipse.jetty.util.Callback; +import org.eclipse.jetty.util.FuturePromise; +import org.eclipse.jetty.util.Promise; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ResponseTrailerTest extends AbstractTest +{ + @Test + public void testEmptyTrailersWithoutContent() throws Exception + { + testEmptyTrailers(null); + } + + @Test + public void testEmptyTrailersWithContent() throws Exception + { + testEmptyTrailers("data"); + } + + public void testEmptyTrailers(String data) throws Exception + { + start(new EmptyServerHandler() + { + @Override + protected void service(String target, Request jettyRequest, HttpServletRequest request, HttpServletResponse response) throws IOException + { + // Send empty response trailers. + Response jettyResponse = jettyRequest.getResponse(); + jettyResponse.setTrailers(HttpFields::new); + if (data != null) + response.getOutputStream().write(data.getBytes(StandardCharsets.US_ASCII)); + } + }); + + HTTP2Client http2Client = new HTTP2Client(); + http2Client.start(); + try + { + String host = "localhost"; + int port = connector.getLocalPort(); + InetSocketAddress address = new InetSocketAddress(host, port); + FuturePromise sessionPromise = new FuturePromise<>(); + http2Client.connect(address, new Session.Listener.Adapter(), sessionPromise); + Session session = sessionPromise.get(5, TimeUnit.SECONDS); + + HttpURI uri = new HttpURI("http://" + host + ":" + port + "/"); + MetaData.Request request = new MetaData.Request(HttpMethod.GET.asString(), uri, HttpVersion.HTTP_2, new HttpFields()); + HeadersFrame frame = new HeadersFrame(request, null, true); + BlockingQueue headers = new LinkedBlockingQueue<>(); + CountDownLatch latch = new CountDownLatch(1); + session.newStream(frame, new Promise.Adapter<>(), new Stream.Listener.Adapter() + { + @Override + public void onHeaders(Stream stream, HeadersFrame frame) + { + headers.offer(frame); + if (frame.isEndStream()) + latch.countDown(); + } + + @Override + public void onData(Stream stream, DataFrame frame, Callback callback) + { + super.onData(stream, frame, callback); + if (frame.isEndStream()) + latch.countDown(); + } + }); + + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertEquals(1, headers.size()); + frame = headers.poll(); + assertNotNull(frame); + assertTrue(frame.getMetaData().isResponse()); + } + finally + { + http2Client.stop(); + } + } +} diff --git a/jetty-http2/http2-http-client-transport/src/test/resources/jetty-logging.properties b/jetty-http2/http2-http-client-transport/src/test/resources/jetty-logging.properties index 287d28319e0..34929219c9d 100644 --- a/jetty-http2/http2-http-client-transport/src/test/resources/jetty-logging.properties +++ b/jetty-http2/http2-http-client-transport/src/test/resources/jetty-logging.properties @@ -1,4 +1,5 @@ org.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.StdErrLog +#org.eclipse.jetty.LEVEL=DEBUG #org.eclipse.jetty.client.LEVEL=DEBUG org.eclipse.jetty.http2.hpack.LEVEL=INFO #org.eclipse.jetty.http2.LEVEL=DEBUG diff --git a/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java b/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java index 7b86321ac83..2df6adb62e3 100644 --- a/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java +++ b/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java @@ -119,9 +119,17 @@ public class HttpTransportOverHTTP2 implements HttpTransport { if (lastContent) { - Supplier trailers = info.getTrailerSupplier(); - if (transportCallback.start(new SendTrailers(getCallback(), trailers), false)) - sendDataFrame(content, true, trailers == null, transportCallback); + HttpFields trailers = retrieveTrailers(); + if (trailers != null) + { + if (transportCallback.start(new SendTrailers(getCallback(), trailers), false)) + sendDataFrame(content, true, false, transportCallback); + } + else + { + if (transportCallback.start(getCallback(), false)) + sendDataFrame(content, true, true, transportCallback); + } } else { @@ -137,9 +145,17 @@ public class HttpTransportOverHTTP2 implements HttpTransport { if (lastContent) { - Supplier trailers = info.getTrailerSupplier(); - if (transportCallback.start(new SendTrailers(callback, trailers), true)) - sendHeadersFrame(info, trailers == null, transportCallback); + HttpFields trailers = retrieveTrailers(); + if (trailers != null) + { + if (transportCallback.start(new SendTrailers(callback, trailers), true)) + sendHeadersFrame(info, false, transportCallback); + } + else + { + if (transportCallback.start(callback, true)) + sendHeadersFrame(info, true, transportCallback); + } } else { @@ -160,16 +176,24 @@ public class HttpTransportOverHTTP2 implements HttpTransport { if (lastContent) { - Supplier trailers = metaData.getTrailerSupplier(); - SendTrailers sendTrailers = new SendTrailers(callback, trailers); - if (hasContent || trailers == null) + HttpFields trailers = retrieveTrailers(); + if (trailers != null) { - if (transportCallback.start(sendTrailers, false)) - sendDataFrame(content, true, trailers == null, transportCallback); + SendTrailers sendTrailers = new SendTrailers(callback, trailers); + if (hasContent) + { + if (transportCallback.start(sendTrailers, false)) + sendDataFrame(content, true, false, transportCallback); + } + else + { + sendTrailers.succeeded(); + } } else { - sendTrailers.succeeded(); + if (transportCallback.start(callback, false)) + sendDataFrame(content, true, true, transportCallback); } } else @@ -185,6 +209,15 @@ public class HttpTransportOverHTTP2 implements HttpTransport } } + private HttpFields retrieveTrailers() + { + Supplier supplier = metaData.getTrailerSupplier(); + if (supplier == null) + return null; + HttpFields trailers = supplier.get(); + return trailers.size() == 0 ? null : trailers; + } + @Override public boolean isPushSupported() { @@ -420,9 +453,9 @@ public class HttpTransportOverHTTP2 implements HttpTransport private class SendTrailers extends Callback.Nested { - private final Supplier trailers; + private final HttpFields trailers; - private SendTrailers(Callback callback, Supplier trailers) + private SendTrailers(Callback callback, HttpFields trailers) { super(callback); this.trailers = trailers; @@ -431,15 +464,8 @@ public class HttpTransportOverHTTP2 implements HttpTransport @Override public void succeeded() { - if (trailers != null) - { - if (transportCallback.start(getCallback(), false)) - sendTrailersFrame(new MetaData(HttpVersion.HTTP_2, trailers.get()), transportCallback); - } - else - { - super.succeeded(); - } + if (transportCallback.start(getCallback(), false)) + sendTrailersFrame(new MetaData(HttpVersion.HTTP_2, trailers), transportCallback); } } } From 9f84c1cb8ed5ff2230271efcd0f21cdfb2979975 Mon Sep 17 00:00:00 2001 From: Simone Bordet Date: Thu, 11 Jul 2019 15:06:15 +0200 Subject: [PATCH 02/33] Fixes #3829 - Avoid sending empty trailer frames for http/2 responses. Added guard against the supplier of trailers returning null. Signed-off-by: Simone Bordet --- .../org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java b/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java index 2df6adb62e3..b19123bb8f1 100644 --- a/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java +++ b/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java @@ -215,6 +215,8 @@ public class HttpTransportOverHTTP2 implements HttpTransport if (supplier == null) return null; HttpFields trailers = supplier.get(); + if (trailers == null) + return null; return trailers.size() == 0 ? null : trailers; } From ec9f4ef392a59fec0f375f81a4f10294908c9642 Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Tue, 16 Jul 2019 07:07:55 -0500 Subject: [PATCH 03/33] Issue #3876 - WebSocket Partial Listener testcases and fix Signed-off-by: Joakim Erdfelt --- .../tests/server/PartialListenerTest.java | 239 ++++++++++++++++++ .../events/JettyListenerEventDriver.java | 83 +++++- 2 files changed, 312 insertions(+), 10 deletions(-) create mode 100644 jetty-websocket/jetty-websocket-tests/src/test/java/org/eclipse/jetty/websocket/tests/server/PartialListenerTest.java diff --git a/jetty-websocket/jetty-websocket-tests/src/test/java/org/eclipse/jetty/websocket/tests/server/PartialListenerTest.java b/jetty-websocket/jetty-websocket-tests/src/test/java/org/eclipse/jetty/websocket/tests/server/PartialListenerTest.java new file mode 100644 index 00000000000..4fd215103c4 --- /dev/null +++ b/jetty-websocket/jetty-websocket-tests/src/test/java/org/eclipse/jetty/websocket/tests/server/PartialListenerTest.java @@ -0,0 +1,239 @@ +// +// ======================================================================== +// Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.websocket.tests.server; + +import java.net.URI; +import java.nio.ByteBuffer; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; + +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.BufferUtil; +import org.eclipse.jetty.util.log.StacklessLogging; +import org.eclipse.jetty.websocket.api.RemoteEndpoint; +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.WebSocketPartialListener; +import org.eclipse.jetty.websocket.api.util.WSURI; +import org.eclipse.jetty.websocket.client.ClientUpgradeRequest; +import org.eclipse.jetty.websocket.client.WebSocketClient; +import org.eclipse.jetty.websocket.common.WebSocketSession; +import org.eclipse.jetty.websocket.common.util.TextUtil; +import org.eclipse.jetty.websocket.servlet.ServletUpgradeRequest; +import org.eclipse.jetty.websocket.servlet.ServletUpgradeResponse; +import org.eclipse.jetty.websocket.servlet.WebSocketCreator; +import org.eclipse.jetty.websocket.servlet.WebSocketServlet; +import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; +import org.eclipse.jetty.websocket.tests.CloseTrackingEndpoint; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +public class PartialListenerTest +{ + private Server server; + private PartialCreator partialCreator; + private WebSocketClient client; + + @BeforeEach + public void startServer() throws Exception + { + server = new Server(); + + ServerConnector connector = new ServerConnector(server); + server.addConnector(connector); + + ServletContextHandler context = new ServletContextHandler(); + context.setContextPath("/"); + + ServletHolder closeEndpoint = new ServletHolder(new WebSocketServlet() + { + @Override + public void configure(WebSocketServletFactory factory) + { + factory.getPolicy().setIdleTimeout(SECONDS.toMillis(2)); + partialCreator = new PartialCreator(); + factory.setCreator(partialCreator); + } + }); + context.addServlet(closeEndpoint, "/ws"); + + HandlerList handlers = new HandlerList(); + handlers.addHandler(context); + handlers.addHandler(new DefaultHandler()); + + server.setHandler(handlers); + + server.start(); + } + + @AfterEach + public void stopServer() throws Exception + { + server.stop(); + } + + @BeforeEach + public void startClient() throws Exception + { + client = new WebSocketClient(); + client.start(); + } + + @AfterEach + public void stopClient() throws Exception + { + client.stop(); + } + + private void close(Session session) + { + if (session != null) + { + session.close(); + } + } + + @Test + public void testPartialText() throws Exception + { + ClientUpgradeRequest request = new ClientUpgradeRequest(); + CloseTrackingEndpoint clientEndpoint = new CloseTrackingEndpoint(); + + URI wsUri = WSURI.toWebsocket(server.getURI().resolve("/ws")); + Future futSession = client.connect(clientEndpoint, wsUri, request); + + Session session = null; + try (StacklessLogging ignore = new StacklessLogging(WebSocketSession.class)) + { + session = futSession.get(5, SECONDS); + + RemoteEndpoint clientRemote = session.getRemote(); + clientRemote.sendPartialString("hello", false); + clientRemote.sendPartialString(" ", false); + clientRemote.sendPartialString("world", true); + + PartialEndpoint serverEndpoint = partialCreator.partialEndpoint; + + String event = serverEndpoint.partialEvents.poll(5, SECONDS); + assertThat("Event", event, is("TEXT[payload=hello, fin=false]")); + event = serverEndpoint.partialEvents.poll(5, SECONDS); + assertThat("Event", event, is("TEXT[payload= , fin=false]")); + event = serverEndpoint.partialEvents.poll(5, SECONDS); + assertThat("Event", event, is("TEXT[payload=world, fin=true]")); + } + finally + { + close(session); + } + } + + @Test + public void testPartialBinary() throws Exception + { + ClientUpgradeRequest request = new ClientUpgradeRequest(); + CloseTrackingEndpoint clientEndpoint = new CloseTrackingEndpoint(); + + URI wsUri = WSURI.toWebsocket(server.getURI().resolve("/ws")); + Future futSession = client.connect(clientEndpoint, wsUri, request); + + Session session = null; + try (StacklessLogging ignore = new StacklessLogging(WebSocketSession.class)) + { + session = futSession.get(5, SECONDS); + + RemoteEndpoint clientRemote = session.getRemote(); + clientRemote.sendPartialBytes(BufferUtil.toBuffer("hello"), false); + clientRemote.sendPartialBytes(BufferUtil.toBuffer(" "), false); + clientRemote.sendPartialBytes(BufferUtil.toBuffer("world"), true); + + PartialEndpoint serverEndpoint = partialCreator.partialEndpoint; + + String event = serverEndpoint.partialEvents.poll(5, SECONDS); + assertThat("Event", event, is("BINARY[payload=<<>>, fin=false]")); + event = serverEndpoint.partialEvents.poll(5, SECONDS); + assertThat("Event", event, is("BINARY[payload=<<< >>>, fin=false]")); + event = serverEndpoint.partialEvents.poll(5, SECONDS); + assertThat("Event", event, is("BINARY[payload=<<>>, fin=true]")); + } + finally + { + close(session); + } + } + + public static class PartialCreator implements WebSocketCreator + { + public PartialEndpoint partialEndpoint; + + @Override + public Object createWebSocket(ServletUpgradeRequest req, ServletUpgradeResponse resp) + { + partialEndpoint = new PartialEndpoint(); + return partialEndpoint; + } + } + + public static class PartialEndpoint implements WebSocketPartialListener + { + public Session session; + public CountDownLatch closeLatch = new CountDownLatch(1); + public LinkedBlockingQueue partialEvents = new LinkedBlockingQueue<>(); + + @Override + public void onWebSocketClose(int statusCode, String reason) + { + closeLatch.countDown(); + } + + @Override + public void onWebSocketConnect(Session session) + { + this.session = session; + } + + @Override + public void onWebSocketError(Throwable cause) + { + cause.printStackTrace(System.err); + } + + @Override + public void onWebSocketPartialBinary(ByteBuffer payload, boolean fin) + { + // our testcases always send bytes limited in the US-ASCII range. + partialEvents.offer(String.format("BINARY[payload=<<<%s>>>, fin=%b]", BufferUtil.toUTF8String(payload), fin)); + } + + @Override + public void onWebSocketPartialText(String payload, boolean fin) + { + partialEvents.offer(String.format("TEXT[payload=%s, fin=%b]", TextUtil.maxStringLength(30, payload), fin)); + } + } +} diff --git a/jetty-websocket/websocket-common/src/main/java/org/eclipse/jetty/websocket/common/events/JettyListenerEventDriver.java b/jetty-websocket/websocket-common/src/main/java/org/eclipse/jetty/websocket/common/events/JettyListenerEventDriver.java index 244460fadf8..125b68db913 100644 --- a/jetty-websocket/websocket-common/src/main/java/org/eclipse/jetty/websocket/common/events/JettyListenerEventDriver.java +++ b/jetty-websocket/websocket-common/src/main/java/org/eclipse/jetty/websocket/common/events/JettyListenerEventDriver.java @@ -23,6 +23,7 @@ import java.io.InputStream; import java.io.Reader; import java.nio.ByteBuffer; +import org.eclipse.jetty.util.Utf8StringBuilder; import org.eclipse.jetty.util.log.Log; import org.eclipse.jetty.util.log.Logger; import org.eclipse.jetty.websocket.api.WebSocketConnectionListener; @@ -37,16 +38,21 @@ import org.eclipse.jetty.websocket.common.CloseInfo; import org.eclipse.jetty.websocket.common.frames.ReadOnlyDelegatedFrame; import org.eclipse.jetty.websocket.common.message.SimpleBinaryMessage; import org.eclipse.jetty.websocket.common.message.SimpleTextMessage; -import org.eclipse.jetty.websocket.common.util.Utf8PartialBuilder; /** * Handler for {@link WebSocketListener} based User WebSocket implementations. */ public class JettyListenerEventDriver extends AbstractEventDriver { + private enum PartialMode + { + NONE, TEXT, BINARY + } + private static final Logger LOG = Log.getLogger(JettyListenerEventDriver.class); private final WebSocketConnectionListener listener; - private Utf8PartialBuilder utf8Partial; + private Utf8StringBuilder utf8Partial; + private PartialMode partialMode = PartialMode.NONE; private boolean hasCloseBeenCalled = false; public JettyListenerEventDriver(WebSocketPolicy policy, WebSocketConnectionListener listener) @@ -70,7 +76,22 @@ public class JettyListenerEventDriver extends AbstractEventDriver if (listener instanceof WebSocketPartialListener) { - ((WebSocketPartialListener)listener).onWebSocketPartialBinary(buffer.slice().asReadOnlyBuffer(), fin); + switch (partialMode) + { + case NONE: + partialMode = PartialMode.BINARY; + // fallthru + case BINARY: + ((WebSocketPartialListener)listener).onWebSocketPartialBinary(buffer.slice().asReadOnlyBuffer(), fin); + break; + case TEXT: + throw new IOException("Out of order binary frame encountered"); + } + + if (fin) + { + partialMode = PartialMode.NONE; + } } } @@ -160,18 +181,39 @@ public class JettyListenerEventDriver extends AbstractEventDriver if (listener instanceof WebSocketPartialListener) { - if (utf8Partial == null) + switch (partialMode) { - utf8Partial = new Utf8PartialBuilder(); + case NONE: + partialMode = PartialMode.TEXT; + // fallthru + case TEXT: + if (utf8Partial == null) + { + utf8Partial = new Utf8StringBuilder(); + } + + String partial = ""; + + if (buffer != null) + { + utf8Partial.append(buffer); + partial = utf8Partial.takePartialString(); + } + + ((WebSocketPartialListener)listener).onWebSocketPartialText(partial, fin); + + if (fin) + { + utf8Partial = null; + } + break; + case BINARY: + throw new IOException("Out of order text frame encountered"); } - String partial = utf8Partial.toPartialString(buffer); - - ((WebSocketPartialListener)listener).onWebSocketPartialText(partial, fin); - if (fin) { - partial = null; + partialMode = PartialMode.NONE; } } } @@ -190,6 +232,27 @@ public class JettyListenerEventDriver extends AbstractEventDriver } } + public void onContinuationFrame(ByteBuffer buffer, boolean fin) throws IOException + { + if (listener instanceof WebSocketPartialListener) + { + switch (partialMode) + { + case NONE: + throw new IOException("Out of order Continuation frame encountered"); + case TEXT: + onTextFrame(buffer, fin); + break; + case BINARY: + onBinaryFrame(buffer, fin); + break; + } + return; + } + + super.onContinuationFrame(buffer, fin); + } + @Override public String toString() { From f20d4034809dc11beb9359df7781731963fec7dc Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Tue, 16 Jul 2019 13:04:19 -0500 Subject: [PATCH 04/33] Issue #3876 - Additional testcase Signed-off-by: Joakim Erdfelt --- .../tests/server/PartialListenerTest.java | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/jetty-websocket/jetty-websocket-tests/src/test/java/org/eclipse/jetty/websocket/tests/server/PartialListenerTest.java b/jetty-websocket/jetty-websocket-tests/src/test/java/org/eclipse/jetty/websocket/tests/server/PartialListenerTest.java index 4fd215103c4..0a895a8c456 100644 --- a/jetty-websocket/jetty-websocket-tests/src/test/java/org/eclipse/jetty/websocket/tests/server/PartialListenerTest.java +++ b/jetty-websocket/jetty-websocket-tests/src/test/java/org/eclipse/jetty/websocket/tests/server/PartialListenerTest.java @@ -187,6 +187,68 @@ public class PartialListenerTest } } + /** + * Test to ensure that the internal state tracking the partial messages is reset after each complete message. + */ + @Test + public void testPartial_TextBinaryText() throws Exception + { + ClientUpgradeRequest request = new ClientUpgradeRequest(); + CloseTrackingEndpoint clientEndpoint = new CloseTrackingEndpoint(); + + URI wsUri = WSURI.toWebsocket(server.getURI().resolve("/ws")); + Future futSession = client.connect(clientEndpoint, wsUri, request); + + Session session = null; + try (StacklessLogging ignore = new StacklessLogging(WebSocketSession.class)) + { + session = futSession.get(5, SECONDS); + + RemoteEndpoint clientRemote = session.getRemote(); + clientRemote.sendPartialString("hello", false); + clientRemote.sendPartialString(" ", false); + clientRemote.sendPartialString("world", true); + + clientRemote.sendPartialBytes(BufferUtil.toBuffer("greetings"), false); + clientRemote.sendPartialBytes(BufferUtil.toBuffer(" "), false); + clientRemote.sendPartialBytes(BufferUtil.toBuffer("mars"), true); + + clientRemote.sendPartialString("salutations", false); + clientRemote.sendPartialString(" ", false); + clientRemote.sendPartialString("phobos", true); + + PartialEndpoint serverEndpoint = partialCreator.partialEndpoint; + + String event; + + event = serverEndpoint.partialEvents.poll(5, SECONDS); + assertThat("Event", event, is("TEXT[payload=hello, fin=false]")); + event = serverEndpoint.partialEvents.poll(5, SECONDS); + assertThat("Event", event, is("TEXT[payload= , fin=false]")); + event = serverEndpoint.partialEvents.poll(5, SECONDS); + assertThat("Event", event, is("TEXT[payload=world, fin=true]")); + + event = serverEndpoint.partialEvents.poll(5, SECONDS); + assertThat("Event", event, is("BINARY[payload=<<>>, fin=false]")); + event = serverEndpoint.partialEvents.poll(5, SECONDS); + assertThat("Event", event, is("BINARY[payload=<<< >>>, fin=false]")); + event = serverEndpoint.partialEvents.poll(5, SECONDS); + assertThat("Event", event, is("BINARY[payload=<<>>, fin=true]")); + + event = serverEndpoint.partialEvents.poll(5, SECONDS); + assertThat("Event", event, is("TEXT[payload=salutations, fin=false]")); + event = serverEndpoint.partialEvents.poll(5, SECONDS); + assertThat("Event", event, is("TEXT[payload= , fin=false]")); + event = serverEndpoint.partialEvents.poll(5, SECONDS); + assertThat("Event", event, is("TEXT[payload=phobos, fin=true]")); + } + finally + { + close(session); + } + } + + public static class PartialCreator implements WebSocketCreator { public PartialEndpoint partialEndpoint; From aef58168d06f3e1edd95e2a007be9df5f1f4ccb2 Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Thu, 18 Jul 2019 11:47:37 -0500 Subject: [PATCH 05/33] Issue #3888 - Adding testcase to demonstrate Huge file Resource issue Signed-off-by: Joakim Erdfelt --- .../jetty/webapp/HugeResourceTest.java | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 jetty-webapp/src/test/java/org/eclipse/jetty/webapp/HugeResourceTest.java diff --git a/jetty-webapp/src/test/java/org/eclipse/jetty/webapp/HugeResourceTest.java b/jetty-webapp/src/test/java/org/eclipse/jetty/webapp/HugeResourceTest.java new file mode 100644 index 00000000000..09d9747772f --- /dev/null +++ b/jetty-webapp/src/test/java/org/eclipse/jetty/webapp/HugeResourceTest.java @@ -0,0 +1,189 @@ +// +// ======================================================================== +// Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.webapp; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; + +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.toolchain.test.FS; +import org.eclipse.jetty.toolchain.test.MavenTestingUtils; +import org.eclipse.jetty.util.IO; +import org.eclipse.jetty.util.resource.PathResource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +public class HugeResourceTest +{ + private static final long KB = 1024; + private static final long MB = 1024 * KB; + private static final long GB = 1024 * MB; + public static Path hugeStaticBase; + public static Path outputDir; + + public Server server; + + @BeforeAll + public static void prepareStaticFiles() throws IOException + { + hugeStaticBase = MavenTestingUtils.getTargetTestingPath(HugeResourceTest.class.getSimpleName() + "-huge-static-base"); + FS.ensureDirExists(hugeStaticBase); + + makeStaticFile(hugeStaticBase.resolve("test-1g.dat"), 1 * GB); + makeStaticFile(hugeStaticBase.resolve("test-4g.dat"), 4 * GB); + makeStaticFile(hugeStaticBase.resolve("test-10g.dat"), 10 * GB); + + outputDir = MavenTestingUtils.getTargetTestingPath(HugeResourceTest.class.getSimpleName() + "-huge-static-outputdir"); + FS.ensureEmpty(outputDir); + } + + @AfterAll + public static void cleanupHugeStaticFiles() + { + FS.ensureDeleted(hugeStaticBase); + FS.ensureDeleted(outputDir); + } + + private static void makeStaticFile(Path staticFile, long size) throws IOException + { + byte[] buf = new byte[(int)(1 * MB)]; + Arrays.fill(buf, (byte)'x'); + ByteBuffer src = ByteBuffer.wrap(buf); + + if (Files.exists(staticFile) && Files.size(staticFile) == size) + { + // all done, nothing left to do. + return; + } + + System.err.printf("Creating %,d byte file: %s ...%n", size, staticFile.getFileName()); + try (SeekableByteChannel channel = Files.newByteChannel(staticFile, StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) + { + long remaining = size; + while (remaining > 0) + { + ByteBuffer slice = src.slice(); + int len = buf.length; + if (remaining < Integer.MAX_VALUE) + { + len = Math.min(buf.length, (int)remaining); + slice.limit(len); + } + + channel.write(slice); + remaining -= len; + } + } + System.err.println(" Done"); + } + + @BeforeEach + public void startServer() throws Exception + { + server = new Server(); + ServerConnector connector = new ServerConnector(server); + connector.setPort(0); + server.addConnector(connector); + + WebAppContext context = new WebAppContext(); + context.setContextPath("/"); + context.setBaseResource(new PathResource(hugeStaticBase)); + + HandlerList handlers = new HandlerList(); + handlers.addHandler(context); + handlers.addHandler(new DefaultHandler()); + + server.setHandler(handlers); + server.start(); + } + + @AfterEach + public void stopServer() throws Exception + { + server.stop(); + } + + @Test + public void testDownload_1G() throws IOException + { + download(server.getURI().resolve("/test-1g.dat"), 1 * GB); + } + + @Test + public void testDownload_4G() throws IOException + { + download(server.getURI().resolve("/test-4g.dat"), 4 * GB); + } + + @Test + public void testDownload_10G() throws IOException + { + download(server.getURI().resolve("/test-10g.dat"), 10 * GB); + } + + private void download(URI destUri, long expectedSize) throws IOException + { + HttpURLConnection http = (HttpURLConnection)destUri.toURL().openConnection(); + assertThat("HTTP Response Code", http.getResponseCode(), is(200)); + + // if a Content-Length is provided, test it + String contentLength = http.getHeaderField("Content-Length"); + if (contentLength != null) + { + long contentLengthLong = Long.parseLong(contentLength); + assertThat("Http Response Header: \"Content-Length: " + contentLength + "\"", contentLengthLong, is(expectedSize)); + } + + // Download the file + String filename = destUri.getPath(); + int idx = filename.lastIndexOf('/'); + if (idx >= 0) + { + filename = filename.substring(idx + 1); + } + + Path outputFile = outputDir.resolve(filename); + try (OutputStream out = Files.newOutputStream(outputFile); + InputStream in = http.getInputStream()) + { + IO.copy(in, out); + } + + // Verify the file download size + assertThat("Downloaded Files Size: " + filename, Files.size(outputFile), is(expectedSize)); + } +} From 36294ef0da95f63545c027ce78a9159f153dca82 Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Thu, 18 Jul 2019 12:13:35 -0500 Subject: [PATCH 06/33] Issue #3888 - Fixing for truncated long to int + Allowing CachedHttpContent._contentLengthValue actually hold the `long` resource size (for the 4G variant on test) + Allowing BufferUtil to not throw Exception if resource length is a positive value, but exceeds Integer.MAX_VALUE, opting instead to return a null to prevent excessive memory usage. (fixes the 10G variant of test) Signed-off-by: Joakim Erdfelt --- .../eclipse/jetty/server/CachedContentFactory.java | 6 +++--- .../java/org/eclipse/jetty/util/BufferUtil.java | 14 +++++++++++--- .../org/eclipse/jetty/webapp/HugeResourceTest.java | 14 ++++++++++++++ 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/CachedContentFactory.java b/jetty-server/src/main/java/org/eclipse/jetty/server/CachedContentFactory.java index faa20fe8da0..5f76fcdb436 100644 --- a/jetty-server/src/main/java/org/eclipse/jetty/server/CachedContentFactory.java +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/CachedContentFactory.java @@ -386,7 +386,7 @@ public class CachedContentFactory implements HttpContent.ContentFactory { private final String _key; private final Resource _resource; - private final int _contentLengthValue; + private final long _contentLengthValue; private final HttpField _contentType; private final String _characterEncoding; private final MimeTypes.Type _mimeType; @@ -415,7 +415,7 @@ public class CachedContentFactory implements HttpContent.ContentFactory _lastModified = _lastModifiedValue == -1 ? null : new PreEncodedHttpField(HttpHeader.LAST_MODIFIED, DateGenerator.formatDate(_lastModifiedValue)); - _contentLengthValue = exists ? (int)resource.length() : 0; + _contentLengthValue = exists ? resource.length() : 0; _contentLength = new PreEncodedHttpField(HttpHeader.CONTENT_LENGTH, Long.toString(_contentLengthValue)); if (_cachedFiles.incrementAndGet() > _maxCachedFiles) @@ -558,7 +558,7 @@ public class CachedContentFactory implements HttpContent.ContentFactory ByteBuffer buffer2 = CachedContentFactory.this.getIndirectBuffer(_resource); if (buffer2 == null) - LOG.warn("Could not load " + this); + LOG.warn("Could not load indirect buffer from " + this); else if (_indirectBuffer.compareAndSet(null, buffer2)) { buffer = buffer2; diff --git a/jetty-util/src/main/java/org/eclipse/jetty/util/BufferUtil.java b/jetty-util/src/main/java/org/eclipse/jetty/util/BufferUtil.java index 7b948213a3f..a0eceef6307 100644 --- a/jetty-util/src/main/java/org/eclipse/jetty/util/BufferUtil.java +++ b/jetty-util/src/main/java/org/eclipse/jetty/util/BufferUtil.java @@ -971,11 +971,19 @@ public class BufferUtil public static ByteBuffer toBuffer(Resource resource, boolean direct) throws IOException { - int len = (int)resource.length(); + long len = resource.length(); if (len < 0) throw new IllegalArgumentException("invalid resource: " + resource + " len=" + len); - ByteBuffer buffer = direct ? BufferUtil.allocateDirect(len) : BufferUtil.allocate(len); + if (len > Integer.MAX_VALUE) + { + // This method cannot handle resources of this size. + return null; + } + + int ilen = (int)len; + + ByteBuffer buffer = direct ? BufferUtil.allocateDirect(ilen) : BufferUtil.allocate(ilen); int pos = BufferUtil.flipToFill(buffer); if (resource.getFile() != null) @@ -984,7 +992,7 @@ public class BufferUtil { try (InputStream is = resource.getInputStream()) { - BufferUtil.readFrom(is, len, buffer); + BufferUtil.readFrom(is, ilen, buffer); } } BufferUtil.flipToFlush(buffer, pos); diff --git a/jetty-webapp/src/test/java/org/eclipse/jetty/webapp/HugeResourceTest.java b/jetty-webapp/src/test/java/org/eclipse/jetty/webapp/HugeResourceTest.java index 09d9747772f..5a7afe13813 100644 --- a/jetty-webapp/src/test/java/org/eclipse/jetty/webapp/HugeResourceTest.java +++ b/jetty-webapp/src/test/java/org/eclipse/jetty/webapp/HugeResourceTest.java @@ -160,6 +160,8 @@ public class HugeResourceTest HttpURLConnection http = (HttpURLConnection)destUri.toURL().openConnection(); assertThat("HTTP Response Code", http.getResponseCode(), is(200)); + dumpResponseHeaders(http); + // if a Content-Length is provided, test it String contentLength = http.getHeaderField("Content-Length"); if (contentLength != null) @@ -186,4 +188,16 @@ public class HugeResourceTest // Verify the file download size assertThat("Downloaded Files Size: " + filename, Files.size(outputFile), is(expectedSize)); } + + private void dumpResponseHeaders(HttpURLConnection http) + { + int i = 0; + String value; + while ((value = http.getHeaderField(i)) != null) + { + String key = http.getHeaderFieldKey(i); + System.err.printf(" %s: %s%n", key, value); + i++; + } + } } From f856ad41da2c20817d124066a75452d54a13f0d5 Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Fri, 19 Jul 2019 10:25:04 -0500 Subject: [PATCH 07/33] Issue #3888 - For Resources exceeding maxCachedFileSize do not allocate + Only used Mapped ByteBuffers, ReadableByteChannel, or InputStream Signed-off-by: Joakim Erdfelt --- .../org/eclipse/jetty/server/CachedContentFactory.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/CachedContentFactory.java b/jetty-server/src/main/java/org/eclipse/jetty/server/CachedContentFactory.java index 5f76fcdb436..59059d91cd6 100644 --- a/jetty-server/src/main/java/org/eclipse/jetty/server/CachedContentFactory.java +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/CachedContentFactory.java @@ -552,6 +552,11 @@ public class CachedContentFactory implements HttpContent.ContentFactory @Override public ByteBuffer getIndirectBuffer() { + if (_resource.length() > (long)_maxCachedFileSize) + { + return null; + } + ByteBuffer buffer = _indirectBuffer.get(); if (buffer == null) { @@ -589,7 +594,7 @@ public class CachedContentFactory implements HttpContent.ContentFactory else buffer = _mappedBuffer.get(); } - else + else if (_resource.length() <= (long)_maxCachedFileSize) { ByteBuffer direct = CachedContentFactory.this.getDirectBuffer(_resource); if (direct != null) From 8afc4464d5f8fe6d6e4e1ee59d413f99f2765b87 Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Thu, 28 Mar 2019 11:08:05 -0500 Subject: [PATCH 08/33] Issue #3503 - Adding testcases for huge Content-Length behaviors Signed-off-by: Joakim Erdfelt --- .../org/eclipse/jetty/server/RequestTest.java | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/jetty-server/src/test/java/org/eclipse/jetty/server/RequestTest.java b/jetty-server/src/test/java/org/eclipse/jetty/server/RequestTest.java index fc0288e3a8c..a0df55079c0 100644 --- a/jetty-server/src/test/java/org/eclipse/jetty/server/RequestTest.java +++ b/jetty-server/src/test/java/org/eclipse/jetty/server/RequestTest.java @@ -639,6 +639,53 @@ public class RequestTest assertThat(responses, startsWith("HTTP/1.1 200")); } + @Test + public void testContentLength_ExceedsMaxInteger() throws Exception + { + final long HUGE_LENGTH = (long) Integer.MAX_VALUE * 10L; + + _handler._checker = (request, response) -> + request.getContentLength() == (-1) // per HttpServletRequest javadoc this must return (-1); + && request.getContentLengthLong() == HUGE_LENGTH; + + //Send a request with encoded form content + String request="POST / HTTP/1.1\r\n"+ + "Host: whatever\r\n"+ + "Content-Type: application/octet-stream\n"+ + "Content-Length: " + HUGE_LENGTH + "\n"+ + "Connection: close\n"+ + "\n"+ + "\n"; + + System.out.println(request); + + String responses=_connector.getResponse(request); + assertThat(responses,startsWith("HTTP/1.1 200")); + } + + @Test + public void testContentLength_ExceedsMaxLong() throws Exception + { + String HUGE_LENGTH = Long.MAX_VALUE + "0"; + + _handler._checker = (request, response) -> + request.getHeader("Content-Length").equals(HUGE_LENGTH) + && request.getContentLength() == (-1) // per HttpServletRequest javadoc this must return (-1); + && request.getContentLengthLong() == (-1); // exact behavior here not specified in Servlet javadoc + + //Send a request with encoded form content + String request="POST / HTTP/1.1\r\n"+ + "Host: whatever\r\n"+ + "Content-Type: application/octet-stream\n"+ + "Content-Length: " + HUGE_LENGTH + "\n"+ + "Connection: close\n"+ + "\n"+ + "\n"; + + String responses=_connector.getResponse(request); + assertThat(responses,startsWith("HTTP/1.1 200")); + } + @Test public void testIdentityParamExtraction() throws Exception { From 446c3d7202c2d8c6223547f47dbdf7f0ea4aeccd Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Fri, 19 Jul 2019 10:35:35 -0500 Subject: [PATCH 09/33] Issue #3888 - Fixing ServletRequest.getContentLength() behavior + For Content-Length values exceeding Integer.MAX_VALUE the return must be -1. Signed-off-by: Joakim Erdfelt --- .../org/eclipse/jetty/server/Request.java | 10 +++++++- .../org/eclipse/jetty/server/RequestTest.java | 23 +++++++++++-------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/Request.java b/jetty-server/src/main/java/org/eclipse/jetty/server/Request.java index 99e2e8deafe..274155dc0e6 100644 --- a/jetty-server/src/main/java/org/eclipse/jetty/server/Request.java +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/Request.java @@ -682,7 +682,15 @@ public class Request implements HttpServletRequest if (metadata == null) return -1; if (metadata.getContentLength() != Long.MIN_VALUE) - return (int)metadata.getContentLength(); + { + if (metadata.getContentLength() > (long)Integer.MAX_VALUE) + { + // Per ServletRequest#getContentLength() javadoc this must return -1 for values exceeding Integer.MAX_VALUE + return -1; + } + else + return (int)metadata.getContentLength(); + } return (int)metadata.getFields().getLongField(HttpHeader.CONTENT_LENGTH.toString()); } diff --git a/jetty-server/src/test/java/org/eclipse/jetty/server/RequestTest.java b/jetty-server/src/test/java/org/eclipse/jetty/server/RequestTest.java index a0df55079c0..69a29dc6b71 100644 --- a/jetty-server/src/test/java/org/eclipse/jetty/server/RequestTest.java +++ b/jetty-server/src/test/java/org/eclipse/jetty/server/RequestTest.java @@ -663,27 +663,30 @@ public class RequestTest assertThat(responses,startsWith("HTTP/1.1 200")); } + /** + * The Servlet spec and API cannot parse Content-Length that exceeds Long.MAX_VALUE + */ @Test public void testContentLength_ExceedsMaxLong() throws Exception { String HUGE_LENGTH = Long.MAX_VALUE + "0"; _handler._checker = (request, response) -> - request.getHeader("Content-Length").equals(HUGE_LENGTH) - && request.getContentLength() == (-1) // per HttpServletRequest javadoc this must return (-1); - && request.getContentLengthLong() == (-1); // exact behavior here not specified in Servlet javadoc + request.getHeader("Content-Length").equals(HUGE_LENGTH) + && request.getContentLength() == (-1) // per HttpServletRequest javadoc this must return (-1); + && request.getContentLengthLong() == (-1); // exact behavior here not specified in Servlet javadoc //Send a request with encoded form content String request="POST / HTTP/1.1\r\n"+ - "Host: whatever\r\n"+ - "Content-Type: application/octet-stream\n"+ - "Content-Length: " + HUGE_LENGTH + "\n"+ - "Connection: close\n"+ - "\n"+ - "\n"; + "Host: whatever\r\n"+ + "Content-Type: application/octet-stream\n"+ + "Content-Length: " + HUGE_LENGTH + "\n"+ + "Connection: close\n"+ + "\n"+ + "\n"; String responses=_connector.getResponse(request); - assertThat(responses,startsWith("HTTP/1.1 200")); + assertThat(responses, startsWith("HTTP/1.1 400")); } @Test From 122f994aaf5699a4fb08d16633bfb185150ced77 Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Fri, 19 Jul 2019 10:38:15 -0500 Subject: [PATCH 10/33] Issue #3888 - Removing excessive "huge" per PR review Signed-off-by: Joakim Erdfelt --- .../jetty/webapp/HugeResourceTest.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/jetty-webapp/src/test/java/org/eclipse/jetty/webapp/HugeResourceTest.java b/jetty-webapp/src/test/java/org/eclipse/jetty/webapp/HugeResourceTest.java index 5a7afe13813..2eefebf8fc8 100644 --- a/jetty-webapp/src/test/java/org/eclipse/jetty/webapp/HugeResourceTest.java +++ b/jetty-webapp/src/test/java/org/eclipse/jetty/webapp/HugeResourceTest.java @@ -52,7 +52,7 @@ public class HugeResourceTest private static final long KB = 1024; private static final long MB = 1024 * KB; private static final long GB = 1024 * MB; - public static Path hugeStaticBase; + public static Path staticBase; public static Path outputDir; public Server server; @@ -60,21 +60,21 @@ public class HugeResourceTest @BeforeAll public static void prepareStaticFiles() throws IOException { - hugeStaticBase = MavenTestingUtils.getTargetTestingPath(HugeResourceTest.class.getSimpleName() + "-huge-static-base"); - FS.ensureDirExists(hugeStaticBase); + staticBase = MavenTestingUtils.getTargetTestingPath(HugeResourceTest.class.getSimpleName() + "-static-base"); + FS.ensureDirExists(staticBase); - makeStaticFile(hugeStaticBase.resolve("test-1g.dat"), 1 * GB); - makeStaticFile(hugeStaticBase.resolve("test-4g.dat"), 4 * GB); - makeStaticFile(hugeStaticBase.resolve("test-10g.dat"), 10 * GB); + makeStaticFile(staticBase.resolve("test-1g.dat"), 1 * GB); + makeStaticFile(staticBase.resolve("test-4g.dat"), 4 * GB); + makeStaticFile(staticBase.resolve("test-10g.dat"), 10 * GB); - outputDir = MavenTestingUtils.getTargetTestingPath(HugeResourceTest.class.getSimpleName() + "-huge-static-outputdir"); + outputDir = MavenTestingUtils.getTargetTestingPath(HugeResourceTest.class.getSimpleName() + "-outputdir"); FS.ensureEmpty(outputDir); } @AfterAll - public static void cleanupHugeStaticFiles() + public static void cleanupStaticFiles() { - FS.ensureDeleted(hugeStaticBase); + FS.ensureDeleted(staticBase); FS.ensureDeleted(outputDir); } @@ -121,7 +121,7 @@ public class HugeResourceTest WebAppContext context = new WebAppContext(); context.setContextPath("/"); - context.setBaseResource(new PathResource(hugeStaticBase)); + context.setBaseResource(new PathResource(staticBase)); HandlerList handlers = new HandlerList(); handlers.addHandler(context); From 6fd8aeefde9f23a8efb9e7238beb5aa9da7578c9 Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Tue, 23 Jul 2019 15:24:33 -0500 Subject: [PATCH 11/33] Issue #3888 - Adding HttpClient tests + Also applying changes from review Signed-off-by: Joakim Erdfelt --- .../jetty/http/ResourceHttpContent.java | 4 +- .../jetty/server/CachedContentFactory.java | 36 +-- .../org/eclipse/jetty/server/Request.java | 14 +- jetty-webapp/pom.xml | 16 +- .../jetty/webapp/HugeResourceTest.java | 220 ++++++++++++++---- 5 files changed, 222 insertions(+), 68 deletions(-) diff --git a/jetty-http/src/main/java/org/eclipse/jetty/http/ResourceHttpContent.java b/jetty-http/src/main/java/org/eclipse/jetty/http/ResourceHttpContent.java index 5441a0d7b7a..22e404a0df8 100644 --- a/jetty-http/src/main/java/org/eclipse/jetty/http/ResourceHttpContent.java +++ b/jetty-http/src/main/java/org/eclipse/jetty/http/ResourceHttpContent.java @@ -125,7 +125,7 @@ public class ResourceHttpContent implements HttpContent @Override public ByteBuffer getDirectBuffer() { - if (_resource.length() <= 0 || _maxBuffer > 0 && _maxBuffer < _resource.length()) + if (_resource.length() <= 0 || _maxBuffer > 0 && _resource.length() > _maxBuffer) return null; try { @@ -152,7 +152,7 @@ public class ResourceHttpContent implements HttpContent @Override public ByteBuffer getIndirectBuffer() { - if (_resource.length() <= 0 || _maxBuffer > 0 && _maxBuffer < _resource.length()) + if (_resource.length() <= 0 || _maxBuffer > 0 && _resource.length() > _maxBuffer) return null; try { diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/CachedContentFactory.java b/jetty-server/src/main/java/org/eclipse/jetty/server/CachedContentFactory.java index 59059d91cd6..7e3c0bbe3fa 100644 --- a/jetty-server/src/main/java/org/eclipse/jetty/server/CachedContentFactory.java +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/CachedContentFactory.java @@ -335,13 +335,14 @@ public class CachedContentFactory implements HttpContent.ContentFactory { try { - return BufferUtil.toBuffer(resource, true); + return BufferUtil.toBuffer(resource, false); } catch (IOException | IllegalArgumentException e) { - LOG.warn(e); - return null; + if (LOG.isDebugEnabled()) + LOG.debug(e); } + return null; } protected ByteBuffer getMappedBuffer(Resource resource) @@ -355,7 +356,8 @@ public class CachedContentFactory implements HttpContent.ContentFactory } catch (IOException | IllegalArgumentException e) { - LOG.warn(e); + if (LOG.isDebugEnabled()) + LOG.debug(e); } return null; } @@ -368,7 +370,8 @@ public class CachedContentFactory implements HttpContent.ContentFactory } catch (IOException | IllegalArgumentException e) { - LOG.warn(e); + if (LOG.isDebugEnabled()) + LOG.debug(e); } return null; } @@ -552,7 +555,7 @@ public class CachedContentFactory implements HttpContent.ContentFactory @Override public ByteBuffer getIndirectBuffer() { - if (_resource.length() > (long)_maxCachedFileSize) + if (_resource.length() > _maxCachedFileSize) { return null; } @@ -561,21 +564,25 @@ public class CachedContentFactory implements HttpContent.ContentFactory if (buffer == null) { ByteBuffer buffer2 = CachedContentFactory.this.getIndirectBuffer(_resource); - if (buffer2 == null) - LOG.warn("Could not load indirect buffer from " + this); - else if (_indirectBuffer.compareAndSet(null, buffer2)) + { + if (LOG.isDebugEnabled()) + LOG.debug("Could not load indirect buffer from " + this); + return null; + } + + if (_indirectBuffer.compareAndSet(null, buffer2)) { buffer = buffer2; if (_cachedSize.addAndGet(BufferUtil.length(buffer)) > _maxCacheSize) shrinkCache(); } else + { buffer = _indirectBuffer.get(); + } } - if (buffer == null) - return null; - return buffer.slice(); + return buffer == null ? null : buffer.asReadOnlyBuffer(); } @Override @@ -594,7 +601,7 @@ public class CachedContentFactory implements HttpContent.ContentFactory else buffer = _mappedBuffer.get(); } - else if (_resource.length() <= (long)_maxCachedFileSize) + else if (_resource.length() > _maxCachedFileSize) { ByteBuffer direct = CachedContentFactory.this.getDirectBuffer(_resource); if (direct != null) @@ -612,7 +619,8 @@ public class CachedContentFactory implements HttpContent.ContentFactory } else { - LOG.warn("Could not load " + this); + if (LOG.isDebugEnabled()) + LOG.debug("Could not load " + this); } } } diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/Request.java b/jetty-server/src/main/java/org/eclipse/jetty/server/Request.java index 274155dc0e6..a27c1df5483 100644 --- a/jetty-server/src/main/java/org/eclipse/jetty/server/Request.java +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/Request.java @@ -681,17 +681,21 @@ public class Request implements HttpServletRequest MetaData.Request metadata = _metaData; if (metadata == null) return -1; - if (metadata.getContentLength() != Long.MIN_VALUE) + + long contentLength = metadata.getContentLength(); + if (contentLength != Long.MIN_VALUE) { - if (metadata.getContentLength() > (long)Integer.MAX_VALUE) + if (contentLength > Integer.MAX_VALUE) { // Per ServletRequest#getContentLength() javadoc this must return -1 for values exceeding Integer.MAX_VALUE return -1; } else - return (int)metadata.getContentLength(); + { + return (int)contentLength; + } } - return (int)metadata.getFields().getLongField(HttpHeader.CONTENT_LENGTH.toString()); + return (int)metadata.getFields().getLongField(HttpHeader.CONTENT_LENGTH.asString()); } /* @@ -705,7 +709,7 @@ public class Request implements HttpServletRequest return -1L; if (metadata.getContentLength() != Long.MIN_VALUE) return metadata.getContentLength(); - return metadata.getFields().getLongField(HttpHeader.CONTENT_LENGTH.toString()); + return metadata.getFields().getLongField(HttpHeader.CONTENT_LENGTH.asString()); } public long getContentRead() diff --git a/jetty-webapp/pom.xml b/jetty-webapp/pom.xml index fa12f47ddd9..49c025fdc1f 100644 --- a/jetty-webapp/pom.xml +++ b/jetty-webapp/pom.xml @@ -52,11 +52,6 @@ jetty-xml ${project.version} - - org.eclipse.jetty.toolchain - jetty-test-helper - test - org.eclipse.jetty jetty-servlet @@ -68,6 +63,17 @@ ${project.version} true + + org.eclipse.jetty + jetty-client + ${project.version} + test + + + org.eclipse.jetty.toolchain + jetty-test-helper + test + diff --git a/jetty-webapp/src/test/java/org/eclipse/jetty/webapp/HugeResourceTest.java b/jetty-webapp/src/test/java/org/eclipse/jetty/webapp/HugeResourceTest.java index 2eefebf8fc8..5c228a59444 100644 --- a/jetty-webapp/src/test/java/org/eclipse/jetty/webapp/HugeResourceTest.java +++ b/jetty-webapp/src/test/java/org/eclipse/jetty/webapp/HugeResourceTest.java @@ -21,19 +21,37 @@ package org.eclipse.jetty.webapp; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.net.HttpURLConnection; +import java.io.PrintWriter; import java.net.URI; import java.nio.ByteBuffer; import java.nio.channels.SeekableByteChannel; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.ArrayList; import java.util.Arrays; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; +import javax.servlet.MultipartConfigElement; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.client.api.ContentResponse; +import org.eclipse.jetty.client.api.Request; +import org.eclipse.jetty.client.api.Response; +import org.eclipse.jetty.client.util.InputStreamResponseListener; +import org.eclipse.jetty.client.util.MultiPartContentProvider; +import org.eclipse.jetty.client.util.PathContentProvider; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.http.HttpMethod; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.handler.DefaultHandler; import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.servlet.ServletHolder; import org.eclipse.jetty.toolchain.test.FS; import org.eclipse.jetty.toolchain.test.MavenTestingUtils; import org.eclipse.jetty.util.IO; @@ -42,9 +60,12 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; public class HugeResourceTest @@ -54,8 +75,10 @@ public class HugeResourceTest private static final long GB = 1024 * MB; public static Path staticBase; public static Path outputDir; + public static Path multipartTempDir; public Server server; + public HttpClient client; @BeforeAll public static void prepareStaticFiles() throws IOException @@ -69,6 +92,20 @@ public class HugeResourceTest outputDir = MavenTestingUtils.getTargetTestingPath(HugeResourceTest.class.getSimpleName() + "-outputdir"); FS.ensureEmpty(outputDir); + + multipartTempDir = MavenTestingUtils.getTargetTestingPath(HugeResourceTest.class.getSimpleName() + "-multipart-tmp"); + FS.ensureEmpty(multipartTempDir); + } + + public static Stream staticFiles() + { + ArrayList ret = new ArrayList<>(); + + ret.add(Arguments.of("test-1g.dat", 1 * GB)); + ret.add(Arguments.of("test-4g.dat", 4 * GB)); + ret.add(Arguments.of("test-10g.dat", 10 * GB)); + + return ret.stream(); } @AfterAll @@ -123,6 +160,17 @@ public class HugeResourceTest context.setContextPath("/"); context.setBaseResource(new PathResource(staticBase)); + context.addServlet(PostServlet.class, "/post"); + + String location = multipartTempDir.toString(); + long maxFileSize = Long.MAX_VALUE; + long maxRequestSize = Long.MAX_VALUE; + int fileSizeThreshold = (int)(2 * MB); + + MultipartConfigElement multipartConfig = new MultipartConfigElement(location, maxFileSize, maxRequestSize, fileSizeThreshold); + ServletHolder holder = context.addServlet(MultipartServlet.class, "/multipart"); + holder.getRegistration().setMultipartConfig(multipartConfig); + HandlerList handlers = new HandlerList(); handlers.addHandler(context); handlers.addHandler(new DefaultHandler()); @@ -137,67 +185,155 @@ public class HugeResourceTest server.stop(); } - @Test - public void testDownload_1G() throws IOException + @BeforeEach + public void startClient() throws Exception { - download(server.getURI().resolve("/test-1g.dat"), 1 * GB); + client = new HttpClient(); + client.start(); } - @Test - public void testDownload_4G() throws IOException + @AfterEach + public void stopClient() throws Exception { - download(server.getURI().resolve("/test-4g.dat"), 4 * GB); + client.stop(); } - @Test - public void testDownload_10G() throws IOException + @ParameterizedTest + @MethodSource("staticFiles") + public void testDownload(String filename, long expectedSize) throws Exception { - download(server.getURI().resolve("/test-10g.dat"), 10 * GB); - } + URI destUri = server.getURI().resolve("/" + filename); + InputStreamResponseListener responseListener = new InputStreamResponseListener(); - private void download(URI destUri, long expectedSize) throws IOException - { - HttpURLConnection http = (HttpURLConnection)destUri.toURL().openConnection(); - assertThat("HTTP Response Code", http.getResponseCode(), is(200)); + Request request = client.newRequest(destUri) + .method(HttpMethod.GET); + request.send(responseListener); + Response response = responseListener.get(5, TimeUnit.SECONDS); - dumpResponseHeaders(http); + assertThat("HTTP Response Code", response.getStatus(), is(200)); + dumpResponse(response); - // if a Content-Length is provided, test it - String contentLength = http.getHeaderField("Content-Length"); - if (contentLength != null) - { - long contentLengthLong = Long.parseLong(contentLength); - assertThat("Http Response Header: \"Content-Length: " + contentLength + "\"", contentLengthLong, is(expectedSize)); - } - - // Download the file - String filename = destUri.getPath(); - int idx = filename.lastIndexOf('/'); - if (idx >= 0) - { - filename = filename.substring(idx + 1); - } + String contentLength = response.getHeaders().get(HttpHeader.CONTENT_LENGTH); + long contentLengthLong = Long.parseLong(contentLength); + assertThat("Http Response Header: \"Content-Length: " + contentLength + "\"", contentLengthLong, is(expectedSize)); Path outputFile = outputDir.resolve(filename); try (OutputStream out = Files.newOutputStream(outputFile); - InputStream in = http.getInputStream()) + InputStream in = responseListener.getInputStream()) { IO.copy(in, out); } - - // Verify the file download size assertThat("Downloaded Files Size: " + filename, Files.size(outputFile), is(expectedSize)); } - private void dumpResponseHeaders(HttpURLConnection http) + @ParameterizedTest + @MethodSource("staticFiles") + public void testUpload(String filename, long expectedSize) throws Exception { - int i = 0; - String value; - while ((value = http.getHeaderField(i)) != null) + Path inputFile = staticBase.resolve(filename); + + PathContentProvider pathContentProvider = new PathContentProvider(inputFile); + URI destUri = server.getURI().resolve("/post"); + Request request = client.newRequest(destUri).method(HttpMethod.POST).content(pathContentProvider); + ContentResponse response = request.send(); + assertThat("HTTP Response Code", response.getStatus(), is(200)); + dumpResponse(response); + + String responseBody = response.getContentAsString(); + assertThat("Response", responseBody, containsString("bytes-received=" + expectedSize)); + } + + @ParameterizedTest + @MethodSource("staticFiles") + public void testUpload_Multipart(String filename, long expectedSize) throws Exception + { + MultiPartContentProvider multipart = new MultiPartContentProvider(); + Path inputFile = staticBase.resolve(filename); + String name = String.format("file-%d", expectedSize); + multipart.addFilePart(name, filename, new PathContentProvider(inputFile), null); + + URI destUri = server.getURI().resolve("/multipart"); + Request request = client.newRequest(destUri).method(HttpMethod.POST).content(multipart); + ContentResponse response = request.send(); + assertThat("HTTP Response Code", response.getStatus(), is(200)); + dumpResponse(response); + + String responseBody = response.getContentAsString(); + String expectedResponse = String.format("part[%s].size=%d", name, expectedSize); + assertThat("Response", responseBody, containsString(expectedResponse)); + } + + private void dumpResponse(Response response) + { + System.out.printf(" %s %d %s%n", response.getVersion(), response.getStatus(), response.getReason()); + response.getHeaders().forEach((field) -> System.out.printf(" %s%n", field)); + } + + public static class ByteCountingOutputStream extends OutputStream + { + private long count = 0; + + public long getCount() { - String key = http.getHeaderFieldKey(i); - System.err.printf(" %s: %s%n", key, value); - i++; + return count; + } + + @Override + public void write(int b) + { + count++; + } + + @Override + public void write(byte[] b) + { + count += b.length; + } + + @Override + public void write(byte[] b, int off, int len) + { + count += len; + } + } + + public static class PostServlet extends HttpServlet + { + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException + { + ByteCountingOutputStream byteCounting = new ByteCountingOutputStream(); + IO.copy(req.getInputStream(), byteCounting); + resp.setContentType("text/plain"); + resp.setCharacterEncoding("utf-8"); + resp.getWriter().printf("bytes-received=%d%n", byteCounting.getCount()); + } + } + + public static class MultipartServlet extends HttpServlet + { + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException + { + resp.setContentType("text/plain"); + resp.setCharacterEncoding("utf-8"); + PrintWriter out = resp.getWriter(); + + req.getParts().forEach((part) -> + { + out.printf("part[%s].filename=%s%n", part.getName(), part.getSubmittedFileName()); + out.printf("part[%s].size=%d%n", part.getName(), part.getSize()); + try (InputStream inputStream = part.getInputStream(); + ByteCountingOutputStream byteCounting = new ByteCountingOutputStream()) + { + IO.copy(inputStream, byteCounting); + out.printf("part[%s].inputStream.length=%d%n", part.getName(), byteCounting.getCount()); + } + catch (IOException e) + { + e.printStackTrace(out); + } + }); } } } From 9e90c60f0e78331be86a50ae198f03b5a5c784bd Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Wed, 24 Jul 2019 11:26:49 -0500 Subject: [PATCH 12/33] Issue #3888 - Fixing implementation of CachedContentFactory + ResourceCacheTest identified a bug in CachedContentFactory (a bad/reversed if statement) + Updated ResourceCacheTest to not rely on jetty-util src/test/resources Signed-off-by: Joakim Erdfelt --- .../jetty/server/CachedContentFactory.java | 2 +- .../jetty/server/ResourceCacheTest.java | 137 +++++++++++++----- 2 files changed, 99 insertions(+), 40 deletions(-) diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/CachedContentFactory.java b/jetty-server/src/main/java/org/eclipse/jetty/server/CachedContentFactory.java index 7e3c0bbe3fa..8cfd3fd206e 100644 --- a/jetty-server/src/main/java/org/eclipse/jetty/server/CachedContentFactory.java +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/CachedContentFactory.java @@ -601,7 +601,7 @@ public class CachedContentFactory implements HttpContent.ContentFactory else buffer = _mappedBuffer.get(); } - else if (_resource.length() > _maxCachedFileSize) + else if (_resource.length() < _maxCachedFileSize) { ByteBuffer direct = CachedContentFactory.this.getDirectBuffer(_resource); if (direct != null) diff --git a/jetty-server/src/test/java/org/eclipse/jetty/server/ResourceCacheTest.java b/jetty-server/src/test/java/org/eclipse/jetty/server/ResourceCacheTest.java index b252af7fba0..61bdcb4e71c 100644 --- a/jetty-server/src/test/java/org/eclipse/jetty/server/ResourceCacheTest.java +++ b/jetty-server/src/test/java/org/eclipse/jetty/server/ResourceCacheTest.java @@ -18,34 +18,106 @@ package org.eclipse.jetty.server; -import java.io.BufferedReader; +import java.io.BufferedWriter; import java.io.File; import java.io.FileOutputStream; -import java.io.InputStreamReader; +import java.io.IOException; import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import org.eclipse.jetty.http.CompressedContentFormat; import org.eclipse.jetty.http.HttpContent; import org.eclipse.jetty.http.MimeTypes; import org.eclipse.jetty.http.ResourceHttpContent; +import org.eclipse.jetty.toolchain.test.FS; +import org.eclipse.jetty.toolchain.test.jupiter.WorkDir; +import org.eclipse.jetty.toolchain.test.jupiter.WorkDirExtension; import org.eclipse.jetty.util.BufferUtil; +import org.eclipse.jetty.util.resource.PathResource; import org.eclipse.jetty.util.resource.Resource; import org.eclipse.jetty.util.resource.ResourceCollection; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +@ExtendWith(WorkDirExtension.class) public class ResourceCacheTest { + public WorkDir workDir; + + public Path createUtilTestResources(Path basePath) throws IOException + { + // root + makeFile(basePath.resolve("resource.txt"), "this is test data"); + + // - one/ + Path one = basePath.resolve("one"); + FS.ensureDirExists(one); + makeFile(one.resolve("1.txt"), "1 - one"); + + // - one/dir/ + Path oneDir = one.resolve("dir"); + FS.ensureDirExists(oneDir); + makeFile(oneDir.resolve("1.txt"), "1 - one"); + + // - two/ + Path two = basePath.resolve("two"); + FS.ensureDirExists(two); + makeFile(two.resolve("1.txt"), "1 - two"); + makeFile(two.resolve("2.txt"), "2 - two"); + + // - two/dir/ + Path twoDir = two.resolve("dir"); + FS.ensureDirExists(twoDir); + makeFile(twoDir.resolve("2.txt"), "2 - two"); + + // - three/ + Path three = basePath.resolve("three"); + FS.ensureDirExists(three); + makeFile(three.resolve("2.txt"), "2 - three"); + makeFile(three.resolve("3.txt"), "3 - three"); + + // - three/dir/ + Path threeDir = three.resolve("dir"); + FS.ensureDirExists(threeDir); + makeFile(threeDir.resolve("3.txt"), "3 - three"); + + // - four/ + Path four = basePath.resolve("four"); + FS.ensureDirExists(four); + makeFile(four.resolve("four"), "4 - four (no extension)"); + makeFile(four.resolve("four.txt"), "4 - four"); + + return basePath; + } + + private void makeFile(Path file, String contents) throws IOException + { + try (BufferedWriter writer = Files.newBufferedWriter(file, UTF_8, StandardOpenOption.CREATE_NEW)) + { + writer.write(contents); + writer.flush(); + } + } + @Test public void testMutlipleSources1() throws Exception { - ResourceCollection rc = new ResourceCollection(new String[]{ - "../jetty-util/src/test/resources/org/eclipse/jetty/util/resource/one/", - "../jetty-util/src/test/resources/org/eclipse/jetty/util/resource/two/", - "../jetty-util/src/test/resources/org/eclipse/jetty/util/resource/three/" - }); + Path basePath = createUtilTestResources(workDir.getEmptyPathDir()); + + ResourceCollection rc = new ResourceCollection( + new PathResource(basePath.resolve("one")), + new PathResource(basePath.resolve("two")), + new PathResource(basePath.resolve("three"))); Resource[] r = rc.getResources(); MimeTypes mime = new MimeTypes(); @@ -70,11 +142,12 @@ public class ResourceCacheTest @Test public void testUncacheable() throws Exception { - ResourceCollection rc = new ResourceCollection(new String[]{ - "../jetty-util/src/test/resources/org/eclipse/jetty/util/resource/one/", - "../jetty-util/src/test/resources/org/eclipse/jetty/util/resource/two/", - "../jetty-util/src/test/resources/org/eclipse/jetty/util/resource/three/" - }); + Path basePath = createUtilTestResources(workDir.getEmptyPathDir()); + + ResourceCollection rc = new ResourceCollection( + new PathResource(basePath.resolve("one")), + new PathResource(basePath.resolve("two")), + new PathResource(basePath.resolve("three"))); Resource[] r = rc.getResources(); MimeTypes mime = new MimeTypes(); @@ -112,19 +185,21 @@ public class ResourceCacheTest String[] names = new String[files.length]; CachedContentFactory cache; + Path basePath = workDir.getEmptyPathDir(); + for (int i = 0; i < files.length; i++) { - files[i] = File.createTempFile("R-" + i + "-", ".txt"); - files[i].deleteOnExit(); - names[i] = files[i].getName(); - try (OutputStream out = new FileOutputStream(files[i])) + Path tmpFile = basePath.resolve("R-" + i + ".txt"); + try (BufferedWriter writer = Files.newBufferedWriter(tmpFile, UTF_8, StandardOpenOption.CREATE_NEW)) { for (int j = 0; j < (i * 10 - 1); j++) { - out.write(' '); + writer.write(' '); } - out.write('\n'); + writer.write('\n'); } + files[i] = tmpFile.toFile(); + names[i] = tmpFile.getFileName().toString(); } directory = Resource.newResource(files[0].getParentFile().getAbsolutePath()); @@ -141,7 +216,7 @@ public class ResourceCacheTest HttpContent content; content = cache.getContent(names[8], 4096); - assertTrue(content != null); + assertThat(content, is(not(nullValue()))); assertEquals(80, content.getContentLengthValue()); assertEquals(0, cache.getCachedSize()); @@ -274,36 +349,20 @@ public class ResourceCacheTest @Test public void testNoextension() throws Exception { - ResourceCollection rc = new ResourceCollection(new String[]{ - "../jetty-util/src/test/resources/org/eclipse/jetty/util/resource/four/" - }); + Path basePath = createUtilTestResources(workDir.getEmptyPathDir()); - Resource[] resources = rc.getResources(); + Resource resource = new PathResource(basePath.resolve("four")); MimeTypes mime = new MimeTypes(); - CachedContentFactory cache = new CachedContentFactory(null, resources[0], mime, false, false, CompressedContentFormat.NONE); + CachedContentFactory cache = new CachedContentFactory(null, resource, mime, false, false, CompressedContentFormat.NONE); assertEquals(getContent(cache, "four.txt"), "4 - four"); assertEquals(getContent(cache, "four"), "4 - four (no extension)"); } - static String getContent(Resource r, String path) throws Exception - { - StringBuilder buffer = new StringBuilder(); - String line = null; - try (BufferedReader br = new BufferedReader(new InputStreamReader(r.addPath(path).getURL().openStream()))) - { - while ((line = br.readLine()) != null) - { - buffer.append(line); - } - } - return buffer.toString(); - } - static String getContent(CachedContentFactory rc, String path) throws Exception { - HttpContent content = rc.lookup(path); + HttpContent content = rc.getContent(path, rc.getMaxCachedFileSize()); if (content == null) return null; From 24b2ca4c328fcc22a1c1534d39ea2970920b1ac4 Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Thu, 25 Jul 2019 11:43:51 -0500 Subject: [PATCH 13/33] Issue #3906 - Introducing Testcase to demonstrate issue Signed-off-by: Joakim Erdfelt --- .../jetty/util/resource/PathResourceTest.java | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 jetty-util/src/test/java/org/eclipse/jetty/util/resource/PathResourceTest.java diff --git a/jetty-util/src/test/java/org/eclipse/jetty/util/resource/PathResourceTest.java b/jetty-util/src/test/java/org/eclipse/jetty/util/resource/PathResourceTest.java new file mode 100644 index 00000000000..e12646f5bdc --- /dev/null +++ b/jetty-util/src/test/java/org/eclipse/jetty/util/resource/PathResourceTest.java @@ -0,0 +1,71 @@ +// +// ======================================================================== +// Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.util.resource; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.channels.ReadableByteChannel; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +import org.eclipse.jetty.toolchain.test.MavenTestingUtils; +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; + +public class PathResourceTest +{ + @Test + public void testNonDefaultFileSystem_GetInputStream() throws URISyntaxException, IOException + { + Path exampleJar = MavenTestingUtils.getTestResourcePathFile("example.jar"); + + URI uri = new URI("jar", exampleJar.toUri().toASCIIString(), null); + System.err.println("URI = " + uri); + + Map env = new HashMap<>(); + env.put("multi-release", "runtime"); + + try (FileSystem zipfs = FileSystems.newFileSystem(uri, env)) + { + Path manifestPath = zipfs.getPath("/META-INF/MANIFEST.MF"); + assertThat(manifestPath, is(not(nullValue()))); + + PathResource resource = new PathResource(manifestPath); + + try (ReadableByteChannel channel = resource.getReadableByteChannel()) + { + assertThat("ReadableByteChannel", channel, is(not(nullValue()))); + } + + try (InputStream inputStream = resource.getInputStream()) + { + assertThat("InputStream", inputStream, is(not(nullValue()))); + } + } + } +} From 3be03db5e6f072e2e6ee908159d5b3194df1ec28 Mon Sep 17 00:00:00 2001 From: Simone Bordet Date: Fri, 26 Jul 2019 11:52:08 +0200 Subject: [PATCH 14/33] Improved logging. Signed-off-by: Simone Bordet --- .../org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java b/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java index b19123bb8f1..a8a9476423f 100644 --- a/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java +++ b/jetty-http2/http2-server/src/main/java/org/eclipse/jetty/http2/server/HttpTransportOverHTTP2.java @@ -441,7 +441,7 @@ public class HttpTransportOverHTTP2 implements HttpTransport } } if (LOG.isDebugEnabled()) - LOG.debug(String.format("HTTP2 Response #%d/%h idle timeout", stream.getId(), stream.getSession()), failure); + LOG.debug(String.format("HTTP2 Response #%d/%h idle timeout %s", stream.getId(), stream.getSession(), result ? "expired" : "ignored"), failure); if (result) callback.failed(failure); return result; From 843168693979e789384d56e314f1609134fe8c11 Mon Sep 17 00:00:00 2001 From: Simone Bordet Date: Fri, 26 Jul 2019 14:51:53 +0200 Subject: [PATCH 15/33] Issue #3883 - Client: support embedding threadlocal data (e.g. distributed tracing data) into the request. Added test case that shows how to do it. Signed-off-by: Simone Bordet --- .../client/HttpClientCorrelationDataTest.java | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientCorrelationDataTest.java diff --git a/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientCorrelationDataTest.java b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientCorrelationDataTest.java new file mode 100644 index 00000000000..ac7e58d6eb6 --- /dev/null +++ b/jetty-client/src/test/java/org/eclipse/jetty/client/HttpClientCorrelationDataTest.java @@ -0,0 +1,70 @@ +// +// ======================================================================== +// Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.client; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.eclipse.jetty.client.api.ContentResponse; +import org.eclipse.jetty.client.api.Request; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ArgumentsSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class HttpClientCorrelationDataTest extends AbstractHttpClientServerTest +{ + @ParameterizedTest + @ArgumentsSource(ScenarioProvider.class) + public void testCorrelationData(Scenario scenario) throws Exception + { + String correlationName = "X-Correlation-Data"; + String correlationData = "123456"; + ThreadLocal correlation = new ThreadLocal<>(); + + start(scenario, new EmptyServerHandler() + { + @Override + protected void service(String target, org.eclipse.jetty.server.Request jettyRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException + { + assertEquals(correlationData, request.getHeader(correlationName)); + } + }); + client.getRequestListeners().add(new Request.Listener.Adapter() + { + @Override + public void onQueued(Request request) + { + request.header(correlationName, correlation.get()); + } + }); + + correlation.set(correlationData); + + ContentResponse response = client.newRequest("localhost", connector.getLocalPort()) + .scheme(scenario.getScheme()) + .timeout(5, TimeUnit.SECONDS) + .send(); + + assertEquals(200, response.getStatus()); + } +} From 4e0e9627473669673753d4f197522e0b10c6a91e Mon Sep 17 00:00:00 2001 From: Simone Bordet Date: Fri, 26 Jul 2019 15:57:53 +0200 Subject: [PATCH 16/33] Fixes #3822 - trustAll will not work on some servers. Made TRUST_ALL_CERTS implement X509ExtendedTrustManager, so that it does not get wrapped in sun.security.ssl.AbstractTrustManagerWrapper, which performs additional trust checks. Signed-off-by: Simone Bordet --- .../jetty/util/ssl/SslContextFactory.java | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/jetty-util/src/main/java/org/eclipse/jetty/util/ssl/SslContextFactory.java b/jetty-util/src/main/java/org/eclipse/jetty/util/ssl/SslContextFactory.java index 8fbb2cf8f12..adb4e271765 100644 --- a/jetty-util/src/main/java/org/eclipse/jetty/util/ssl/SslContextFactory.java +++ b/jetty-util/src/main/java/org/eclipse/jetty/util/ssl/SslContextFactory.java @@ -22,6 +22,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.Socket; import java.security.InvalidAlgorithmParameterException; import java.security.KeyStore; import java.security.NoSuchAlgorithmException; @@ -74,6 +75,7 @@ import javax.net.ssl.StandardConstants; import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.X509ExtendedKeyManager; +import javax.net.ssl.X509ExtendedTrustManager; import javax.net.ssl.X509TrustManager; import org.eclipse.jetty.util.StringUtil; @@ -98,21 +100,41 @@ import org.eclipse.jetty.util.security.Password; public class SslContextFactory extends AbstractLifeCycle implements Dumpable { public static final TrustManager[] TRUST_ALL_CERTS = new X509TrustManager[]{ - new X509TrustManager() + new X509ExtendedTrustManager() { @Override - public java.security.cert.X509Certificate[] getAcceptedIssuers() + public X509Certificate[] getAcceptedIssuers() { - return new java.security.cert.X509Certificate[]{}; + return new X509Certificate[0]; } @Override - public void checkClientTrusted(java.security.cert.X509Certificate[] certs, String authType) + public void checkClientTrusted(X509Certificate[] certs, String authType) { } @Override - public void checkServerTrusted(java.security.cert.X509Certificate[] certs, String authType) + public void checkClientTrusted(X509Certificate[] chain, String authType, Socket socket) + { + } + + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType, SSLEngine engine) + { + } + + @Override + public void checkServerTrusted(X509Certificate[] certs, String authType) + { + } + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType, Socket socket) + { + } + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType, SSLEngine engine) { } } From e5bce5f7cd203dec9e7063e2b0a8f54a23bbe337 Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Fri, 26 Jul 2019 12:36:31 -0500 Subject: [PATCH 17/33] Issue #3840 - Static resource byte-range support performance + Reverting toFile().getInputStream() on PathResource + Adding RangeWriter concept for managing open resource across multiple range writes + RangeWriter implementation delegates to HttpContent behaviors Lookup is : - Direct Buffer - Indirect Buffer - ReadableByteChannel (as SeekableByteChannel) - InputStream + Adding unit tests for all RangeWriter implementation to ensure that they behave the same way everywhere. + Making ResourceService use new RangeWriter implementation + Existing DefaultServletRangeTest still works as-is Signed-off-by: Joakim Erdfelt --- .../eclipse/jetty/server/ResourceService.java | 45 ++--- .../resource/ByteBufferRangeWriter.java | 64 +++++++ .../resource/HttpContentRangeWriter.java | 83 ++++++++++ .../resource/InputStreamRangeWriter.java | 93 +++++++++++ .../jetty/server/resource/RangeWriter.java | 38 +++++ .../SeekableByteChannelRangeWriter.java | 66 ++++++++ .../server/resource/RangeWriterTest.java | 156 ++++++++++++++++++ .../jetty/util/resource/PathResource.java | 8 +- 8 files changed, 514 insertions(+), 39 deletions(-) create mode 100644 jetty-server/src/main/java/org/eclipse/jetty/server/resource/ByteBufferRangeWriter.java create mode 100644 jetty-server/src/main/java/org/eclipse/jetty/server/resource/HttpContentRangeWriter.java create mode 100644 jetty-server/src/main/java/org/eclipse/jetty/server/resource/InputStreamRangeWriter.java create mode 100644 jetty-server/src/main/java/org/eclipse/jetty/server/resource/RangeWriter.java create mode 100644 jetty-server/src/main/java/org/eclipse/jetty/server/resource/SeekableByteChannelRangeWriter.java create mode 100644 jetty-server/src/test/java/org/eclipse/jetty/server/resource/RangeWriterTest.java diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/ResourceService.java b/jetty-server/src/main/java/org/eclipse/jetty/server/ResourceService.java index 21591a08f03..dad1223087d 100644 --- a/jetty-server/src/main/java/org/eclipse/jetty/server/ResourceService.java +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/ResourceService.java @@ -20,7 +20,6 @@ package org.eclipse.jetty.server; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -47,9 +46,10 @@ import org.eclipse.jetty.http.PreEncodedHttpField; import org.eclipse.jetty.http.QuotedCSV; import org.eclipse.jetty.http.QuotedQualityCSV; import org.eclipse.jetty.io.WriterOutputStream; +import org.eclipse.jetty.server.resource.HttpContentRangeWriter; +import org.eclipse.jetty.server.resource.RangeWriter; import org.eclipse.jetty.util.BufferUtil; import org.eclipse.jetty.util.Callback; -import org.eclipse.jetty.util.IO; import org.eclipse.jetty.util.MultiPartOutputStream; import org.eclipse.jetty.util.URIUtil; import org.eclipse.jetty.util.log.Log; @@ -779,9 +779,6 @@ public class ResourceService ctp = "multipart/byteranges; boundary="; response.setContentType(ctp + multi.getBoundary()); - InputStream in = content.getResource().getInputStream(); - long pos = 0; - // calculate the content-length int length = 0; String[] header = new String[ranges.size()]; @@ -801,39 +798,21 @@ public class ResourceService length += 2 + 2 + multi.getBoundary().length() + 2 + 2; response.setContentLength(length); - i = 0; - for (InclusiveByteRange ibr : ranges) + try (RangeWriter rangeWriter = HttpContentRangeWriter.newRangeWriter(content)) { - multi.startPart(mimetype, new String[]{HttpHeader.CONTENT_RANGE + ": " + header[i]}); - - long start = ibr.getFirst(); - long size = ibr.getSize(); - if (in != null) + i = 0; + for (InclusiveByteRange ibr : ranges) { - // Handle non cached resource - if (start < pos) - { - in.close(); - in = content.getResource().getInputStream(); - pos = 0; - } - if (pos < start) - { - in.skip(start - pos); - pos = start; - } + multi.startPart(mimetype, new String[]{HttpHeader.CONTENT_RANGE + ": " + header[i]}); - IO.copy(in, multi, size); - pos += size; + long start = ibr.getFirst(); + long size = ibr.getSize(); + + rangeWriter.writeTo(multi, start, size); + i++; } - else - // Handle cached resource - content.getResource().writeTo(multi, start, size); - - i++; } - if (in != null) - in.close(); + multi.close(); } return true; diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/resource/ByteBufferRangeWriter.java b/jetty-server/src/main/java/org/eclipse/jetty/server/resource/ByteBufferRangeWriter.java new file mode 100644 index 00000000000..8593fa5a94a --- /dev/null +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/resource/ByteBufferRangeWriter.java @@ -0,0 +1,64 @@ +// +// ======================================================================== +// Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.server.resource; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +import org.eclipse.jetty.util.BufferUtil; + +/** + * ByteBuffer based RangeWriter + */ +public class ByteBufferRangeWriter implements RangeWriter +{ + private final ByteBuffer buffer; + private boolean closed = false; + + public ByteBufferRangeWriter(ByteBuffer buffer) + { + this.buffer = buffer.asReadOnlyBuffer(); + } + + @Override + public void close() throws IOException + { + closed = true; + } + + @Override + public void writeTo(OutputStream outputStream, long skipTo, long length) throws IOException + { + if (skipTo > Integer.MAX_VALUE) + { + throw new IllegalArgumentException("Unsupported skipTo " + skipTo + " > " + Integer.MAX_VALUE); + } + + if (length > Integer.MAX_VALUE) + { + throw new IllegalArgumentException("Unsupported length " + skipTo + " > " + Integer.MAX_VALUE); + } + + ByteBuffer src = buffer.slice(); + src.position((int)skipTo); + src.limit(Math.addExact((int)skipTo, (int)length)); + BufferUtil.writeTo(src, outputStream); + } +} diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/resource/HttpContentRangeWriter.java b/jetty-server/src/main/java/org/eclipse/jetty/server/resource/HttpContentRangeWriter.java new file mode 100644 index 00000000000..08a5c3805c6 --- /dev/null +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/resource/HttpContentRangeWriter.java @@ -0,0 +1,83 @@ +// +// ======================================================================== +// Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.server.resource; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.SeekableByteChannel; +import java.util.Objects; + +import org.eclipse.jetty.http.HttpContent; +import org.eclipse.jetty.util.log.Log; +import org.eclipse.jetty.util.log.Logger; + +/** + * Range Writer selection for HttpContent + */ +public class HttpContentRangeWriter +{ + private static final Logger LOG = Log.getLogger(HttpContentRangeWriter.class); + + /** + * Obtain a new RangeWriter for the supplied HttpContent. + * + * @param content the HttpContent to base RangeWriter on + * @return the RangeWriter best suited for the supplied HttpContent + */ + public static RangeWriter newRangeWriter(HttpContent content) + { + Objects.requireNonNull(content, "HttpContent"); + + // Try direct buffer + ByteBuffer buffer = content.getDirectBuffer(); + if (buffer == null) + { + buffer = content.getIndirectBuffer(); + } + if (buffer != null) + { + return new ByteBufferRangeWriter(buffer); + } + + try + { + ReadableByteChannel channel = content.getReadableByteChannel(); + if (channel != null) + { + if (channel instanceof SeekableByteChannel) + { + SeekableByteChannel seekableByteChannel = (SeekableByteChannel)channel; + return new SeekableByteChannelRangeWriter(seekableByteChannel); + } + + if (LOG.isDebugEnabled()) + LOG.debug("Skipping non-SeekableByteChannel option " + channel + " from content " + content); + channel.close(); + } + } + catch (IOException e) + { + if (LOG.isDebugEnabled()) + LOG.debug("Skipping ReadableByteChannel option", e); + } + + return new InputStreamRangeWriter(() -> content.getInputStream()); + } +} diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/resource/InputStreamRangeWriter.java b/jetty-server/src/main/java/org/eclipse/jetty/server/resource/InputStreamRangeWriter.java new file mode 100644 index 00000000000..0ab44b20613 --- /dev/null +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/resource/InputStreamRangeWriter.java @@ -0,0 +1,93 @@ +// +// ======================================================================== +// Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.server.resource; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import org.eclipse.jetty.util.IO; + +/** + * Default Range Writer for InputStream + */ +public class InputStreamRangeWriter implements RangeWriter +{ + public interface InputStreamSupplier + { + InputStream newInputStream() throws IOException; + } + + private final InputStreamSupplier inputStreamSupplier; + private boolean closed = false; + private InputStream inputStream; + private long pos; + + /** + * Create InputStremRangeWriter + * + * @param inputStreamSupplier Supplier of the InputStream. If the stream needs to be regenerated, such as the next + * requested range being before the current position, then the current InputStream is closed and a new one obtained + * from this supplier. + */ + public InputStreamRangeWriter(InputStreamSupplier inputStreamSupplier) + { + this.inputStreamSupplier = inputStreamSupplier; + } + + @Override + public void close() throws IOException + { + closed = true; + if (inputStream != null) + { + inputStream.close(); + } + } + + @Override + public void writeTo(OutputStream outputStream, long skipTo, long length) throws IOException + { + if (closed) + { + throw new IOException("RangeWriter is closed"); + } + + if (inputStream == null) + { + inputStream = inputStreamSupplier.newInputStream(); + pos = 0; + } + + if (skipTo < pos) + { + inputStream.close(); + inputStream = inputStreamSupplier.newInputStream(); + pos = 0; + } + if (pos < skipTo) + { + inputStream.skip(skipTo - pos); + pos = skipTo; + } + + IO.copy(inputStream, outputStream, length); + pos += length; + } +} diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/resource/RangeWriter.java b/jetty-server/src/main/java/org/eclipse/jetty/server/resource/RangeWriter.java new file mode 100644 index 00000000000..80cb9c6b606 --- /dev/null +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/resource/RangeWriter.java @@ -0,0 +1,38 @@ +// +// ======================================================================== +// Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.server.resource; + +import java.io.Closeable; +import java.io.IOException; +import java.io.OutputStream; + +/** + * Interface for writing sections (ranges) of a single resource (SeekableByteChannel, Resource, etc) to an outputStream. + */ +public interface RangeWriter extends Closeable +{ + /** + * Write the specific range (start, size) to the outputStream. + * + * @param outputStream the stream to write to + * @param skipTo the offset / skip-to / seek-to / position in the resource to start the write from + * @param length the size of the section to write + */ + void writeTo(OutputStream outputStream, long skipTo, long length) throws IOException; +} diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/resource/SeekableByteChannelRangeWriter.java b/jetty-server/src/main/java/org/eclipse/jetty/server/resource/SeekableByteChannelRangeWriter.java new file mode 100644 index 00000000000..1a2e86a8fdb --- /dev/null +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/resource/SeekableByteChannelRangeWriter.java @@ -0,0 +1,66 @@ +// +// ======================================================================== +// Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.server.resource; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; + +import org.eclipse.jetty.util.BufferUtil; +import org.eclipse.jetty.util.IO; + +public class SeekableByteChannelRangeWriter implements RangeWriter +{ + private final SeekableByteChannel channel; + private final int bufSize; + private final ByteBuffer buffer; + + public SeekableByteChannelRangeWriter(SeekableByteChannel seekableByteChannel) + { + this.channel = seekableByteChannel; + this.bufSize = IO.bufferSize; + this.buffer = BufferUtil.allocate(this.bufSize); + } + + @Override + public void close() throws IOException + { + this.channel.close(); + } + + @Override + public void writeTo(OutputStream outputStream, long skipTo, long length) throws IOException + { + this.channel.position(skipTo); + + // copy from channel to output stream + long readTotal = 0; + while (readTotal < length) + { + BufferUtil.flipToFill(buffer); + int size = (int)Math.min(bufSize, length - readTotal); + buffer.limit(size); + int readLen = channel.read(buffer); + BufferUtil.flipToFlush(buffer, 0); + BufferUtil.writeTo(buffer, outputStream); + readTotal += readLen; + } + } +} diff --git a/jetty-server/src/test/java/org/eclipse/jetty/server/resource/RangeWriterTest.java b/jetty-server/src/test/java/org/eclipse/jetty/server/resource/RangeWriterTest.java new file mode 100644 index 00000000000..e073fa55734 --- /dev/null +++ b/jetty-server/src/test/java/org/eclipse/jetty/server/resource/RangeWriterTest.java @@ -0,0 +1,156 @@ +// +// ======================================================================== +// Copyright (c) 1995-2019 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.server.resource; + +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.stream.Stream; + +import org.eclipse.jetty.toolchain.test.FS; +import org.eclipse.jetty.toolchain.test.MavenTestingUtils; +import org.eclipse.jetty.util.BufferUtil; +import org.eclipse.jetty.util.resource.PathResource; +import org.eclipse.jetty.util.resource.Resource; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +public class RangeWriterTest +{ + public static final String DATA = "01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWZYZ!@#$%^&*()_+/.,[]"; + + public static Path initDataFile() throws IOException + { + Path testDir = MavenTestingUtils.getTargetTestingPath(RangeWriterTest.class.getSimpleName()); + FS.ensureEmpty(testDir); + + Path dataFile = testDir.resolve("data.dat"); + try (BufferedWriter writer = Files.newBufferedWriter(dataFile, UTF_8, StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) + { + writer.write(DATA); + writer.flush(); + } + + return dataFile; + } + + public static Stream impls() throws IOException + { + Resource resource = new PathResource(initDataFile()); + + return Stream.of( + Arguments.of(new ByteBufferRangeWriter(BufferUtil.toBuffer(resource, true))), + Arguments.of(new ByteBufferRangeWriter(BufferUtil.toBuffer(resource, false))), + Arguments.of(new SeekableByteChannelRangeWriter((SeekableByteChannel)resource.getReadableByteChannel())), + Arguments.of(new InputStreamRangeWriter(() -> resource.getInputStream())) + ); + } + + @ParameterizedTest + @MethodSource("impls") + public void testSimpleRange(RangeWriter rangeWriter) throws IOException + { + ByteArrayOutputStream outputStream; + + outputStream = new ByteArrayOutputStream(); + rangeWriter.writeTo(outputStream, 10, 50); + assertThat("Range: 10 (len=50)", new String(outputStream.toByteArray(), UTF_8), is(DATA.substring(10, 60))); + } + + @ParameterizedTest + @MethodSource("impls") + public void testSameRange_MultipleTimes(RangeWriter rangeWriter) throws IOException + { + ByteArrayOutputStream outputStream; + + outputStream = new ByteArrayOutputStream(); + rangeWriter.writeTo(outputStream, 10, 50); + assertThat("Range(a): 10 (len=50)", new String(outputStream.toByteArray(), UTF_8), is(DATA.substring(10, 60))); + + outputStream = new ByteArrayOutputStream(); + rangeWriter.writeTo(outputStream, 10, 50); + assertThat("Range(b): 10 (len=50)", new String(outputStream.toByteArray(), UTF_8), is(DATA.substring(10, 60))); + } + + @ParameterizedTest + @MethodSource("impls") + public void testMultipleRanges_Ordered(RangeWriter rangeWriter) throws IOException + { + ByteArrayOutputStream outputStream; + + outputStream = new ByteArrayOutputStream(); + rangeWriter.writeTo(outputStream, 10, 20); + assertThat("Range(a): 10 (len=20)", new String(outputStream.toByteArray(), UTF_8), is(DATA.substring(10, 10 + 20))); + + outputStream = new ByteArrayOutputStream(); + rangeWriter.writeTo(outputStream, 35, 10); + assertThat("Range(b): 35 (len=10)", new String(outputStream.toByteArray(), UTF_8), is(DATA.substring(35, 35 + 10))); + + outputStream = new ByteArrayOutputStream(); + rangeWriter.writeTo(outputStream, 55, 10); + assertThat("Range(b): 55 (len=10)", new String(outputStream.toByteArray(), UTF_8), is(DATA.substring(55, 55 + 10))); + } + + @ParameterizedTest + @MethodSource("impls") + public void testMultipleRanges_Overlapping(RangeWriter rangeWriter) throws IOException + { + ByteArrayOutputStream outputStream; + + outputStream = new ByteArrayOutputStream(); + rangeWriter.writeTo(outputStream, 10, 20); + assertThat("Range(a): 10 (len=20)", new String(outputStream.toByteArray(), UTF_8), is(DATA.substring(10, 10 + 20))); + + outputStream = new ByteArrayOutputStream(); + rangeWriter.writeTo(outputStream, 15, 20); + assertThat("Range(b): 15 (len=20)", new String(outputStream.toByteArray(), UTF_8), is(DATA.substring(15, 15 + 20))); + + outputStream = new ByteArrayOutputStream(); + rangeWriter.writeTo(outputStream, 20, 20); + assertThat("Range(b): 20 (len=20)", new String(outputStream.toByteArray(), UTF_8), is(DATA.substring(20, 20 + 20))); + } + + @ParameterizedTest + @MethodSource("impls") + public void testMultipleRanges_ReverseOrder(RangeWriter rangeWriter) throws IOException + { + ByteArrayOutputStream outputStream; + + outputStream = new ByteArrayOutputStream(); + rangeWriter.writeTo(outputStream, 55, 10); + assertThat("Range(b): 55 (len=10)", new String(outputStream.toByteArray(), UTF_8), is(DATA.substring(55, 55 + 10))); + + outputStream = new ByteArrayOutputStream(); + rangeWriter.writeTo(outputStream, 35, 10); + assertThat("Range(b): 35 (len=10)", new String(outputStream.toByteArray(), UTF_8), is(DATA.substring(35, 35 + 10))); + + outputStream = new ByteArrayOutputStream(); + rangeWriter.writeTo(outputStream, 10, 20); + assertThat("Range(a): 10 (len=20)", new String(outputStream.toByteArray(), UTF_8), is(DATA.substring(10, 10 + 20))); + } +} diff --git a/jetty-util/src/main/java/org/eclipse/jetty/util/resource/PathResource.java b/jetty-util/src/main/java/org/eclipse/jetty/util/resource/PathResource.java index 2e2c327b06a..7530f13e5d9 100644 --- a/jetty-util/src/main/java/org/eclipse/jetty/util/resource/PathResource.java +++ b/jetty-util/src/main/java/org/eclipse/jetty/util/resource/PathResource.java @@ -19,14 +19,12 @@ package org.eclipse.jetty.util.resource; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; -import java.nio.channels.FileChannel; import java.nio.channels.ReadableByteChannel; import java.nio.file.DirectoryIteratorException; import java.nio.file.DirectoryStream; @@ -379,9 +377,7 @@ public class PathResource extends Resource @Override public InputStream getInputStream() throws IOException { - // Use a FileInputStream rather than Files.newInputStream(path) - // since it produces a stream with a fast skip implementation - return new FileInputStream(getFile()); + return Files.newInputStream(path, StandardOpenOption.READ, StandardOpenOption.SPARSE); } @Override @@ -393,7 +389,7 @@ public class PathResource extends Resource @Override public ReadableByteChannel getReadableByteChannel() throws IOException { - return FileChannel.open(path, StandardOpenOption.READ); + return Files.newByteChannel(path, StandardOpenOption.READ, StandardOpenOption.SPARSE); } @Override From 8601baa3cc020aeac3566da63ac21048db13943c Mon Sep 17 00:00:00 2001 From: Joakim Erdfelt Date: Fri, 26 Jul 2019 13:32:03 -0500 Subject: [PATCH 18/33] Issue #3804 - PathResource should not use SPARSE hint always + SPARSE hint only applies to real os file systems or default file systems, not for all file systems. Signed-off-by: Joakim Erdfelt --- .../jetty/util/resource/PathResource.java | 6 ++-- .../jetty/util/resource/PathResourceTest.java | 30 +++++++++++++++---- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/jetty-util/src/main/java/org/eclipse/jetty/util/resource/PathResource.java b/jetty-util/src/main/java/org/eclipse/jetty/util/resource/PathResource.java index 7530f13e5d9..3253fd2507e 100644 --- a/jetty-util/src/main/java/org/eclipse/jetty/util/resource/PathResource.java +++ b/jetty-util/src/main/java/org/eclipse/jetty/util/resource/PathResource.java @@ -377,7 +377,8 @@ public class PathResource extends Resource @Override public InputStream getInputStream() throws IOException { - return Files.newInputStream(path, StandardOpenOption.READ, StandardOpenOption.SPARSE); + // TODO: investigate if SPARSE use for default FileSystem usages is worth it + return Files.newInputStream(path, StandardOpenOption.READ); } @Override @@ -389,7 +390,8 @@ public class PathResource extends Resource @Override public ReadableByteChannel getReadableByteChannel() throws IOException { - return Files.newByteChannel(path, StandardOpenOption.READ, StandardOpenOption.SPARSE); + // TODO: investigate if SPARSE use for default FileSystem usages is worth it + return Files.newByteChannel(path, StandardOpenOption.READ); } @Override diff --git a/jetty-util/src/test/java/org/eclipse/jetty/util/resource/PathResourceTest.java b/jetty-util/src/test/java/org/eclipse/jetty/util/resource/PathResourceTest.java index e12646f5bdc..a0fd66f0049 100644 --- a/jetty-util/src/test/java/org/eclipse/jetty/util/resource/PathResourceTest.java +++ b/jetty-util/src/test/java/org/eclipse/jetty/util/resource/PathResourceTest.java @@ -57,15 +57,35 @@ public class PathResourceTest PathResource resource = new PathResource(manifestPath); - try (ReadableByteChannel channel = resource.getReadableByteChannel()) - { - assertThat("ReadableByteChannel", channel, is(not(nullValue()))); - } - try (InputStream inputStream = resource.getInputStream()) { assertThat("InputStream", inputStream, is(not(nullValue()))); } } } + + @Test + public void testNonDefaultFileSystem_GetReadableByteChannel() throws URISyntaxException, IOException + { + Path exampleJar = MavenTestingUtils.getTestResourcePathFile("example.jar"); + + URI uri = new URI("jar", exampleJar.toUri().toASCIIString(), null); + System.err.println("URI = " + uri); + + Map env = new HashMap<>(); + env.put("multi-release", "runtime"); + + try (FileSystem zipfs = FileSystems.newFileSystem(uri, env)) + { + Path manifestPath = zipfs.getPath("/META-INF/MANIFEST.MF"); + assertThat(manifestPath, is(not(nullValue()))); + + PathResource resource = new PathResource(manifestPath); + + try (ReadableByteChannel channel = resource.getReadableByteChannel()) + { + assertThat("ReadableByteChannel", channel, is(not(nullValue()))); + } + } + } } From bb639817a3db2b3ea16ba1162a634ffedfcd8804 Mon Sep 17 00:00:00 2001 From: mpe85 Date: Mon, 29 Jul 2019 06:33:40 +0200 Subject: [PATCH 19/33] Made loading of blobs from database more robust (#3895) Signed-off-by: mpe85 --- .../jetty/server/session/DatabaseAdaptor.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/session/DatabaseAdaptor.java b/jetty-server/src/main/java/org/eclipse/jetty/server/session/DatabaseAdaptor.java index 46624566c3f..b01556c4034 100644 --- a/jetty-server/src/main/java/org/eclipse/jetty/server/session/DatabaseAdaptor.java +++ b/jetty-server/src/main/java/org/eclipse/jetty/server/session/DatabaseAdaptor.java @@ -27,6 +27,7 @@ import java.sql.Driver; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; import java.util.Locale; import javax.naming.InitialContext; import javax.naming.NamingException; @@ -164,8 +165,16 @@ public class DatabaseAdaptor return new ByteArrayInputStream(bytes); } - Blob blob = result.getBlob(columnName); - return blob.getBinaryStream(); + try + { + Blob blob = result.getBlob(columnName); + return blob.getBinaryStream(); + } + catch (SQLFeatureNotSupportedException ex) + { + byte[] bytes = result.getBytes(columnName); + return new ByteArrayInputStream(bytes); + } } public boolean isEmptyStringNull() From 1b6bf389d62e4f825f91dfe5ab7ce187461a3657 Mon Sep 17 00:00:00 2001 From: Olivier Lamy Date: Tue, 30 Jul 2019 11:13:54 +1000 Subject: [PATCH 20/33] Jetty 9.4.x pax exam upgrade for central repo https, pre download all artifacts for maven its (#3902) * ensure we use local repository first to avoid download, upgrade pax versions pre download artifacts to avoid download by maven invoker its Signed-off-by: olivier lamy * force using local repository first see https://issues.apache.org/jira/browse/MINVOKER-249 Signed-off-by: olivier lamy * use local setttings to use mirrors Signed-off-by: olivier lamy * use https Signed-off-by: olivier lamy * install build resources first to avoid checkstyle build issue Signed-off-by: olivier lamy * downgrade pax-swissbox-framework as the pax exam seems to use this one and want to download it... Signed-off-by: olivier lamy * fix enforcer rule RequireUpperBoundDeps... Signed-off-by: olivier lamy * not used anymore Signed-off-by: olivier lamy * force pax url to use provided settings via Jenkins withMvn Signed-off-by: olivier lamy * renove debugging System.out Signed-off-by: olivier lamy * version is defined in the project Signed-off-by: olivier lamy * cleanup after Jan review comments Signed-off-by: olivier lamy --- Jenkinsfile | 5 ++--- jetty-maven-plugin/src/it/it-parent-pom/pom.xml | 10 ++++++++++ .../postbuild.groovy | 2 +- jetty-maven-plugin/src/it/settings.xml | 7 +++++++ jetty-osgi/test-jetty-osgi/pom.xml | 5 +++-- .../org/eclipse/jetty/osgi/test/TestOSGiUtil.java | 14 +++++++++++++- pom.xml | 14 ++++++++++---- 7 files changed, 46 insertions(+), 11 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 77a8c47fb48..dd81c80846a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -73,6 +73,7 @@ pipeline { agent { node { label 'linux' } } options { timeout(time: 30, unit: 'MINUTES') } steps { + mavenBuild("jdk11", "install -f build-resources", "maven3", true) mavenBuild("jdk11", "install checkstyle:check -DskipTests", "maven3", true) recordIssues( enabledForFailure: true, aggregatingResults: true, @@ -137,15 +138,13 @@ def slackNotif() { * @return the Jenkinsfile step representing a maven build */ def mavenBuild(jdk, cmdline, mvnName, junitPublishDisabled) { - def localRepo = "${env.JENKINS_HOME}/${env.EXECUTOR_NUMBER}" // ".repository" // - def settingsName = 'oss-settings.xml' + def localRepo = ".repository" def mavenOpts = '-Xms1g -Xmx4g -Djava.awt.headless=true' withMaven( maven: mvnName, jdk: "$jdk", publisherStrategy: 'EXPLICIT', - globalMavenSettingsConfig: settingsName, options: [junitPublisher(disabled: junitPublishDisabled),mavenLinkerPublisher(disabled: false),pipelineGraphPublisher(disabled: false)], mavenOpts: mavenOpts, mavenLocalRepo: localRepo) { diff --git a/jetty-maven-plugin/src/it/it-parent-pom/pom.xml b/jetty-maven-plugin/src/it/it-parent-pom/pom.xml index 6350cf41786..e18e1028903 100644 --- a/jetty-maven-plugin/src/it/it-parent-pom/pom.xml +++ b/jetty-maven-plugin/src/it/it-parent-pom/pom.xml @@ -144,6 +144,16 @@ maven-war-plugin @maven.war.plugin.version@ + + org.apache.maven.plugins + maven-install-plugin + @maven.install.plugin.version@ + + + org.apache.maven.plugins + maven-deploy-plugin + @maven.deploy.plugin.version@ + diff --git a/jetty-maven-plugin/src/it/jetty-run-mojo-multi-module-single-war-it/postbuild.groovy b/jetty-maven-plugin/src/it/jetty-run-mojo-multi-module-single-war-it/postbuild.groovy index b8b8b5e2599..bd9510f83fc 100644 --- a/jetty-maven-plugin/src/it/jetty-run-mojo-multi-module-single-war-it/postbuild.groovy +++ b/jetty-maven-plugin/src/it/jetty-run-mojo-multi-module-single-war-it/postbuild.groovy @@ -20,7 +20,7 @@ File buildLog = new File( basedir, 'build.log' ) assert buildLog.text.contains( 'Started Jetty Server' ) assert buildLog.text.contains( '(1a) >> javax.servlet.ServletContextListener loaded from jar:' ) -assert buildLog.text.contains( 'local-repo/javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar!/javax/servlet/ServletContextListener.class << (1b)' ) +assert buildLog.text.contains( 'javax/servlet/javax.servlet-api/3.1.0/javax.servlet-api-3.1.0.jar!/javax/servlet/ServletContextListener.class << (1b)' ) assert buildLog.text.contains( '(2a) >> mca.common.CommonService loaded from file:' ) assert buildLog.text.contains( 'common/target/classes/mca/common/CommonService.class << (2b)' ) diff --git a/jetty-maven-plugin/src/it/settings.xml b/jetty-maven-plugin/src/it/settings.xml index d64bdb89034..c78e01dff9d 100644 --- a/jetty-maven-plugin/src/it/settings.xml +++ b/jetty-maven-plugin/src/it/settings.xml @@ -1,6 +1,13 @@ + + + local.mirror + file://@localRepo@ + central + + it-repo diff --git a/jetty-osgi/test-jetty-osgi/pom.xml b/jetty-osgi/test-jetty-osgi/pom.xml index 0122c6de984..8befd094b12 100644 --- a/jetty-osgi/test-jetty-osgi/pom.xml +++ b/jetty-osgi/test-jetty-osgi/pom.xml @@ -14,8 +14,8 @@ ${project.groupId}.boot.test.osgi http://download.eclipse.org/jetty/orbit/ target/distribution - 4.12.0 - 2.5.2 + 4.13.1 + 2.6.1 1.0 true @@ -449,6 +449,7 @@ ${skipTests} ${settings.localRepository} + ${env.GLOBAL_MVN_SETTINGS} diff --git a/jetty-osgi/test-jetty-osgi/src/test/java/org/eclipse/jetty/osgi/test/TestOSGiUtil.java b/jetty-osgi/test-jetty-osgi/src/test/java/org/eclipse/jetty/osgi/test/TestOSGiUtil.java index 606bf4b6831..641d79d805c 100644 --- a/jetty-osgi/test-jetty-osgi/src/test/java/org/eclipse/jetty/osgi/test/TestOSGiUtil.java +++ b/jetty-osgi/test-jetty-osgi/src/test/java/org/eclipse/jetty/osgi/test/TestOSGiUtil.java @@ -38,6 +38,7 @@ import org.eclipse.jetty.util.StringUtil; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.ops4j.pax.exam.CoreOptions; import org.ops4j.pax.exam.Option; +import org.ops4j.pax.url.mvn.internal.AetherBasedResolver; import org.osgi.framework.Bundle; import org.osgi.framework.BundleContext; import org.osgi.framework.ServiceReference; @@ -106,11 +107,22 @@ public class TestOSGiUtil public static List ci + + ${env.GLOBAL_MVN_SETTINGS} + aggregates/jetty-all