From b3eb881849b289f624515ec96bddc72e66817a29 Mon Sep 17 00:00:00 2001 From: Simone Bordet Date: Mon, 2 Sep 2013 17:59:54 +0200 Subject: [PATCH] More work on generation/parsing. --- .../java/org/eclipse/jetty/fcgi/FCGI.java | 5 + .../jetty/fcgi/generator/ClientGenerator.java | 18 ++- .../jetty/fcgi/generator/Generator.java | 47 ++++++- .../jetty/fcgi/parser/ClientParser.java | 13 +- .../jetty/fcgi/parser/ContentParser.java | 5 + .../jetty/fcgi/parser/HeaderParser.java | 5 + .../fcgi/parser/ParamsContentParser.java | 39 ++++-- .../org/eclipse/jetty/fcgi/parser/Parser.java | 20 +++ .../fcgi/parser/StreamContentParser.java | 115 ++++++++++++++++++ .../fcgi/generator/ClientGeneratorTest.java | 70 ++++++++++- 10 files changed, 303 insertions(+), 34 deletions(-) create mode 100644 src/main/java/org/eclipse/jetty/fcgi/parser/StreamContentParser.java diff --git a/src/main/java/org/eclipse/jetty/fcgi/FCGI.java b/src/main/java/org/eclipse/jetty/fcgi/FCGI.java index e8f4a3f3376..9bab54c966e 100644 --- a/src/main/java/org/eclipse/jetty/fcgi/FCGI.java +++ b/src/main/java/org/eclipse/jetty/fcgi/FCGI.java @@ -73,4 +73,9 @@ public class FCGI this.code = code; } } + + public enum StreamType + { + STD_IN, STD_OUT, STD_ERR + } } diff --git a/src/main/java/org/eclipse/jetty/fcgi/generator/ClientGenerator.java b/src/main/java/org/eclipse/jetty/fcgi/generator/ClientGenerator.java index 55583ae4b13..5db08e35560 100644 --- a/src/main/java/org/eclipse/jetty/fcgi/generator/ClientGenerator.java +++ b/src/main/java/org/eclipse/jetty/fcgi/generator/ClientGenerator.java @@ -36,16 +36,14 @@ public class ClientGenerator extends Generator // 0x7F_FF - 4 (the 4 is to make room for the name (or value) length). public static final int MAX_PARAM_LENGTH = 0x7F_FF - 4; - private final ByteBufferPool byteBufferPool; - public ClientGenerator(ByteBufferPool byteBufferPool) { - this.byteBufferPool = byteBufferPool; + super(byteBufferPool); } - public Result generateRequestHeaders(int id, Fields fields, Callback callback) + public Result generateRequestHeaders(int request, Fields fields, Callback callback) { - id = id & 0xFF_FF; + request &= 0xFF_FF; Charset utf8 = Charset.forName("UTF-8"); List bytes = new ArrayList<>(fields.size() * 2); @@ -86,7 +84,7 @@ public class ClientGenerator extends Generator result.add(beginRequestBuffer, true); // Generate the FCGI_BEGIN_REQUEST frame - beginRequestBuffer.putInt(0x01_01_00_00 + id); + beginRequestBuffer.putInt(0x01_01_00_00 + request); beginRequestBuffer.putInt(0x00_08_00_00); beginRequestBuffer.putLong(0x00_01_01_00_00_00_00_00L); beginRequestBuffer.flip(); @@ -100,7 +98,7 @@ public class ClientGenerator extends Generator result.add(buffer, true); // Generate the FCGI_PARAMS frame - buffer.putInt(0x01_04_00_00 + id); + buffer.putInt(0x01_04_00_00 + request); buffer.putShort((short)0); buffer.putShort((short)0); capacity -= 8; @@ -138,7 +136,7 @@ public class ClientGenerator extends Generator result.add(lastParamsBuffer, true); // Generate the last FCGI_PARAMS frame - lastParamsBuffer.putInt(0x01_04_00_00 + id); + lastParamsBuffer.putInt(0x01_04_00_00 + request); lastParamsBuffer.putInt(0x00_00_00_00); lastParamsBuffer.flip(); @@ -160,8 +158,8 @@ public class ClientGenerator extends Generator return length > 127 ? 4 : 1; } - public ByteBuffer generateRequestContent(ByteBuffer content) + public Result generateRequestContent(int request, ByteBuffer content, boolean lastContent, Callback callback) { - return content == null ? generateContent(FCGI.FrameType.STDIN, 0) : generateContent(FCGI.FrameType.STDIN, content.remaining()); + return generateContent(request, content, lastContent, callback, FCGI.FrameType.STDIN); } } diff --git a/src/main/java/org/eclipse/jetty/fcgi/generator/Generator.java b/src/main/java/org/eclipse/jetty/fcgi/generator/Generator.java index dd861fbcc80..d289e3fea7f 100644 --- a/src/main/java/org/eclipse/jetty/fcgi/generator/Generator.java +++ b/src/main/java/org/eclipse/jetty/fcgi/generator/Generator.java @@ -27,9 +27,52 @@ import org.eclipse.jetty.util.Callback; public class Generator { - protected ByteBuffer generateContent(FCGI.FrameType frameType, int length) + public static final int MAX_CONTENT_LENGTH = 0xFF_FF; + + protected final ByteBufferPool byteBufferPool; + + public Generator(ByteBufferPool byteBufferPool) { - return BufferUtil.EMPTY_BUFFER; + this.byteBufferPool = byteBufferPool; + } + + protected Result generateContent(int id, ByteBuffer content, boolean lastContent, Callback callback, FCGI.FrameType frameType) + { + id &= 0xFF_FF; + + int remaining = content == null ? 0 : content.remaining(); + int frames = 2 * (remaining / MAX_CONTENT_LENGTH + 1) + (lastContent ? 1 : 0); + Result result = new Result(byteBufferPool, callback, frames); + + while (remaining > 0 || lastContent) + { + ByteBuffer buffer = byteBufferPool.acquire(8, false); + BufferUtil.clearToFill(buffer); + result.add(buffer, true); + + // Generate the frame header + buffer.put((byte)0x01); + buffer.put((byte)frameType.code); + buffer.putShort((short)id); + int length = Math.min(MAX_CONTENT_LENGTH, remaining); + buffer.putShort((short)length); + buffer.putShort((short)0); + buffer.flip(); + + if (remaining == 0) + break; + + // Slice to content to avoid copying + int limit = content.limit(); + content.limit(content.position() + length); + ByteBuffer slice = content.slice(); + result.add(slice, false); + content.position(content.limit()); + content.limit(limit); + remaining -= length; + } + + return result; } public static class Result implements Callback diff --git a/src/main/java/org/eclipse/jetty/fcgi/parser/ClientParser.java b/src/main/java/org/eclipse/jetty/fcgi/parser/ClientParser.java index 66f74a55739..6349dcee1aa 100644 --- a/src/main/java/org/eclipse/jetty/fcgi/parser/ClientParser.java +++ b/src/main/java/org/eclipse/jetty/fcgi/parser/ClientParser.java @@ -32,6 +32,7 @@ public class ClientParser extends Parser this.listener = listener; contentParsers.put(FCGI.FrameType.BEGIN_REQUEST, new BeginRequestContentParser(headerParser)); contentParsers.put(FCGI.FrameType.PARAMS, new ParamsContentParser(headerParser, listener)); + contentParsers.put(FCGI.FrameType.STDIN, new StreamContentParser(headerParser, FCGI.StreamType.STD_IN, listener)); } @Override @@ -40,21 +41,21 @@ public class ClientParser extends Parser return contentParsers.get(frameType); } - public interface Listener + public interface Listener extends Parser.Listener { - public void onParam(String name, String value); + public void onParam(int request, String name, String value); - public void onParams(); + public void onParams(int request); - public static class Adapter implements Listener + public static class Adapter extends Parser.Listener.Adapter implements Listener { @Override - public void onParam(String name, String value) + public void onParam(int request, String name, String value) { } @Override - public void onParams() + public void onParams(int request) { } } diff --git a/src/main/java/org/eclipse/jetty/fcgi/parser/ContentParser.java b/src/main/java/org/eclipse/jetty/fcgi/parser/ContentParser.java index d3cfb6e0f70..9775551d6f6 100644 --- a/src/main/java/org/eclipse/jetty/fcgi/parser/ContentParser.java +++ b/src/main/java/org/eclipse/jetty/fcgi/parser/ContentParser.java @@ -36,6 +36,11 @@ public abstract class ContentParser throw new IllegalStateException(); } + protected int getRequest() + { + return headerParser.getRequest(); + } + protected int getContentLength() { return headerParser.getContentLength(); diff --git a/src/main/java/org/eclipse/jetty/fcgi/parser/HeaderParser.java b/src/main/java/org/eclipse/jetty/fcgi/parser/HeaderParser.java index 4a3eeea149b..8176e777dca 100644 --- a/src/main/java/org/eclipse/jetty/fcgi/parser/HeaderParser.java +++ b/src/main/java/org/eclipse/jetty/fcgi/parser/HeaderParser.java @@ -119,6 +119,11 @@ public class HeaderParser return FCGI.FrameType.from(type); } + public int getRequest() + { + return request; + } + public int getContentLength() { return length; diff --git a/src/main/java/org/eclipse/jetty/fcgi/parser/ParamsContentParser.java b/src/main/java/org/eclipse/jetty/fcgi/parser/ParamsContentParser.java index 6639f39555d..03032cbda70 100644 --- a/src/main/java/org/eclipse/jetty/fcgi/parser/ParamsContentParser.java +++ b/src/main/java/org/eclipse/jetty/fcgi/parser/ParamsContentParser.java @@ -21,8 +21,13 @@ package org.eclipse.jetty.fcgi.parser; import java.nio.ByteBuffer; import java.nio.charset.Charset; +import org.eclipse.jetty.util.log.Log; +import org.eclipse.jetty.util.log.Logger; + public class ParamsContentParser extends ContentParser { + private static final Logger logger = Log.getLogger(ParamsContentParser.class); + private final ClientParser.Listener listener; private State state = State.LENGTH; private int cursor; @@ -192,22 +197,36 @@ public class ParamsContentParser extends ContentParser return false; } - protected void onParam(String name, String value) - { - listener.onParam(name, value); - } - - protected void onParams() - { - listener.onParams(); - } - @Override public void noContent() { onParams(); } + protected void onParam(String name, String value) + { + try + { + listener.onParam(getRequest(), name, value); + } + catch (Throwable x) + { + logger.debug("Exception while invoking listener " + listener, x); + } + } + + protected void onParams() + { + try + { + listener.onParams(getRequest()); + } + catch (Throwable x) + { + logger.debug("Exception while invoking listener " + listener, x); + } + } + private boolean isLargeLength(ByteBuffer buffer) { return (buffer.get(buffer.position()) & 0x80) == 0x80; diff --git a/src/main/java/org/eclipse/jetty/fcgi/parser/Parser.java b/src/main/java/org/eclipse/jetty/fcgi/parser/Parser.java index ba67f2aeb07..a9982559a77 100644 --- a/src/main/java/org/eclipse/jetty/fcgi/parser/Parser.java +++ b/src/main/java/org/eclipse/jetty/fcgi/parser/Parser.java @@ -89,6 +89,26 @@ public abstract class Parser padding = 0; } + public interface Listener + { + public void onContent(int request, FCGI.StreamType stream, ByteBuffer buffer); + + public void onEnd(int request); + + public static class Adapter implements Listener + { + @Override + public void onContent(int request, FCGI.StreamType stream, ByteBuffer buffer) + { + } + + @Override + public void onEnd(int request) + { + } + } + } + private enum State { HEADER, CONTENT, PADDING diff --git a/src/main/java/org/eclipse/jetty/fcgi/parser/StreamContentParser.java b/src/main/java/org/eclipse/jetty/fcgi/parser/StreamContentParser.java new file mode 100644 index 00000000000..419254544cd --- /dev/null +++ b/src/main/java/org/eclipse/jetty/fcgi/parser/StreamContentParser.java @@ -0,0 +1,115 @@ +// +// ======================================================================== +// Copyright (c) 1995-2013 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.fcgi.parser; + +import java.nio.ByteBuffer; + +import org.eclipse.jetty.fcgi.FCGI; +import org.eclipse.jetty.util.log.Log; +import org.eclipse.jetty.util.log.Logger; + +public class StreamContentParser extends ContentParser +{ + private static final Logger logger = Log.getLogger(StreamContentParser.class); + + private final FCGI.StreamType streamType; + private final Parser.Listener listener; + private State state = State.LENGTH; + private int contentLength; + + public StreamContentParser(HeaderParser headerParser, FCGI.StreamType streamType, Parser.Listener listener) + { + super(headerParser); + this.streamType = streamType; + this.listener = listener; + } + + @Override + public boolean parse(ByteBuffer buffer) + { + while (buffer.hasRemaining()) + { + switch (state) + { + case LENGTH: + { + contentLength = getContentLength(); + state = State.CONTENT; + break; + } + case CONTENT: + { + int length = Math.min(contentLength, buffer.remaining()); + int limit = buffer.limit(); + buffer.limit(buffer.position() + length); + ByteBuffer slice = buffer.slice(); + onContent(slice); + buffer.position(buffer.limit()); + buffer.limit(limit); + contentLength -= length; + if (contentLength > 0) + break; + state = State.LENGTH; + return true; + } + default: + { + throw new IllegalStateException(); + } + } + } + return false; + } + + @Override + public void noContent() + { + if (streamType == FCGI.StreamType.STD_IN) + onEnd(); + } + + protected void onContent(ByteBuffer buffer) + { + try + { + listener.onContent(getRequest(), streamType, buffer); + } + catch (Throwable x) + { + logger.debug("Exception while invoking listener " + listener, x); + } + } + + protected void onEnd() + { + try + { + listener.onEnd(getRequest()); + } + catch (Throwable x) + { + logger.debug("Exception while invoking listener " + listener, x); + } + } + + private enum State + { + LENGTH, CONTENT + } +} diff --git a/src/test/java/org/eclipse/jetty/fcgi/generator/ClientGeneratorTest.java b/src/test/java/org/eclipse/jetty/fcgi/generator/ClientGeneratorTest.java index f9a0efd6db5..9c6bee81f35 100644 --- a/src/test/java/org/eclipse/jetty/fcgi/generator/ClientGeneratorTest.java +++ b/src/test/java/org/eclipse/jetty/fcgi/generator/ClientGeneratorTest.java @@ -22,6 +22,7 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.concurrent.atomic.AtomicInteger; +import org.eclipse.jetty.fcgi.FCGI; import org.eclipse.jetty.fcgi.parser.ClientParser; import org.eclipse.jetty.io.ByteBufferPool; import org.eclipse.jetty.io.MappedByteBufferPool; @@ -32,7 +33,7 @@ import org.junit.Test; public class ClientGeneratorTest { @Test - public void testGenerateRequestWithoutContent() throws Exception + public void testGenerateRequestHeaders() throws Exception { Fields fields = new Fields(); @@ -57,12 +58,13 @@ public class ClientGeneratorTest char[] chars = new char[ClientGenerator.MAX_PARAM_LENGTH]; Arrays.fill(chars, 'z'); final String longLongName = new String(chars); - final String longLongValue = longLongName; + final String longLongValue = new String(chars); fields.put(new Fields.Field(longLongName, longLongValue)); ByteBufferPool byteBufferPool = new MappedByteBufferPool(); ClientGenerator generator = new ClientGenerator(byteBufferPool); - Generator.Result result = generator.generateRequestHeaders(13, fields, null); + final int id = 13; + Generator.Result result = generator.generateRequestHeaders(id, fields, null); // Use the fundamental theorem of arithmetic to test the results. // This way we know onParam() has been called the right number of @@ -76,8 +78,9 @@ public class ClientGeneratorTest ClientParser parser = new ClientParser(new ClientParser.Listener.Adapter() { @Override - public void onParam(String name, String value) + public void onParam(int request, String name, String value) { + Assert.assertEquals(id, request); switch (name) { case shortShortName: @@ -101,8 +104,9 @@ public class ClientGeneratorTest } @Override - public void onParams() + public void onParams(int request) { + Assert.assertEquals(id, request); params.set(params.get() * primes[4]); } }); @@ -117,7 +121,6 @@ public class ClientGeneratorTest // Parse again byte by byte params.set(1); - for (ByteBuffer buffer : result.getByteBuffers()) { buffer.flip(); @@ -128,4 +131,59 @@ public class ClientGeneratorTest Assert.assertEquals(value, params.get()); } + + @Test + public void testGenerateSmallRequestContent() throws Exception + { + testGenerateRequestContent(1024); + } + + @Test + public void testGenerateLargeRequestContent() throws Exception + { + testGenerateRequestContent(128 * 1024); + } + + private void testGenerateRequestContent(final int contentLength) throws Exception + { + ByteBuffer content = ByteBuffer.allocate(contentLength); + + ByteBufferPool byteBufferPool = new MappedByteBufferPool(); + ClientGenerator generator = new ClientGenerator(byteBufferPool); + final int id = 13; + Generator.Result result = generator.generateRequestContent(id, content, true, null); + + final AtomicInteger length = new AtomicInteger(); + ClientParser parser = new ClientParser(new ClientParser.Listener.Adapter() + { + @Override + public void onContent(int request, FCGI.StreamType stream, ByteBuffer buffer) + { + Assert.assertEquals(id, request); + length.addAndGet(buffer.remaining()); + } + + @Override + public void onEnd(int request) + { + Assert.assertEquals(id, request); + Assert.assertEquals(contentLength, length.get()); + } + }); + + for (ByteBuffer buffer : result.getByteBuffers()) + { + parser.parse(buffer); + Assert.assertFalse(buffer.hasRemaining()); + } + + // Parse again one byte at a time + for (ByteBuffer buffer : result.getByteBuffers()) + { + buffer.flip(); + while (buffer.hasRemaining()) + parser.parse(ByteBuffer.wrap(new byte[]{buffer.get()})); + Assert.assertFalse(buffer.hasRemaining()); + } + } }