Implemented parser and generator for DATA frame.

This commit is contained in:
Simone Bordet 2014-06-05 18:11:22 +02:00
parent 2b0f983b9c
commit a7e5963dfa
8 changed files with 822 additions and 0 deletions

View File

@ -13,6 +13,11 @@
<bundle-symbolic-name>${project.groupId}.http</bundle-symbolic-name>
</properties>
<dependencies>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-io</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>

View File

@ -0,0 +1,52 @@
//
// ========================================================================
// Copyright (c) 1995-2014 Mort Bay Consulting Pty. Ltd.
// ------------------------------------------------------------------------
// All rights reserved. This program and the accompanying materials
// are made available under the terms of the Eclipse Public License v1.0
// and Apache License v2.0 which accompanies this distribution.
//
// The Eclipse Public License is available at
// http://www.eclipse.org/legal/epl-v10.html
//
// The Apache License v2.0 is available at
// http://www.opensource.org/licenses/apache2.0.php
//
// You may elect to redistribute this code under either of these licenses.
// ========================================================================
//
package org.eclipse.jetty.http2.frames;
import java.nio.ByteBuffer;
public class DataFrame
{
public static final int MAX_LENGTH = 0x3F_FF;
private final int streamId;
private final ByteBuffer data;
private boolean end;
public DataFrame(int streamId, ByteBuffer data, boolean end)
{
this.streamId = streamId;
this.data = data;
this.end = end;
}
public int getStreamId()
{
return streamId;
}
public boolean isEnd()
{
return end;
}
public ByteBuffer getData()
{
return data;
}
}

View File

@ -0,0 +1,155 @@
//
// ========================================================================
// Copyright (c) 1995-2014 Mort Bay Consulting Pty. Ltd.
// ------------------------------------------------------------------------
// All rights reserved. This program and the accompanying materials
// are made available under the terms of the Eclipse Public License v1.0
// and Apache License v2.0 which accompanies this distribution.
//
// The Eclipse Public License is available at
// http://www.eclipse.org/legal/epl-v10.html
//
// The Apache License v2.0 is available at
// http://www.opensource.org/licenses/apache2.0.php
//
// You may elect to redistribute this code under either of these licenses.
// ========================================================================
//
package org.eclipse.jetty.http2.generator;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import org.eclipse.jetty.http2.frames.DataFrame;
import org.eclipse.jetty.io.ByteBufferPool;
import org.eclipse.jetty.util.BufferUtil;
public class Generator
{
private final ByteBufferPool byteBufferPool;
public Generator(ByteBufferPool byteBufferPool)
{
this.byteBufferPool = byteBufferPool;
}
public Result generateContent(int streamId, int paddingLength, ByteBuffer data, boolean last, boolean compress)
{
if (streamId < 0)
throw new IllegalArgumentException("Invalid stream id: " + streamId);
// Leave space for at least one byte of content.
if (paddingLength > DataFrame.MAX_LENGTH - 3)
throw new IllegalArgumentException("Invalid padding length: " + paddingLength);
int paddingBytes = paddingLength > 0xFF ? 2 : paddingLength > 0 ? 1 : 0;
// TODO: here we should compress the data, and then reason on the data length !
int dataLength = data.remaining();
Result result = new Result(byteBufferPool);
// Can we fit just one frame ?
if (dataLength + paddingBytes + paddingLength <= DataFrame.MAX_LENGTH)
{
generateFrame(result, streamId, paddingBytes, paddingLength, data, last, compress);
}
else
{
int dataBytesPerFrame = DataFrame.MAX_LENGTH - paddingBytes - paddingLength;
int frames = dataLength / dataBytesPerFrame;
if (frames * dataBytesPerFrame != dataLength)
{
++frames;
}
int limit = data.limit();
for (int i = 1; i <= frames; ++i)
{
data.limit(Math.min(dataBytesPerFrame * i, limit));
ByteBuffer slice = data.slice();
data.position(data.limit());
generateFrame(result, streamId, paddingBytes, paddingLength, slice, i == frames && last, compress);
}
}
return result;
}
private void generateFrame(Result result, int streamId, int paddingBytes, int paddingLength, ByteBuffer data, boolean last, boolean compress)
{
ByteBuffer header = byteBufferPool.acquire(8 + paddingBytes, true);
BufferUtil.clearToFill(header);
int length = paddingBytes + data.remaining() + paddingLength;
header.putShort((short)length);
// Frame type for DATA frames is 0.
header.put((byte)0);
int flags = 0;
if (last)
flags |= 0x01;
if (paddingBytes > 0)
flags |= 0x08;
if (paddingBytes > 1)
flags |= 0x10;
if (compress)
flags |= 0x20;
header.put((byte)flags);
header.putInt(streamId);
if (paddingBytes == 2)
header.putShort((short)paddingLength);
else if (paddingBytes == 1)
header.put((byte)paddingLength);
BufferUtil.flipToFlush(header, 0);
result.add(header, true);
result.add(data, false);
if (paddingBytes > 0)
{
ByteBuffer padding = byteBufferPool.acquire(paddingLength, true);
BufferUtil.clearToFill(padding);
padding.position(paddingLength);
BufferUtil.flipToFlush(padding, 0);
result.add(padding, true);
}
}
public static class Result
{
private final ByteBufferPool byteBufferPool;
private final List<ByteBuffer> buffers;
private final List<Boolean> recycles;
public Result(ByteBufferPool byteBufferPool)
{
this.byteBufferPool = byteBufferPool;
this.buffers = new ArrayList<>();
this.recycles = new ArrayList<>();
}
public void add(ByteBuffer buffer, boolean recycle)
{
buffers.add(buffer);
recycles.add(recycle);
}
public List<ByteBuffer> getByteBuffers()
{
return buffers;
}
public Result merge(Result that)
{
assert byteBufferPool == that.byteBufferPool;
buffers.addAll(that.buffers);
recycles.addAll(that.recycles);
return this;
}
}
}

View File

@ -0,0 +1,36 @@
//
// ========================================================================
// Copyright (c) 1995-2014 Mort Bay Consulting Pty. Ltd.
// ------------------------------------------------------------------------
// All rights reserved. This program and the accompanying materials
// are made available under the terms of the Eclipse Public License v1.0
// and Apache License v2.0 which accompanies this distribution.
//
// The Eclipse Public License is available at
// http://www.eclipse.org/legal/epl-v10.html
//
// The Apache License v2.0 is available at
// http://www.opensource.org/licenses/apache2.0.php
//
// You may elect to redistribute this code under either of these licenses.
// ========================================================================
//
package org.eclipse.jetty.http2.parser;
import java.nio.ByteBuffer;
import org.eclipse.jetty.util.log.Log;
import org.eclipse.jetty.util.log.Logger;
public abstract class BodyParser
{
protected static final Logger LOG = Log.getLogger(BodyParser.class);
public abstract Result parse(ByteBuffer buffer);
public enum Result
{
PENDING, ASYNC, COMPLETE
}
}

View File

@ -0,0 +1,156 @@
//
// ========================================================================
// Copyright (c) 1995-2014 Mort Bay Consulting Pty. Ltd.
// ------------------------------------------------------------------------
// All rights reserved. This program and the accompanying materials
// are made available under the terms of the Eclipse Public License v1.0
// and Apache License v2.0 which accompanies this distribution.
//
// The Eclipse Public License is available at
// http://www.eclipse.org/legal/epl-v10.html
//
// The Apache License v2.0 is available at
// http://www.opensource.org/licenses/apache2.0.php
//
// You may elect to redistribute this code under either of these licenses.
// ========================================================================
//
package org.eclipse.jetty.http2.parser;
import java.nio.ByteBuffer;
import org.eclipse.jetty.http2.frames.DataFrame;
public class DataBodyParser extends BodyParser
{
private final HeaderParser headerParser;
private final Parser.Listener listener;
private State state = State.PREPARE;
private int paddingLength;
private int length;
public DataBodyParser(HeaderParser headerParser, Parser.Listener listener)
{
this.headerParser = headerParser;
this.listener = listener;
}
private void reset()
{
headerParser.reset();
state = State.PREPARE;
paddingLength = 0;
length = 0;
}
@Override
public Result parse(ByteBuffer buffer)
{
while (buffer.hasRemaining())
{
switch (state)
{
case PREPARE:
{
length = headerParser.getLength();
if (headerParser.isPaddingHigh())
{
state = State.PADDING_HIGH;
}
else if (headerParser.isPaddingLow())
{
state = State.PADDING_LOW;
}
else
{
state = State.DATA;
}
break;
}
case PADDING_HIGH:
{
paddingLength = (buffer.get() & 0xFF) << 8;
length -= 1;
state = State.PADDING_LOW;
break;
}
case PADDING_LOW:
{
paddingLength += buffer.get() & 0xFF;
length -= 1;
length -= paddingLength;
state = State.DATA;
break;
}
case DATA:
{
int size = Math.min(buffer.remaining(), length);
int position = buffer.position();
int limit = buffer.limit();
buffer.limit(position + size);
ByteBuffer slice = buffer.slice();
buffer.limit(limit);
buffer.position(position + size);
length -= size;
if (length == 0)
{
state = State.PADDING;
if (onData(slice, false))
{
return Result.ASYNC;
}
}
else
{
// We got partial data, fake the frame.
if (onData(slice, true))
{
return Result.ASYNC;
}
}
break;
}
case PADDING:
{
int size = Math.min(buffer.remaining(), paddingLength);
buffer.position(buffer.position() + size);
paddingLength -= size;
if (paddingLength == 0)
{
reset();
return Result.COMPLETE;
}
break;
}
}
}
return Result.PENDING;
}
private boolean onData(ByteBuffer buffer, boolean fragment)
{
boolean end = headerParser.isEndStream();
DataFrame frame = new DataFrame(headerParser.getStreamId(), buffer, fragment ? false : end);
return notifyDataFrame(frame);
}
protected boolean notifyDataFrame(DataFrame frame)
{
try
{
return listener.onDataFrame(frame);
}
catch (Throwable x)
{
LOG.info("Failure while notifying listener " + listener, x);
return false;
}
}
private enum State
{
PREPARE, PADDING_HIGH, PADDING_LOW, DATA, PADDING
}
}

View File

@ -0,0 +1,148 @@
//
// ========================================================================
// Copyright (c) 1995-2014 Mort Bay Consulting Pty. Ltd.
// ------------------------------------------------------------------------
// All rights reserved. This program and the accompanying materials
// are made available under the terms of the Eclipse Public License v1.0
// and Apache License v2.0 which accompanies this distribution.
//
// The Eclipse Public License is available at
// http://www.eclipse.org/legal/epl-v10.html
//
// The Apache License v2.0 is available at
// http://www.opensource.org/licenses/apache2.0.php
//
// You may elect to redistribute this code under either of these licenses.
// ========================================================================
//
package org.eclipse.jetty.http2.parser;
import java.nio.ByteBuffer;
import org.eclipse.jetty.http2.frames.DataFrame;
public class HeaderParser
{
private State state = State.LENGTH;
private int cursor;
private int length;
private int type;
private int flags;
private int streamId;
protected void reset()
{
state = State.LENGTH;
cursor = 0;
length = 0;
type = 0;
flags = 0;
streamId = 0;
}
public boolean parse(ByteBuffer buffer)
{
while (buffer.hasRemaining())
{
switch (state)
{
case LENGTH:
{
int halfShort = buffer.get() & 0xFF;
length = (length << 8) + halfShort;
if (++cursor == 2)
{
// First 2 most significant bits MUST be ignored as per specification.
length &= DataFrame.MAX_LENGTH;
state = State.TYPE;
}
break;
}
case TYPE:
{
type = buffer.get() & 0xFF;
state = State.FLAGS;
break;
}
case FLAGS:
{
flags = buffer.get() & 0xFF;
state = State.STREAM_ID;
break;
}
case STREAM_ID:
{
if (buffer.remaining() >= 4)
{
streamId = buffer.getInt();
// Most significant bit MUST be ignored as per specification.
streamId &= 0x7F_FF_FF_FF;
return true;
}
else
{
state = State.STREAM_ID_BYTES;
cursor = 4;
}
break;
}
case STREAM_ID_BYTES:
{
int currByte = buffer.get() & 0xFF;
--cursor;
streamId += currByte << (8 * cursor);
if (cursor == 0)
{
// Most significant bit MUST be ignored as per specification.
streamId &= 0x7F_FF_FF_FF;
return true;
}
break;
}
default:
{
throw new IllegalStateException();
}
}
}
return false;
}
public int getLength()
{
return length;
}
public int getFrameType()
{
return type;
}
public boolean isPaddingHigh()
{
return (flags & 0x10) == 0x10;
}
public boolean isPaddingLow()
{
return (flags & 0x08) == 0x08;
}
public boolean isEndStream()
{
return (flags & 0x01) == 0x01;
}
public int getStreamId()
{
return streamId;
}
private enum State
{
LENGTH, TYPE, FLAGS, STREAM_ID, STREAM_ID_BYTES
}
}

View File

@ -0,0 +1,96 @@
//
// ========================================================================
// Copyright (c) 1995-2014 Mort Bay Consulting Pty. Ltd.
// ------------------------------------------------------------------------
// All rights reserved. This program and the accompanying materials
// are made available under the terms of the Eclipse Public License v1.0
// and Apache License v2.0 which accompanies this distribution.
//
// The Eclipse Public License is available at
// http://www.eclipse.org/legal/epl-v10.html
//
// The Apache License v2.0 is available at
// http://www.opensource.org/licenses/apache2.0.php
//
// You may elect to redistribute this code under either of these licenses.
// ========================================================================
//
package org.eclipse.jetty.http2.parser;
import java.nio.ByteBuffer;
import org.eclipse.jetty.http2.frames.DataFrame;
public class Parser
{
private final HeaderParser headerParser = new HeaderParser();
private final BodyParser[] bodyParsers = new BodyParser[1];
private State state = State.HEADER;
private BodyParser bodyParser;
public Parser(Listener listener)
{
bodyParsers[0] = new DataBodyParser(headerParser, listener);
}
private void reset()
{
state = State.HEADER;
}
public boolean parse(ByteBuffer buffer)
{
while (buffer.hasRemaining())
{
switch (state)
{
case HEADER:
{
if (headerParser.parse(buffer))
{
int type = headerParser.getFrameType();
bodyParser = bodyParsers[type];
state = State.BODY;
}
break;
}
case BODY:
{
BodyParser.Result result = bodyParser.parse(buffer);
if (result == BodyParser.Result.ASYNC)
{
// The content will be processed asynchronously, stop parsing;
// the asynchronous operation will eventually resume parsing.
return true;
}
else if (result == BodyParser.Result.COMPLETE)
{
reset();
}
break;
}
}
}
return false;
}
public interface Listener
{
public boolean onDataFrame(DataFrame frame);
public static class Adapter implements Listener
{
@Override
public boolean onDataFrame(DataFrame frame)
{
return false;
}
}
}
private enum State
{
HEADER, BODY
}
}

View File

@ -0,0 +1,174 @@
//
// ========================================================================
// Copyright (c) 1995-2014 Mort Bay Consulting Pty. Ltd.
// ------------------------------------------------------------------------
// All rights reserved. This program and the accompanying materials
// are made available under the terms of the Eclipse Public License v1.0
// and Apache License v2.0 which accompanies this distribution.
//
// The Eclipse Public License is available at
// http://www.eclipse.org/legal/epl-v10.html
//
// The Apache License v2.0 is available at
// http://www.opensource.org/licenses/apache2.0.php
//
// You may elect to redistribute this code under either of these licenses.
// ========================================================================
//
package org.eclipse.jetty.http2.frames;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import org.eclipse.jetty.http2.generator.Generator;
import org.eclipse.jetty.http2.parser.Parser;
import org.eclipse.jetty.io.ByteBufferPool;
import org.eclipse.jetty.io.MappedByteBufferPool;
import org.junit.Assert;
import org.junit.Test;
public class DataGenerateParseTest
{
private final byte[] smallContent = new byte[128];
private final byte[] largeContent = new byte[128 * 1024];
private final ByteBufferPool byteBufferPool = new MappedByteBufferPool();
public DataGenerateParseTest()
{
Random random = new Random();
random.nextBytes(smallContent);
random.nextBytes(largeContent);
}
@Test
public void testGenerateParseSmallContentNoPadding()
{
testGenerateParseSmallContent(0);
}
@Test
public void testGenerateParseSmallContentSmallPadding()
{
testGenerateParseSmallContent(128);
}
@Test
public void testGenerateParseSmallContentLargePadding()
{
testGenerateParseSmallContent(1024);
}
private void testGenerateParseSmallContent(int paddingLength)
{
ByteBuffer content = ByteBuffer.wrap(smallContent);
List<DataFrame> frames = testGenerateParse(paddingLength, content);
Assert.assertEquals(1, frames.size());
DataFrame frame = frames.get(0);
Assert.assertTrue(frame.getStreamId() != 0);
Assert.assertTrue(frame.isEnd());
Assert.assertEquals(content, frame.getData());
}
@Test
public void testGenerateParseLargeContent()
{
testGenerateParseLargeContent(0);
}
@Test
public void testGenerateParseLargeContentSmallPadding()
{
testGenerateParseLargeContent(128);
}
@Test
public void testGenerateParseLargeContentLargePadding()
{
testGenerateParseLargeContent(1024);
}
private void testGenerateParseLargeContent(int paddingLength)
{
ByteBuffer content = ByteBuffer.wrap(largeContent);
List<DataFrame> frames = testGenerateParse(paddingLength, content);
Assert.assertEquals(9, frames.size());
ByteBuffer aggregate = ByteBuffer.allocate(content.remaining());
for (int i = 1; i <= frames.size(); ++i)
{
DataFrame frame = frames.get(i - 1);
Assert.assertTrue(frame.getStreamId() != 0);
Assert.assertEquals(i == frames.size(), frame.isEnd());
aggregate.put(frame.getData());
}
aggregate.flip();
Assert.assertEquals(content, aggregate);
}
private List<DataFrame> testGenerateParse(int paddingLength, ByteBuffer... data)
{
Generator generator = new Generator(byteBufferPool);
// Iterate a few times to be sure generator
// and parser are properly reset.
final List<DataFrame> frames = new ArrayList<>();
for (int i = 0; i < 2; ++i)
{
Generator.Result result = new Generator.Result(byteBufferPool);
for (int j = 1; j <= data.length; ++j)
{
result = result.merge(generator.generateContent(13, paddingLength, data[j - 1].slice(), j == data.length, false));
}
Parser parser = new Parser(new Parser.Listener.Adapter()
{
@Override
public boolean onDataFrame(DataFrame frame)
{
frames.add(frame);
return false;
}
});
frames.clear();
for (ByteBuffer buffer : result.getByteBuffers())
{
parser.parse(buffer);
}
}
return frames;
}
@Test
public void testGenerateParseOneByteAtATime()
{
Generator generator = new Generator(byteBufferPool);
Generator.Result result = new Generator.Result(byteBufferPool);
result = result.merge(generator.generateContent(13, 1024, ByteBuffer.wrap(largeContent).slice(), true, false));
final List<DataFrame> frames = new ArrayList<>();
Parser parser = new Parser(new Parser.Listener.Adapter()
{
@Override
public boolean onDataFrame(DataFrame frame)
{
frames.add(frame);
return false;
}
});
for (ByteBuffer buffer : result.getByteBuffers())
{
while (buffer.hasRemaining())
{
parser.parse(ByteBuffer.wrap(new byte[]{buffer.get()}));
}
}
Assert.assertEquals(largeContent.length, frames.size());
}
}