Merge pull request #9798 from eclipse/jetty-10.0.x-QPACK-encoding

review and cleanup of HTTP/3 QPACK Integer and String encoding
This commit is contained in:
Lachlan 2023-05-29 16:43:05 +10:00 committed by GitHub
commit 6567a4478e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 160 additions and 206 deletions

View File

@ -25,98 +25,67 @@ public class NBitIntegerEncoder
}
/**
* @param n the prefix used to encode this long.
* @param i the integer to encode.
* @param prefix the prefix used to encode this long.
* @param value the integer to encode.
* @return the number of octets it would take to encode the long.
*/
public static int octetsNeeded(int n, long i)
public static int octetsNeeded(int prefix, long value)
{
if (n == 8)
{
int nbits = 0xFF;
i = i - nbits;
if (i < 0)
return 1;
if (i == 0)
return 2;
int lz = Long.numberOfLeadingZeros(i);
int log = 64 - lz;
return 1 + (log + 6) / 7;
}
if (prefix <= 0 || prefix > 8)
throw new IllegalArgumentException();
int nbits = 0xFF >>> (8 - n);
i = i - nbits;
if (i < 0)
return 0;
if (i == 0)
int nbits = 0xFF >>> (8 - prefix);
value = value - nbits;
if (value < 0)
return 1;
int lz = Long.numberOfLeadingZeros(i);
if (value == 0)
return 2;
int lz = Long.numberOfLeadingZeros(value);
int log = 64 - lz;
return (log + 6) / 7;
// The return value is 1 for the prefix + the number of 7-bit groups necessary to encode the value.
return 1 + (log + 6) / 7;
}
/**
*
* @param buf the buffer to encode into.
* @param n the prefix used to encode this long.
* @param i the long to encode into the buffer.
* @param buffer the buffer to encode into.
* @param prefix the prefix used to encode this long.
* @param value the long to encode into the buffer.
*/
public static void encode(ByteBuffer buf, int n, long i)
public static void encode(ByteBuffer buffer, int prefix, long value)
{
if (n == 8)
if (prefix <= 0 || prefix > 8)
throw new IllegalArgumentException();
// If prefix is 8 we add an empty byte as we initially modify last byte from the buffer.
if (prefix == 8)
buffer.put((byte)0x00);
int bits = 0xFF >>> (8 - prefix);
int p = buffer.position() - 1;
if (value < bits)
{
if (i < 0xFF)
{
buf.put((byte)i);
buffer.put(p, (byte)((buffer.get(p) & ~bits) | value));
}
else
{
buf.put((byte)0xFF);
long length = i - 0xFF;
buffer.put(p, (byte)(buffer.get(p) | bits));
long length = value - bits;
while (true)
{
if ((length & ~0x7F) == 0)
// The value of ~0x7F is different to 0x80 because of all the 1s from the MSB.
if ((length & ~0x7FL) == 0)
{
buf.put((byte)length);
buffer.put((byte)length);
return;
}
else
{
buf.put((byte)((length & 0x7F) | 0x80));
length >>>= 7;
}
}
}
}
else
{
int p = buf.position() - 1;
int bits = 0xFF >>> (8 - n);
if (i < bits)
{
buf.put(p, (byte)((buf.get(p) & ~bits) | i));
}
else
{
buf.put(p, (byte)(buf.get(p) | bits));
long length = i - bits;
while (true)
{
if ((length & ~0x7F) == 0)
{
buf.put((byte)length);
return;
}
else
{
buf.put((byte)((length & 0x7F) | 0x80));
buffer.put((byte)((length & 0x7F) | 0x80));
length >>>= 7;
}
}
}
}
}
}

View File

@ -0,0 +1,77 @@
//
// ========================================================================
// Copyright (c) 1995 Mort Bay Consulting Pty Ltd and others.
//
// This program and the accompanying materials are made available under the
// terms of the Eclipse Public License v. 2.0 which is available at
// https://www.eclipse.org/legal/epl-2.0, or the Apache License, Version 2.0
// which is available at https://www.apache.org/licenses/LICENSE-2.0.
//
// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
// ========================================================================
//
package org.eclipse.jetty.http.compression;
import java.nio.ByteBuffer;
import org.eclipse.jetty.http.HttpTokens;
public class NBitStringEncoder
{
private NBitStringEncoder()
{
}
public static int octetsNeeded(int prefix, String value, boolean huffman)
{
if (prefix <= 0 || prefix > 8)
throw new IllegalArgumentException();
int contentPrefix = (prefix == 1) ? 8 : prefix - 1;
int encodedValueSize = huffman ? HuffmanEncoder.octetsNeeded(value) : value.length();
int encodedLengthSize = NBitIntegerEncoder.octetsNeeded(contentPrefix, encodedValueSize);
// If prefix was 1, then we count an extra byte needed for the prefix.
return encodedLengthSize + encodedValueSize + (prefix == 1 ? 1 : 0);
}
public static void encode(ByteBuffer buffer, int prefix, String value, boolean huffman)
{
if (prefix <= 0 || prefix > 8)
throw new IllegalArgumentException();
byte huffmanFlag = huffman ? (byte)(0x01 << (prefix - 1)) : (byte)0x00;
if (prefix == 8)
{
buffer.put(huffmanFlag);
}
else
{
int p = buffer.position() - 1;
buffer.put(p, (byte)(buffer.get(p) | huffmanFlag));
}
// Start encoding size & content in rest of prefix.
// If prefix was 1 we set it back to 8 to indicate to start on a new byte.
prefix = (prefix == 1) ? 8 : prefix - 1;
if (huffman)
{
int encodedValueSize = HuffmanEncoder.octetsNeeded(value);
NBitIntegerEncoder.encode(buffer, prefix, encodedValueSize);
HuffmanEncoder.encode(buffer, value);
}
else
{
int encodedValueSize = value.length();
NBitIntegerEncoder.encode(buffer, prefix, encodedValueSize);
for (int i = 0; i < encodedValueSize; i++)
{
char c = value.charAt(i);
c = HttpTokens.sanitizeFieldVchar(c);
buffer.put((byte)c);
}
}
}
}

View File

@ -18,6 +18,7 @@ import java.nio.ByteBuffer;
import org.eclipse.jetty.http.compression.NBitIntegerDecoder;
import org.eclipse.jetty.http.compression.NBitIntegerEncoder;
import org.eclipse.jetty.util.BufferUtil;
import org.eclipse.jetty.util.StringUtil;
import org.eclipse.jetty.util.TypeUtil;
import org.junit.jupiter.api.Test;
@ -31,17 +32,17 @@ public class NBitIntegerTest
@Test
public void testOctetsNeeded()
{
assertEquals(0, NBitIntegerEncoder.octetsNeeded(5, 10));
assertEquals(2, NBitIntegerEncoder.octetsNeeded(5, 1337));
assertEquals(1, NBitIntegerEncoder.octetsNeeded(5, 10));
assertEquals(3, NBitIntegerEncoder.octetsNeeded(5, 1337));
assertEquals(1, NBitIntegerEncoder.octetsNeeded(8, 42));
assertEquals(3, NBitIntegerEncoder.octetsNeeded(8, 1337));
assertEquals(0, NBitIntegerEncoder.octetsNeeded(6, 62));
assertEquals(1, NBitIntegerEncoder.octetsNeeded(6, 63));
assertEquals(1, NBitIntegerEncoder.octetsNeeded(6, 64));
assertEquals(2, NBitIntegerEncoder.octetsNeeded(6, 63 + 0x00 + 0x80 * 0x01));
assertEquals(3, NBitIntegerEncoder.octetsNeeded(6, 63 + 0x00 + 0x80 * 0x80));
assertEquals(4, NBitIntegerEncoder.octetsNeeded(6, 63 + 0x00 + 0x80 * 0x80 * 0x80));
assertEquals(1, NBitIntegerEncoder.octetsNeeded(6, 62));
assertEquals(2, NBitIntegerEncoder.octetsNeeded(6, 63));
assertEquals(2, NBitIntegerEncoder.octetsNeeded(6, 64));
assertEquals(3, NBitIntegerEncoder.octetsNeeded(6, 63 + 0x00 + 0x80 * 0x01));
assertEquals(4, NBitIntegerEncoder.octetsNeeded(6, 63 + 0x00 + 0x80 * 0x80));
assertEquals(5, NBitIntegerEncoder.octetsNeeded(6, 63 + 0x00 + 0x80 * 0x80 * 0x80));
}
@Test
@ -87,7 +88,7 @@ public class NBitIntegerTest
String r = TypeUtil.toHexString(BufferUtil.toArray(buf));
assertEquals(expected, r);
assertEquals(expected.length() / 2, (n < 8 ? 1 : 0) + NBitIntegerEncoder.octetsNeeded(n, i));
assertEquals(expected.length() / 2, NBitIntegerEncoder.octetsNeeded(n, i));
}
@Test
@ -163,8 +164,7 @@ public class NBitIntegerTest
NBitIntegerEncoder.encode(buf, 5, 1337);
BufferUtil.flipToFlush(buf, p);
String r = TypeUtil.toHexString(BufferUtil.toArray(buf));
String r = StringUtil.toHexString(BufferUtil.toArray(buf));
assertEquals("881f9a0a", r);
}

View File

@ -464,7 +464,7 @@ public class HpackContext
if (huffmanLen < 0)
throw new IllegalStateException("bad value");
int lenLen = NBitIntegerEncoder.octetsNeeded(7, huffmanLen);
_huffmanValue = new byte[1 + lenLen + huffmanLen];
_huffmanValue = new byte[lenLen + huffmanLen];
ByteBuffer buffer = ByteBuffer.wrap(_huffmanValue);
// Indicate Huffman

View File

@ -25,12 +25,12 @@ import org.eclipse.jetty.http.HttpHeader;
import org.eclipse.jetty.http.HttpMethod;
import org.eclipse.jetty.http.HttpScheme;
import org.eclipse.jetty.http.HttpStatus;
import org.eclipse.jetty.http.HttpTokens;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.http.MetaData;
import org.eclipse.jetty.http.PreEncodedHttpField;
import org.eclipse.jetty.http.compression.HuffmanEncoder;
import org.eclipse.jetty.http.compression.NBitIntegerEncoder;
import org.eclipse.jetty.http.compression.NBitStringEncoder;
import org.eclipse.jetty.http2.hpack.HpackContext.Entry;
import org.eclipse.jetty.http2.hpack.HpackContext.StaticEntry;
import org.eclipse.jetty.util.BufferUtil;
@ -337,7 +337,7 @@ public class HpackEncoder
buffer.put((byte)0x80);
NBitIntegerEncoder.encode(buffer, 7, index);
if (_debug)
encoding = "IdxField" + (entry.isStatic() ? "S" : "") + (1 + NBitIntegerEncoder.octetsNeeded(7, index));
encoding = "IdxField" + (entry.isStatic() ? "S" : "") + NBitIntegerEncoder.octetsNeeded(7, index);
}
}
else
@ -470,25 +470,6 @@ public class HpackEncoder
static void encodeValue(ByteBuffer buffer, boolean huffman, String value)
{
if (huffman)
{
// huffman literal value
buffer.put((byte)0x80);
int needed = HuffmanEncoder.octetsNeeded(value);
NBitIntegerEncoder.encode(buffer, 7, needed);
HuffmanEncoder.encode(buffer, value);
}
else
{
// add literal assuming iso_8859_1
buffer.put((byte)0x00).mark();
NBitIntegerEncoder.encode(buffer, 7, value.length());
for (int i = 0; i < value.length(); i++)
{
char c = value.charAt(i);
c = HttpTokens.sanitizeFieldVchar(c);
buffer.put((byte)c);
}
}
NBitStringEncoder.encode(buffer, 8, value, huffman);
}
}

View File

@ -14,14 +14,13 @@
package org.eclipse.jetty.http3.qpack.internal;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Objects;
import org.eclipse.jetty.http.HttpField;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.http.PreEncodedHttpField;
import org.eclipse.jetty.http.compression.HuffmanEncoder;
import org.eclipse.jetty.http.compression.NBitIntegerEncoder;
import org.eclipse.jetty.http.compression.NBitStringEncoder;
import org.eclipse.jetty.http3.qpack.internal.table.Entry;
public abstract class EncodableEntry
@ -96,19 +95,19 @@ public abstract class EncodableEntry
{
// Indexed Field Line with Static Reference.
int relativeIndex = _entry.getIndex();
return 1 + NBitIntegerEncoder.octetsNeeded(6, relativeIndex);
return NBitIntegerEncoder.octetsNeeded(6, relativeIndex);
}
else if (_entry.getIndex() < base)
{
// Indexed Field Line with Dynamic Reference.
int relativeIndex = base - (_entry.getIndex() + 1);
return 1 + NBitIntegerEncoder.octetsNeeded(6, relativeIndex);
return NBitIntegerEncoder.octetsNeeded(6, relativeIndex);
}
else
{
// Indexed Field Line with Post-Base Index.
int relativeIndex = _entry.getIndex() - base;
return 1 + NBitIntegerEncoder.octetsNeeded(4, relativeIndex);
return NBitIntegerEncoder.octetsNeeded(4, relativeIndex);
}
}
@ -163,27 +162,12 @@ public abstract class EncodableEntry
}
// Encode the value.
String value = getValue();
if (_huffman)
{
buffer.put((byte)0x80);
NBitIntegerEncoder.encode(buffer, 7, HuffmanEncoder.octetsNeeded(value));
HuffmanEncoder.encode(buffer, value);
}
else
{
buffer.put((byte)0x00);
NBitIntegerEncoder.encode(buffer, 7, value.length());
buffer.put(value.getBytes(StandardCharsets.ISO_8859_1));
}
NBitStringEncoder.encode(buffer, 8, getValue(), _huffman);
}
@Override
public int getRequiredSize(int base)
{
String value = getValue();
int valueLength = _huffman ? HuffmanEncoder.octetsNeeded(value) : value.length();
int nameOctets;
if (_nameEntry.isStatic())
{
@ -201,7 +185,7 @@ public abstract class EncodableEntry
nameOctets = NBitIntegerEncoder.octetsNeeded(3, relativeIndex);
}
return 1 + nameOctets + 1 + NBitIntegerEncoder.octetsNeeded(7, valueLength) + valueLength;
return nameOctets + NBitStringEncoder.octetsNeeded(8, getValue(), _huffman);
}
@Override
@ -232,38 +216,19 @@ public abstract class EncodableEntry
public void encode(ByteBuffer buffer, int base)
{
byte allowIntermediary = 0x00; // TODO: this is 0x10 bit, when should this be set?
String name = getName();
String value = getValue();
// Encode the prefix code and the name.
if (_huffman)
{
buffer.put((byte)(0x28 | allowIntermediary));
NBitIntegerEncoder.encode(buffer, 3, HuffmanEncoder.octetsNeeded(name));
HuffmanEncoder.encode(buffer, name);
buffer.put((byte)0x80);
NBitIntegerEncoder.encode(buffer, 7, HuffmanEncoder.octetsNeeded(value));
HuffmanEncoder.encode(buffer, value);
}
else
{
buffer.put((byte)(0x20 | allowIntermediary));
NBitIntegerEncoder.encode(buffer, 3, name.length());
buffer.put(name.getBytes(StandardCharsets.ISO_8859_1));
buffer.put((byte)0x00);
NBitIntegerEncoder.encode(buffer, 7, value.length());
buffer.put(value.getBytes(StandardCharsets.ISO_8859_1));
}
NBitStringEncoder.encode(buffer, 4, getName(), _huffman);
NBitStringEncoder.encode(buffer, 8, getValue(), _huffman);
}
@Override
public int getRequiredSize(int base)
{
String name = getName();
String value = getValue();
int nameLength = _huffman ? HuffmanEncoder.octetsNeeded(name) : name.length();
int valueLength = _huffman ? HuffmanEncoder.octetsNeeded(value) : value.length();
return 2 + NBitIntegerEncoder.octetsNeeded(3, nameLength) + nameLength + NBitIntegerEncoder.octetsNeeded(7, valueLength) + valueLength;
int encodedNameSize = NBitStringEncoder.octetsNeeded(4, getName(), _huffman);
int encodedValueSize = NBitStringEncoder.octetsNeeded(8, getValue(), _huffman);
return encodedNameSize + encodedValueSize;
}
@Override

View File

@ -37,7 +37,7 @@ public class DuplicateInstruction implements Instruction
@Override
public void encode(ByteBufferPool.Lease lease)
{
int size = NBitIntegerEncoder.octetsNeeded(5, _index) + 1;
int size = NBitIntegerEncoder.octetsNeeded(5, _index);
ByteBuffer buffer = lease.acquire(size, false);
buffer.put((byte)0x00);
NBitIntegerEncoder.encode(buffer, 5, _index);

View File

@ -14,10 +14,9 @@
package org.eclipse.jetty.http3.qpack.internal.instruction;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import org.eclipse.jetty.http.compression.HuffmanEncoder;
import org.eclipse.jetty.http.compression.NBitIntegerEncoder;
import org.eclipse.jetty.http.compression.NBitStringEncoder;
import org.eclipse.jetty.http3.qpack.Instruction;
import org.eclipse.jetty.io.ByteBufferPool;
import org.eclipse.jetty.util.BufferUtil;
@ -55,27 +54,14 @@ public class IndexedNameEntryInstruction implements Instruction
@Override
public void encode(ByteBufferPool.Lease lease)
{
int size = NBitIntegerEncoder.octetsNeeded(6, _index) + (_huffman ? HuffmanEncoder.octetsNeeded(_value) : _value.length()) + 2;
int size = NBitIntegerEncoder.octetsNeeded(6, _index) + NBitStringEncoder.octetsNeeded(8, _value, _huffman);
ByteBuffer buffer = lease.acquire(size, false);
// First bit indicates the instruction, second bit is whether it is a dynamic table reference or not.
buffer.put((byte)(0x80 | (_dynamic ? 0x00 : 0x40)));
NBitIntegerEncoder.encode(buffer, 6, _index);
// We will not huffman encode the string.
if (_huffman)
{
buffer.put((byte)(0x80));
NBitIntegerEncoder.encode(buffer, 7, HuffmanEncoder.octetsNeeded(_value));
HuffmanEncoder.encode(buffer, _value);
}
else
{
buffer.put((byte)(0x00));
NBitIntegerEncoder.encode(buffer, 7, _value.length());
buffer.put(_value.getBytes(StandardCharsets.ISO_8859_1));
}
NBitStringEncoder.encode(buffer, 8, _value, _huffman);
BufferUtil.flipToFlush(buffer, 0);
lease.append(buffer, true);
}

View File

@ -37,7 +37,7 @@ public class InsertCountIncrementInstruction implements Instruction
@Override
public void encode(ByteBufferPool.Lease lease)
{
int size = NBitIntegerEncoder.octetsNeeded(6, _increment) + 1;
int size = NBitIntegerEncoder.octetsNeeded(6, _increment);
ByteBuffer buffer = lease.acquire(size, false);
buffer.put((byte)0x00);
NBitIntegerEncoder.encode(buffer, 6, _increment);

View File

@ -14,11 +14,9 @@
package org.eclipse.jetty.http3.qpack.internal.instruction;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import org.eclipse.jetty.http.HttpField;
import org.eclipse.jetty.http.compression.HuffmanEncoder;
import org.eclipse.jetty.http.compression.NBitIntegerEncoder;
import org.eclipse.jetty.http.compression.NBitStringEncoder;
import org.eclipse.jetty.http3.qpack.Instruction;
import org.eclipse.jetty.io.ByteBufferPool;
import org.eclipse.jetty.util.BufferUtil;
@ -56,35 +54,13 @@ public class LiteralNameEntryInstruction implements Instruction
@Override
public void encode(ByteBufferPool.Lease lease)
{
int size = (_huffmanName ? HuffmanEncoder.octetsNeeded(_name) : _name.length()) +
(_huffmanValue ? HuffmanEncoder.octetsNeeded(_value) : _value.length()) + 2;
int size = NBitStringEncoder.octetsNeeded(6, _name, _huffmanName) +
NBitStringEncoder.octetsNeeded(8, _value, _huffmanValue);
ByteBuffer buffer = lease.acquire(size, false);
if (_huffmanName)
{
buffer.put((byte)(0x40 | 0x20));
NBitIntegerEncoder.encode(buffer, 5, HuffmanEncoder.octetsNeeded(_name));
HuffmanEncoder.encode(buffer, _name);
}
else
{
buffer.put((byte)(0x40));
NBitIntegerEncoder.encode(buffer, 5, _name.length());
buffer.put(_name.getBytes(StandardCharsets.ISO_8859_1));
}
if (_huffmanValue)
{
buffer.put((byte)(0x80));
NBitIntegerEncoder.encode(buffer, 7, HuffmanEncoder.octetsNeeded(_value));
HuffmanEncoder.encode(buffer, _value);
}
else
{
buffer.put((byte)(0x00));
NBitIntegerEncoder.encode(buffer, 7, _value.length());
buffer.put(_value.getBytes(StandardCharsets.ISO_8859_1));
}
buffer.put((byte)0x40); // Instruction Pattern.
NBitStringEncoder.encode(buffer, 6, _name, _huffmanName);
NBitStringEncoder.encode(buffer, 8, _value, _huffmanValue);
BufferUtil.flipToFlush(buffer, 0);
lease.append(buffer, true);

View File

@ -37,7 +37,7 @@ public class SectionAcknowledgmentInstruction implements Instruction
@Override
public void encode(ByteBufferPool.Lease lease)
{
int size = NBitIntegerEncoder.octetsNeeded(7, _streamId) + 1;
int size = NBitIntegerEncoder.octetsNeeded(7, _streamId);
ByteBuffer buffer = lease.acquire(size, false);
buffer.put((byte)0x80);
NBitIntegerEncoder.encode(buffer, 7, _streamId);

View File

@ -37,7 +37,7 @@ public class SetCapacityInstruction implements Instruction
@Override
public void encode(ByteBufferPool.Lease lease)
{
int size = NBitIntegerEncoder.octetsNeeded(5, _capacity) + 1;
int size = NBitIntegerEncoder.octetsNeeded(5, _capacity);
ByteBuffer buffer = lease.acquire(size, false);
buffer.put((byte)0x20);
NBitIntegerEncoder.encode(buffer, 5, _capacity);

View File

@ -32,7 +32,7 @@ public class StreamCancellationInstruction implements Instruction
@Override
public void encode(ByteBufferPool.Lease lease)
{
int size = NBitIntegerEncoder.octetsNeeded(6, _streamId) + 1;
int size = NBitIntegerEncoder.octetsNeeded(6, _streamId);
ByteBuffer buffer = lease.acquire(size, false);
buffer.put((byte)0x40);
NBitIntegerEncoder.encode(buffer, 6, _streamId);

View File

@ -120,7 +120,7 @@ public class Entry
if (huffmanLen < 0)
throw new IllegalStateException("bad value");
int lenLen = NBitIntegerEncoder.octetsNeeded(7, huffmanLen);
_huffmanValue = new byte[1 + lenLen + huffmanLen];
_huffmanValue = new byte[lenLen + huffmanLen];
ByteBuffer buffer = ByteBuffer.wrap(_huffmanValue);
// Indicate Huffman