Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

upgrade LZ4 to operate directly on ByteBuffers

This commit is contained in:
Xavier Léauté 2014-12-02 23:49:34 -08:00
parent 1566999a6a
commit c21a82a697
2 changed files with 8 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -383,7 +383,7 @@
<dependency> <dependency>
<groupId>net.jpountz.lz4</groupId> <groupId>net.jpountz.lz4</groupId>
<artifactId>lz4</artifactId> <artifactId>lz4</artifactId>
<version>1.2.0</version> <version>1.3.0</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.google.protobuf</groupId> <groupId>com.google.protobuf</groupId>

37
processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java
View File

@ -19,9 +19,7 @@
package io.druid.segment.data; package io.druid.segment.data;
import com.google.common.base.Throwables;
import com.google.common.collect.Maps; import com.google.common.collect.Maps;
import com.metamx.common.guava.CloseQuietly;
import com.metamx.common.logger.Logger; import com.metamx.common.logger.Logger;
import com.ning.compress.lzf.ChunkEncoder; import com.ning.compress.lzf.ChunkEncoder;
import com.ning.compress.lzf.LZFChunk; import com.ning.compress.lzf.LZFChunk;
@ -34,7 +32,6 @@ import net.jpountz.lz4.LZ4SafeDecompressor;
import java.io.IOException; import java.io.IOException;
import java.nio.Buffer; import java.nio.Buffer;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.nio.ByteOrder; import java.nio.ByteOrder;
import java.util.Map; import java.util.Map;
@ -233,38 +230,18 @@ public class CompressedObjectStrategy<T extends Buffer> implements ObjectStrateg
@Override @Override
public void decompress(ByteBuffer in, int numBytes, ByteBuffer out) public void decompress(ByteBuffer in, int numBytes, ByteBuffer out)
{ {
final byte[] bytes = new byte[numBytes]; // Since decompressed size is NOT known, must use lz4Safe
in.get(bytes); // lz4Safe.decompress does not modify buffer positions
final int numDecompressedBytes = lz4Safe.decompress(in, in.position(), numBytes, out, out.position(), out.remaining());
try (final ResourceHolder<byte[]> outputBytesHolder = CompressedPools.getOutputBytes()) { out.limit(out.position() + numDecompressedBytes);
final byte[] outputBytes = outputBytesHolder.get();
// Since decompressed size is NOT known, must use lz4Safe
final int numDecompressedBytes = lz4Safe.decompress(bytes, outputBytes);
out.put(outputBytes, 0, numDecompressedBytes);
out.flip();
}
catch (IOException e) {
log.error(e, "IOException thrown while closing ChunkEncoder.");
}
} }
@Override @Override
public void decompress(ByteBuffer in, int numBytes, ByteBuffer out, int decompressedSize) public void decompress(ByteBuffer in, int numBytes, ByteBuffer out, int decompressedSize)
{ {
final byte[] bytes = new byte[numBytes]; // lz4Fast.decompress does not modify buffer positions
in.get(bytes); lz4Fast.decompress(in, in.position(), out, out.position(), decompressedSize);
out.limit(out.position() + decompressedSize);
// TODO: Upgrade this to ByteBuffer once https://github.com/jpountz/lz4-java/issues/9 is in mainline code for lz4-java
try (final ResourceHolder<byte[]> outputBytesHolder = CompressedPools.getOutputBytes()) {
final byte[] outputBytes = outputBytesHolder.get();
lz4Fast.decompress(bytes, 0, outputBytes, 0, decompressedSize);
out.put(outputBytes, 0, decompressedSize);
out.flip();
}
catch (IOException e) {
log.error(e, "IOException thrown while closing ChunkEncoder.");
}
} }
} }