mirror of https://github.com/apache/druid.git
make LZ4 the default compression strategy
- LZ4 is now hardwired to be the default strategy - Rework tests to test all available compression strategies
This commit is contained in:
parent
c40a315c81
commit
991e1828b0
|
@ -53,7 +53,7 @@ public class FloatMetricColumnSerializer implements MetricColumnSerializer
|
||||||
{
|
{
|
||||||
writer = CompressedFloatsSupplierSerializer.create(
|
writer = CompressedFloatsSupplierSerializer.create(
|
||||||
ioPeon, String.format("%s_little", metricName), IndexIO.BYTE_ORDER,
|
ioPeon, String.format("%s_little", metricName), IndexIO.BYTE_ORDER,
|
||||||
CompressedObjectStrategy.CompressionStrategy.LZ4 // TODO define this somewhere else
|
CompressedObjectStrategy.DEFAULT_COMPRESSION_STRATEGY
|
||||||
);
|
);
|
||||||
|
|
||||||
writer.open();
|
writer.open();
|
||||||
|
|
|
@ -596,7 +596,7 @@ public class IndexMerger
|
||||||
|
|
||||||
CompressedLongsSupplierSerializer timeWriter = CompressedLongsSupplierSerializer.create(
|
CompressedLongsSupplierSerializer timeWriter = CompressedLongsSupplierSerializer.create(
|
||||||
ioPeon, "little_end_time", IndexIO.BYTE_ORDER,
|
ioPeon, "little_end_time", IndexIO.BYTE_ORDER,
|
||||||
CompressedObjectStrategy.CompressionStrategy.LZ4 // TODO define this somewhere else
|
CompressedObjectStrategy.DEFAULT_COMPRESSION_STRATEGY
|
||||||
);
|
);
|
||||||
|
|
||||||
timeWriter.open();
|
timeWriter.open();
|
||||||
|
|
|
@ -42,6 +42,8 @@ import java.util.Map;
|
||||||
*/
|
*/
|
||||||
public class CompressedObjectStrategy<T extends Buffer> implements ObjectStrategy<ResourceHolder<T>>
|
public class CompressedObjectStrategy<T extends Buffer> implements ObjectStrategy<ResourceHolder<T>>
|
||||||
{
|
{
|
||||||
|
public static final CompressionStrategy DEFAULT_COMPRESSION_STRATEGY = CompressionStrategy.LZ4;
|
||||||
|
|
||||||
public static enum CompressionStrategy {
|
public static enum CompressionStrategy {
|
||||||
LZF ((byte)0x0)
|
LZF ((byte)0x0)
|
||||||
{
|
{
|
||||||
|
|
|
@ -37,7 +37,7 @@ import java.util.List;
|
||||||
*/
|
*/
|
||||||
public class InMemoryCompressedFloats implements IndexedFloats
|
public class InMemoryCompressedFloats implements IndexedFloats
|
||||||
{
|
{
|
||||||
public static final CompressedObjectStrategy.CompressionStrategy COMPRESSION = CompressedObjectStrategy.CompressionStrategy.LZ4;
|
public static final CompressedObjectStrategy.CompressionStrategy COMPRESSION = CompressedObjectStrategy.DEFAULT_COMPRESSION_STRATEGY;
|
||||||
private final CompressedFloatBufferObjectStrategy strategy;
|
private final CompressedFloatBufferObjectStrategy strategy;
|
||||||
private final int sizePer;
|
private final int sizePer;
|
||||||
|
|
||||||
|
|
|
@ -38,7 +38,7 @@ import java.util.List;
|
||||||
*/
|
*/
|
||||||
public class InMemoryCompressedLongs implements IndexedLongs
|
public class InMemoryCompressedLongs implements IndexedLongs
|
||||||
{
|
{
|
||||||
public static final CompressedObjectStrategy.CompressionStrategy COMPRESSION = CompressedObjectStrategy.CompressionStrategy.LZ4;
|
public static final CompressedObjectStrategy.CompressionStrategy COMPRESSION = CompressedObjectStrategy.DEFAULT_COMPRESSION_STRATEGY;
|
||||||
private final CompressedLongBufferObjectStrategy strategy;
|
private final CompressedLongBufferObjectStrategy strategy;
|
||||||
private final int sizePer;
|
private final int sizePer;
|
||||||
|
|
||||||
|
|
|
@ -25,6 +25,8 @@ import org.junit.After;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
import org.junit.runner.RunWith;
|
||||||
|
import org.junit.runners.Parameterized;
|
||||||
|
|
||||||
import java.io.ByteArrayOutputStream;
|
import java.io.ByteArrayOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
@ -36,10 +38,14 @@ import java.util.concurrent.CountDownLatch;
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
import java.util.concurrent.atomic.AtomicReference;
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
|
|
||||||
/**
|
@RunWith(Parameterized.class)
|
||||||
*/
|
public class CompressedFloatsIndexedSupplierTest extends CompressionStrategyTest
|
||||||
public class CompressedFloatsIndexedSupplierTest
|
|
||||||
{
|
{
|
||||||
|
public CompressedFloatsIndexedSupplierTest(CompressedObjectStrategy.CompressionStrategy compressionStrategy)
|
||||||
|
{
|
||||||
|
super(compressionStrategy);
|
||||||
|
}
|
||||||
|
|
||||||
private IndexedFloats indexed;
|
private IndexedFloats indexed;
|
||||||
private CompressedFloatsIndexedSupplier supplier;
|
private CompressedFloatsIndexedSupplier supplier;
|
||||||
private float[] vals;
|
private float[] vals;
|
||||||
|
@ -69,7 +75,7 @@ public class CompressedFloatsIndexedSupplierTest
|
||||||
FloatBuffer.wrap(vals),
|
FloatBuffer.wrap(vals),
|
||||||
5,
|
5,
|
||||||
ByteOrder.nativeOrder(),
|
ByteOrder.nativeOrder(),
|
||||||
CompressedObjectStrategy.CompressionStrategy.LZ4
|
compressionStrategy
|
||||||
);
|
);
|
||||||
|
|
||||||
indexed = supplier.get();
|
indexed = supplier.get();
|
||||||
|
@ -83,7 +89,7 @@ public class CompressedFloatsIndexedSupplierTest
|
||||||
|
|
||||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||||
final CompressedFloatsIndexedSupplier theSupplier = CompressedFloatsIndexedSupplier.fromFloatBuffer(
|
final CompressedFloatsIndexedSupplier theSupplier = CompressedFloatsIndexedSupplier.fromFloatBuffer(
|
||||||
FloatBuffer.wrap(vals), 5, ByteOrder.nativeOrder(), CompressedObjectStrategy.CompressionStrategy.LZ4
|
FloatBuffer.wrap(vals), 5, ByteOrder.nativeOrder(), compressionStrategy
|
||||||
);
|
);
|
||||||
theSupplier.writeToChannel(Channels.newChannel(baos));
|
theSupplier.writeToChannel(Channels.newChannel(baos));
|
||||||
|
|
||||||
|
|
|
@ -23,6 +23,8 @@ import com.google.common.io.OutputSupplier;
|
||||||
import io.druid.collections.ResourceHolder;
|
import io.druid.collections.ResourceHolder;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
import org.junit.runner.RunWith;
|
||||||
|
import org.junit.runners.Parameterized;
|
||||||
|
|
||||||
import java.io.ByteArrayOutputStream;
|
import java.io.ByteArrayOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
@ -31,10 +33,14 @@ import java.nio.ByteBuffer;
|
||||||
import java.nio.ByteOrder;
|
import java.nio.ByteOrder;
|
||||||
import java.nio.FloatBuffer;
|
import java.nio.FloatBuffer;
|
||||||
|
|
||||||
/**
|
@RunWith(Parameterized.class)
|
||||||
*/
|
public class CompressedFloatsSupplierSerializerTest extends CompressionStrategyTest
|
||||||
public class CompressedFloatsSupplierSerializerTest
|
|
||||||
{
|
{
|
||||||
|
public CompressedFloatsSupplierSerializerTest(CompressedObjectStrategy.CompressionStrategy compressionStrategy)
|
||||||
|
{
|
||||||
|
super(compressionStrategy);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSanity() throws Exception
|
public void testSanity() throws Exception
|
||||||
{
|
{
|
||||||
|
@ -47,11 +53,11 @@ public class CompressedFloatsSupplierSerializerTest
|
||||||
"test",
|
"test",
|
||||||
CompressedFloatBufferObjectStrategy.getBufferForOrder(
|
CompressedFloatBufferObjectStrategy.getBufferForOrder(
|
||||||
order,
|
order,
|
||||||
CompressedObjectStrategy.CompressionStrategy.LZ4,
|
compressionStrategy,
|
||||||
sizePer
|
sizePer
|
||||||
)
|
)
|
||||||
),
|
),
|
||||||
CompressedObjectStrategy.CompressionStrategy.LZ4
|
compressionStrategy
|
||||||
);
|
);
|
||||||
serializer.open();
|
serializer.open();
|
||||||
|
|
||||||
|
|
|
@ -38,8 +38,13 @@ import java.util.concurrent.atomic.AtomicReference;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
*/
|
*/
|
||||||
public class CompressedLongsIndexedSupplierTest
|
public class CompressedLongsIndexedSupplierTest extends CompressionStrategyTest
|
||||||
{
|
{
|
||||||
|
public CompressedLongsIndexedSupplierTest(CompressedObjectStrategy.CompressionStrategy compressionStrategy)
|
||||||
|
{
|
||||||
|
super(compressionStrategy);
|
||||||
|
}
|
||||||
|
|
||||||
private IndexedLongs indexed;
|
private IndexedLongs indexed;
|
||||||
private CompressedLongsIndexedSupplier supplier;
|
private CompressedLongsIndexedSupplier supplier;
|
||||||
private long[] vals;
|
private long[] vals;
|
||||||
|
@ -67,7 +72,7 @@ public class CompressedLongsIndexedSupplierTest
|
||||||
LongBuffer.wrap(vals),
|
LongBuffer.wrap(vals),
|
||||||
5,
|
5,
|
||||||
ByteOrder.nativeOrder(),
|
ByteOrder.nativeOrder(),
|
||||||
CompressedObjectStrategy.CompressionStrategy.LZ4
|
compressionStrategy
|
||||||
);
|
);
|
||||||
|
|
||||||
indexed = supplier.get();
|
indexed = supplier.get();
|
||||||
|
@ -79,7 +84,7 @@ public class CompressedLongsIndexedSupplierTest
|
||||||
|
|
||||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||||
final CompressedLongsIndexedSupplier theSupplier = CompressedLongsIndexedSupplier.fromLongBuffer(
|
final CompressedLongsIndexedSupplier theSupplier = CompressedLongsIndexedSupplier.fromLongBuffer(
|
||||||
LongBuffer.wrap(vals), 5, ByteOrder.nativeOrder(), CompressedObjectStrategy.CompressionStrategy.LZ4
|
LongBuffer.wrap(vals), 5, ByteOrder.nativeOrder(), compressionStrategy
|
||||||
);
|
);
|
||||||
theSupplier.writeToChannel(Channels.newChannel(baos));
|
theSupplier.writeToChannel(Channels.newChannel(baos));
|
||||||
|
|
||||||
|
|
|
@ -23,6 +23,8 @@ import com.google.common.io.OutputSupplier;
|
||||||
import io.druid.collections.ResourceHolder;
|
import io.druid.collections.ResourceHolder;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
import org.junit.runner.RunWith;
|
||||||
|
import org.junit.runners.Parameterized;
|
||||||
|
|
||||||
import java.io.ByteArrayOutputStream;
|
import java.io.ByteArrayOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
@ -31,10 +33,14 @@ import java.nio.ByteBuffer;
|
||||||
import java.nio.ByteOrder;
|
import java.nio.ByteOrder;
|
||||||
import java.nio.LongBuffer;
|
import java.nio.LongBuffer;
|
||||||
|
|
||||||
/**
|
@RunWith(Parameterized.class)
|
||||||
*/
|
public class CompressedLongsSupplierSerializerTest extends CompressionStrategyTest
|
||||||
public class CompressedLongsSupplierSerializerTest
|
|
||||||
{
|
{
|
||||||
|
public CompressedLongsSupplierSerializerTest(CompressedObjectStrategy.CompressionStrategy compressionStrategy)
|
||||||
|
{
|
||||||
|
super(compressionStrategy);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSanity() throws Exception
|
public void testSanity() throws Exception
|
||||||
{
|
{
|
||||||
|
@ -45,9 +51,9 @@ public class CompressedLongsSupplierSerializerTest
|
||||||
new GenericIndexedWriter<ResourceHolder<LongBuffer>>(
|
new GenericIndexedWriter<ResourceHolder<LongBuffer>>(
|
||||||
new IOPeonForTesting(),
|
new IOPeonForTesting(),
|
||||||
"test",
|
"test",
|
||||||
CompressedLongBufferObjectStrategy.getBufferForOrder(order, CompressedObjectStrategy.CompressionStrategy.LZ4, sizePer)
|
CompressedLongBufferObjectStrategy.getBufferForOrder(order, compressionStrategy, sizePer)
|
||||||
),
|
),
|
||||||
CompressedObjectStrategy.CompressionStrategy.LZ4
|
compressionStrategy
|
||||||
);
|
);
|
||||||
serializer.open();
|
serializer.open();
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,52 @@
|
||||||
|
/*
|
||||||
|
* Druid - a distributed column store.
|
||||||
|
* Copyright (C) 2012, 2013, 2014 Metamarkets Group Inc.
|
||||||
|
*
|
||||||
|
* This program is free software; you can redistribute it and/or
|
||||||
|
* modify it under the terms of the GNU General Public License
|
||||||
|
* as published by the Free Software Foundation; either version 2
|
||||||
|
* of the License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU General Public License
|
||||||
|
* along with this program; if not, write to the Free Software
|
||||||
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package io.druid.segment.data;
|
||||||
|
|
||||||
|
import com.google.common.base.Function;
|
||||||
|
import com.google.common.collect.Iterables;
|
||||||
|
import org.junit.runners.Parameterized;
|
||||||
|
|
||||||
|
import java.util.Arrays;
|
||||||
|
|
||||||
|
public class CompressionStrategyTest
|
||||||
|
{
|
||||||
|
@Parameterized.Parameters
|
||||||
|
public static Iterable<Object[]> compressionStrategies()
|
||||||
|
{
|
||||||
|
return Iterables.transform(
|
||||||
|
Arrays.asList(CompressedObjectStrategy.CompressionStrategy.values()),
|
||||||
|
new Function<CompressedObjectStrategy.CompressionStrategy, Object[]>()
|
||||||
|
{
|
||||||
|
@Override
|
||||||
|
public Object[] apply(CompressedObjectStrategy.CompressionStrategy compressionStrategy)
|
||||||
|
{
|
||||||
|
return new Object[]{compressionStrategy};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected final CompressedObjectStrategy.CompressionStrategy compressionStrategy;
|
||||||
|
|
||||||
|
public CompressionStrategyTest(CompressedObjectStrategy.CompressionStrategy compressionStrategy)
|
||||||
|
{
|
||||||
|
this.compressionStrategy = compressionStrategy;
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue