mirror of https://github.com/apache/druid.git
make LZ4 the default compression strategy
- LZ4 is now hardwired to be the default strategy - Rework tests to test all available compression strategies
This commit is contained in:
parent
c40a315c81
commit
991e1828b0
|
@ -53,7 +53,7 @@ public class FloatMetricColumnSerializer implements MetricColumnSerializer
|
|||
{
|
||||
writer = CompressedFloatsSupplierSerializer.create(
|
||||
ioPeon, String.format("%s_little", metricName), IndexIO.BYTE_ORDER,
|
||||
CompressedObjectStrategy.CompressionStrategy.LZ4 // TODO define this somewhere else
|
||||
CompressedObjectStrategy.DEFAULT_COMPRESSION_STRATEGY
|
||||
);
|
||||
|
||||
writer.open();
|
||||
|
|
|
@ -596,7 +596,7 @@ public class IndexMerger
|
|||
|
||||
CompressedLongsSupplierSerializer timeWriter = CompressedLongsSupplierSerializer.create(
|
||||
ioPeon, "little_end_time", IndexIO.BYTE_ORDER,
|
||||
CompressedObjectStrategy.CompressionStrategy.LZ4 // TODO define this somewhere else
|
||||
CompressedObjectStrategy.DEFAULT_COMPRESSION_STRATEGY
|
||||
);
|
||||
|
||||
timeWriter.open();
|
||||
|
|
|
@ -42,6 +42,8 @@ import java.util.Map;
|
|||
*/
|
||||
public class CompressedObjectStrategy<T extends Buffer> implements ObjectStrategy<ResourceHolder<T>>
|
||||
{
|
||||
public static final CompressionStrategy DEFAULT_COMPRESSION_STRATEGY = CompressionStrategy.LZ4;
|
||||
|
||||
public static enum CompressionStrategy {
|
||||
LZF ((byte)0x0)
|
||||
{
|
||||
|
|
|
@ -37,7 +37,7 @@ import java.util.List;
|
|||
*/
|
||||
public class InMemoryCompressedFloats implements IndexedFloats
|
||||
{
|
||||
public static final CompressedObjectStrategy.CompressionStrategy COMPRESSION = CompressedObjectStrategy.CompressionStrategy.LZ4;
|
||||
public static final CompressedObjectStrategy.CompressionStrategy COMPRESSION = CompressedObjectStrategy.DEFAULT_COMPRESSION_STRATEGY;
|
||||
private final CompressedFloatBufferObjectStrategy strategy;
|
||||
private final int sizePer;
|
||||
|
||||
|
|
|
@ -38,7 +38,7 @@ import java.util.List;
|
|||
*/
|
||||
public class InMemoryCompressedLongs implements IndexedLongs
|
||||
{
|
||||
public static final CompressedObjectStrategy.CompressionStrategy COMPRESSION = CompressedObjectStrategy.CompressionStrategy.LZ4;
|
||||
public static final CompressedObjectStrategy.CompressionStrategy COMPRESSION = CompressedObjectStrategy.DEFAULT_COMPRESSION_STRATEGY;
|
||||
private final CompressedLongBufferObjectStrategy strategy;
|
||||
private final int sizePer;
|
||||
|
||||
|
|
|
@ -25,6 +25,8 @@ import org.junit.After;
|
|||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
|
@ -36,10 +38,14 @@ import java.util.concurrent.CountDownLatch;
|
|||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class CompressedFloatsIndexedSupplierTest
|
||||
@RunWith(Parameterized.class)
|
||||
public class CompressedFloatsIndexedSupplierTest extends CompressionStrategyTest
|
||||
{
|
||||
public CompressedFloatsIndexedSupplierTest(CompressedObjectStrategy.CompressionStrategy compressionStrategy)
|
||||
{
|
||||
super(compressionStrategy);
|
||||
}
|
||||
|
||||
private IndexedFloats indexed;
|
||||
private CompressedFloatsIndexedSupplier supplier;
|
||||
private float[] vals;
|
||||
|
@ -69,7 +75,7 @@ public class CompressedFloatsIndexedSupplierTest
|
|||
FloatBuffer.wrap(vals),
|
||||
5,
|
||||
ByteOrder.nativeOrder(),
|
||||
CompressedObjectStrategy.CompressionStrategy.LZ4
|
||||
compressionStrategy
|
||||
);
|
||||
|
||||
indexed = supplier.get();
|
||||
|
@ -83,7 +89,7 @@ public class CompressedFloatsIndexedSupplierTest
|
|||
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
final CompressedFloatsIndexedSupplier theSupplier = CompressedFloatsIndexedSupplier.fromFloatBuffer(
|
||||
FloatBuffer.wrap(vals), 5, ByteOrder.nativeOrder(), CompressedObjectStrategy.CompressionStrategy.LZ4
|
||||
FloatBuffer.wrap(vals), 5, ByteOrder.nativeOrder(), compressionStrategy
|
||||
);
|
||||
theSupplier.writeToChannel(Channels.newChannel(baos));
|
||||
|
||||
|
|
|
@ -23,6 +23,8 @@ import com.google.common.io.OutputSupplier;
|
|||
import io.druid.collections.ResourceHolder;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
|
@ -31,10 +33,14 @@ import java.nio.ByteBuffer;
|
|||
import java.nio.ByteOrder;
|
||||
import java.nio.FloatBuffer;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class CompressedFloatsSupplierSerializerTest
|
||||
@RunWith(Parameterized.class)
|
||||
public class CompressedFloatsSupplierSerializerTest extends CompressionStrategyTest
|
||||
{
|
||||
public CompressedFloatsSupplierSerializerTest(CompressedObjectStrategy.CompressionStrategy compressionStrategy)
|
||||
{
|
||||
super(compressionStrategy);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSanity() throws Exception
|
||||
{
|
||||
|
@ -47,11 +53,11 @@ public class CompressedFloatsSupplierSerializerTest
|
|||
"test",
|
||||
CompressedFloatBufferObjectStrategy.getBufferForOrder(
|
||||
order,
|
||||
CompressedObjectStrategy.CompressionStrategy.LZ4,
|
||||
compressionStrategy,
|
||||
sizePer
|
||||
)
|
||||
),
|
||||
CompressedObjectStrategy.CompressionStrategy.LZ4
|
||||
compressionStrategy
|
||||
);
|
||||
serializer.open();
|
||||
|
||||
|
|
|
@ -38,8 +38,13 @@ import java.util.concurrent.atomic.AtomicReference;
|
|||
|
||||
/**
|
||||
*/
|
||||
public class CompressedLongsIndexedSupplierTest
|
||||
public class CompressedLongsIndexedSupplierTest extends CompressionStrategyTest
|
||||
{
|
||||
public CompressedLongsIndexedSupplierTest(CompressedObjectStrategy.CompressionStrategy compressionStrategy)
|
||||
{
|
||||
super(compressionStrategy);
|
||||
}
|
||||
|
||||
private IndexedLongs indexed;
|
||||
private CompressedLongsIndexedSupplier supplier;
|
||||
private long[] vals;
|
||||
|
@ -67,7 +72,7 @@ public class CompressedLongsIndexedSupplierTest
|
|||
LongBuffer.wrap(vals),
|
||||
5,
|
||||
ByteOrder.nativeOrder(),
|
||||
CompressedObjectStrategy.CompressionStrategy.LZ4
|
||||
compressionStrategy
|
||||
);
|
||||
|
||||
indexed = supplier.get();
|
||||
|
@ -79,7 +84,7 @@ public class CompressedLongsIndexedSupplierTest
|
|||
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
final CompressedLongsIndexedSupplier theSupplier = CompressedLongsIndexedSupplier.fromLongBuffer(
|
||||
LongBuffer.wrap(vals), 5, ByteOrder.nativeOrder(), CompressedObjectStrategy.CompressionStrategy.LZ4
|
||||
LongBuffer.wrap(vals), 5, ByteOrder.nativeOrder(), compressionStrategy
|
||||
);
|
||||
theSupplier.writeToChannel(Channels.newChannel(baos));
|
||||
|
||||
|
|
|
@ -23,6 +23,8 @@ import com.google.common.io.OutputSupplier;
|
|||
import io.druid.collections.ResourceHolder;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
|
@ -31,10 +33,14 @@ import java.nio.ByteBuffer;
|
|||
import java.nio.ByteOrder;
|
||||
import java.nio.LongBuffer;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class CompressedLongsSupplierSerializerTest
|
||||
@RunWith(Parameterized.class)
|
||||
public class CompressedLongsSupplierSerializerTest extends CompressionStrategyTest
|
||||
{
|
||||
public CompressedLongsSupplierSerializerTest(CompressedObjectStrategy.CompressionStrategy compressionStrategy)
|
||||
{
|
||||
super(compressionStrategy);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSanity() throws Exception
|
||||
{
|
||||
|
@ -45,9 +51,9 @@ public class CompressedLongsSupplierSerializerTest
|
|||
new GenericIndexedWriter<ResourceHolder<LongBuffer>>(
|
||||
new IOPeonForTesting(),
|
||||
"test",
|
||||
CompressedLongBufferObjectStrategy.getBufferForOrder(order, CompressedObjectStrategy.CompressionStrategy.LZ4, sizePer)
|
||||
CompressedLongBufferObjectStrategy.getBufferForOrder(order, compressionStrategy, sizePer)
|
||||
),
|
||||
CompressedObjectStrategy.CompressionStrategy.LZ4
|
||||
compressionStrategy
|
||||
);
|
||||
serializer.open();
|
||||
|
||||
|
|
|
@ -0,0 +1,52 @@
|
|||
/*
|
||||
* Druid - a distributed column store.
|
||||
* Copyright (C) 2012, 2013, 2014 Metamarkets Group Inc.
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
package io.druid.segment.data;
|
||||
|
||||
import com.google.common.base.Function;
|
||||
import com.google.common.collect.Iterables;
|
||||
import org.junit.runners.Parameterized;
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
public class CompressionStrategyTest
|
||||
{
|
||||
@Parameterized.Parameters
|
||||
public static Iterable<Object[]> compressionStrategies()
|
||||
{
|
||||
return Iterables.transform(
|
||||
Arrays.asList(CompressedObjectStrategy.CompressionStrategy.values()),
|
||||
new Function<CompressedObjectStrategy.CompressionStrategy, Object[]>()
|
||||
{
|
||||
@Override
|
||||
public Object[] apply(CompressedObjectStrategy.CompressionStrategy compressionStrategy)
|
||||
{
|
||||
return new Object[]{compressionStrategy};
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
protected final CompressedObjectStrategy.CompressionStrategy compressionStrategy;
|
||||
|
||||
public CompressionStrategyTest(CompressedObjectStrategy.CompressionStrategy compressionStrategy)
|
||||
{
|
||||
this.compressionStrategy = compressionStrategy;
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue