make LZ4 the default compression strategy

- LZ4 is now hardwired to be the default strategy
- Rework tests to test all available compression strategies
This commit is contained in:
Xavier Léauté 2014-08-06 15:50:04 -07:00
parent c40a315c81
commit 991e1828b0
10 changed files with 99 additions and 22 deletions

View File

@ -53,7 +53,7 @@ public class FloatMetricColumnSerializer implements MetricColumnSerializer
{
writer = CompressedFloatsSupplierSerializer.create(
ioPeon, String.format("%s_little", metricName), IndexIO.BYTE_ORDER,
CompressedObjectStrategy.CompressionStrategy.LZ4 // TODO define this somewhere else
CompressedObjectStrategy.DEFAULT_COMPRESSION_STRATEGY
);
writer.open();

View File

@ -596,7 +596,7 @@ public class IndexMerger
CompressedLongsSupplierSerializer timeWriter = CompressedLongsSupplierSerializer.create(
ioPeon, "little_end_time", IndexIO.BYTE_ORDER,
CompressedObjectStrategy.CompressionStrategy.LZ4 // TODO define this somewhere else
CompressedObjectStrategy.DEFAULT_COMPRESSION_STRATEGY
);
timeWriter.open();

View File

@ -42,6 +42,8 @@ import java.util.Map;
*/
public class CompressedObjectStrategy<T extends Buffer> implements ObjectStrategy<ResourceHolder<T>>
{
public static final CompressionStrategy DEFAULT_COMPRESSION_STRATEGY = CompressionStrategy.LZ4;
public static enum CompressionStrategy {
LZF ((byte)0x0)
{

View File

@ -37,7 +37,7 @@ import java.util.List;
*/
public class InMemoryCompressedFloats implements IndexedFloats
{
public static final CompressedObjectStrategy.CompressionStrategy COMPRESSION = CompressedObjectStrategy.CompressionStrategy.LZ4;
public static final CompressedObjectStrategy.CompressionStrategy COMPRESSION = CompressedObjectStrategy.DEFAULT_COMPRESSION_STRATEGY;
private final CompressedFloatBufferObjectStrategy strategy;
private final int sizePer;

View File

@ -38,7 +38,7 @@ import java.util.List;
*/
public class InMemoryCompressedLongs implements IndexedLongs
{
public static final CompressedObjectStrategy.CompressionStrategy COMPRESSION = CompressedObjectStrategy.CompressionStrategy.LZ4;
public static final CompressedObjectStrategy.CompressionStrategy COMPRESSION = CompressedObjectStrategy.DEFAULT_COMPRESSION_STRATEGY;
private final CompressedLongBufferObjectStrategy strategy;
private final int sizePer;

View File

@ -25,6 +25,8 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@ -36,10 +38,14 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
/**
*/
public class CompressedFloatsIndexedSupplierTest
@RunWith(Parameterized.class)
public class CompressedFloatsIndexedSupplierTest extends CompressionStrategyTest
{
public CompressedFloatsIndexedSupplierTest(CompressedObjectStrategy.CompressionStrategy compressionStrategy)
{
super(compressionStrategy);
}
private IndexedFloats indexed;
private CompressedFloatsIndexedSupplier supplier;
private float[] vals;
@ -69,7 +75,7 @@ public class CompressedFloatsIndexedSupplierTest
FloatBuffer.wrap(vals),
5,
ByteOrder.nativeOrder(),
CompressedObjectStrategy.CompressionStrategy.LZ4
compressionStrategy
);
indexed = supplier.get();
@ -83,7 +89,7 @@ public class CompressedFloatsIndexedSupplierTest
ByteArrayOutputStream baos = new ByteArrayOutputStream();
final CompressedFloatsIndexedSupplier theSupplier = CompressedFloatsIndexedSupplier.fromFloatBuffer(
FloatBuffer.wrap(vals), 5, ByteOrder.nativeOrder(), CompressedObjectStrategy.CompressionStrategy.LZ4
FloatBuffer.wrap(vals), 5, ByteOrder.nativeOrder(), compressionStrategy
);
theSupplier.writeToChannel(Channels.newChannel(baos));

View File

@ -23,6 +23,8 @@ import com.google.common.io.OutputSupplier;
import io.druid.collections.ResourceHolder;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@ -31,10 +33,14 @@ import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
/**
*/
public class CompressedFloatsSupplierSerializerTest
@RunWith(Parameterized.class)
public class CompressedFloatsSupplierSerializerTest extends CompressionStrategyTest
{
public CompressedFloatsSupplierSerializerTest(CompressedObjectStrategy.CompressionStrategy compressionStrategy)
{
super(compressionStrategy);
}
@Test
public void testSanity() throws Exception
{
@ -47,11 +53,11 @@ public class CompressedFloatsSupplierSerializerTest
"test",
CompressedFloatBufferObjectStrategy.getBufferForOrder(
order,
CompressedObjectStrategy.CompressionStrategy.LZ4,
compressionStrategy,
sizePer
)
),
CompressedObjectStrategy.CompressionStrategy.LZ4
compressionStrategy
);
serializer.open();

View File

@ -38,8 +38,13 @@ import java.util.concurrent.atomic.AtomicReference;
/**
*/
public class CompressedLongsIndexedSupplierTest
public class CompressedLongsIndexedSupplierTest extends CompressionStrategyTest
{
public CompressedLongsIndexedSupplierTest(CompressedObjectStrategy.CompressionStrategy compressionStrategy)
{
super(compressionStrategy);
}
private IndexedLongs indexed;
private CompressedLongsIndexedSupplier supplier;
private long[] vals;
@ -67,7 +72,7 @@ public class CompressedLongsIndexedSupplierTest
LongBuffer.wrap(vals),
5,
ByteOrder.nativeOrder(),
CompressedObjectStrategy.CompressionStrategy.LZ4
compressionStrategy
);
indexed = supplier.get();
@ -79,7 +84,7 @@ public class CompressedLongsIndexedSupplierTest
ByteArrayOutputStream baos = new ByteArrayOutputStream();
final CompressedLongsIndexedSupplier theSupplier = CompressedLongsIndexedSupplier.fromLongBuffer(
LongBuffer.wrap(vals), 5, ByteOrder.nativeOrder(), CompressedObjectStrategy.CompressionStrategy.LZ4
LongBuffer.wrap(vals), 5, ByteOrder.nativeOrder(), compressionStrategy
);
theSupplier.writeToChannel(Channels.newChannel(baos));

View File

@ -23,6 +23,8 @@ import com.google.common.io.OutputSupplier;
import io.druid.collections.ResourceHolder;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@ -31,10 +33,14 @@ import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.LongBuffer;
/**
*/
public class CompressedLongsSupplierSerializerTest
@RunWith(Parameterized.class)
public class CompressedLongsSupplierSerializerTest extends CompressionStrategyTest
{
public CompressedLongsSupplierSerializerTest(CompressedObjectStrategy.CompressionStrategy compressionStrategy)
{
super(compressionStrategy);
}
@Test
public void testSanity() throws Exception
{
@ -45,9 +51,9 @@ public class CompressedLongsSupplierSerializerTest
new GenericIndexedWriter<ResourceHolder<LongBuffer>>(
new IOPeonForTesting(),
"test",
CompressedLongBufferObjectStrategy.getBufferForOrder(order, CompressedObjectStrategy.CompressionStrategy.LZ4, sizePer)
CompressedLongBufferObjectStrategy.getBufferForOrder(order, compressionStrategy, sizePer)
),
CompressedObjectStrategy.CompressionStrategy.LZ4
compressionStrategy
);
serializer.open();

View File

@ -0,0 +1,52 @@
/*
* Druid - a distributed column store.
* Copyright (C) 2012, 2013, 2014 Metamarkets Group Inc.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package io.druid.segment.data;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
import org.junit.runners.Parameterized;
import java.util.Arrays;
public class CompressionStrategyTest
{
@Parameterized.Parameters
public static Iterable<Object[]> compressionStrategies()
{
return Iterables.transform(
Arrays.asList(CompressedObjectStrategy.CompressionStrategy.values()),
new Function<CompressedObjectStrategy.CompressionStrategy, Object[]>()
{
@Override
public Object[] apply(CompressedObjectStrategy.CompressionStrategy compressionStrategy)
{
return new Object[]{compressionStrategy};
}
}
);
}
protected final CompressedObjectStrategy.CompressionStrategy compressionStrategy;
public CompressionStrategyTest(CompressedObjectStrategy.CompressionStrategy compressionStrategy)
{
this.compressionStrategy = compressionStrategy;
}
}