Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix travis timeout in BufferHashGrouperTest (#6713) 

* Fix travis timeout in BufferHashGrouperTest

* adjust buffer size

* adjust bufferSize and loadFactor

* increase memory

* add debug code

* cat error

* after script

* print logs

* print per 2 min

* use direct mem

* clean up
This commit is contained in:
Jihoon Son 2018-12-06 20:05:27 -08:00 committed by Benedict Jin
parent cc44a4a28f
commit d525e5b18e
2 changed files with 20 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
processing/pom.xml
View File

@ -195,7 +195,7 @@
<!-- set default options -->
<argLine>
-Xmx512m
-XX:MaxDirectMemorySize=1500m
-XX:MaxDirectMemorySize=2500m
-Duser.language=en
-Duser.GroupByQueryRunnerTest.javacountry=US
-Dfile.encoding=UTF-8

66
processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java
View File

@ -20,16 +20,13 @@
package org.apache.druid.query.groupby.epinephelinae;
import com.google.common.base.Suppliers;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Ordering;
import com.google.common.io.Files;
import com.google.common.primitives.Ints;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.data.input.MapBasedRow;
import org.apache.druid.java.util.common.ByteBufferUtils;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
@ -39,12 +36,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
@ -112,7 +104,7 @@ public class BufferHashGrouperTest
public void testGrowing()
{
final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory, 10000, 2);
final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory, 10000, 2, 0.75f);
final int expectedMaxSize = NullHandling.replaceWithDefault() ? 219 : 210;
columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L)));
@ -137,38 +129,29 @@ public class BufferHashGrouperTest
}
@Test
public void testGrowing2()
public void testGrowingOverflowingInteger()
{
final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory, 2_000_000_000, 2);
final int expectedMaxSize = NullHandling.replaceWithDefault() ? 40988516 : 39141224;
// This test checks the bug reported in https://github.com/apache/incubator-druid/pull/4333 only when
// NullHandling.replaceWithDefault() is true
if (NullHandling.replaceWithDefault()) {
final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
// the buffer size below is chosen to test integer overflow in ByteBufferHashTable.adjustTableWhenFull().
final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory, 1_900_000_000, 2, 0.3f);
final int expectedMaxSize = 15323979;
columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L)));
for (int i = 0; i < expectedMaxSize; i++) {
Assert.assertTrue(String.valueOf(i), grouper.aggregate(i).isOk());
columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L)));
for (int i = 0; i < expectedMaxSize; i++) {
Assert.assertTrue(String.valueOf(i), grouper.aggregate(i).isOk());
}
Assert.assertFalse(grouper.aggregate(expectedMaxSize).isOk());
}
Assert.assertFalse(grouper.aggregate(expectedMaxSize).isOk());
}
@Test
public void testGrowing3()
{
final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory, Integer.MAX_VALUE, 2);
final int expectedMaxSize = NullHandling.replaceWithDefault() ? 44938972 : 42955456;
columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L)));
for (int i = 0; i < expectedMaxSize; i++) {
Assert.assertTrue(String.valueOf(i), grouper.aggregate(i).isOk());
}
Assert.assertFalse(grouper.aggregate(expectedMaxSize).isOk());
}
@Test
public void testNoGrowing()
{
final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory, 10000, Integer.MAX_VALUE);
final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory, 10000, Integer.MAX_VALUE, 0.75f);
final int expectedMaxSize = NullHandling.replaceWithDefault() ? 267 : 258;
columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L)));
@ -195,22 +178,11 @@ public class BufferHashGrouperTest
private BufferHashGrouper<Integer> makeGrouper(
TestColumnSelectorFactory columnSelectorFactory,
int bufferSize,
int initialBuckets
int initialBuckets,
float maxLoadFactor
)
{
final MappedByteBuffer buffer;
try {
final File file = temporaryFolder.newFile();
try (final FileChannel channel = new FileOutputStream(file).getChannel()) {
channel.truncate(bufferSize);
}
buffer = Files.map(file, FileChannel.MapMode.READ_WRITE, bufferSize);
closerRule.closeLater(() -> ByteBufferUtils.unmap(buffer));
}
catch (IOException e) {
throw Throwables.propagate(e);
}
final ByteBuffer buffer = ByteBuffer.allocateDirect(bufferSize);
final BufferHashGrouper<Integer> grouper = new BufferHashGrouper<>(
Suppliers.ofInstance(buffer),
@ -221,7 +193,7 @@ public class BufferHashGrouperTest
new CountAggregatorFactory("count")
},
Integer.MAX_VALUE,
0.75f,
maxLoadFactor,
initialBuckets,
true
);