From 3a5c4804055f10aadcae34277cb1345c7d51874d Mon Sep 17 00:00:00 2001 From: Jihoon Son Date: Thu, 22 Jun 2017 12:55:51 +0900 Subject: [PATCH] Split IndexMergerTest and ImmutableConciseSetTest (#4427) --- .../ImmutableConciseSetIntersectionTest.java | 594 ++++++++++++++++++ .../intset/ImmutableConciseSetTest.java | 565 +---------------- .../segment/ConciseBitmapIndexMergerTest.java | 40 ++ .../ConciseBitmapIndexMergerV9Test.java | 40 ++ .../java/io/druid/segment/IndexIOTest.java | 2 +- ...rgerTest.java => IndexMergerTestBase.java} | 147 ++--- .../IndexMergerV9CompatibilityTest.java | 2 +- .../QueryableIndexIndexableAdapterTest.java | 2 +- .../segment/RoaringBitmapIndexMergerTest.java | 40 ++ .../RoaringBitmapIndexMergerV9Test.java | 40 ++ 10 files changed, 823 insertions(+), 649 deletions(-) create mode 100644 extendedset/src/test/java/io/druid/extendedset/intset/ImmutableConciseSetIntersectionTest.java create mode 100644 processing/src/test/java/io/druid/segment/ConciseBitmapIndexMergerTest.java create mode 100644 processing/src/test/java/io/druid/segment/ConciseBitmapIndexMergerV9Test.java rename processing/src/test/java/io/druid/segment/{IndexMergerTest.java => IndexMergerTestBase.java} (96%) create mode 100644 processing/src/test/java/io/druid/segment/RoaringBitmapIndexMergerTest.java create mode 100644 processing/src/test/java/io/druid/segment/RoaringBitmapIndexMergerV9Test.java diff --git a/extendedset/src/test/java/io/druid/extendedset/intset/ImmutableConciseSetIntersectionTest.java b/extendedset/src/test/java/io/druid/extendedset/intset/ImmutableConciseSetIntersectionTest.java new file mode 100644 index 00000000000..221099e82bb --- /dev/null +++ b/extendedset/src/test/java/io/druid/extendedset/intset/ImmutableConciseSetIntersectionTest.java @@ -0,0 +1,594 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.extendedset.intset; + +import com.google.common.collect.Lists; +import junit.framework.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +@RunWith(Parameterized.class) +public class ImmutableConciseSetIntersectionTest +{ + @Parameterized.Parameters + public static List parameters() + { + return Arrays.asList(new Object[] {false}, new Object[] {true}); + } + + private boolean compact; + + public ImmutableConciseSetIntersectionTest(boolean compact) + { + this.compact = compact; + } + + /** + * Testing basic intersection of similar sets + */ + @Test + public void testIntersection1() + { + final int[] ints1 = {33, 100000}; + final int[] ints2 = {33, 100000}; + List expected = Arrays.asList(33, 100000); + + ConciseSet set1 = new ConciseSet(); + for (int i : ints1) { + set1.add(i); + } + ConciseSet set2 = new ConciseSet(); + for (int i : ints2) { + set2.add(i); + } + + verifyIntersection(expected, set1, set2); + } + + /** + * Set1: literal, zero fill with flip bit, literal + * Set2: literal, zero fill with different flip bit, literal + */ + @Test + public void testIntersection2() + { + final int[] ints1 = {33, 100000}; + final int[] ints2 = {34, 100000}; + List expected = Collections.singletonList(100000); + + ConciseSet set1 = new ConciseSet(); + for (int i : ints1) { + set1.add(i); + } + ConciseSet set2 = new ConciseSet(); + for (int i : ints2) { + set2.add(i); + } + + verifyIntersection(expected, set1, set2); + } + + /** + * Testing intersection of one fills + */ + @Test + public void testIntersection3() + { + List expected = Lists.newArrayList(); + ConciseSet set1 = new ConciseSet(); + ConciseSet set2 = new ConciseSet(); + for (int i = 0; i < 1000; i++) { + set1.add(i); + set2.add(i); + expected.add(i); + } + + verifyIntersection(expected, set1, set2); + } + + /** + * Similar to previous test with one bit in the sequence set to zero + */ + @Test + public void testIntersection4() + { + List expected = Lists.newArrayList(); + ConciseSet set1 = new ConciseSet(); + ConciseSet set2 = new ConciseSet(); + for (int i = 0; i < 1000; i++) { + set1.add(i); + if (i != 500) { + set2.add(i); + expected.add(i); + } + } + + verifyIntersection(expected, set1, set2); + } + + /** + * Testing with disjoint sets + */ + @Test + public void testIntersection5() + { + final int[] ints1 = {33, 100000}; + final int[] ints2 = {34, 200000}; + List expected = Lists.newArrayList(); + + ConciseSet set1 = new ConciseSet(); + for (int i : ints1) { + set1.add(i); + } + ConciseSet set2 = new ConciseSet(); + for (int i : ints2) { + set2.add(i); + } + + verifyIntersection(expected, set1, set2); + } + + /** + * Set 1: literal, zero fill, literal + * Set 2: one fill, literal that falls within the zero fill above, one fill + */ + @Test + public void testIntersection6() + { + List expected = Lists.newArrayList(); + ConciseSet set1 = new ConciseSet(); + for (int i = 0; i < 5; i++) { + set1.add(i); + } + for (int i = 1000; i < 1005; i++) { + set1.add(i); + } + + ConciseSet set2 = new ConciseSet(); + for (int i = 800; i < 805; i++) { + set2.add(i); + } + for (int i = 806; i < 1005; i++) { + set2.add(i); + } + + for (int i = 1000; i < 1005; i++) { + expected.add(i); + } + + verifyIntersection(expected, set1, set2); + } + + @Test + public void testIntersection7() + { + ConciseSet set1 = new ConciseSet(); + for (int i = 0; i < 3100; i++) { + set1.add(i); + } + + ConciseSet set2 = new ConciseSet(); + set2.add(100); + set2.add(500); + for (int i = 600; i < 700; i++) { + set2.add(i); + } + + List expected = Lists.newArrayList(); + expected.add(100); + expected.add(500); + for (int i = 600; i < 700; i++) { + expected.add(i); + } + + verifyIntersection(expected, set1, set2); + } + + @Test + public void testIntersection8() + { + ConciseSet set1 = new ConciseSet(); + for (int i = 0; i < 3100; i++) { + set1.add(i); + } + set1.add(4001); + + ConciseSet set2 = new ConciseSet(); + set2.add(100); + set2.add(500); + for (int i = 600; i < 700; i++) { + set2.add(i); + } + set2.add(4001); + + List expected = Lists.newArrayList(); + expected.add(100); + expected.add(500); + for (int i = 600; i < 700; i++) { + expected.add(i); + } + expected.add(4001); + + verifyIntersection(expected, set1, set2); + } + + @Test + public void testIntersection9() + { + ConciseSet set1 = new ConciseSet(); + set1.add(2005); + set1.add(3005); + set1.add(3008); + + ConciseSet set2 = new ConciseSet(); + for (int i = 0; i < 3007; i++) { + set2.add(i); + } + + List expected = Lists.newArrayList(); + expected.add(2005); + expected.add(3005); + + verifyIntersection(expected, set1, set2); + } + + @Test + public void testIntersection10() + { + ConciseSet set1 = new ConciseSet(); + for (int i = 0; i < 3100; i++) { + set1.add(i); + } + + ConciseSet set2 = new ConciseSet(); + + set2.add(500); + set2.add(600); + set2.add(4001); + + List expected = Lists.newArrayList(); + expected.add(500); + expected.add(600); + + verifyIntersection(expected, set1, set2); + } + + @Test + public void testIntersection11() + { + ConciseSet set1 = new ConciseSet(); + set1.add(2005); + for (int i = 2800; i < 3500; i++) { + set1.add(i); + } + + ConciseSet set2 = new ConciseSet(); + for (int i = 0; i < 3007; i++) { + set2.add(i); + } + + List expected = Lists.newArrayList(); + expected.add(2005); + for (int i = 2800; i < 3007; i++) { + expected.add(i); + } + + verifyIntersection(expected, set1, set2); + } + + @Test + public void testIntersection12() + { + ConciseSet set1 = new ConciseSet(); + set1.add(2005); + for (int i = 2800; i < 3500; i++) { + set1.add(i); + } + set1.add(10005); + + ConciseSet set2 = new ConciseSet(); + for (int i = 0; i < 3007; i++) { + set2.add(i); + } + set2.add(10005); + + List expected = Lists.newArrayList(); + expected.add(2005); + for (int i = 2800; i < 3007; i++) { + expected.add(i); + } + expected.add(10005); + + verifyIntersection(expected, set1, set2); + } + + @Test + public void testIntersection13() + { + ConciseSet set1 = new ConciseSet(); + set1.add(2005); + + ConciseSet set2 = new ConciseSet(); + for (int i = 0; i < 100; i++) { + set2.add(i); + } + + List expected = Lists.newArrayList(); + + verifyIntersection(expected, set1, set2); + } + + @Test + public void testIntersection14() + { + ConciseSet set1 = new ConciseSet(); + for (int i = 0; i < 1000; i++) { + set1.add(i); + } + + ConciseSet set2 = new ConciseSet(); + set2.add(0); + set2.add(3); + set2.add(5); + set2.add(100); + set2.add(101); + + List expected = Lists.newArrayList(); + expected.add(0); + expected.add(3); + expected.add(5); + expected.add(100); + expected.add(101); + + verifyIntersection(expected, set1, set2); + } + + @Test + public void testIntersection15() + { + ConciseSet set1 = new ConciseSet(); + for (int i = 0; i < 1000; i++) { + set1.add(i); + } + + ConciseSet set2 = new ConciseSet(); + set2.add(0); + set2.add(3); + set2.add(5); + for (int i = 100; i < 500; i++) { + set2.add(i); + } + + List expected = Lists.newArrayList(); + expected.add(0); + expected.add(3); + expected.add(5); + for (int i = 100; i < 500; i++) { + expected.add(i); + } + + verifyIntersection(expected, set1, set2); + } + + @Test + public void testIntersection16() + { + ConciseSet set1 = new ConciseSet(); + set1.add(2005); + + ConciseSet set2 = new ConciseSet(); + set2.add(0); + set2.add(3); + set2.add(5); + set2.add(100); + set2.add(101); + + List expected = Lists.newArrayList(); + + verifyIntersection(expected, set1, set2); + } + + @Test + public void testIntersection17() + { + ConciseSet set1 = new ConciseSet(); + for (int i = 0; i < 4002; i++) { + set1.add(i); + } + + ConciseSet set2 = new ConciseSet(); + set2.add(4001); + + List expected = Lists.newArrayList(); + expected.add(4001); + + verifyIntersection(expected, set1, set2); + } + + @Test + public void testIntersection18() + { + ConciseSet set1 = new ConciseSet(); + for (int i = 32; i < 93; i++) { + set1.add(i); + } + + ConciseSet set2 = new ConciseSet(); + for (int i = 0; i < 62; i++) { + set2.add(i); + } + + List expected = Lists.newArrayList(); + for (int i = 32; i < 62; i++) { + expected.add(i); + } + + verifyIntersection(expected, set1, set2); + } + + @Test + public void testIntersection19() + { + ConciseSet set1 = new ConciseSet(); + set1.add(2005); + + ConciseSet set2 = new ConciseSet(); + for (int i = 0; i < 10000; i++) { + set2.add(i); + } + + List expected = Lists.newArrayList(); + expected.add(2005); + + verifyIntersection(expected, set1, set2); + } + + @Test + public void testIntersectionLiteralAndOneFill() + { + ConciseSet set1 = new ConciseSet(); + for (int i = 0; i < 31; i += 2) { + set1.add(i); + } + ConciseSet set2 = new ConciseSet(); + for (int i = 0; i < 1000; i++) { + if (i != 2) { + set2.add(i); + } + } + verifyIntersection(set1, set2); + } + + @Test + public void testIntersectionZeroSequenceRemovedFromQueue() + { + // Seems that it is impossible to test this case with naturally constructed ConciseSet, because the end of the + // sequence is defined by the last set bit, then naturally constructed ConciseSet won't have the last word as zero + // sequence, it will be a literal or one sequence. + int zeroSequence = 1; // Zero sequence of length 62 + ConciseSet set1 = new ConciseSet(new int[] {zeroSequence}, false); + + ConciseSet set2 = new ConciseSet(); + for (int i = 0; i < 1000; i++) { + set2.add(i); + } + verifyIntersection(set1, set2); + } + + @Test + public void testIntersectionOneFillAndOneFillWithFlipBit() + { + ConciseSet set1 = new ConciseSet(); + for (int i = 0; i < 100; i++) { + set1.add(i); + } + ConciseSet set2 = new ConciseSet(); + for (int i = 0; i < 1000; i++) { + if (i != 2) { + set2.add(i); + } + } + verifyIntersection(set1, set2); + } + + @Test + public void testIntersectionSecondOneFillRemovedFromQueue() + { + ConciseSet set1 = new ConciseSet(); + for (int i = 0; i < 31 * 2; i++) { + set1.add(i); + } + set1.add(100); + + ConciseSet set2 = new ConciseSet(); + for (int i = 0; i < 31 * 3; i++) { + set2.add(i); + } + + verifyIntersection(set1, set2); + } + + @Test + public void testIntersectionFirstOneFillRemovedFromQueue() + { + ConciseSet set1 = new ConciseSet(); + for (int i = 0; i < 31 * 2; i++) { + set1.add(i); + } + + ConciseSet set2 = new ConciseSet(); + for (int i = 0; i < 31 * 3; i++) { + set2.add(i); + } + + verifyIntersection(set1, set2); + } + + @Test + public void testIntersectionTerminates() throws Exception + { + verifyIntersection(Collections.emptyList(), Arrays.asList(new ImmutableConciseSet(), new ImmutableConciseSet())); + } + + private static List toList(ConciseSet set) + { + List list1 = new ArrayList<>(); + for (IntSet.IntIterator it = set.iterator(); it.hasNext(); ) { + list1.add(it.next()); + } + return list1; + } + + private void verifyIntersection(ConciseSet set1, ConciseSet set2) + { + List expectedIntersection = toList(set1); + expectedIntersection.retainAll(toList(set2)); + verifyIntersection(expectedIntersection, set1, set2); + } + + private void verifyIntersection(List expected, ConciseSet set1, ConciseSet set2) + { + ImmutableConciseSet immutableSet1 = ImmutableConciseSet.newImmutableFromMutable(set1); + ImmutableConciseSet immutableSet2 = ImmutableConciseSet.newImmutableFromMutable(set2); + if (compact) { + immutableSet1 = ImmutableConciseSet.compact(immutableSet1); + immutableSet2 = ImmutableConciseSet.compact(immutableSet2); + } + List immutableSets = Arrays.asList(immutableSet1, immutableSet2); + verifyIntersection(expected, immutableSets); + } + + private void verifyIntersection(List expected, List sets) + { + List actual = Lists.newArrayList(); + ImmutableConciseSet set = ImmutableConciseSet.intersection(sets); + IntSet.IntIterator itr = set.iterator(); + while (itr.hasNext()) { + actual.add(itr.next()); + } + Assert.assertEquals(expected, actual); + } +} diff --git a/extendedset/src/test/java/io/druid/extendedset/intset/ImmutableConciseSetTest.java b/extendedset/src/test/java/io/druid/extendedset/intset/ImmutableConciseSetTest.java index b2084b5f0aa..794e6a7cb66 100755 --- a/extendedset/src/test/java/io/druid/extendedset/intset/ImmutableConciseSetTest.java +++ b/extendedset/src/test/java/io/druid/extendedset/intset/ImmutableConciseSetTest.java @@ -19,13 +19,10 @@ package io.druid.extendedset.intset; import com.google.common.collect.Lists; import junit.framework.Assert; import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; import java.nio.IntBuffer; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.NoSuchElementException; @@ -34,24 +31,10 @@ import java.util.Set; /** */ -@RunWith(Parameterized.class) public class ImmutableConciseSetTest { public static final int NO_COMPLEMENT_LENGTH = -1; - @Parameterized.Parameters - public static List parameters() - { - return Arrays.asList(new Object[] {false}, new Object[] {true}); - } - - private boolean compact; - - public ImmutableConciseSetTest(boolean compact) - { - this.compact = compact; - } - @Test public void testWordIteratorNext1() { @@ -1152,552 +1135,6 @@ public class ImmutableConciseSetTest Assert.assertEquals(expected, actual); } - /** - * Testing basic intersection of similar sets - */ - @Test - public void testIntersection1() - { - final int[] ints1 = {33, 100000}; - final int[] ints2 = {33, 100000}; - List expected = Arrays.asList(33, 100000); - - ConciseSet set1 = new ConciseSet(); - for (int i : ints1) { - set1.add(i); - } - ConciseSet set2 = new ConciseSet(); - for (int i : ints2) { - set2.add(i); - } - - verifyIntersection(expected, set1, set2); - } - - /** - * Set1: literal, zero fill with flip bit, literal - * Set2: literal, zero fill with different flip bit, literal - */ - @Test - public void testIntersection2() - { - final int[] ints1 = {33, 100000}; - final int[] ints2 = {34, 100000}; - List expected = Collections.singletonList(100000); - - ConciseSet set1 = new ConciseSet(); - for (int i : ints1) { - set1.add(i); - } - ConciseSet set2 = new ConciseSet(); - for (int i : ints2) { - set2.add(i); - } - - verifyIntersection(expected, set1, set2); - } - - /** - * Testing intersection of one fills - */ - @Test - public void testIntersection3() - { - List expected = Lists.newArrayList(); - ConciseSet set1 = new ConciseSet(); - ConciseSet set2 = new ConciseSet(); - for (int i = 0; i < 1000; i++) { - set1.add(i); - set2.add(i); - expected.add(i); - } - - verifyIntersection(expected, set1, set2); - } - - /** - * Similar to previous test with one bit in the sequence set to zero - */ - @Test - public void testIntersection4() - { - List expected = Lists.newArrayList(); - ConciseSet set1 = new ConciseSet(); - ConciseSet set2 = new ConciseSet(); - for (int i = 0; i < 1000; i++) { - set1.add(i); - if (i != 500) { - set2.add(i); - expected.add(i); - } - } - - verifyIntersection(expected, set1, set2); - } - - /** - * Testing with disjoint sets - */ - @Test - public void testIntersection5() - { - final int[] ints1 = {33, 100000}; - final int[] ints2 = {34, 200000}; - List expected = Lists.newArrayList(); - - ConciseSet set1 = new ConciseSet(); - for (int i : ints1) { - set1.add(i); - } - ConciseSet set2 = new ConciseSet(); - for (int i : ints2) { - set2.add(i); - } - - verifyIntersection(expected, set1, set2); - } - - /** - * Set 1: literal, zero fill, literal - * Set 2: one fill, literal that falls within the zero fill above, one fill - */ - @Test - public void testIntersection6() - { - List expected = Lists.newArrayList(); - ConciseSet set1 = new ConciseSet(); - for (int i = 0; i < 5; i++) { - set1.add(i); - } - for (int i = 1000; i < 1005; i++) { - set1.add(i); - } - - ConciseSet set2 = new ConciseSet(); - for (int i = 800; i < 805; i++) { - set2.add(i); - } - for (int i = 806; i < 1005; i++) { - set2.add(i); - } - - for (int i = 1000; i < 1005; i++) { - expected.add(i); - } - - verifyIntersection(expected, set1, set2); - } - - @Test - public void testIntersection7() - { - ConciseSet set1 = new ConciseSet(); - for (int i = 0; i < 3100; i++) { - set1.add(i); - } - - ConciseSet set2 = new ConciseSet(); - set2.add(100); - set2.add(500); - for (int i = 600; i < 700; i++) { - set2.add(i); - } - - List expected = Lists.newArrayList(); - expected.add(100); - expected.add(500); - for (int i = 600; i < 700; i++) { - expected.add(i); - } - - verifyIntersection(expected, set1, set2); - } - - @Test - public void testIntersection8() - { - ConciseSet set1 = new ConciseSet(); - for (int i = 0; i < 3100; i++) { - set1.add(i); - } - set1.add(4001); - - ConciseSet set2 = new ConciseSet(); - set2.add(100); - set2.add(500); - for (int i = 600; i < 700; i++) { - set2.add(i); - } - set2.add(4001); - - List expected = Lists.newArrayList(); - expected.add(100); - expected.add(500); - for (int i = 600; i < 700; i++) { - expected.add(i); - } - expected.add(4001); - - verifyIntersection(expected, set1, set2); - } - - @Test - public void testIntersection9() - { - ConciseSet set1 = new ConciseSet(); - set1.add(2005); - set1.add(3005); - set1.add(3008); - - ConciseSet set2 = new ConciseSet(); - for (int i = 0; i < 3007; i++) { - set2.add(i); - } - - List expected = Lists.newArrayList(); - expected.add(2005); - expected.add(3005); - - verifyIntersection(expected, set1, set2); - } - - @Test - public void testIntersection10() - { - ConciseSet set1 = new ConciseSet(); - for (int i = 0; i < 3100; i++) { - set1.add(i); - } - - ConciseSet set2 = new ConciseSet(); - - set2.add(500); - set2.add(600); - set2.add(4001); - - List expected = Lists.newArrayList(); - expected.add(500); - expected.add(600); - - verifyIntersection(expected, set1, set2); - } - - @Test - public void testIntersection11() - { - ConciseSet set1 = new ConciseSet(); - set1.add(2005); - for (int i = 2800; i < 3500; i++) { - set1.add(i); - } - - ConciseSet set2 = new ConciseSet(); - for (int i = 0; i < 3007; i++) { - set2.add(i); - } - - List expected = Lists.newArrayList(); - expected.add(2005); - for (int i = 2800; i < 3007; i++) { - expected.add(i); - } - - verifyIntersection(expected, set1, set2); - } - - @Test - public void testIntersection12() - { - ConciseSet set1 = new ConciseSet(); - set1.add(2005); - for (int i = 2800; i < 3500; i++) { - set1.add(i); - } - set1.add(10005); - - ConciseSet set2 = new ConciseSet(); - for (int i = 0; i < 3007; i++) { - set2.add(i); - } - set2.add(10005); - - List expected = Lists.newArrayList(); - expected.add(2005); - for (int i = 2800; i < 3007; i++) { - expected.add(i); - } - expected.add(10005); - - verifyIntersection(expected, set1, set2); - } - - @Test - public void testIntersection13() - { - ConciseSet set1 = new ConciseSet(); - set1.add(2005); - - ConciseSet set2 = new ConciseSet(); - for (int i = 0; i < 100; i++) { - set2.add(i); - } - - List expected = Lists.newArrayList(); - - verifyIntersection(expected, set1, set2); - } - - @Test - public void testIntersection14() - { - ConciseSet set1 = new ConciseSet(); - for (int i = 0; i < 1000; i++) { - set1.add(i); - } - - ConciseSet set2 = new ConciseSet(); - set2.add(0); - set2.add(3); - set2.add(5); - set2.add(100); - set2.add(101); - - List expected = Lists.newArrayList(); - expected.add(0); - expected.add(3); - expected.add(5); - expected.add(100); - expected.add(101); - - verifyIntersection(expected, set1, set2); - } - - @Test - public void testIntersection15() - { - ConciseSet set1 = new ConciseSet(); - for (int i = 0; i < 1000; i++) { - set1.add(i); - } - - ConciseSet set2 = new ConciseSet(); - set2.add(0); - set2.add(3); - set2.add(5); - for (int i = 100; i < 500; i++) { - set2.add(i); - } - - List expected = Lists.newArrayList(); - expected.add(0); - expected.add(3); - expected.add(5); - for (int i = 100; i < 500; i++) { - expected.add(i); - } - - verifyIntersection(expected, set1, set2); - } - - @Test - public void testIntersection16() - { - ConciseSet set1 = new ConciseSet(); - set1.add(2005); - - ConciseSet set2 = new ConciseSet(); - set2.add(0); - set2.add(3); - set2.add(5); - set2.add(100); - set2.add(101); - - List expected = Lists.newArrayList(); - - verifyIntersection(expected, set1, set2); - } - - @Test - public void testIntersection17() - { - ConciseSet set1 = new ConciseSet(); - for (int i = 0; i < 4002; i++) { - set1.add(i); - } - - ConciseSet set2 = new ConciseSet(); - set2.add(4001); - - List expected = Lists.newArrayList(); - expected.add(4001); - - verifyIntersection(expected, set1, set2); - } - - @Test - public void testIntersection18() - { - ConciseSet set1 = new ConciseSet(); - for (int i = 32; i < 93; i++) { - set1.add(i); - } - - ConciseSet set2 = new ConciseSet(); - for (int i = 0; i < 62; i++) { - set2.add(i); - } - - List expected = Lists.newArrayList(); - for (int i = 32; i < 62; i++) { - expected.add(i); - } - - verifyIntersection(expected, set1, set2); - } - - @Test - public void testIntersection19() - { - ConciseSet set1 = new ConciseSet(); - set1.add(2005); - - ConciseSet set2 = new ConciseSet(); - for (int i = 0; i < 10000; i++) { - set2.add(i); - } - - List expected = Lists.newArrayList(); - expected.add(2005); - - verifyIntersection(expected, set1, set2); - } - - @Test - public void testIntersectionLiteralAndOneFill() - { - ConciseSet set1 = new ConciseSet(); - for (int i = 0; i < 31; i += 2) { - set1.add(i); - } - ConciseSet set2 = new ConciseSet(); - for (int i = 0; i < 1000; i++) { - if (i != 2) { - set2.add(i); - } - } - verifyIntersection(set1, set2); - } - - @Test - public void testIntersectionZeroSequenceRemovedFromQueue() - { - // Seems that it is impossible to test this case with naturally constructed ConciseSet, because the end of the - // sequence is defined by the last set bit, then naturally constructed ConciseSet won't have the last word as zero - // sequence, it will be a literal or one sequence. - int zeroSequence = 1; // Zero sequence of length 62 - ConciseSet set1 = new ConciseSet(new int[] {zeroSequence}, false); - - ConciseSet set2 = new ConciseSet(); - for (int i = 0; i < 1000; i++) { - set2.add(i); - } - verifyIntersection(set1, set2); - } - - @Test - public void testIntersectionOneFillAndOneFillWithFlipBit() - { - ConciseSet set1 = new ConciseSet(); - for (int i = 0; i < 100; i++) { - set1.add(i); - } - ConciseSet set2 = new ConciseSet(); - for (int i = 0; i < 1000; i++) { - if (i != 2) { - set2.add(i); - } - } - verifyIntersection(set1, set2); - } - - @Test - public void testIntersectionSecondOneFillRemovedFromQueue() - { - ConciseSet set1 = new ConciseSet(); - for (int i = 0; i < 31 * 2; i++) { - set1.add(i); - } - set1.add(100); - - ConciseSet set2 = new ConciseSet(); - for (int i = 0; i < 31 * 3; i++) { - set2.add(i); - } - - verifyIntersection(set1, set2); - } - - @Test - public void testIntersectionFirstOneFillRemovedFromQueue() - { - ConciseSet set1 = new ConciseSet(); - for (int i = 0; i < 31 * 2; i++) { - set1.add(i); - } - - ConciseSet set2 = new ConciseSet(); - for (int i = 0; i < 31 * 3; i++) { - set2.add(i); - } - - verifyIntersection(set1, set2); - } - - @Test - public void testIntersectionTerminates() throws Exception - { - verifyIntersection(Collections.emptyList(), Arrays.asList(new ImmutableConciseSet(), new ImmutableConciseSet())); - } - - private void verifyIntersection(ConciseSet set1, ConciseSet set2) - { - List expectedIntersection = toList(set1); - expectedIntersection.retainAll(toList(set2)); - verifyIntersection(expectedIntersection, set1, set2); - } - - private static List toList(ConciseSet set) - { - List list1 = new ArrayList<>(); - for (IntSet.IntIterator it = set.iterator(); it.hasNext(); ) { - list1.add(it.next()); - } - return list1; - } - - private void verifyIntersection(List expected, ConciseSet set1, ConciseSet set2) - { - ImmutableConciseSet immutableSet1 = ImmutableConciseSet.newImmutableFromMutable(set1); - ImmutableConciseSet immutableSet2 = ImmutableConciseSet.newImmutableFromMutable(set2); - if (compact) { - immutableSet1 = ImmutableConciseSet.compact(immutableSet1); - immutableSet2 = ImmutableConciseSet.compact(immutableSet2); - } - List immutableSets = Arrays.asList(immutableSet1, immutableSet2); - verifyIntersection(expected, immutableSets); - } - - private void verifyIntersection(List expected, List sets) - { - List actual = Lists.newArrayList(); - ImmutableConciseSet set = ImmutableConciseSet.intersection(sets); - IntSet.IntIterator itr = set.iterator(); - while (itr.hasNext()) { - actual.add(itr.next()); - } - Assert.assertEquals(expected, actual); - } - /** * Basic complement with no length */ @@ -1891,7 +1328,7 @@ public class ImmutableConciseSetTest ) ); final Random random = new Random(701534702L); - for (int i = 0; i < 10; ++i) { + for (int i = 0; i < 5; ++i) { lengths.add(random.nextInt(ConciseSetUtils.MAX_ALLOWED_INTEGER + 1)); } final ImmutableConciseSet emptySet = new ImmutableConciseSet(); diff --git a/processing/src/test/java/io/druid/segment/ConciseBitmapIndexMergerTest.java b/processing/src/test/java/io/druid/segment/ConciseBitmapIndexMergerTest.java new file mode 100644 index 00000000000..aba188de6ec --- /dev/null +++ b/processing/src/test/java/io/druid/segment/ConciseBitmapIndexMergerTest.java @@ -0,0 +1,40 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import io.druid.segment.data.CompressedObjectStrategy.CompressionStrategy; +import io.druid.segment.data.CompressionFactory.LongEncodingStrategy; +import io.druid.segment.data.ConciseBitmapSerdeFactory; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class ConciseBitmapIndexMergerTest extends IndexMergerTestBase +{ + public ConciseBitmapIndexMergerTest( + CompressionStrategy compressionStrategy, + CompressionStrategy dimCompressionStrategy, + LongEncodingStrategy longEncodingStrategy + ) + { + super(new ConciseBitmapSerdeFactory(), compressionStrategy, dimCompressionStrategy, longEncodingStrategy); + indexMerger = TestHelper.getTestIndexMerger(); + } +} diff --git a/processing/src/test/java/io/druid/segment/ConciseBitmapIndexMergerV9Test.java b/processing/src/test/java/io/druid/segment/ConciseBitmapIndexMergerV9Test.java new file mode 100644 index 00000000000..595a1e80e04 --- /dev/null +++ b/processing/src/test/java/io/druid/segment/ConciseBitmapIndexMergerV9Test.java @@ -0,0 +1,40 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import io.druid.segment.data.CompressedObjectStrategy.CompressionStrategy; +import io.druid.segment.data.CompressionFactory.LongEncodingStrategy; +import io.druid.segment.data.ConciseBitmapSerdeFactory; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class ConciseBitmapIndexMergerV9Test extends IndexMergerTestBase +{ + public ConciseBitmapIndexMergerV9Test( + CompressionStrategy compressionStrategy, + CompressionStrategy dimCompressionStrategy, + LongEncodingStrategy longEncodingStrategy + ) + { + super(new ConciseBitmapSerdeFactory(), compressionStrategy, dimCompressionStrategy, longEncodingStrategy); + indexMerger = TestHelper.getTestIndexMergerV9(); + } +} diff --git a/processing/src/test/java/io/druid/segment/IndexIOTest.java b/processing/src/test/java/io/druid/segment/IndexIOTest.java index c2b85534137..ac8289cd7fe 100644 --- a/processing/src/test/java/io/druid/segment/IndexIOTest.java +++ b/processing/src/test/java/io/druid/segment/IndexIOTest.java @@ -63,7 +63,7 @@ import java.util.Map; public class IndexIOTest { private static Interval DEFAULT_INTERVAL = Interval.parse("1970-01-01/2000-01-01"); - private static final IndexSpec INDEX_SPEC = IndexMergerTest.makeIndexSpec( + private static final IndexSpec INDEX_SPEC = IndexMergerTestBase.makeIndexSpec( new ConciseBitmapSerdeFactory(), CompressedObjectStrategy.CompressionStrategy.LZ4, CompressedObjectStrategy.CompressionStrategy.LZ4, diff --git a/processing/src/test/java/io/druid/segment/IndexMergerTest.java b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java similarity index 96% rename from processing/src/test/java/io/druid/segment/IndexMergerTest.java rename to processing/src/test/java/io/druid/segment/IndexMergerTestBase.java index aad7daf0456..0313116efb1 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerTest.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerTestBase.java @@ -50,10 +50,8 @@ import io.druid.segment.column.SimpleDictionaryEncodedColumn; import io.druid.segment.data.BitmapSerdeFactory; import io.druid.segment.data.CompressedObjectStrategy; import io.druid.segment.data.CompressionFactory; -import io.druid.segment.data.ConciseBitmapSerdeFactory; import io.druid.segment.data.IncrementalIndexTest; import io.druid.segment.data.IndexedInts; -import io.druid.segment.data.RoaringBitmapSerdeFactory; import io.druid.segment.incremental.IncrementalIndex; import io.druid.segment.incremental.IncrementalIndexAdapter; import io.druid.segment.incremental.IncrementalIndexSchema; @@ -64,7 +62,6 @@ import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import javax.annotation.Nullable; @@ -81,29 +78,21 @@ import java.util.EnumSet; import java.util.List; import java.util.Map; -@RunWith(Parameterized.class) -public class IndexMergerTest +public class IndexMergerTestBase { + private final static IndexIO INDEX_IO = TestHelper.getTestIndexIO(); + @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); - protected IndexMerger INDEX_MERGER; - private final static IndexIO INDEX_IO = TestHelper.getTestIndexIO(); + protected IndexMerger indexMerger; - @Parameterized.Parameters(name = "{index}: useV9={0}, bitmap={1}, metric compression={2}, dimension compression={3}, long encoding={4}") + @Parameterized.Parameters(name = "{index}: metric compression={0}, dimension compression={1}, long encoding={2}") public static Collection data() { return Collections2.transform( Sets.cartesianProduct( ImmutableList.of( - ImmutableSet.of( - true, - false - ), - ImmutableSet.of( - new RoaringBitmapSerdeFactory(null), - new ConciseBitmapSerdeFactory() - ), EnumSet.allOf(CompressedObjectStrategy.CompressionStrategy.class), ImmutableSet.copyOf(CompressedObjectStrategy.CompressionStrategy.noNoneValues()), EnumSet.allOf(CompressionFactory.LongEncodingStrategy.class) @@ -143,8 +132,7 @@ public class IndexMergerTest @Rule public final CloserRule closer = new CloserRule(false); - public IndexMergerTest( - boolean useV9, + protected IndexMergerTestBase( BitmapSerdeFactory bitmapSerdeFactory, CompressedObjectStrategy.CompressionStrategy compressionStrategy, CompressedObjectStrategy.CompressionStrategy dimCompressionStrategy, @@ -152,11 +140,6 @@ public class IndexMergerTest ) { this.indexSpec = makeIndexSpec(bitmapSerdeFactory, compressionStrategy, dimCompressionStrategy, longEncodingStrategy); - if (useV9) { - INDEX_MERGER = TestHelper.getTestIndexMergerV9(); - } else { - INDEX_MERGER = TestHelper.getTestIndexMerger(); - } } @Test @@ -170,7 +153,7 @@ public class IndexMergerTest final File tempDir = temporaryFolder.newFolder(); QueryableIndex index = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersist, tempDir, indexSpec @@ -217,7 +200,7 @@ public class IndexMergerTest final File tempDir = temporaryFolder.newFolder(); QueryableIndex index = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersist, tempDir, indexSpec @@ -259,7 +242,7 @@ public class IndexMergerTest final File tempDir = temporaryFolder.newFolder(); QueryableIndex index = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersist, tempDir, indexSpec @@ -319,7 +302,7 @@ public class IndexMergerTest QueryableIndex index1 = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersist1, tempDir1, indexSpec @@ -333,7 +316,7 @@ public class IndexMergerTest QueryableIndex index2 = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersist2, tempDir2, indexSpec @@ -350,7 +333,7 @@ public class IndexMergerTest }; QueryableIndex merged = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex( + indexMerger.mergeQueryableIndex( Arrays.asList(index1, index2), true, mergedAggregators, @@ -408,7 +391,7 @@ public class IndexMergerTest final QueryableIndex index1 = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersist1, tmpDir1, indexSpec @@ -417,7 +400,7 @@ public class IndexMergerTest ); final QueryableIndex index2 = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersist2, tmpDir2, indexSpec @@ -426,7 +409,7 @@ public class IndexMergerTest ); final QueryableIndex merged = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex( + indexMerger.mergeQueryableIndex( Arrays.asList(index1, index2), true, new AggregatorFactory[]{}, @@ -468,7 +451,7 @@ public class IndexMergerTest QueryableIndex index1 = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersist1, tempDir1, indexSpec @@ -488,7 +471,7 @@ public class IndexMergerTest QueryableIndex merged = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex( + indexMerger.mergeQueryableIndex( ImmutableList.of(index1), true, new AggregatorFactory[]{new CountAggregatorFactory("count")}, @@ -526,7 +509,7 @@ public class IndexMergerTest QueryableIndex index1 = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.append( + indexMerger.append( ImmutableList.of(incrementalAdapter), null, tempDir1, indexSpec ) ) @@ -547,7 +530,7 @@ public class IndexMergerTest AggregatorFactory[] mergedAggregators = new AggregatorFactory[]{new CountAggregatorFactory("count")}; QueryableIndex merged = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex( + indexMerger.mergeQueryableIndex( ImmutableList.of(index1), true, mergedAggregators, @@ -590,7 +573,7 @@ public class IndexMergerTest QueryableIndex index1 = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersist1, tempDir1, indexSpec @@ -623,7 +606,7 @@ public class IndexMergerTest AggregatorFactory[] mergedAggregators = new AggregatorFactory[]{new CountAggregatorFactory("count")}; QueryableIndex merged = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex( + indexMerger.mergeQueryableIndex( ImmutableList.of(index1), true, mergedAggregators, @@ -669,7 +652,7 @@ public class IndexMergerTest ); QueryableIndex index1 = closer.closeLater( - INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersist1, tempDir1, indexSpec)) + INDEX_IO.loadIndex(indexMerger.persist(toPersist1, tempDir1, indexSpec)) ); final IndexableAdapter queryableAdapter = new QueryableIndexIndexableAdapter(index1); @@ -683,7 +666,7 @@ public class IndexMergerTest QueryableIndex converted = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.convert( + indexMerger.convert( tempDir1, convertDir, indexSpec @@ -733,7 +716,7 @@ public class IndexMergerTest QueryableIndex index1 = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersist1, tempDir1, indexSpec @@ -766,7 +749,7 @@ public class IndexMergerTest QueryableIndex converted = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.convert( + indexMerger.convert( tempDir1, convertDir, newSpec @@ -840,7 +823,7 @@ public class IndexMergerTest QueryableIndex index1 = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersist1, tmpDir, indexSpec @@ -850,7 +833,7 @@ public class IndexMergerTest QueryableIndex index2 = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersist2, tmpDir2, indexSpec @@ -860,7 +843,7 @@ public class IndexMergerTest QueryableIndex index3 = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersist3, tmpDir3, indexSpec @@ -871,7 +854,7 @@ public class IndexMergerTest final QueryableIndex merged = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex( + indexMerger.mergeQueryableIndex( Arrays.asList(index1, index2, index3), true, new AggregatorFactory[]{new CountAggregatorFactory("count")}, @@ -948,7 +931,7 @@ public class IndexMergerTest QueryableIndex index1 = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersist1, tmpDir, indexSpec @@ -958,7 +941,7 @@ public class IndexMergerTest QueryableIndex index2 = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersist2, tmpDir2, indexSpec @@ -968,7 +951,7 @@ public class IndexMergerTest QueryableIndex index3 = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersist3, tmpDir3, indexSpec @@ -978,7 +961,7 @@ public class IndexMergerTest final QueryableIndex merged = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex( + indexMerger.mergeQueryableIndex( Arrays.asList(index1, index2, index3), true, new AggregatorFactory[]{new CountAggregatorFactory("count")}, @@ -1029,7 +1012,7 @@ public class IndexMergerTest QueryableIndex indexA = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersistA, tmpDirA, indexSpec @@ -1039,7 +1022,7 @@ public class IndexMergerTest QueryableIndex indexB = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersistB, tmpDirB, indexSpec @@ -1049,7 +1032,7 @@ public class IndexMergerTest QueryableIndex indexB2 = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersistB2, tmpDirB2, indexSpec @@ -1059,7 +1042,7 @@ public class IndexMergerTest final QueryableIndex merged = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex( + indexMerger.mergeQueryableIndex( Arrays.asList(indexA, indexB), true, new AggregatorFactory[]{new CountAggregatorFactory("count")}, @@ -1071,7 +1054,7 @@ public class IndexMergerTest final QueryableIndex merged2 = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex( + indexMerger.mergeQueryableIndex( Arrays.asList(indexA, indexB2), true, new AggregatorFactory[]{new CountAggregatorFactory("count")}, @@ -1197,7 +1180,7 @@ public class IndexMergerTest QueryableIndex indexA = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersistA, tmpDirA, indexSpec @@ -1207,7 +1190,7 @@ public class IndexMergerTest QueryableIndex indexB = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersistB, tmpDirB, indexSpec @@ -1217,7 +1200,7 @@ public class IndexMergerTest final QueryableIndex merged = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex( + indexMerger.mergeQueryableIndex( Arrays.asList(indexA, indexB), true, new AggregatorFactory[]{new CountAggregatorFactory("count")}, @@ -1339,7 +1322,7 @@ public class IndexMergerTest QueryableIndex indexA = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersistA, tmpDirA, indexSpec @@ -1349,7 +1332,7 @@ public class IndexMergerTest QueryableIndex indexB = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersistB, tmpDirB, indexSpec @@ -1359,7 +1342,7 @@ public class IndexMergerTest final QueryableIndex merged = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex( + indexMerger.mergeQueryableIndex( Arrays.asList(indexA, indexB), true, new AggregatorFactory[]{new CountAggregatorFactory("count")}, @@ -1480,7 +1463,7 @@ public class IndexMergerTest QueryableIndex indexA = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersistA, tmpDirA, indexSpec @@ -1490,7 +1473,7 @@ public class IndexMergerTest QueryableIndex indexB = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersistB, tmpDirB, indexSpec @@ -1500,7 +1483,7 @@ public class IndexMergerTest final QueryableIndex merged = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex( + indexMerger.mergeQueryableIndex( Arrays.asList(indexA, indexB), false, new AggregatorFactory[]{new CountAggregatorFactory("count")}, @@ -1590,7 +1573,7 @@ public class IndexMergerTest QueryableIndex indexA = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersistA, tmpDirA, indexSpec @@ -1600,7 +1583,7 @@ public class IndexMergerTest QueryableIndex indexB = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersistB, tmpDirB, indexSpec @@ -1610,7 +1593,7 @@ public class IndexMergerTest QueryableIndex indexBA = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersistBA, tmpDirBA, indexSpec @@ -1620,7 +1603,7 @@ public class IndexMergerTest QueryableIndex indexBA2 = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersistBA2, tmpDirBA2, indexSpec @@ -1630,7 +1613,7 @@ public class IndexMergerTest QueryableIndex indexC = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersistC, tmpDirC, indexSpec @@ -1640,7 +1623,7 @@ public class IndexMergerTest final QueryableIndex merged = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex( + indexMerger.mergeQueryableIndex( Arrays.asList(indexA, indexB, indexBA, indexBA2), true, new AggregatorFactory[]{new CountAggregatorFactory("count")}, @@ -1652,7 +1635,7 @@ public class IndexMergerTest final QueryableIndex merged2 = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex( + indexMerger.mergeQueryableIndex( Arrays.asList(indexA, indexB, indexBA, indexC), true, new AggregatorFactory[]{new CountAggregatorFactory("count")}, @@ -1772,7 +1755,7 @@ public class IndexMergerTest final File tmpDirMerged = temporaryFolder.newFolder(); - INDEX_MERGER.merge( + indexMerger.merge( toMerge, true, new AggregatorFactory[]{ @@ -1824,7 +1807,7 @@ public class IndexMergerTest final File tmpDirMerged = temporaryFolder.newFolder(); - File merged = INDEX_MERGER.merge( + File merged = indexMerger.merge( toMerge, true, new AggregatorFactory[]{ @@ -1895,7 +1878,7 @@ public class IndexMergerTest final File tmpDirMerged = temporaryFolder.newFolder(); - File merged = INDEX_MERGER.merge( + File merged = indexMerger.merge( toMerge, true, new AggregatorFactory[]{ @@ -1956,7 +1939,7 @@ public class IndexMergerTest final File tmpDirMerged = temporaryFolder.newFolder(); - File merged = INDEX_MERGER.merge( + File merged = indexMerger.merge( toMerge, true, new AggregatorFactory[]{ @@ -2001,7 +1984,7 @@ public class IndexMergerTest final File tmpDirMerged = temporaryFolder.newFolder(); - final File merged = INDEX_MERGER.merge( + final File merged = indexMerger.merge( toMerge, true, new AggregatorFactory[]{ @@ -2029,7 +2012,7 @@ public class IndexMergerTest QueryableIndex index1 = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersist1, tmpDir, indexSpec @@ -2039,7 +2022,7 @@ public class IndexMergerTest QueryableIndex index2 = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( toPersist2, tmpDir2, indexSpec @@ -2049,7 +2032,7 @@ public class IndexMergerTest final QueryableIndex merged = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.mergeQueryableIndex( + indexMerger.mergeQueryableIndex( Arrays.asList(index1, index2), true, new AggregatorFactory[]{new CountAggregatorFactory("count")}, @@ -2160,7 +2143,7 @@ public class IndexMergerTest final File tempDir = temporaryFolder.newFolder(); QueryableIndex index = closer.closeLater( INDEX_IO.loadIndex( - INDEX_MERGER.persist( + indexMerger.persist( index1, tempDir, indexSpec @@ -2301,7 +2284,7 @@ public class IndexMergerTest final File v9TmpDir = new File(tempDir, "v9-tmp"); try { - INDEX_MERGER.persist( + indexMerger.persist( toPersist, tempDir, indexSpec @@ -2423,6 +2406,6 @@ public class IndexMergerTest } final File tempDir = temporaryFolder.newFolder(); - return closer.closeLater(INDEX_IO.loadIndex(INDEX_MERGER.persist(toPersist, tempDir, indexSpec))); + return closer.closeLater(INDEX_IO.loadIndex(indexMerger.persist(toPersist, tempDir, indexSpec))); } } diff --git a/processing/src/test/java/io/druid/segment/IndexMergerV9CompatibilityTest.java b/processing/src/test/java/io/druid/segment/IndexMergerV9CompatibilityTest.java index 63742ef3d50..a5906f2f2cd 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerV9CompatibilityTest.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerV9CompatibilityTest.java @@ -72,7 +72,7 @@ public class IndexMergerV9CompatibilityTest private static final IndexIO INDEX_IO = TestHelper.getTestIndexIO(); private static final IndexMerger INDEX_MERGER = TestHelper.getTestIndexMerger(); - private static final IndexSpec INDEX_SPEC = IndexMergerTest.makeIndexSpec( + private static final IndexSpec INDEX_SPEC = IndexMergerTestBase.makeIndexSpec( new ConciseBitmapSerdeFactory(), CompressedObjectStrategy.CompressionStrategy.LZ4, CompressedObjectStrategy.CompressionStrategy.LZ4, diff --git a/processing/src/test/java/io/druid/segment/QueryableIndexIndexableAdapterTest.java b/processing/src/test/java/io/druid/segment/QueryableIndexIndexableAdapterTest.java index e3a02fabac2..5a79a0eecbd 100644 --- a/processing/src/test/java/io/druid/segment/QueryableIndexIndexableAdapterTest.java +++ b/processing/src/test/java/io/druid/segment/QueryableIndexIndexableAdapterTest.java @@ -36,7 +36,7 @@ public class QueryableIndexIndexableAdapterTest { private final static IndexMerger INDEX_MERGER = TestHelper.getTestIndexMerger(); private final static IndexIO INDEX_IO = TestHelper.getTestIndexIO(); - private static final IndexSpec INDEX_SPEC = IndexMergerTest.makeIndexSpec( + private static final IndexSpec INDEX_SPEC = IndexMergerTestBase.makeIndexSpec( new ConciseBitmapSerdeFactory(), CompressedObjectStrategy.CompressionStrategy.LZ4, CompressedObjectStrategy.CompressionStrategy.LZ4, diff --git a/processing/src/test/java/io/druid/segment/RoaringBitmapIndexMergerTest.java b/processing/src/test/java/io/druid/segment/RoaringBitmapIndexMergerTest.java new file mode 100644 index 00000000000..539b2208089 --- /dev/null +++ b/processing/src/test/java/io/druid/segment/RoaringBitmapIndexMergerTest.java @@ -0,0 +1,40 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import io.druid.segment.data.CompressedObjectStrategy.CompressionStrategy; +import io.druid.segment.data.CompressionFactory.LongEncodingStrategy; +import io.druid.segment.data.RoaringBitmapSerdeFactory; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class RoaringBitmapIndexMergerTest extends IndexMergerTestBase +{ + public RoaringBitmapIndexMergerTest( + CompressionStrategy compressionStrategy, + CompressionStrategy dimCompressionStrategy, + LongEncodingStrategy longEncodingStrategy + ) + { + super(new RoaringBitmapSerdeFactory(null), compressionStrategy, dimCompressionStrategy, longEncodingStrategy); + indexMerger = TestHelper.getTestIndexMerger(); + } +} diff --git a/processing/src/test/java/io/druid/segment/RoaringBitmapIndexMergerV9Test.java b/processing/src/test/java/io/druid/segment/RoaringBitmapIndexMergerV9Test.java new file mode 100644 index 00000000000..c5e59470be9 --- /dev/null +++ b/processing/src/test/java/io/druid/segment/RoaringBitmapIndexMergerV9Test.java @@ -0,0 +1,40 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.segment; + +import io.druid.segment.data.CompressedObjectStrategy.CompressionStrategy; +import io.druid.segment.data.CompressionFactory.LongEncodingStrategy; +import io.druid.segment.data.RoaringBitmapSerdeFactory; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class RoaringBitmapIndexMergerV9Test extends IndexMergerTestBase +{ + public RoaringBitmapIndexMergerV9Test( + CompressionStrategy compressionStrategy, + CompressionStrategy dimCompressionStrategy, + LongEncodingStrategy longEncodingStrategy + ) + { + super(new RoaringBitmapSerdeFactory(null), compressionStrategy, dimCompressionStrategy, longEncodingStrategy); + indexMerger = TestHelper.getTestIndexMergerV9(); + } +}