diff --git a/processing/src/main/java/io/druid/segment/IndexIO.java b/processing/src/main/java/io/druid/segment/IndexIO.java index b9ba5bf0b17..8fb1cc39b3b 100644 --- a/processing/src/main/java/io/druid/segment/IndexIO.java +++ b/processing/src/main/java/io/druid/segment/IndexIO.java @@ -36,10 +36,15 @@ import com.google.common.primitives.Ints; import com.google.inject.Binder; import com.google.inject.Injector; import com.google.inject.Module; +import com.metamx.collections.bitmap.BitmapFactory; import com.metamx.collections.bitmap.ConciseBitmapFactory; import com.metamx.collections.bitmap.ImmutableBitmap; import com.metamx.collections.bitmap.MutableBitmap; +import com.metamx.collections.spatial.ImmutablePoint; import com.metamx.collections.spatial.ImmutableRTree; +import com.metamx.collections.spatial.RTree; +import com.metamx.collections.spatial.RTreeUtils; +import com.metamx.collections.spatial.split.LinearGutmanSplitStrategy; import com.metamx.common.IAE; import com.metamx.common.ISE; import com.metamx.common.guava.FunctionalIterable; @@ -441,10 +446,25 @@ public class IndexIO VSizeIndexedInts singleValCol = null; VSizeIndexed multiValCol = VSizeIndexed.readFromByteBuffer(dimBuffer.asReadOnlyBuffer()); GenericIndexed bitmaps = bitmapIndexes.get(dimension); + ImmutableRTree spatialIndex = spatialIndexes.get(dimension); + // TODO: this is some UGLY shizzle // All V8 segments use concise sets for bitmap indexes. Starting in V9, we can optionally choose other // methods to store and compress these bitmap methods. + final BitmapFactory bitmapFactory = bitmapSerdeFactory.getBitmapFactory(); if (!(bitmapSerdeFactory instanceof ConciseBitmapSerdeFactory)) { + if (spatialIndex != null) { + RTree convertedTree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bitmapFactory), bitmapFactory); + + for (ImmutablePoint point : RTreeUtils.getBitmaps(spatialIndex)) { + IntIterator iterator = point.getImmutableBitmap().iterator(); + while (iterator.hasNext()) { + convertedTree.insert(point.getCoords(), iterator.next()); + } + } + spatialIndex = ImmutableRTree.newImmutableFromMutable(convertedTree); + } + bitmaps = GenericIndexed.fromIterable( FunctionalIterable.create( bitmaps @@ -457,13 +477,16 @@ public class IndexIO ImmutableBitmap bitmap ) { + if (bitmap == null) { + return bitmapFactory.makeEmptyImmutableBitmap(); + } IntIterator intIter = bitmap.iterator(); - MutableBitmap mutableBitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyMutableBitmap(); + MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap(); // TODO: is there a faster way to do this? I don't think so while (intIter.hasNext()) { mutableBitmap.add(intIter.next()); } - return bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(mutableBitmap); + return bitmapFactory.makeImmutableBitmap(mutableBitmap); } } ), @@ -471,8 +494,6 @@ public class IndexIO ); } - ImmutableRTree spatialIndex = spatialIndexes.get(dimension); - boolean onlyOneValue = true; MutableBitmap nullsSet = null; for (int i = 0; i < multiValCol.size(); ++i) { @@ -485,7 +506,7 @@ public class IndexIO } if (rowValue.size() == 0) { if (nullsSet == null) { - nullsSet = bitmapSerdeFactory.getBitmapFactory().makeEmptyMutableBitmap(); + nullsSet = bitmapFactory.makeEmptyMutableBitmap(); } nullsSet.add(i); } @@ -496,7 +517,7 @@ public class IndexIO final boolean bumpedDictionary; if (nullsSet != null) { log.info("Dimension[%s] has null rows.", dimension); - final ImmutableBitmap theNullSet = bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(nullsSet); + final ImmutableBitmap theNullSet = bitmapFactory.makeImmutableBitmap(nullsSet); if (dictionary.get(0) != null) { log.info("Dimension[%s] has no null value in the dictionary, expanding...", dimension); @@ -518,8 +539,8 @@ public class IndexIO bitmaps = GenericIndexed.fromIterable( Iterables.concat( Arrays.asList( - bitmapSerdeFactory.getBitmapFactory() - .union(Arrays.asList(theNullSet, bitmaps.get(0))) + bitmapFactory + .union(Arrays.asList(theNullSet, bitmaps.get(0))) ), Iterables.skip(bitmaps, 1) ), diff --git a/processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java b/processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java index 09db6f5fff0..c6cd2884dfc 100644 --- a/processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java +++ b/processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java @@ -26,7 +26,7 @@ import com.metamx.collections.bitmap.ImmutableBitmap; /** */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = ConciseBitmapSerdeFactory.class) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = RoaringBitmapSerdeFactory.class) @JsonSubTypes(value = { @JsonSubTypes.Type(name = "concise", value = ConciseBitmapSerdeFactory.class), @JsonSubTypes.Type(name = "roaring", value = RoaringBitmapSerdeFactory.class) diff --git a/processing/src/main/java/io/druid/segment/filter/SpatialFilter.java b/processing/src/main/java/io/druid/segment/filter/SpatialFilter.java index f25cc054f9e..c2f77690fd4 100644 --- a/processing/src/main/java/io/druid/segment/filter/SpatialFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/SpatialFilter.java @@ -44,7 +44,11 @@ public class SpatialFilter implements Filter @Override public ImmutableBitmap getBitmapIndex(final BitmapIndexSelector selector) { - return selector.getBitmapFactory().union(selector.getSpatialIndex(dimension).search(bound)); + Iterable search = selector.getSpatialIndex(dimension).search(bound); + for (ImmutableBitmap immutableBitmap : search) { + System.out.println(immutableBitmap); + } + return selector.getBitmapFactory().union(search); } @Override diff --git a/server/src/main/java/io/druid/segment/indexing/granularity/ArbitraryGranularitySpec.java b/server/src/main/java/io/druid/segment/indexing/granularity/ArbitraryGranularitySpec.java index 807764e515c..677f7a2d181 100644 --- a/server/src/main/java/io/druid/segment/indexing/granularity/ArbitraryGranularitySpec.java +++ b/server/src/main/java/io/druid/segment/indexing/granularity/ArbitraryGranularitySpec.java @@ -50,6 +50,10 @@ public class ArbitraryGranularitySpec implements GranularitySpec this.queryGranularity = queryGranularity; this.intervals = Sets.newTreeSet(Comparators.intervalsByStartThenEnd()); + if (inputIntervals == null) { + inputIntervals = Lists.newArrayList(); + } + // Insert all intervals for (final Interval inputInterval : inputIntervals) { intervals.add(inputInterval);