Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

a whole bunch of fixes and some ugly conversion code

This commit is contained in:
fjy 2014-11-07 16:03:22 -08:00
parent 253009d208
commit 3c21f62afd
4 changed files with 39 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
processing/src/main/java/io/druid/segment/IndexIO.java
View File

@ -36,10 +36,15 @@ import com.google.common.primitives.Ints;
import com.google.inject.Binder;
import com.google.inject.Injector;
import com.google.inject.Module;
import com.metamx.collections.bitmap.BitmapFactory;
import com.metamx.collections.bitmap.ConciseBitmapFactory;
import com.metamx.collections.bitmap.ImmutableBitmap;
import com.metamx.collections.bitmap.MutableBitmap;
import com.metamx.collections.spatial.ImmutablePoint;
import com.metamx.collections.spatial.ImmutableRTree;
import com.metamx.collections.spatial.RTree;
import com.metamx.collections.spatial.RTreeUtils;
import com.metamx.collections.spatial.split.LinearGutmanSplitStrategy;
import com.metamx.common.IAE;
import com.metamx.common.ISE;
import com.metamx.common.guava.FunctionalIterable;
@ -441,10 +446,25 @@ public class IndexIO
VSizeIndexedInts singleValCol = null;
VSizeIndexed multiValCol = VSizeIndexed.readFromByteBuffer(dimBuffer.asReadOnlyBuffer());
GenericIndexed<ImmutableBitmap> bitmaps = bitmapIndexes.get(dimension);
ImmutableRTree spatialIndex = spatialIndexes.get(dimension);
// TODO: this is some UGLY shizzle
// All V8 segments use concise sets for bitmap indexes. Starting in V9, we can optionally choose other
// methods to store and compress these bitmap methods.
final BitmapFactory bitmapFactory = bitmapSerdeFactory.getBitmapFactory();
if (!(bitmapSerdeFactory instanceof ConciseBitmapSerdeFactory)) {
if (spatialIndex != null) {
RTree convertedTree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bitmapFactory), bitmapFactory);
for (ImmutablePoint point : RTreeUtils.getBitmaps(spatialIndex)) {
IntIterator iterator = point.getImmutableBitmap().iterator();
while (iterator.hasNext()) {
convertedTree.insert(point.getCoords(), iterator.next());
}
}
spatialIndex = ImmutableRTree.newImmutableFromMutable(convertedTree);
}
bitmaps = GenericIndexed.fromIterable(
FunctionalIterable.create(
bitmaps
@ -457,13 +477,16 @@ public class IndexIO
ImmutableBitmap bitmap
)
{
if (bitmap == null) {
return bitmapFactory.makeEmptyImmutableBitmap();
}
IntIterator intIter = bitmap.iterator();
MutableBitmap mutableBitmap = bitmapSerdeFactory.getBitmapFactory().makeEmptyMutableBitmap();
MutableBitmap mutableBitmap = bitmapFactory.makeEmptyMutableBitmap();
// TODO: is there a faster way to do this? I don't think so
while (intIter.hasNext()) {
mutableBitmap.add(intIter.next());
}
return bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(mutableBitmap);
return bitmapFactory.makeImmutableBitmap(mutableBitmap);
}
}
),
@ -471,8 +494,6 @@ public class IndexIO
);
}
ImmutableRTree spatialIndex = spatialIndexes.get(dimension);
boolean onlyOneValue = true;
MutableBitmap nullsSet = null;
for (int i = 0; i < multiValCol.size(); ++i) {
@ -485,7 +506,7 @@ public class IndexIO
}
if (rowValue.size() == 0) {
if (nullsSet == null) {
nullsSet = bitmapSerdeFactory.getBitmapFactory().makeEmptyMutableBitmap();
nullsSet = bitmapFactory.makeEmptyMutableBitmap();
}
nullsSet.add(i);
}
@ -496,7 +517,7 @@ public class IndexIO
final boolean bumpedDictionary;
if (nullsSet != null) {
log.info("Dimension[%s] has null rows.", dimension);
final ImmutableBitmap theNullSet = bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(nullsSet);
final ImmutableBitmap theNullSet = bitmapFactory.makeImmutableBitmap(nullsSet);
if (dictionary.get(0) != null) {
log.info("Dimension[%s] has no null value in the dictionary, expanding...", dimension);
@ -518,8 +539,8 @@ public class IndexIO
bitmaps = GenericIndexed.fromIterable(
Iterables.concat(
Arrays.asList(
bitmapSerdeFactory.getBitmapFactory()
.union(Arrays.asList(theNullSet, bitmaps.get(0)))
bitmapFactory
.union(Arrays.asList(theNullSet, bitmaps.get(0)))
),
Iterables.skip(bitmaps, 1)
),

2
processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java
View File

@ -26,7 +26,7 @@ import com.metamx.collections.bitmap.ImmutableBitmap;
/**
*/
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = ConciseBitmapSerdeFactory.class)
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = RoaringBitmapSerdeFactory.class)
@JsonSubTypes(value = {
@JsonSubTypes.Type(name = "concise", value = ConciseBitmapSerdeFactory.class),
@JsonSubTypes.Type(name = "roaring", value = RoaringBitmapSerdeFactory.class)

6
processing/src/main/java/io/druid/segment/filter/SpatialFilter.java
View File

@ -44,7 +44,11 @@ public class SpatialFilter implements Filter
@Override
public ImmutableBitmap getBitmapIndex(final BitmapIndexSelector selector)
{
return selector.getBitmapFactory().union(selector.getSpatialIndex(dimension).search(bound));
Iterable<ImmutableBitmap> search = selector.getSpatialIndex(dimension).search(bound);
for (ImmutableBitmap immutableBitmap : search) {
System.out.println(immutableBitmap);
}
return selector.getBitmapFactory().union(search);
}
@Override

4
server/src/main/java/io/druid/segment/indexing/granularity/ArbitraryGranularitySpec.java
View File

@ -50,6 +50,10 @@ public class ArbitraryGranularitySpec implements GranularitySpec
this.queryGranularity = queryGranularity;
this.intervals = Sets.newTreeSet(Comparators.intervalsByStartThenEnd());
if (inputIntervals == null) {
inputIntervals = Lists.newArrayList();
}
// Insert all intervals
for (final Interval inputInterval : inputIntervals) {
intervals.add(inputInterval);