diff --git a/client/src/main/java/com/metamx/druid/query/filter/DimFilter.java b/client/src/main/java/com/metamx/druid/query/filter/DimFilter.java index 1f1464791b3..26f9bf8519c 100644 --- a/client/src/main/java/com/metamx/druid/query/filter/DimFilter.java +++ b/client/src/main/java/com/metamx/druid/query/filter/DimFilter.java @@ -33,7 +33,8 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(name="extraction", value=ExtractionDimFilter.class), @JsonSubTypes.Type(name="regex", value=RegexDimFilter.class), @JsonSubTypes.Type(name="search", value=SearchQueryDimFilter.class), - @JsonSubTypes.Type(name="javascript", value=JavaScriptDimFilter.class) + @JsonSubTypes.Type(name="javascript", value=JavaScriptDimFilter.class), + @JsonSubTypes.Type(name="spatial", value=SpatialDimFilter.class) }) public interface DimFilter { diff --git a/client/src/main/java/com/metamx/druid/query/filter/DimFilterCacheHelper.java b/client/src/main/java/com/metamx/druid/query/filter/DimFilterCacheHelper.java index 8497a53b206..02d20ea528c 100644 --- a/client/src/main/java/com/metamx/druid/query/filter/DimFilterCacheHelper.java +++ b/client/src/main/java/com/metamx/druid/query/filter/DimFilterCacheHelper.java @@ -35,6 +35,7 @@ class DimFilterCacheHelper static final byte REGEX_CACHE_ID = 0x5; static final byte SEARCH_QUERY_TYPE_ID = 0x6; static final byte JAVASCRIPT_CACHE_ID = 0x7; + static final byte SPATIAL_CACHE_ID = 0x8; static byte[] computeCacheKey(byte cacheIdKey, List filters) { diff --git a/client/src/main/java/com/metamx/druid/query/filter/SpatialDimFilter.java b/client/src/main/java/com/metamx/druid/query/filter/SpatialDimFilter.java new file mode 100644 index 00000000000..fac08ce2161 --- /dev/null +++ b/client/src/main/java/com/metamx/druid/query/filter/SpatialDimFilter.java @@ -0,0 +1,101 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package com.metamx.druid.query.filter; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import com.metamx.common.spatial.rtree.search.Bound; + +import java.nio.ByteBuffer; + +/** + */ +public class SpatialDimFilter implements DimFilter +{ + private final String dimension; + private final Bound bound; + + @JsonCreator + public SpatialDimFilter( + @JsonProperty("dimension") String dimension, + @JsonProperty("bound") Bound bound + ) + { + Preconditions.checkArgument(dimension != null, "dimension must not be null"); + Preconditions.checkArgument(dimension.endsWith(".geo"), "must filter over geo dimension!"); + Preconditions.checkArgument(bound != null, "bound must not be null"); + + this.dimension = dimension; + this.bound = bound; + } + + @Override + public byte[] getCacheKey() + { + byte[] boundBytes = bound.getCacheKey(); + + return ByteBuffer.allocate(1 + boundBytes.length) + .put(DimFilterCacheHelper.SPATIAL_CACHE_ID) + .put(boundBytes) + .array(); + } + + @JsonProperty + public String getDimension() + { + return dimension; + } + + @JsonProperty + public Bound getBound() + { + return bound; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + SpatialDimFilter that = (SpatialDimFilter) o; + + if (bound != null ? !bound.equals(that.bound) : that.bound != null) { + return false; + } + if (dimension != null ? !dimension.equals(that.dimension) : that.dimension != null) { + return false; + } + + return true; + } + + @Override + public int hashCode() + { + int result = dimension != null ? dimension.hashCode() : 0; + result = 31 * result + (bound != null ? bound.hashCode() : 0); + return result; + } +} diff --git a/index-common/src/main/java/com/metamx/druid/index/column/AbstractColumn.java b/index-common/src/main/java/com/metamx/druid/index/column/AbstractColumn.java index 978a3822071..c44c59a5e47 100644 --- a/index-common/src/main/java/com/metamx/druid/index/column/AbstractColumn.java +++ b/index-common/src/main/java/com/metamx/druid/index/column/AbstractColumn.java @@ -29,6 +29,12 @@ public abstract class AbstractColumn implements Column throw new UnsupportedOperationException(); } + @Override + public SpatialIndex getSpatialIndex() + { + throw new UnsupportedOperationException(); + } + @Override public ComplexColumn getComplexColumn() { diff --git a/index-common/src/main/java/com/metamx/druid/index/column/BitmapIndex.java b/index-common/src/main/java/com/metamx/druid/index/column/BitmapIndex.java index 6873292fee2..8b3309b3421 100644 --- a/index-common/src/main/java/com/metamx/druid/index/column/BitmapIndex.java +++ b/index-common/src/main/java/com/metamx/druid/index/column/BitmapIndex.java @@ -28,4 +28,5 @@ public interface BitmapIndex public int getCardinality(); public String getValue(int index); public ImmutableConciseSet getConciseSet(String value); + public ImmutableConciseSet getConciseSet(int idx); } diff --git a/index-common/src/main/java/com/metamx/druid/index/column/Column.java b/index-common/src/main/java/com/metamx/druid/index/column/Column.java index fa418a3398a..8de06439af7 100644 --- a/index-common/src/main/java/com/metamx/druid/index/column/Column.java +++ b/index-common/src/main/java/com/metamx/druid/index/column/Column.java @@ -31,4 +31,5 @@ public interface Column public GenericColumn getGenericColumn(); public ComplexColumn getComplexColumn(); public BitmapIndex getBitmapIndex(); + public SpatialIndex getSpatialIndex(); } diff --git a/index-common/src/main/java/com/metamx/druid/index/column/ColumnBuilder.java b/index-common/src/main/java/com/metamx/druid/index/column/ColumnBuilder.java index 7bf77e4fa10..180099f108f 100644 --- a/index-common/src/main/java/com/metamx/druid/index/column/ColumnBuilder.java +++ b/index-common/src/main/java/com/metamx/druid/index/column/ColumnBuilder.java @@ -34,6 +34,7 @@ public class ColumnBuilder private Supplier genericColumn = null; private Supplier complexColumn = null; private Supplier bitmapIndex = null; + private Supplier spatialIndex = null; public ColumnBuilder setType(ValueType type) { @@ -77,6 +78,12 @@ public class ColumnBuilder return this; } + public ColumnBuilder setSpatialIndex(Supplier spatialIndex) + { + this.spatialIndex = spatialIndex; + return this; + } + public Column build() { Preconditions.checkState(type != null, "Type must be set."); @@ -86,6 +93,7 @@ public class ColumnBuilder .setType(type) .setDictionaryEncoded(dictionaryEncodedColumn != null) .setHasBitmapIndexes(bitmapIndex != null) + .setHasSpatialIndexes(spatialIndex != null) .setRunLengthEncoded(runLengthColumn != null) .setHasMultipleValues(hasMultipleValues) , @@ -93,7 +101,8 @@ public class ColumnBuilder runLengthColumn, genericColumn, complexColumn, - bitmapIndex + bitmapIndex, + spatialIndex ); } } diff --git a/index-common/src/main/java/com/metamx/druid/index/column/ColumnCapabilities.java b/index-common/src/main/java/com/metamx/druid/index/column/ColumnCapabilities.java index 8e2ee8d1af4..3fbd439ffca 100644 --- a/index-common/src/main/java/com/metamx/druid/index/column/ColumnCapabilities.java +++ b/index-common/src/main/java/com/metamx/druid/index/column/ColumnCapabilities.java @@ -28,5 +28,6 @@ public interface ColumnCapabilities public boolean isDictionaryEncoded(); public boolean isRunLengthEncoded(); public boolean hasBitmapIndexes(); + public boolean hasSpatialIndexes(); public boolean hasMultipleValues(); } diff --git a/index-common/src/main/java/com/metamx/druid/index/column/ColumnCapabilitiesImpl.java b/index-common/src/main/java/com/metamx/druid/index/column/ColumnCapabilitiesImpl.java index c2c28c4d4d6..1e1554c030d 100644 --- a/index-common/src/main/java/com/metamx/druid/index/column/ColumnCapabilitiesImpl.java +++ b/index-common/src/main/java/com/metamx/druid/index/column/ColumnCapabilitiesImpl.java @@ -29,6 +29,7 @@ public class ColumnCapabilitiesImpl implements ColumnCapabilities private boolean dictionaryEncoded = false; private boolean runLengthEncoded = false; private boolean hasInvertedIndexes = false; + private boolean hasSpatialIndexes = false; private boolean hasMultipleValues = false; @Override @@ -83,6 +84,19 @@ public class ColumnCapabilitiesImpl implements ColumnCapabilities return this; } + @Override + @JsonProperty("hasSpatialIndexes") + public boolean hasSpatialIndexes() + { + return hasSpatialIndexes; + } + + public ColumnCapabilitiesImpl setHasSpatialIndexes(boolean hasSpatialIndexes) + { + this.hasSpatialIndexes = hasSpatialIndexes; + return this; + } + @Override @JsonProperty("hasMultipleValues") public boolean hasMultipleValues() diff --git a/index-common/src/main/java/com/metamx/druid/index/column/SimpleColumn.java b/index-common/src/main/java/com/metamx/druid/index/column/SimpleColumn.java index 8179cd623e4..5fd9f1c381e 100644 --- a/index-common/src/main/java/com/metamx/druid/index/column/SimpleColumn.java +++ b/index-common/src/main/java/com/metamx/druid/index/column/SimpleColumn.java @@ -32,6 +32,7 @@ class SimpleColumn implements Column private final Supplier genericColumn; private final Supplier complexColumn; private final Supplier bitmapIndex; + private final Supplier spatialIndex; SimpleColumn( ColumnCapabilities capabilities, @@ -39,7 +40,8 @@ class SimpleColumn implements Column Supplier runLengthColumn, Supplier genericColumn, Supplier complexColumn, - Supplier bitmapIndex + Supplier bitmapIndex, + Supplier spatialIndex ) { this.capabilities = capabilities; @@ -48,6 +50,7 @@ class SimpleColumn implements Column this.genericColumn = genericColumn; this.complexColumn = complexColumn; this.bitmapIndex = bitmapIndex; + this.spatialIndex = spatialIndex; } @Override @@ -98,4 +101,10 @@ class SimpleColumn implements Column { return bitmapIndex == null ? null : bitmapIndex.get(); } + + @Override + public SpatialIndex getSpatialIndex() + { + return spatialIndex == null ? null : spatialIndex.get(); + } } diff --git a/index-common/src/main/java/com/metamx/druid/index/column/SpatialIndex.java b/index-common/src/main/java/com/metamx/druid/index/column/SpatialIndex.java new file mode 100644 index 00000000000..800ab293234 --- /dev/null +++ b/index-common/src/main/java/com/metamx/druid/index/column/SpatialIndex.java @@ -0,0 +1,30 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package com.metamx.druid.index.column; + +import com.metamx.common.spatial.rtree.ImmutableRTree; + +/** + */ +public interface SpatialIndex +{ + public int getCardinality(); + public String getValue(int index); + public ImmutableRTree getRTree(String value); +} diff --git a/index-common/src/main/java/com/metamx/druid/index/serde/BitmapIndexColumnPartSupplier.java b/index-common/src/main/java/com/metamx/druid/index/serde/BitmapIndexColumnPartSupplier.java index dba6b7fcf91..12f5fb4f354 100644 --- a/index-common/src/main/java/com/metamx/druid/index/serde/BitmapIndexColumnPartSupplier.java +++ b/index-common/src/main/java/com/metamx/druid/index/serde/BitmapIndexColumnPartSupplier.java @@ -63,11 +63,17 @@ public class BitmapIndexColumnPartSupplier implements Supplier { final int index = dictionary.indexOf(value); - if (index < 0) { + return getConciseSet(index); + } + + @Override + public ImmutableConciseSet getConciseSet(int idx) + { + if (idx < 0) { return EMPTY_SET; } - final ImmutableConciseSet bitmap = bitmaps.get(index); + final ImmutableConciseSet bitmap = bitmaps.get(idx); return bitmap == null ? EMPTY_SET : bitmap; } }; diff --git a/index-common/src/main/java/com/metamx/druid/index/serde/DictionaryEncodedColumnPartSerde.java b/index-common/src/main/java/com/metamx/druid/index/serde/DictionaryEncodedColumnPartSerde.java index 5f7a38014d6..ddd5b2f5356 100644 --- a/index-common/src/main/java/com/metamx/druid/index/serde/DictionaryEncodedColumnPartSerde.java +++ b/index-common/src/main/java/com/metamx/druid/index/serde/DictionaryEncodedColumnPartSerde.java @@ -22,10 +22,12 @@ package com.metamx.druid.index.serde; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.metamx.common.IAE; +import com.metamx.common.spatial.rtree.ImmutableRTree; import com.metamx.druid.index.column.ColumnBuilder; import com.metamx.druid.index.column.ValueType; import com.metamx.druid.kv.ConciseCompressedIndexedInts; import com.metamx.druid.kv.GenericIndexed; +import com.metamx.druid.kv.IndexedRTree; import com.metamx.druid.kv.VSizeIndexed; import com.metamx.druid.kv.VSizeIndexedInts; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; @@ -35,7 +37,7 @@ import java.nio.ByteBuffer; import java.nio.channels.WritableByteChannel; /** -*/ + */ public class DictionaryEncodedColumnPartSerde implements ColumnPartSerde { @JsonCreator @@ -48,6 +50,7 @@ public class DictionaryEncodedColumnPartSerde implements ColumnPartSerde private final VSizeIndexedInts singleValuedColumn; private final VSizeIndexed multiValuedColumn; private final GenericIndexed bitmaps; + private final GenericIndexed spatialIndex; private final int size; @@ -55,25 +58,26 @@ public class DictionaryEncodedColumnPartSerde implements ColumnPartSerde GenericIndexed dictionary, VSizeIndexedInts singleValCol, VSizeIndexed multiValCol, - GenericIndexed bitmaps + GenericIndexed bitmaps, + GenericIndexed spatialIndex ) { this.dictionary = dictionary; this.singleValuedColumn = singleValCol; this.multiValuedColumn = multiValCol; this.bitmaps = bitmaps; + this.spatialIndex = spatialIndex; int size = dictionary.getSerializedSize(); if (singleValCol != null && multiValCol == null) { size += singleValCol.getSerializedSize(); - } - else if (singleValCol == null && multiValCol != null) { + } else if (singleValCol == null && multiValCol != null) { size += multiValCol.getSerializedSize(); - } - else { + } else { throw new IAE("Either singleValCol[%s] or multiValCol[%s] must be set", singleValCol, multiValCol); } size += bitmaps.getSerializedSize(); + size += spatialIndex.getSerializedSize(); this.size = size; } @@ -84,6 +88,7 @@ public class DictionaryEncodedColumnPartSerde implements ColumnPartSerde singleValuedColumn = null; multiValuedColumn = null; bitmaps = null; + spatialIndex = null; size = 0; } @@ -106,11 +111,11 @@ public class DictionaryEncodedColumnPartSerde implements ColumnPartSerde dictionary.writeToChannel(channel); if (isSingleValued()) { singleValuedColumn.writeToChannel(channel); - } - else { + } else { multiValuedColumn.writeToChannel(channel); } bitmaps.writeToChannel(channel); + spatialIndex.writeToChannel(channel); } @Override @@ -128,8 +133,7 @@ public class DictionaryEncodedColumnPartSerde implements ColumnPartSerde multiValuedColumn = null; builder.setHasMultipleValues(false) .setDictionaryEncodedColumn(new DictionaryEncodedColumnSupplier(dictionary, singleValuedColumn, null)); - } - else { + } else { singleValuedColumn = null; multiValuedColumn = VSizeIndexed.readFromByteBuffer(buffer); builder.setHasMultipleValues(true) @@ -140,8 +144,19 @@ public class DictionaryEncodedColumnPartSerde implements ColumnPartSerde buffer, ConciseCompressedIndexedInts.objectStrategy ); - builder.setBitmapIndex(new BitmapIndexColumnPartSupplier(bitmaps, dictionary)); + GenericIndexed spatialIndex = GenericIndexed.read( + buffer, IndexedRTree.objectStrategy + ); - return new DictionaryEncodedColumnPartSerde(dictionary, singleValuedColumn, multiValuedColumn, bitmaps); + builder.setBitmapIndex(new BitmapIndexColumnPartSupplier(bitmaps, dictionary)); + builder.setSpatialIndex(new SpatialIndexColumnPartSupplier(spatialIndex, dictionary)); + + return new DictionaryEncodedColumnPartSerde( + dictionary, + singleValuedColumn, + multiValuedColumn, + bitmaps, + spatialIndex + ); } } diff --git a/index-common/src/main/java/com/metamx/druid/index/serde/SpatialIndexColumnPartSupplier.java b/index-common/src/main/java/com/metamx/druid/index/serde/SpatialIndexColumnPartSupplier.java new file mode 100644 index 00000000000..413d936b8ca --- /dev/null +++ b/index-common/src/main/java/com/metamx/druid/index/serde/SpatialIndexColumnPartSupplier.java @@ -0,0 +1,74 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package com.metamx.druid.index.serde; + +import com.google.common.base.Supplier; +import com.metamx.common.spatial.rtree.ImmutableRTree; +import com.metamx.druid.index.column.SpatialIndex; +import com.metamx.druid.kv.GenericIndexed; + +/** + */ +public class SpatialIndexColumnPartSupplier implements Supplier +{ + private static final ImmutableRTree EMPTY_SET = new ImmutableRTree(); + + private final GenericIndexed indexedTree; + private final GenericIndexed dictionary; + + public SpatialIndexColumnPartSupplier( + GenericIndexed indexedTree, + GenericIndexed dictionary + ) { + this.indexedTree = indexedTree; + this.dictionary = dictionary; + } + + @Override + public SpatialIndex get() + { + return new SpatialIndex() + { + @Override + public int getCardinality() + { + return dictionary.size(); + } + + @Override + public String getValue(int index) + { + return dictionary.get(index); + } + + @Override + public ImmutableRTree getRTree(String value) + { + final int index = dictionary.indexOf(value); + + if (index < 0) { + return EMPTY_SET; + } + + final ImmutableRTree rTree = indexedTree.get(index); + return rTree == null ? EMPTY_SET : rTree; + } + }; + } +} diff --git a/index-common/src/main/java/com/metamx/druid/index/v1/IncrementalIndex.java b/index-common/src/main/java/com/metamx/druid/index/v1/IncrementalIndex.java index 624d6d4b375..b71cadbc58a 100644 --- a/index-common/src/main/java/com/metamx/druid/index/v1/IncrementalIndex.java +++ b/index-common/src/main/java/com/metamx/druid/index/v1/IncrementalIndex.java @@ -20,6 +20,7 @@ package com.metamx.druid.index.v1; import com.google.common.base.Function; +import com.google.common.base.Joiner; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import com.google.common.collect.ImmutableList; @@ -64,6 +65,7 @@ import java.util.concurrent.CopyOnWriteArrayList; public class IncrementalIndex implements Iterable { private static final Logger log = new Logger(IncrementalIndex.class); + private static final Joiner JOINER = Joiner.on(","); private final long minTimestamp; private final QueryGranularity gran; @@ -134,6 +136,12 @@ public class IncrementalIndex implements Iterable List overflow = null; for (String dimension : rowDimensions) { dimension = dimension.toLowerCase(); + List dimensionValues = row.getDimension(dimension); + + // FIXME: HACK!!! Rewrite this when this file is rewritten + if (dimension.endsWith(".geo")) { + dimensionValues = Arrays.asList(JOINER.join(dimensionValues)); + } final Integer index = dimensionOrder.get(dimension); if (index == null) { @@ -143,9 +151,9 @@ public class IncrementalIndex implements Iterable if (overflow == null) { overflow = Lists.newArrayList(); } - overflow.add(getDimVals(dimValues.add(dimension), row.getDimension(dimension))); + overflow.add(getDimVals(dimValues.add(dimension), dimensionValues)); } else { - dims[index] = getDimVals(dimValues.get(dimension), row.getDimension(dimension)); + dims[index] = getDimVals(dimValues.get(dimension), dimensionValues); } } @@ -409,8 +417,7 @@ public class IncrementalIndex implements Iterable if (holder == null) { holder = new DimDim(); dimensions.put(dimension, holder); - } - else { + } else { throw new ISE("dimension[%s] already existed even though add() was called!?", dimension); } return holder; diff --git a/index-common/src/main/java/com/metamx/druid/index/v1/Index.java b/index-common/src/main/java/com/metamx/druid/index/v1/Index.java index 883fc38474c..bdacd3380ad 100644 --- a/index-common/src/main/java/com/metamx/druid/index/v1/Index.java +++ b/index-common/src/main/java/com/metamx/druid/index/v1/Index.java @@ -20,9 +20,11 @@ package com.metamx.druid.index.v1; import com.metamx.common.logger.Logger; +import com.metamx.common.spatial.rtree.ImmutableRTree; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; import org.joda.time.Interval; +import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; @@ -45,6 +47,7 @@ public class Index final Map> dimIdLookup; final Map reverseDimLookup; final Map indexes; + final Map spatialIndexes; final Map dimensionValues; /* @@ -79,6 +82,7 @@ public class Index Map> dimIdLookup, Map reverseDimLookup, Map indexes, + Map spatialIndexes, Map dimensionValues ) { @@ -90,6 +94,7 @@ public class Index this.dimIdLookup = dimIdLookup; this.reverseDimLookup = reverseDimLookup; this.indexes = indexes; + this.spatialIndexes = spatialIndexes; this.dimensionValues = dimensionValues; for (int i = 0; i < dimensions.length; i++) { @@ -126,4 +131,37 @@ public class Index return emptySet; } } + + public ImmutableConciseSet getInvertedIndex(String dimension, int valueIndex) + { + try { + return indexes.get(dimension)[valueIndex]; + } + catch (NullPointerException e) { + log.warn( + e, + "NPE on dimension[%s], valueIndex[%d], with index over interval[%s]", + dimension, + valueIndex, + dataInterval + ); + return emptySet; + } + } + + public ImmutableRTree getSpatialIndex(String dimension) + { + try { + return spatialIndexes.get(dimension); + } + catch (NullPointerException e) { + log.warn( + e, + "NPE on dimension[%s] over interval[%s]", + dimension, + dataInterval + ); + return new ImmutableRTree(); + } + } } diff --git a/index-common/src/main/java/com/metamx/druid/index/v1/IndexIO.java b/index-common/src/main/java/com/metamx/druid/index/v1/IndexIO.java index df3836d7945..0dda868e402 100644 --- a/index-common/src/main/java/com/metamx/druid/index/v1/IndexIO.java +++ b/index-common/src/main/java/com/metamx/druid/index/v1/IndexIO.java @@ -38,6 +38,7 @@ import com.metamx.common.io.smoosh.Smoosh; import com.metamx.common.io.smoosh.SmooshedFileMapper; import com.metamx.common.io.smoosh.SmooshedWriter; import com.metamx.common.logger.Logger; +import com.metamx.common.spatial.rtree.ImmutableRTree; import com.metamx.druid.index.QueryableIndex; import com.metamx.druid.index.SimpleQueryableIndex; import com.metamx.druid.index.column.Column; @@ -53,11 +54,13 @@ import com.metamx.druid.index.serde.FloatGenericColumnPartSerde; import com.metamx.druid.index.serde.FloatGenericColumnSupplier; import com.metamx.druid.index.serde.LongGenericColumnPartSerde; import com.metamx.druid.index.serde.LongGenericColumnSupplier; +import com.metamx.druid.index.serde.SpatialIndexColumnPartSupplier; import com.metamx.druid.jackson.DefaultObjectMapper; import com.metamx.druid.kv.ArrayIndexed; import com.metamx.druid.kv.ConciseCompressedIndexedInts; import com.metamx.druid.kv.GenericIndexed; import com.metamx.druid.kv.IndexedIterable; +import com.metamx.druid.kv.IndexedRTree; import com.metamx.druid.kv.VSizeIndexed; import com.metamx.druid.kv.VSizeIndexedInts; import com.metamx.druid.utils.SerializerUtils; @@ -221,7 +224,10 @@ public class IndexIO case 6: case 7: log.info("Old version, re-persisting."); - IndexMerger.append(Arrays.asList(new QueryableIndexIndexableAdapter(loadIndex(toConvert))), converted); + IndexMerger.append( + Arrays.asList(new QueryableIndexIndexableAdapter(loadIndex(toConvert))), + converted + ); return true; case 8: DefaultIndexIOHandler.convertV8toV9(toConvert, converted); @@ -251,6 +257,7 @@ public class IndexIO * @return */ public boolean canBeMapped(File inDir) throws IOException; + public MMappedIndex mapDir(File inDir) throws IOException; /** @@ -265,6 +272,7 @@ public class IndexIO public static class DefaultIndexIOHandler implements IndexIOHandler { private static final Logger log = new Logger(DefaultIndexIOHandler.class); + @Override public Index readIndex(File inDir) { @@ -325,6 +333,7 @@ public class IndexIO Map> dimValueLookups = Maps.newHashMap(); Map dimColumns = Maps.newHashMap(); Map> invertedIndexed = Maps.newHashMap(); + Map> spatialIndexed = Maps.newHashMap(); for (String dimension : IndexedIterable.create(availableDimensions)) { ByteBuffer dimBuffer = smooshedFiles.mapFile(makeDimFile(inDir, dimension).getName()); @@ -348,6 +357,14 @@ public class IndexIO ); } + ByteBuffer spatialBuffer = smooshedFiles.mapFile("spatial.drd"); + for (String dimension : IndexedIterable.create(availableDimensions)) { + spatialIndexed.put( + serializerUtils.readString(spatialBuffer), + GenericIndexed.read(spatialBuffer, IndexedRTree.objectStrategy) + ); + } + final MMappedIndex retVal = new MMappedIndex( availableDimensions, availableMetrics, @@ -356,7 +373,8 @@ public class IndexIO metrics, dimValueLookups, dimColumns, - invertedIndexed + invertedIndexed, + spatialIndexed ); log.debug("Mapped v8 index[%s] in %,d millis", inDir, System.currentTimeMillis() - startTime); @@ -402,6 +420,15 @@ public class IndexIO ); } + Map> spatialIndexes = Maps.newHashMap(); + final ByteBuffer spatialBuffer = v8SmooshedFiles.mapFile("spatial.drd"); + while (spatialBuffer.hasRemaining()) { + spatialIndexes.put( + serializerUtils.readString(spatialBuffer), + GenericIndexed.read(spatialBuffer, IndexedRTree.objectStrategy) + ); + } + final LinkedHashSet skippedFiles = Sets.newLinkedHashSet(); final Set skippedDimensions = Sets.newLinkedHashSet(); for (String filename : v8SmooshedFiles.getInternalFilenames()) { @@ -435,6 +462,7 @@ public class IndexIO VSizeIndexedInts singleValCol = null; VSizeIndexed multiValCol = VSizeIndexed.readFromByteBuffer(dimBuffer.asReadOnlyBuffer()); GenericIndexed bitmaps = bitmapIndexes.get(dimension); + GenericIndexed spatialIndex = spatialIndexes.get(dimension); boolean onlyOneValue = true; ConciseSet nullsSet = null; @@ -476,8 +504,7 @@ public class IndexIO Iterables.concat(Arrays.asList(theNullSet), bitmaps), ConciseCompressedIndexedInts.objectStrategy ); - } - else { + } else { bumpedDictionary = false; bitmaps = GenericIndexed.fromIterable( Iterables.concat( @@ -487,8 +514,7 @@ public class IndexIO ConciseCompressedIndexedInts.objectStrategy ); } - } - else { + } else { bumpedDictionary = false; } @@ -517,7 +543,7 @@ public class IndexIO } builder.addSerde( - new DictionaryEncodedColumnPartSerde(dictionary, singleValCol, multiValCol, bitmaps) + new DictionaryEncodedColumnPartSerde(dictionary, singleValCol, multiValCol, bitmaps, spatialIndex) ); final ColumnDescriptor serdeficator = builder.build(); @@ -673,8 +699,11 @@ public class IndexIO new BitmapIndexColumnPartSupplier( index.getInvertedIndexes().get(dimension), index.getDimValueLookup(dimension) ) + ).setSpatialIndex( + new SpatialIndexColumnPartSupplier( + index.getSpatialIndexes().get(dimension), index.getDimValueLookup(dimension) ) - .build() + ).build() ); } diff --git a/index-common/src/main/java/com/metamx/druid/index/v1/IndexMerger.java b/index-common/src/main/java/com/metamx/druid/index/v1/IndexMerger.java index 3c6ea59fc5c..ddee90e867f 100644 --- a/index-common/src/main/java/com/metamx/druid/index/v1/IndexMerger.java +++ b/index-common/src/main/java/com/metamx/druid/index/v1/IndexMerger.java @@ -21,6 +21,8 @@ package com.metamx.druid.index.v1; import com.google.common.base.Function; import com.google.common.base.Objects; +import com.google.common.base.Splitter; +import com.google.common.base.Stopwatch; import com.google.common.collect.Iterables; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; @@ -39,6 +41,9 @@ import com.metamx.common.guava.MergeIterable; import com.metamx.common.guava.nary.BinaryFn; import com.metamx.common.io.smoosh.Smoosh; import com.metamx.common.logger.Logger; +import com.metamx.common.spatial.rtree.ImmutableRTree; +import com.metamx.common.spatial.rtree.RTree; +import com.metamx.common.spatial.rtree.split.LinearGutmanSplitStrategy; import com.metamx.druid.CombiningIterable; import com.metamx.druid.aggregation.AggregatorFactory; import com.metamx.druid.aggregation.ToLowerCaseAggregatorFactory; @@ -54,6 +59,7 @@ import com.metamx.druid.kv.IOPeon; import com.metamx.druid.kv.Indexed; import com.metamx.druid.kv.IndexedInts; import com.metamx.druid.kv.IndexedIterable; +import com.metamx.druid.kv.IndexedRTree; import com.metamx.druid.kv.TmpFileIOPeon; import com.metamx.druid.kv.VSizeIndexedWriter; import com.metamx.druid.utils.JodaUtils; @@ -88,6 +94,7 @@ public class IndexMerger private static final SerializerUtils serializerUtils = new SerializerUtils(); private static final int INVALID_ROW = -1; + private static final Splitter SPLITTER = Splitter.on(","); public static File persist(final IncrementalIndex index, File outDir) throws IOException { @@ -682,6 +689,59 @@ public class IndexMerger System.currentTimeMillis() - startTime ); + + /************ Create Geographical Indexes *************/ + // FIXME: Rewrite when indexing is updated + Stopwatch stopwatch = new Stopwatch(); + stopwatch.start(); + + final File geoFile = new File(v8OutDir, "spatial.drd"); + Files.touch(geoFile); + out = Files.newOutputStreamSupplier(geoFile, true); + + for (int i = 0; i < mergedDimensions.size(); ++i) { + String dimension = mergedDimensions.get(i); + + File dimOutFile = dimOuts.get(i).getFile(); + final MappedByteBuffer dimValsMapped = Files.map(dimOutFile); + + if (!dimension.equals(serializerUtils.readString(dimValsMapped))) { + throw new ISE("dimensions[%s] didn't equate!? This is a major WTF moment.", dimension); + } + Indexed dimVals = GenericIndexed.read(dimValsMapped, GenericIndexed.stringStrategy); + log.info("Indexing geo dimension[%s] with cardinality[%,d]", dimension, dimVals.size()); + + FlattenedArrayWriter writer = new FlattenedArrayWriter( + ioPeon, dimension, IndexedRTree.objectStrategy + ); + writer.open(); + + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50)); + + if (dimension.endsWith(".geo")) { + int count = 0; + for (String dimVal : IndexedIterable.create(dimVals)) { + progress.progress(); + + List stringCoords = Lists.newArrayList(SPLITTER.split(dimVal)); + float[] coords = new float[stringCoords.size()]; + for (int j = 0; j < coords.length; j++) { + coords[j] = Float.valueOf(stringCoords.get(j)); + } + tree.insert(coords, count); + } + } + writer.write(ImmutableRTree.newImmutableFromMutable(tree)); + writer.close(); + + serializerUtils.writeString(out, dimension); + ByteStreams.copy(writer.combineStreams(), out); + ioPeon.cleanup(); + + log.info("Completed spatial dimension[%s] in %,d millis.", dimension, stopwatch.elapsedMillis()); + } + + /************ Create Inverted Indexes *************/ startTime = System.currentTimeMillis(); @@ -718,7 +778,10 @@ public class IndexMerger } ConciseSet bitset = new ConciseSet(); - for (Integer row : CombiningIterable.createSplatted(convertedInverteds, Ordering.natural().nullsFirst())) { + for (Integer row : CombiningIterable.createSplatted( + convertedInverteds, + Ordering.natural().nullsFirst() + )) { if (row != INVALID_ROW) { bitset.add(row); } @@ -734,12 +797,13 @@ public class IndexMerger log.info("Completed dimension[%s] in %,d millis.", dimension, System.currentTimeMillis() - dimStartTime); } + log.info("outDir[%s] completed inverted.drd in %,d millis.", v8OutDir, System.currentTimeMillis() - startTime); final ArrayList expectedFiles = Lists.newArrayList( Iterables.concat( Arrays.asList( - "index.drd", "inverted.drd", String.format("time_%s.drd", IndexIO.BYTE_ORDER) + "index.drd", "inverted.drd", "spatial.drd", String.format("time_%s.drd", IndexIO.BYTE_ORDER) ), Iterables.transform(mergedDimensions, GuavaUtils.formatFunction("dim_%s.drd")), Iterables.transform( diff --git a/index-common/src/main/java/com/metamx/druid/index/v1/MMappedIndex.java b/index-common/src/main/java/com/metamx/druid/index/v1/MMappedIndex.java index b354778078f..687c7672ec8 100644 --- a/index-common/src/main/java/com/metamx/druid/index/v1/MMappedIndex.java +++ b/index-common/src/main/java/com/metamx/druid/index/v1/MMappedIndex.java @@ -24,6 +24,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.primitives.Ints; import com.metamx.common.logger.Logger; +import com.metamx.common.spatial.rtree.ImmutableRTree; import com.metamx.druid.kv.ConciseCompressedIndexedInts; import com.metamx.druid.kv.GenericIndexed; import com.metamx.druid.kv.Indexed; @@ -54,6 +55,7 @@ public class MMappedIndex final Map> dimValueLookups; final Map dimColumns; final Map> invertedIndexes; + final Map> spatialIndexes; private final Map metricIndexes = Maps.newHashMap(); @@ -65,7 +67,8 @@ public class MMappedIndex Map metrics, Map> dimValueLookups, Map dimColumns, - Map> invertedIndexes + Map> invertedIndexes, + Map> spatialIndexes ) { this.availableDimensions = availableDimensions; @@ -76,6 +79,7 @@ public class MMappedIndex this.dimValueLookups = dimValueLookups; this.dimColumns = dimColumns; this.invertedIndexes = invertedIndexes; + this.spatialIndexes = spatialIndexes; for (int i = 0; i < availableMetrics.size(); i++) { metricIndexes.put(availableMetrics.get(i), i); @@ -143,6 +147,11 @@ public class MMappedIndex return invertedIndexes; } + public Map> getSpatialIndexes() + { + return spatialIndexes; + } + public ImmutableConciseSet getInvertedIndex(String dimension, String value) { final GenericIndexed lookup = dimValueLookups.get(dimension); @@ -159,6 +168,7 @@ public class MMappedIndex return (retVal == null) ? emptySet : retVal; } +/* public static MMappedIndex fromIndex(Index index) { log.info("Converting timestamps"); @@ -177,6 +187,8 @@ public class MMappedIndex Map dimColumns = Maps.newHashMap(); Map> invertedIndexes = Maps.newLinkedHashMap(); + Map> spatialIndexes = Maps.newLinkedHashMap(); + for (String dimension : Arrays.asList(index.dimensions)) { final String[] dimVals = index.reverseDimLookup.get(dimension); final DimensionColumn dimColumn = index.dimensionValues.get(dimension); @@ -258,7 +270,9 @@ public class MMappedIndex index.metricVals, dimValueLookups, dimColumns, - invertedIndexes + invertedIndexes, + spatialIndexes ); } +*/ } diff --git a/index-common/src/main/java/com/metamx/druid/kv/IndexedRTree.java b/index-common/src/main/java/com/metamx/druid/kv/IndexedRTree.java new file mode 100644 index 00000000000..a723d8bc6af --- /dev/null +++ b/index-common/src/main/java/com/metamx/druid/kv/IndexedRTree.java @@ -0,0 +1,84 @@ +package com.metamx.druid.kv; + +import com.google.common.collect.Ordering; +import com.metamx.common.spatial.rtree.ImmutableRTree; + +import java.nio.ByteBuffer; + +/** + */ +public class IndexedRTree implements Comparable +{ + public static ObjectStrategy objectStrategy = + new ImmutableRTreeObjectStrategy(); + + private static Ordering comparator = new Ordering() + { + @Override + public int compare( + ImmutableRTree tree, ImmutableRTree tree1 + ) + { + if (tree.size() == 0 && tree1.size() == 0) { + return 0; + } + if (tree.size() == 0) { + return -1; + } + if (tree1.size() == 0) { + return 1; + } + return tree.compareTo(tree1); + } + }.nullsFirst(); + + private final ImmutableRTree immutableRTree; + + public IndexedRTree(ImmutableRTree immutableRTree) + { + this.immutableRTree = immutableRTree; + } + + @Override + public int compareTo(IndexedRTree spatialIndexedInts) + { + return immutableRTree.compareTo(spatialIndexedInts.getImmutableRTree()); + } + + public ImmutableRTree getImmutableRTree() + { + return immutableRTree; + } + + private static class ImmutableRTreeObjectStrategy + implements ObjectStrategy + { + @Override + public Class getClazz() + { + return ImmutableRTree.class; + } + + @Override + public ImmutableRTree fromByteBuffer(ByteBuffer buffer, int numBytes) + { + buffer.limit(buffer.position() + numBytes); + return new ImmutableRTree(buffer.asReadOnlyBuffer()); + } + + @Override + public byte[] toBytes(ImmutableRTree val) + { + if (val == null || val.size() == 0) { + return new byte[]{}; + } + return val.toBytes(); + } + + @Override + public int compare(ImmutableRTree o1, ImmutableRTree o2) + { + return comparator.compare(o1, o2); + } + } +} diff --git a/pom.xml b/pom.xml index 1b831c817eb..003fa370c38 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ UTF-8 - 0.21.0 + 0.21.1-SNAPSHOT diff --git a/server/src/main/java/com/metamx/druid/BaseStorageAdapter.java b/server/src/main/java/com/metamx/druid/BaseStorageAdapter.java index 293ba9263b1..2aa7e4f59bf 100644 --- a/server/src/main/java/com/metamx/druid/BaseStorageAdapter.java +++ b/server/src/main/java/com/metamx/druid/BaseStorageAdapter.java @@ -44,6 +44,8 @@ public abstract class BaseStorageAdapter implements StorageAdapter public abstract ImmutableConciseSet getInvertedIndex(String dimension, String dimVal); + public abstract ImmutableConciseSet getInvertedIndex(String dimension, int idx); + public abstract Offset getFilterOffset(Filter filter); @Override diff --git a/server/src/main/java/com/metamx/druid/index/brita/BitmapIndexSelector.java b/server/src/main/java/com/metamx/druid/index/brita/BitmapIndexSelector.java index d2e73353a1b..e18c7ef868b 100644 --- a/server/src/main/java/com/metamx/druid/index/brita/BitmapIndexSelector.java +++ b/server/src/main/java/com/metamx/druid/index/brita/BitmapIndexSelector.java @@ -19,6 +19,7 @@ package com.metamx.druid.index.brita; +import com.metamx.common.spatial.rtree.ImmutableRTree; import com.metamx.druid.kv.Indexed; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; @@ -29,4 +30,6 @@ public interface BitmapIndexSelector public Indexed getDimensionValues(String dimension); public int getNumRows(); public ImmutableConciseSet getConciseInvertedIndex(String dimension, String value); + public ImmutableConciseSet getConciseInvertedIndex(String dimension, int idx); + public ImmutableRTree getSpatialIndex(String dimension); } diff --git a/server/src/main/java/com/metamx/druid/index/brita/Filters.java b/server/src/main/java/com/metamx/druid/index/brita/Filters.java index f7c390c44a9..c27940b06fd 100644 --- a/server/src/main/java/com/metamx/druid/index/brita/Filters.java +++ b/server/src/main/java/com/metamx/druid/index/brita/Filters.java @@ -30,6 +30,7 @@ import com.metamx.druid.query.filter.OrDimFilter; import com.metamx.druid.query.filter.RegexDimFilter; import com.metamx.druid.query.filter.SearchQueryDimFilter; import com.metamx.druid.query.filter.SelectorDimFilter; +import com.metamx.druid.query.filter.SpatialDimFilter; import javax.annotation.Nullable; import java.util.List; @@ -89,6 +90,10 @@ public class Filters final JavaScriptDimFilter javaScriptDimFilter = (JavaScriptDimFilter) dimFilter; filter = new JavaScriptFilter(javaScriptDimFilter.getDimension(), javaScriptDimFilter.getFunction()); + } else if (dimFilter instanceof SpatialDimFilter) { + final SpatialDimFilter spatialDimFilter = (SpatialDimFilter) dimFilter; + + filter = new SpatialFilter(spatialDimFilter.getDimension(), spatialDimFilter.getBound()); } return filter; diff --git a/server/src/main/java/com/metamx/druid/index/brita/SpatialFilter.java b/server/src/main/java/com/metamx/druid/index/brita/SpatialFilter.java new file mode 100644 index 00000000000..cccaeead1e7 --- /dev/null +++ b/server/src/main/java/com/metamx/druid/index/brita/SpatialFilter.java @@ -0,0 +1,77 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package com.metamx.druid.index.brita; + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import com.metamx.common.spatial.rtree.search.Bound; +import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; + +/** + */ +public class SpatialFilter implements Filter +{ + private final String dimension; + private final Bound bound; + + public SpatialFilter( + String dimension, + Bound bound + ) + { + this.dimension = dimension; + this.bound = bound; + } + + @Override + public ImmutableConciseSet goConcise(final BitmapIndexSelector selector) + { + Iterable indexes = selector.getSpatialIndex(dimension).search(bound); + return ImmutableConciseSet.union( + Iterables.transform( + indexes, + new Function() + { + @Override + public ImmutableConciseSet apply(Integer input) + { + return selector.getConciseInvertedIndex(dimension, input); + } + } + ) + ); + } + + @Override + public ValueMatcher makeMatcher(ValueMatcherFactory factory) + { + return factory.makeValueMatcher( + dimension, + new Predicate() + { + @Override + public boolean apply(String input) + { + return true; + } + } + ); + } +} diff --git a/server/src/main/java/com/metamx/druid/index/v1/IndexStorageAdapter.java b/server/src/main/java/com/metamx/druid/index/v1/IndexStorageAdapter.java index 2d41ce338aa..04220ec4594 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/IndexStorageAdapter.java +++ b/server/src/main/java/com/metamx/druid/index/v1/IndexStorageAdapter.java @@ -28,6 +28,7 @@ import com.metamx.common.Pair; import com.metamx.common.collect.MoreIterators; import com.metamx.common.guava.FunctionalIterator; import com.metamx.common.logger.Logger; +import com.metamx.common.spatial.rtree.ImmutableRTree; import com.metamx.druid.BaseStorageAdapter; import com.metamx.druid.Capabilities; import com.metamx.druid.QueryGranularity; @@ -349,6 +350,12 @@ public class IndexStorageAdapter extends BaseStorageAdapter return index.getInvertedIndex(dimension.toLowerCase(), dimVal); } + @Override + public ImmutableConciseSet getInvertedIndex(String dimension, int idx) + { + return index.getInvertedIndex(dimension.toLowerCase(), idx); + } + @Override public Offset getFilterOffset(Filter filter) { @@ -519,5 +526,17 @@ public class IndexStorageAdapter extends BaseStorageAdapter { return index.getInvertedIndex(dimension.toLowerCase(), value); } + + @Override + public ImmutableConciseSet getConciseInvertedIndex(String dimension, int idx) + { + return index.getInvertedIndex(dimension.toLowerCase(), idx); + } + + @Override + public ImmutableRTree getSpatialIndex(String dimension) + { + return index.getSpatialIndex(dimension.toLowerCase()); + } } } diff --git a/server/src/main/java/com/metamx/druid/index/v1/QueryableIndexStorageAdapter.java b/server/src/main/java/com/metamx/druid/index/v1/QueryableIndexStorageAdapter.java index 1cde70b5c73..9d54140d241 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/QueryableIndexStorageAdapter.java +++ b/server/src/main/java/com/metamx/druid/index/v1/QueryableIndexStorageAdapter.java @@ -28,6 +28,7 @@ import com.google.common.io.Closeables; import com.metamx.common.collect.MoreIterators; import com.metamx.common.guava.FunctionalIterable; import com.metamx.common.guava.FunctionalIterator; +import com.metamx.common.spatial.rtree.ImmutableRTree; import com.metamx.druid.BaseStorageAdapter; import com.metamx.druid.Capabilities; import com.metamx.druid.QueryGranularity; @@ -52,6 +53,7 @@ import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; import org.joda.time.DateTime; import org.joda.time.Interval; +import java.nio.ByteBuffer; import java.util.Iterator; import java.util.Map; @@ -221,6 +223,30 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter return column.getBitmapIndex().getConciseSet(dimVal); } + @Override + public ImmutableConciseSet getInvertedIndex(String dimension, int idx) + { + final Column column = index.getColumn(dimension.toLowerCase()); + if (column == null) { + return new ImmutableConciseSet(); + } + if (!column.getCapabilities().hasBitmapIndexes()) { + return new ImmutableConciseSet(); + } + + return column.getBitmapIndex().getConciseSet(idx); + } + + public ImmutableRTree getRTreeSpatialIndex(String dimension) + { + final Column column = index.getColumn(dimension.toLowerCase()); + if (column == null || !column.getCapabilities().hasSpatialIndexes()) { + return new ImmutableRTree(); + } + + return column.getSpatialIndex().getRTree(dimension); + } + @Override public Offset getFilterOffset(Filter filter) { @@ -344,8 +370,7 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter return column.lookupId(name); } }; - } - else { + } else { return new DimensionSelector() { @Override @@ -404,8 +429,8 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter if (cachedMetricVals == null) { Column holder = index.getColumn(metricName); if (holder != null && holder.getCapabilities().getType() == ValueType.FLOAT) { - cachedMetricVals = holder.getGenericColumn(); - genericColumnCache.put(metricName, cachedMetricVals); + cachedMetricVals = holder.getGenericColumn(); + genericColumnCache.put(metricName, cachedMetricVals); } } @@ -648,8 +673,7 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter return column.lookupId(name); } }; - } - else { + } else { return new DimensionSelector() { @Override @@ -708,8 +732,8 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter if (cachedMetricVals == null) { Column holder = index.getColumn(metricName); if (holder != null && holder.getCapabilities().getType() == ValueType.FLOAT) { - cachedMetricVals = holder.getGenericColumn(); - genericColumnCache.put(metricName, cachedMetricVals); + cachedMetricVals = holder.getGenericColumn(); + genericColumnCache.put(metricName, cachedMetricVals); } } @@ -863,5 +887,17 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter { return getInvertedIndex(dimension, value); } + + @Override + public ImmutableConciseSet getConciseInvertedIndex(String dimension, int idx) + { + return getInvertedIndex(dimension, idx); + } + + @Override + public ImmutableRTree getSpatialIndex(String dimension) + { + return getRTreeSpatialIndex(dimension); + } } }