Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

make spatial filters work

This commit is contained in:
fjy 2013-05-21 17:10:44 -07:00
parent 69c7131a7b
commit 6e5b893e6d
9 changed files with 66 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
index-common/src/main/java/com/metamx/druid/index/column/BitmapIndex.java
View File

@ -27,6 +27,7 @@ public interface BitmapIndex
{
public int getCardinality();
public String getValue(int index);
public boolean hasNulls();
public ImmutableConciseSet getConciseSet(String value);
public ImmutableConciseSet getConciseSet(int idx);
}

6
index-common/src/main/java/com/metamx/druid/index/serde/BitmapIndexColumnPartSupplier.java
View File

@ -58,6 +58,12 @@ public class BitmapIndexColumnPartSupplier implements Supplier<BitmapIndex>
return dictionary.get(index);
}
@Override
public boolean hasNulls()
{
return dictionary.indexOf(null) >= 0;
}
@Override
public ImmutableConciseSet getConciseSet(String value)
{

3
index-common/src/main/java/com/metamx/druid/index/v1/IndexMerger.java
View File

@ -790,6 +790,9 @@ public class IndexMerger
int count = 0;
for (String dimVal : IndexedIterable.create(dimVals)) {
progress.progress();
if (dimVal == null) {
continue;
}
List<String> stringCoords = Lists.newArrayList(SPLITTER.split(dimVal));
float[] coords = new float[stringCoords.size()];

27
index-common/src/main/java/com/metamx/druid/index/v1/SpatialDimensionRowFormatter.java
View File

@ -26,6 +26,7 @@ import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.primitives.Floats;
import com.metamx.druid.input.InputRow;
import java.util.Arrays;
@ -122,17 +123,33 @@ public class SpatialDimensionRowFormatter
for (SpatialDimensionSchema spatialDimension : spatialDimensions) {
List<String> spatialDimVals = Lists.newArrayList();
for (String partialSpatialDim : spatialDimension.getDims()) {
List<String> dimVals = row.getDimension(partialSpatialDim);
if (dimVals == null || dimVals.isEmpty()) {
return retVal;
if (isSpatialDimValsValid(dimVals)) {
spatialDimVals.addAll(dimVals);
}
spatialDimVals.addAll(dimVals);
}
spatialLookup.put(spatialDimension.getDimName(), Arrays.asList(JOINER.join(spatialDimVals)));
finalDims.add(spatialDimension.getDimName());
if (!spatialDimVals.isEmpty()) {
spatialLookup.put(spatialDimension.getDimName(), Arrays.asList(JOINER.join(spatialDimVals)));
finalDims.add(spatialDimension.getDimName());
}
}
return retVal;
}
private boolean isSpatialDimValsValid(List<String> dimVals)
{
if (dimVals == null || dimVals.isEmpty()) {
return false;
}
for (String dimVal : dimVals) {
if (Floats.tryParse(dimVal) == null) {
return false;
}
}
return true;
}
}

2
pom.xml
View File

@ -162,7 +162,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>11.0.1</version>
<version>14.0.1</version>
</dependency>
<dependency>
<groupId>com.google.inject</groupId>

4
server/src/main/java/com/metamx/druid/index/brita/RegexFilter.java
View File

@ -40,9 +40,9 @@ public class RegexFilter extends DimensionPredicateFilter
Pattern compiled = Pattern.compile(pattern);
@Override
public boolean apply(@Nullable String input)
public boolean apply(String input)
{
return compiled.matcher(input).find();
return (input != null) && compiled.matcher(input).find();
}
}
);

2
server/src/main/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapter.java
View File

@ -227,7 +227,7 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter
numAdvanced++;
}
} else {
Iterators.skip(baseIter, numAdvanced);
Iterators.advance(baseIter, numAdvanced);
if (baseIter.hasNext()) {
currEntry.set(baseIter.next());
}

4
server/src/main/java/com/metamx/druid/index/v1/QueryableIndexStorageAdapter.java
View File

@ -237,8 +237,10 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter
if (!column.getCapabilities().hasBitmapIndexes()) {
return new ImmutableConciseSet();
}
// This is a workaround given the current state of indexing, I feel shame
final int index = column.getBitmapIndex().hasNulls() ? idx + 1 : idx;
return column.getBitmapIndex().getConciseSet(idx);
return column.getBitmapIndex().getConciseSet(index);
}
public ImmutableRTree getRTreeSpatialIndex(String dimension)

28
server/src/test/java/com/metamx/druid/index/brita/SpatialFilterTest.java
View File

@ -61,7 +61,7 @@ import java.util.List;
import java.util.Random;
/**
*/
*/
@RunWith(Parameterized.class)
public class SpatialFilterTest
{
@ -175,6 +175,19 @@ public class SpatialFilterTest
)
)
);
theIndex.add(
new MapBasedInputRow(
new DateTime("2013-01-05").getMillis(),
DIMS,
ImmutableMap.<String, Object>of(
"timestamp", new DateTime("2013-01-05").toString(),
"dim", "foo",
"lat", "_mmx.unknown",
"long", "_mmx.unknown",
"val", 101l
)
)
);
// Add a bunch of random points
Random rand = new Random();
@ -292,6 +305,19 @@ public class SpatialFilterTest
)
)
);
first.add(
new MapBasedInputRow(
new DateTime("2013-01-05").getMillis(),
DIMS,
ImmutableMap.<String, Object>of(
"timestamp", new DateTime("2013-01-05").toString(),
"dim", "foo",
"lat", "_mmx.unknown",
"long", "_mmx.unknown",
"val", 101l
)
)
);
second.add(
new MapBasedInputRow(
new DateTime("2013-01-04").getMillis(),