From c02d887cfe919381da7709477fc133ce107d5a62 Mon Sep 17 00:00:00 2001 From: cheddar Date: Wed, 28 Aug 2013 18:50:40 -0500 Subject: [PATCH] 1) Rewrite SearchQueryRunner to not require StorageAdapter to be "Searchable" 2) Extract SearchQueryRunner out of SearchQueryRunnerFactory 3) Extract ColumnSelectorBitmapIndexSelector out to make it reusable --- .../com/metamx/druid/sql/antlr4/DruidSQL.g4 | 52 ++++- .../main/java/com/metamx/druid/Druids.java | 6 +- .../query/search/FragmentSearchQuerySpec.java | 13 +- .../InsensitiveContainsSearchQuerySpec.java | 13 +- .../druid/query/search/SearchQuery.java | 4 +- .../druid/query/search/SearchQuerySpec.java | 12 -- .../SearchQuerySpecDimExtractionFnTest.java | 3 +- .../com/metamx/druid/BaseStorageAdapter.java | 93 --------- .../v1/ColumnSelectorBitmapIndexSelector.java | 143 +++++++++++++ .../v1/IncrementalIndexStorageAdapter.java | 101 +-------- .../v1/QueryableIndexStorageAdapter.java | 195 +----------------- .../v1/SegmentIdAttachedStorageAdapter.java | 100 --------- .../druid/query/search/SearchQueryRunner.java | 165 +++++++++++++++ .../search/SearchQueryRunnerFactory.java | 54 ----- .../query/search/SearchQueryRunnerTest.java | 12 +- 15 files changed, 381 insertions(+), 585 deletions(-) delete mode 100644 server/src/main/java/com/metamx/druid/BaseStorageAdapter.java create mode 100644 server/src/main/java/com/metamx/druid/index/v1/ColumnSelectorBitmapIndexSelector.java delete mode 100644 server/src/main/java/com/metamx/druid/index/v1/SegmentIdAttachedStorageAdapter.java create mode 100644 server/src/main/java/com/metamx/druid/query/search/SearchQueryRunner.java diff --git a/client/src/main/antlr4/com/metamx/druid/sql/antlr4/DruidSQL.g4 b/client/src/main/antlr4/com/metamx/druid/sql/antlr4/DruidSQL.g4 index 9dd297dc8c0..d1ce41a1cc5 100644 --- a/client/src/main/antlr4/com/metamx/druid/sql/antlr4/DruidSQL.g4 +++ b/client/src/main/antlr4/com/metamx/druid/sql/antlr4/DruidSQL.g4 @@ -1,18 +1,50 @@ grammar DruidSQL; @header { -import com.metamx.druid.aggregation.post.*; -import com.metamx.druid.aggregation.*; -import com.metamx.druid.query.filter.*; -import com.metamx.druid.query.dimension.*; -import com.metamx.druid.*; - -import com.google.common.base.*; +import com.google.common.base.Joiner; import com.google.common.collect.Lists; -import org.joda.time.*; +import com.metamx.druid.aggregation.CountAggregatorFactory; +import com.metamx.druid.aggregation.DoubleSumAggregatorFactory; +import com.metamx.druid.aggregation.MaxAggregatorFactory; +import com.metamx.druid.aggregation.MinAggregatorFactory; +import com.metamx.druid.aggregation.post.ArithmeticPostAggregator; +import com.metamx.druid.aggregation.post.ConstantPostAggregator; +import com.metamx.druid.aggregation.post.FieldAccessPostAggregator; +import com.metamx.druid.aggregation.post.PostAggregator; +import com.metamx.druid.query.dimension.DefaultDimensionSpec; +import com.metamx.druid.query.dimension.DimensionSpec; +import com.metamx.druid.query.filter.AndDimFilter; +import com.metamx.druid.query.filter.DimFilter; +import com.metamx.druid.query.filter.NotDimFilter; +import com.metamx.druid.query.filter.OrDimFilter; +import com.metamx.druid.query.filter.RegexDimFilter; +import com.metamx.druid.query.filter.SelectorDimFilter; +import io.druid.granularity.PeriodGranularity; +import io.druid.granularity.QueryGranularity; +import io.druid.query.aggregation.AggregatorFactory; +import org.antlr.v4.runtime.NoViableAltException; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.TokenStream; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.atn.ATNSimulator; +import org.antlr.v4.runtime.atn.ParserATNSimulator; +import org.antlr.v4.runtime.atn.PredictionContextCache; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.runtime.tree.ParseTreeListener; +import org.antlr.v4.runtime.tree.TerminalNode; +import org.joda.time.DateTime; +import org.joda.time.Period; -import java.text.*; -import java.util.*; +import java.text.NumberFormat; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; } @parser::members { diff --git a/client/src/main/java/com/metamx/druid/Druids.java b/client/src/main/java/com/metamx/druid/Druids.java index dd02ad85d32..53ad282a036 100644 --- a/client/src/main/java/com/metamx/druid/Druids.java +++ b/client/src/main/java/com/metamx/druid/Druids.java @@ -524,7 +524,7 @@ public class Druids querySegmentSpec, dimensions, querySpec, - querySpec.getSearchSortSpec(), + null, context ); } @@ -639,13 +639,13 @@ public class Druids public SearchQueryBuilder query(String q) { - querySpec = new InsensitiveContainsSearchQuerySpec(q, null); + querySpec = new InsensitiveContainsSearchQuerySpec(q); return this; } public SearchQueryBuilder query(Map q) { - querySpec = new InsensitiveContainsSearchQuerySpec((String) q.get("value"), null); + querySpec = new InsensitiveContainsSearchQuerySpec((String) q.get("value")); return this; } diff --git a/client/src/main/java/com/metamx/druid/query/search/FragmentSearchQuerySpec.java b/client/src/main/java/com/metamx/druid/query/search/FragmentSearchQuerySpec.java index c4a7b707712..3abf19d5124 100644 --- a/client/src/main/java/com/metamx/druid/query/search/FragmentSearchQuerySpec.java +++ b/client/src/main/java/com/metamx/druid/query/search/FragmentSearchQuerySpec.java @@ -34,12 +34,10 @@ public class FragmentSearchQuerySpec implements SearchQuerySpec private static final byte CACHE_TYPE_ID = 0x2; private final List values; - private final SearchSortSpec sortSpec; @JsonCreator public FragmentSearchQuerySpec( - @JsonProperty("values") List values, - @JsonProperty("sort") SearchSortSpec sortSpec + @JsonProperty("values") List values ) { this.values = Lists.transform( @@ -53,7 +51,6 @@ public class FragmentSearchQuerySpec implements SearchQuerySpec } } ); - this.sortSpec = (sortSpec == null) ? new LexicographicSearchSortSpec() : sortSpec; } @JsonProperty @@ -62,13 +59,6 @@ public class FragmentSearchQuerySpec implements SearchQuerySpec return values; } - @JsonProperty("sort") - @Override - public SearchSortSpec getSearchSortSpec() - { - return sortSpec; - } - @Override public boolean accept(String dimVal) { @@ -107,7 +97,6 @@ public class FragmentSearchQuerySpec implements SearchQuerySpec { return "FragmentSearchQuerySpec{" + "values=" + values + - ", sortSpec=" + sortSpec + "}"; } } diff --git a/client/src/main/java/com/metamx/druid/query/search/InsensitiveContainsSearchQuerySpec.java b/client/src/main/java/com/metamx/druid/query/search/InsensitiveContainsSearchQuerySpec.java index 80c609cae76..44b0ee07197 100644 --- a/client/src/main/java/com/metamx/druid/query/search/InsensitiveContainsSearchQuerySpec.java +++ b/client/src/main/java/com/metamx/druid/query/search/InsensitiveContainsSearchQuerySpec.java @@ -31,16 +31,13 @@ public class InsensitiveContainsSearchQuerySpec implements SearchQuerySpec private static final byte CACHE_TYPE_ID = 0x1; private final String value; - private final SearchSortSpec sortSpec; @JsonCreator public InsensitiveContainsSearchQuerySpec( - @JsonProperty("value") String value, - @JsonProperty("sort") SearchSortSpec sortSpec + @JsonProperty("value") String value ) { this.value = value.toLowerCase(); - this.sortSpec = (sortSpec == null) ? new LexicographicSearchSortSpec() : sortSpec; } @JsonProperty @@ -49,13 +46,6 @@ public class InsensitiveContainsSearchQuerySpec implements SearchQuerySpec return value; } - @JsonProperty("sort") - @Override - public SearchSortSpec getSearchSortSpec() - { - return sortSpec; - } - @Override public boolean accept(String dimVal) { @@ -81,7 +71,6 @@ public class InsensitiveContainsSearchQuerySpec implements SearchQuerySpec { return "InsensitiveContainsSearchQuerySpec{" + "value=" + value + - ", sortSpec=" + sortSpec + "}"; } } diff --git a/client/src/main/java/com/metamx/druid/query/search/SearchQuery.java b/client/src/main/java/com/metamx/druid/query/search/SearchQuery.java index b81e5e54737..8702156598b 100644 --- a/client/src/main/java/com/metamx/druid/query/search/SearchQuery.java +++ b/client/src/main/java/com/metamx/druid/query/search/SearchQuery.java @@ -62,7 +62,7 @@ public class SearchQuery extends BaseQuery> { super(dataSource, querySegmentSpec, context); this.dimFilter = dimFilter; - this.sortSpec = sortSpec; + this.sortSpec = sortSpec == null ? new LexicographicSearchSortSpec() : sortSpec; this.granularity = granularity == null ? QueryGranularity.ALL : granularity; this.limit = (limit == 0) ? 1000 : limit; this.dimensions = (dimensions == null) ? null : Lists.transform( @@ -159,7 +159,7 @@ public class SearchQuery extends BaseQuery> @JsonProperty("sort") public SearchSortSpec getSort() { - return sortSpec == null ? querySpec.getSearchSortSpec() : sortSpec; + return sortSpec; } public SearchQuery withLimit(int newLimit) diff --git a/client/src/main/java/com/metamx/druid/query/search/SearchQuerySpec.java b/client/src/main/java/com/metamx/druid/query/search/SearchQuerySpec.java index 68eb86dfb73..a54aae35ea8 100644 --- a/client/src/main/java/com/metamx/druid/query/search/SearchQuerySpec.java +++ b/client/src/main/java/com/metamx/druid/query/search/SearchQuerySpec.java @@ -31,18 +31,6 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; }) public interface SearchQuerySpec { - /** - * Deprecated! - * - * This has been moved to the SearchQuery and is only still here for backwards compatibility purposes. Search - * queries should be adjusted to use the sort parameter on the SearchQuery object itself rather than on this - * object. This method will eventually go away. - * - * @return - */ - @Deprecated - public SearchSortSpec getSearchSortSpec(); - public boolean accept(String dimVal); public byte[] getCacheKey(); diff --git a/client/src/test/java/com/metamx/druid/query/extraction/SearchQuerySpecDimExtractionFnTest.java b/client/src/test/java/com/metamx/druid/query/extraction/SearchQuerySpecDimExtractionFnTest.java index 808c3603b6b..06e9cde0ffd 100644 --- a/client/src/test/java/com/metamx/druid/query/extraction/SearchQuerySpecDimExtractionFnTest.java +++ b/client/src/test/java/com/metamx/druid/query/extraction/SearchQuerySpecDimExtractionFnTest.java @@ -21,7 +21,6 @@ package com.metamx.druid.query.extraction; import com.google.common.collect.Sets; import com.metamx.druid.query.search.FragmentSearchQuerySpec; -import com.metamx.druid.query.search.LexicographicSearchSortSpec; import com.metamx.druid.query.search.SearchQuerySpec; import org.junit.Assert; import org.junit.Test; @@ -49,7 +48,7 @@ public class SearchQuerySpecDimExtractionFnTest public void testExtraction() { SearchQuerySpec spec = new FragmentSearchQuerySpec( - Arrays.asList("to", "yo"), new LexicographicSearchSortSpec() + Arrays.asList("to", "yo") ); DimExtractionFn dimExtractionFn = new SearchQuerySpecDimExtractionFn(spec); List expected = Arrays.asList("Kyoto", "Tokyo", "Toyokawa", "Yorktown"); diff --git a/server/src/main/java/com/metamx/druid/BaseStorageAdapter.java b/server/src/main/java/com/metamx/druid/BaseStorageAdapter.java deleted file mode 100644 index 34ec2986b63..00000000000 --- a/server/src/main/java/com/metamx/druid/BaseStorageAdapter.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid; - -import com.google.common.collect.Sets; -import com.metamx.common.guava.FunctionalIterable; -import com.metamx.druid.index.v1.ConciseOffset; -import com.metamx.druid.index.v1.processing.IntersectingOffset; -import com.metamx.druid.index.v1.processing.Offset; -import com.metamx.druid.query.search.SearchHit; -import com.metamx.druid.query.search.SearchQuery; -import com.metamx.druid.query.search.SearchQuerySpec; -import io.druid.data.Indexed; -import io.druid.query.filter.Filter; -import io.druid.segment.StorageAdapter; -import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; - -import java.util.List; -import java.util.TreeSet; - -/** - */ -public abstract class BaseStorageAdapter implements StorageAdapter -{ - public abstract Indexed getAvailableDimensions(); - - public abstract Indexed getDimValueLookup(String dimension); - - public abstract ImmutableConciseSet getInvertedIndex(String dimension, String dimVal); - - public abstract ImmutableConciseSet getInvertedIndex(String dimension, int idx); - - public abstract Offset getFilterOffset(Filter filter); - - @Override - public Iterable searchDimensions(final SearchQuery query, final Filter filter) - { - final List dimensions = query.getDimensions(); - final SearchQuerySpec searchQuerySpec = query.getQuery(); - - final TreeSet retVal = Sets.newTreeSet(query.getSort().getComparator()); - - Iterable dimsToSearch; - if (dimensions == null || dimensions.isEmpty()) { - dimsToSearch = getAvailableDimensions(); - } else { - dimsToSearch = dimensions; - } - - Offset filterOffset = (filter == null) ? null : getFilterOffset(filter); - - for (String dimension : dimsToSearch) { - Iterable dims = getDimValueLookup(dimension); - if (dims != null) { - for (String dimVal : dims) { - dimVal = dimVal == null ? "" : dimVal; - if (searchQuerySpec.accept(dimVal)) { - if (filterOffset != null) { - Offset lhs = new ConciseOffset(getInvertedIndex(dimension, dimVal)); - Offset rhs = filterOffset.clone(); - - if (new IntersectingOffset(lhs, rhs).withinBounds()) { - retVal.add(new SearchHit(dimension, dimVal)); - } - } else { - retVal.add(new SearchHit(dimension, dimVal)); - } - } - } - } - } - - return new FunctionalIterable(retVal).limit(query.getLimit()); - } -} - diff --git a/server/src/main/java/com/metamx/druid/index/v1/ColumnSelectorBitmapIndexSelector.java b/server/src/main/java/com/metamx/druid/index/v1/ColumnSelectorBitmapIndexSelector.java new file mode 100644 index 00000000000..6489655a59a --- /dev/null +++ b/server/src/main/java/com/metamx/druid/index/v1/ColumnSelectorBitmapIndexSelector.java @@ -0,0 +1,143 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package com.metamx.druid.index.v1; + +import com.google.common.io.Closeables; +import com.metamx.collections.spatial.ImmutableRTree; +import com.metamx.druid.kv.IndexedIterable; +import io.druid.data.Indexed; +import io.druid.query.filter.BitmapIndexSelector; +import io.druid.segment.ColumnSelector; +import io.druid.segment.column.Column; +import io.druid.segment.column.DictionaryEncodedColumn; +import io.druid.segment.column.GenericColumn; +import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; + +import java.util.Iterator; + +/** +*/ +public class ColumnSelectorBitmapIndexSelector implements BitmapIndexSelector +{ + private final ColumnSelector index; + + public ColumnSelectorBitmapIndexSelector( + final ColumnSelector index + ) + { + this.index = index; + } + + @Override + public Indexed getDimensionValues(String dimension) + { + final Column columnDesc = index.getColumn(dimension.toLowerCase()); + if (columnDesc == null || !columnDesc.getCapabilities().isDictionaryEncoded()) { + return null; + } + final DictionaryEncodedColumn column = columnDesc.getDictionaryEncoding(); + return new Indexed() + { + @Override + public Class getClazz() + { + return String.class; + } + + @Override + public int size() + { + return column.getCardinality(); + } + + @Override + public String get(int index) + { + return column.lookupName(index); + } + + @Override + public int indexOf(String value) + { + return column.lookupId(value); + } + + @Override + public Iterator iterator() + { + return IndexedIterable.create(this).iterator(); + } + }; + } + + @Override + public int getNumRows() + { + GenericColumn column = null; + try { + column = index.getTimeColumn().getGenericColumn(); + return column.length(); + } + finally { + Closeables.closeQuietly(column); + } + } + + @Override + public ImmutableConciseSet getConciseInvertedIndex(String dimension, String value) + { + final Column column = index.getColumn(dimension.toLowerCase()); + if (column == null) { + return new ImmutableConciseSet(); + } + if (!column.getCapabilities().hasBitmapIndexes()) { + return new ImmutableConciseSet(); + } + + return column.getBitmapIndex().getConciseSet(value); + } + + @Override + public ImmutableConciseSet getConciseInvertedIndex(String dimension, int idx) + { + final Column column = index.getColumn(dimension.toLowerCase()); + if (column == null) { + return new ImmutableConciseSet(); + } + if (!column.getCapabilities().hasBitmapIndexes()) { + return new ImmutableConciseSet(); + } + // This is a workaround given the current state of indexing, I feel shame + final int index1 = column.getBitmapIndex().hasNulls() ? idx + 1 : idx; + + return column.getBitmapIndex().getConciseSet(index1); + } + + @Override + public ImmutableRTree getSpatialIndex(String dimension) + { + final Column column = index.getColumn(dimension.toLowerCase()); + if (column == null || !column.getCapabilities().hasSpatialIndexes()) { + return new ImmutableRTree(); + } + + return column.getSpatialIndex().getRTree(); + } +} diff --git a/server/src/main/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapter.java b/server/src/main/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapter.java index 88fb9dc406f..80a734fe8ea 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapter.java +++ b/server/src/main/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapter.java @@ -25,17 +25,13 @@ import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import com.metamx.collections.spatial.search.Bound; -import com.metamx.common.IAE; -import com.metamx.common.guava.FunctionalIterable; import com.metamx.common.guava.FunctionalIterator; import com.metamx.druid.index.brita.BooleanValueMatcher; import com.metamx.druid.index.v1.serde.ComplexMetricSerde; import com.metamx.druid.index.v1.serde.ComplexMetrics; -import com.metamx.druid.query.search.SearchHit; -import com.metamx.druid.query.search.SearchQuery; -import com.metamx.druid.query.search.SearchQuerySpec; +import com.metamx.druid.kv.ListIndexed; +import io.druid.data.Indexed; import io.druid.data.IndexedInts; import io.druid.granularity.QueryGranularity; import io.druid.query.aggregation.Aggregator; @@ -57,7 +53,6 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.TreeSet; import java.util.concurrent.ConcurrentNavigableMap; /** @@ -87,6 +82,12 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter return index.getInterval(); } + @Override + public Indexed getAvailableDimensions() + { + return new ListIndexed(index.getDimensions(), String.class); + } + @Override public int getDimensionCardinality(String dimension) { @@ -432,92 +433,6 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter }; } - @Override - public Iterable searchDimensions(final SearchQuery query, final Filter filter) - { - final List dimensions = query.getDimensions(); - final int[] dimensionIndexes; - final String[] dimensionNames; - final List dimensionOrder = index.getDimensions(); - if (dimensions == null || dimensions.isEmpty()) { - dimensionIndexes = new int[dimensionOrder.size()]; - dimensionNames = new String[dimensionIndexes.length]; - - Iterator dimensionOrderIter = dimensionOrder.iterator(); - for (int i = 0; i < dimensionIndexes.length; ++i) { - dimensionNames[i] = dimensionOrderIter.next(); - dimensionIndexes[i] = index.getDimensionIndex(dimensionNames[i]); - } - } else { - int[] tmpDimensionIndexes = new int[dimensions.size()]; - String[] tmpDimensionNames = new String[dimensions.size()]; - int i = 0; - for (String dimension : dimensions) { - Integer dimIndex = index.getDimensionIndex(dimension.toLowerCase()); - if (dimIndex != null) { - tmpDimensionNames[i] = dimension; - tmpDimensionIndexes[i] = dimIndex; - ++i; - } - } - - if (i != tmpDimensionIndexes.length) { - dimensionIndexes = new int[i]; - dimensionNames = new String[i]; - System.arraycopy(tmpDimensionIndexes, 0, dimensionIndexes, 0, i); - System.arraycopy(tmpDimensionNames, 0, dimensionNames, 0, i); - } else { - dimensionIndexes = tmpDimensionIndexes; - dimensionNames = tmpDimensionNames; - } - } - - final List queryIntervals = query.getIntervals(); - if (queryIntervals.size() != 1) { - throw new IAE("Can only handle one interval, got query[%s]", query); - } - - final Interval queryInterval = queryIntervals.get(0); - final long intervalStart = queryInterval.getStartMillis(); - final long intervalEnd = queryInterval.getEndMillis(); - - final EntryHolder holder = new EntryHolder(); - final ValueMatcher theMatcher = makeFilterMatcher(filter, holder); - final SearchQuerySpec searchQuerySpec = query.getQuery(); - final TreeSet retVal = Sets.newTreeSet(query.getSort().getComparator()); - - ConcurrentNavigableMap facts = index.getSubMap( - new IncrementalIndex.TimeAndDims(intervalStart, new String[][]{}), - new IncrementalIndex.TimeAndDims(intervalEnd, new String[][]{}) - ); - - for (Map.Entry entry : facts.entrySet()) { - holder.set(entry); - final IncrementalIndex.TimeAndDims key = holder.getKey(); - final long timestamp = key.getTimestamp(); - - if (timestamp >= intervalStart && timestamp < intervalEnd && theMatcher.matches()) { - final String[][] dims = key.getDims(); - - for (int i = 0; i < dimensionIndexes.length; ++i) { - if (dimensionIndexes[i] < dims.length) { - final String[] dimVals = dims[dimensionIndexes[i]]; - if (dimVals != null) { - for (int j = 0; j < dimVals.length; ++j) { - if (searchQuerySpec.accept(dimVals[j])) { - retVal.add(new SearchHit(dimensionNames[i], dimVals[j])); - } - } - } - } - } - } - } - - - return new FunctionalIterable(retVal).limit(query.getLimit()); - } - private ValueMatcher makeFilterMatcher(final Filter filter, final EntryHolder holder) { return filter == null diff --git a/server/src/main/java/com/metamx/druid/index/v1/QueryableIndexStorageAdapter.java b/server/src/main/java/com/metamx/druid/index/v1/QueryableIndexStorageAdapter.java index 8129b57c368..817ac8a9dec 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/QueryableIndexStorageAdapter.java +++ b/server/src/main/java/com/metamx/druid/index/v1/QueryableIndexStorageAdapter.java @@ -24,18 +24,14 @@ import com.google.common.base.Functions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import com.google.common.io.Closeables; -import com.metamx.collections.spatial.ImmutableRTree; import com.metamx.common.collect.MoreIterators; import com.metamx.common.guava.FunctionalIterable; import com.metamx.common.guava.FunctionalIterator; -import com.metamx.druid.BaseStorageAdapter; import com.metamx.druid.index.v1.processing.Offset; -import com.metamx.druid.kv.IndexedIterable; import com.metamx.druid.kv.SingleIndexedInts; import io.druid.data.Indexed; import io.druid.data.IndexedInts; import io.druid.granularity.QueryGranularity; -import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.filter.Filter; import io.druid.segment.Capabilities; import io.druid.segment.ColumnSelector; @@ -45,13 +41,13 @@ import io.druid.segment.DimensionSelector; import io.druid.segment.FloatMetricSelector; import io.druid.segment.ObjectMetricSelector; import io.druid.segment.QueryableIndex; +import io.druid.segment.StorageAdapter; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; import io.druid.segment.column.ComplexColumn; import io.druid.segment.column.DictionaryEncodedColumn; import io.druid.segment.column.GenericColumn; import io.druid.segment.column.ValueType; -import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -61,7 +57,7 @@ import java.util.Map; /** */ -public class QueryableIndexStorageAdapter extends BaseStorageAdapter +public class QueryableIndexStorageAdapter implements StorageAdapter { private final QueryableIndex index; @@ -84,6 +80,12 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter return index.getDataInterval(); } + @Override + public Indexed getAvailableDimensions() + { + return index.getAvailableDimensions(); + } + @Override public int getDimensionCardinality(String dimension) { @@ -155,7 +157,7 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter if (filter == null) { iterable = new NoFilterCursorIterable(index, actualInterval, gran); } else { - Offset offset = new ConciseOffset(filter.goConcise(new MMappedBitmapIndexSelector(index))); + Offset offset = new ConciseOffset(filter.goConcise(new ColumnSelectorBitmapIndexSelector(index))); iterable = new CursorIterable(index, actualInterval, gran, offset); } @@ -163,102 +165,6 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter return FunctionalIterable.create(iterable).keep(Functions.identity()); } - @Override - public Indexed getAvailableDimensions() - { - return index.getAvailableDimensions(); - } - - @Override - public Indexed getDimValueLookup(String dimension) - { - final Column column = index.getColumn(dimension.toLowerCase()); - - if (column == null || !column.getCapabilities().isDictionaryEncoded()) { - return null; - } - - final DictionaryEncodedColumn dictionary = column.getDictionaryEncoding(); - return new Indexed() - { - @Override - public Class getClazz() - { - return String.class; - } - - @Override - public int size() - { - return dictionary.getCardinality(); - } - - @Override - public String get(int index) - { - return dictionary.lookupName(index); - } - - @Override - public int indexOf(String value) - { - return dictionary.lookupId(value); - } - - @Override - public Iterator iterator() - { - return IndexedIterable.create(this).iterator(); - } - }; - } - - @Override - public ImmutableConciseSet getInvertedIndex(String dimension, String dimVal) - { - final Column column = index.getColumn(dimension.toLowerCase()); - if (column == null) { - return new ImmutableConciseSet(); - } - if (!column.getCapabilities().hasBitmapIndexes()) { - return new ImmutableConciseSet(); - } - - return column.getBitmapIndex().getConciseSet(dimVal); - } - - @Override - public ImmutableConciseSet getInvertedIndex(String dimension, int idx) - { - final Column column = index.getColumn(dimension.toLowerCase()); - if (column == null) { - return new ImmutableConciseSet(); - } - if (!column.getCapabilities().hasBitmapIndexes()) { - return new ImmutableConciseSet(); - } - // This is a workaround given the current state of indexing, I feel shame - final int index = column.getBitmapIndex().hasNulls() ? idx + 1 : idx; - - return column.getBitmapIndex().getConciseSet(index); - } - - public ImmutableRTree getRTreeSpatialIndex(String dimension) - { - final Column column = index.getColumn(dimension.toLowerCase()); - if (column == null || !column.getCapabilities().hasSpatialIndexes()) { - return new ImmutableRTree(); - } - - return column.getSpatialIndex().getRTree(); - } - - @Override - public Offset getFilterOffset(Filter filter) - { - return new ConciseOffset(filter.goConcise(new MMappedBitmapIndexSelector(index))); - } - private static class CursorIterable implements Iterable { private final ColumnSelector index; @@ -1081,87 +987,4 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter return 0; } } - - private class MMappedBitmapIndexSelector implements BitmapIndexSelector - { - private final ColumnSelector index; - - public MMappedBitmapIndexSelector(final ColumnSelector index) - { - this.index = index; - } - - @Override - public Indexed getDimensionValues(String dimension) - { - final Column columnDesc = index.getColumn(dimension.toLowerCase()); - if (columnDesc == null || !columnDesc.getCapabilities().isDictionaryEncoded()) { - return null; - } - final DictionaryEncodedColumn column = columnDesc.getDictionaryEncoding(); - return new Indexed() - { - @Override - public Class getClazz() - { - return String.class; - } - - @Override - public int size() - { - return column.getCardinality(); - } - - @Override - public String get(int index) - { - return column.lookupName(index); - } - - @Override - public int indexOf(String value) - { - return column.lookupId(value); - } - - @Override - public Iterator iterator() - { - return IndexedIterable.create(this).iterator(); - } - }; - } - - @Override - public int getNumRows() - { - GenericColumn column = null; - try { - column = index.getTimeColumn().getGenericColumn(); - return column.length(); - } - finally { - Closeables.closeQuietly(column); - } - } - - @Override - public ImmutableConciseSet getConciseInvertedIndex(String dimension, String value) - { - return getInvertedIndex(dimension, value); - } - - @Override - public ImmutableConciseSet getConciseInvertedIndex(String dimension, int idx) - { - return getInvertedIndex(dimension, idx); - } - - @Override - public ImmutableRTree getSpatialIndex(String dimension) - { - return getRTreeSpatialIndex(dimension); - } - } } diff --git a/server/src/main/java/com/metamx/druid/index/v1/SegmentIdAttachedStorageAdapter.java b/server/src/main/java/com/metamx/druid/index/v1/SegmentIdAttachedStorageAdapter.java deleted file mode 100644 index 5842cd136bd..00000000000 --- a/server/src/main/java/com/metamx/druid/index/v1/SegmentIdAttachedStorageAdapter.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.index.v1; - -import com.metamx.druid.query.search.SearchHit; -import com.metamx.druid.query.search.SearchQuery; -import io.druid.granularity.QueryGranularity; -import io.druid.query.filter.Filter; -import io.druid.segment.Capabilities; -import io.druid.segment.Cursor; -import io.druid.segment.StorageAdapter; -import org.joda.time.DateTime; -import org.joda.time.Interval; - -/** - */ -public class SegmentIdAttachedStorageAdapter implements StorageAdapter -{ - private final String segmentId; - private final StorageAdapter delegate; - - public SegmentIdAttachedStorageAdapter( - String segmentId, - StorageAdapter delegate - ) - { - this.segmentId = segmentId; - this.delegate = delegate; - } - - @Override - public String getSegmentIdentifier() - { - return segmentId; - } - - @Override - public Interval getInterval() - { - return delegate.getInterval(); - } - - @Override - public Iterable searchDimensions(SearchQuery query, Filter filter) - { - return delegate.searchDimensions(query, filter); - } - - @Override - public Iterable makeCursors(Filter filter, Interval interval, QueryGranularity gran) - { - return delegate.makeCursors(filter, interval, gran); - } - - @Override - public Capabilities getCapabilities() - { - return delegate.getCapabilities(); - } - - @Override - public DateTime getMaxTime() - { - return delegate.getMaxTime(); - } - - @Override - public DateTime getMinTime() - { - return delegate.getMinTime(); - } - - @Override - public int getDimensionCardinality(String dimension) - { - return delegate.getDimensionCardinality(dimension); - } - - public StorageAdapter getDelegate() - { - return delegate; - } -} diff --git a/server/src/main/java/com/metamx/druid/query/search/SearchQueryRunner.java b/server/src/main/java/com/metamx/druid/query/search/SearchQueryRunner.java new file mode 100644 index 00000000000..2e25720122f --- /dev/null +++ b/server/src/main/java/com/metamx/druid/query/search/SearchQueryRunner.java @@ -0,0 +1,165 @@ +package com.metamx.druid.query.search; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.metamx.common.ISE; +import com.metamx.common.guava.FunctionalIterable; +import com.metamx.common.guava.Sequence; +import com.metamx.common.guava.Sequences; +import com.metamx.druid.index.brita.Filters; +import com.metamx.druid.index.v1.ColumnSelectorBitmapIndexSelector; +import com.metamx.druid.result.Result; +import com.metamx.druid.result.SearchResultValue; +import com.metamx.emitter.EmittingLogger; +import io.druid.data.IndexedInts; +import io.druid.granularity.QueryGranularity; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.filter.Filter; +import io.druid.segment.Cursor; +import io.druid.segment.DimensionSelector; +import io.druid.segment.QueryableIndex; +import io.druid.segment.Segment; +import io.druid.segment.StorageAdapter; +import io.druid.segment.column.BitmapIndex; +import io.druid.segment.column.Column; +import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; + +import java.util.List; +import java.util.Map; +import java.util.TreeSet; + +/** +*/ +public class SearchQueryRunner implements QueryRunner> +{ + private static final EmittingLogger log = new EmittingLogger(SearchQueryRunner.class); + + private final Segment segment; + + public SearchQueryRunner(Segment segment) + { + this.segment = segment; + } + + @Override + public Sequence> run(final Query> input) + { + if (!(input instanceof SearchQuery)) { + throw new ISE("Got a [%s] which isn't a %s", input.getClass(), SearchQuery.class); + } + + final SearchQuery query = (SearchQuery) input; + final Filter filter = Filters.convertDimensionFilters(query.getDimensionsFilter()); + final List dimensions = query.getDimensions(); + final SearchQuerySpec searchQuerySpec = query.getQuery(); + final int limit = query.getLimit(); + + final QueryableIndex index = segment.asQueryableIndex(); + if (index != null) { + final TreeSet retVal = Sets.newTreeSet(query.getSort().getComparator()); + + Iterable dimsToSearch; + if (dimensions == null || dimensions.isEmpty()) { + dimsToSearch = index.getAvailableDimensions(); + } else { + dimsToSearch = dimensions; + } + + + final ImmutableConciseSet baseFilter; + if (filter == null) { + // Accept all + baseFilter = ImmutableConciseSet.complement(new ImmutableConciseSet(), index.getNumRows()); + } + else { + baseFilter = filter.goConcise(new ColumnSelectorBitmapIndexSelector(index)); + } + + for (String dimension : dimsToSearch) { + final Column column = index.getColumn(dimension.toLowerCase()); + if (column == null) { + continue; + } + + final BitmapIndex bitmapIndex = column.getBitmapIndex(); + if (bitmapIndex != null) { + for (int i = 0; i < bitmapIndex.getCardinality(); ++i) { + String dimVal = Strings.nullToEmpty(bitmapIndex.getValue(i)); + if (searchQuerySpec.accept(dimVal) && + ImmutableConciseSet.intersection(baseFilter, bitmapIndex.getConciseSet(i)).size() > 0) { + retVal.add(new SearchHit(dimension, dimVal)); + if (retVal.size() >= limit) { + return makeReturnResult(limit, retVal); + } + } + } + } + } + + return makeReturnResult(limit, retVal); + } + + final StorageAdapter adapter = segment.asStorageAdapter(); + if (adapter != null) { + Iterable dimsToSearch; + if (dimensions == null || dimensions.isEmpty()) { + dimsToSearch = adapter.getAvailableDimensions(); + } else { + dimsToSearch = dimensions; + } + + final TreeSet retVal = Sets.newTreeSet(query.getSort().getComparator()); + + final Iterable cursors = adapter.makeCursors(filter, segment.getDataInterval(), QueryGranularity.ALL); + for (Cursor cursor : cursors) { + Map dimSelectors = Maps.newHashMap(); + for (String dim : dimsToSearch) { + dimSelectors.put(dim, cursor.makeDimensionSelector(dim)); + } + + while (!cursor.isDone()) { + for (Map.Entry entry : dimSelectors.entrySet()) { + final DimensionSelector selector = entry.getValue(); + final IndexedInts vals = selector.getRow(); + for (int i = 0; i < vals.size(); ++i) { + final String dimVal = selector.lookupName(vals.get(i)); + if (searchQuerySpec.accept(dimVal)) { + retVal.add(new SearchHit(entry.getKey(), dimVal)); + if (retVal.size() >= limit) { + return makeReturnResult(limit, retVal); + } + } + } + } + + cursor.advance(); + } + } + + return makeReturnResult(limit, retVal); + } + + log.makeAlert("WTF!? Unable to process search query on segment.") + .addData("segment", segment.getIdentifier()) + .addData("query", query); + return Sequences.empty(); + } + + private Sequence> makeReturnResult(int limit, TreeSet retVal) + { + return Sequences.simple( + ImmutableList.of( + new Result( + segment.getDataInterval().getStart(), + new SearchResultValue( + Lists.newArrayList(new FunctionalIterable(retVal).limit(limit)) + ) + ) + ) + ); + } +} diff --git a/server/src/main/java/com/metamx/druid/query/search/SearchQueryRunnerFactory.java b/server/src/main/java/com/metamx/druid/query/search/SearchQueryRunnerFactory.java index 53f5b560a17..9df93768f2c 100644 --- a/server/src/main/java/com/metamx/druid/query/search/SearchQueryRunnerFactory.java +++ b/server/src/main/java/com/metamx/druid/query/search/SearchQueryRunnerFactory.java @@ -19,24 +19,15 @@ package com.metamx.druid.query.search; -import com.google.common.collect.Iterators; import com.google.inject.Inject; -import com.metamx.common.ISE; -import com.metamx.common.guava.BaseSequence; -import com.metamx.common.guava.Sequence; -import com.metamx.druid.SearchResultBuilder; -import com.metamx.druid.index.brita.Filters; import com.metamx.druid.query.ChainedExecutionQueryRunner; import com.metamx.druid.result.Result; import com.metamx.druid.result.SearchResultValue; -import io.druid.query.Query; import io.druid.query.QueryRunner; import io.druid.query.QueryRunnerFactory; import io.druid.query.QueryToolChest; import io.druid.segment.Segment; -import io.druid.segment.StorageAdapter; -import java.util.Iterator; import java.util.concurrent.ExecutorService; /** @@ -74,49 +65,4 @@ public class SearchQueryRunnerFactory implements QueryRunnerFactory> - { - private final StorageAdapter adapter; - - public SearchQueryRunner(Segment segment) - { - this.adapter = segment.asStorageAdapter(); - } - - @Override - public Sequence> run(final Query> input) - { - if (!(input instanceof SearchQuery)) { - throw new ISE("Got a [%s] which isn't a %s", input.getClass(), SearchQuery.class); - } - - final SearchQuery query = (SearchQuery) input; - - return new BaseSequence, Iterator>>( - new BaseSequence.IteratorMaker, Iterator>>() - { - @Override - public Iterator> make() - { - return Iterators.singletonIterator( - new SearchResultBuilder( - adapter.getInterval().getStart(), - adapter.searchDimensions( - query, - Filters.convertDimensionFilters(query.getDimensionsFilter()) - ) - ).build() - ); - } - - @Override - public void cleanup(Iterator> toClean) - { - - } - } - ); - } - } } diff --git a/server/src/test/java/com/metamx/druid/query/search/SearchQueryRunnerTest.java b/server/src/test/java/com/metamx/druid/query/search/SearchQueryRunnerTest.java index 59184d4da60..b0512978b77 100644 --- a/server/src/test/java/com/metamx/druid/query/search/SearchQueryRunnerTest.java +++ b/server/src/test/java/com/metamx/druid/query/search/SearchQueryRunnerTest.java @@ -94,7 +94,7 @@ public class SearchQueryRunnerTest .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) .intervals(QueryRunnerTestHelper.fullOnInterval) - .query(new FragmentSearchQuerySpec(Arrays.asList("auto", "ve"), null)) + .query(new FragmentSearchQuerySpec(Arrays.asList("auto", "ve"))) .build(); Map> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); @@ -104,7 +104,7 @@ public class SearchQueryRunnerTest } @Test - public void testSearchWithDimension1() + public void testSearchWithDimensionQuality() { Map> expectedResults = new HashMap>(); expectedResults.put( @@ -128,7 +128,7 @@ public class SearchQueryRunnerTest } @Test - public void testSearchWithDimension2() + public void testSearchWithDimensionProvider() { Map> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); expectedResults.put(QueryRunnerTestHelper.providerDimension, new HashSet(Arrays.asList("total_market"))); @@ -146,7 +146,7 @@ public class SearchQueryRunnerTest } @Test - public void testSearchWithDimensions1() + public void testSearchWithDimensionsQualityAndProvider() { Map> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); expectedResults.putAll( @@ -182,7 +182,7 @@ public class SearchQueryRunnerTest } @Test - public void testSearchWithDimensions2() + public void testSearchWithDimensionsPlacementAndProvider() { Map> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); expectedResults.put(QueryRunnerTestHelper.providerDimension, new HashSet(Arrays.asList("total_market"))); @@ -390,7 +390,7 @@ public class SearchQueryRunnerTest for (Map.Entry> entry : expectedResults.entrySet()) { Assert.assertTrue( String.format( - "Dimension %s should have had everything removed, still has[%s]", entry.getKey(), entry.getValue() + "Dimension[%s] should have had everything removed, still has[%s]", entry.getKey(), entry.getValue() ), entry.getValue().isEmpty() );