Added `has_parent` filter (#2243)
The `has_parent` filter accepts a query and a parent type. The query is executed in the parent document space, which is specified by the parent type. This filter return child documents which associated parents have matched. For the rest `has_parent` filter has the same options and works in the same manner as the `has_child` filter. This is an experimental filter. Filter example ################### ``` { "has_parent" : { "parent_type" : "blog" "query" : { "term" : { "tag" : "something" } } } } ``` The `parent_type` field name can also be abbreviated to `type`. Memory considerations ############### With the current implementation, all _id values are loaded to memory (heap) in order to support fast lookups, so make sure there is enough mem for it. This issue originates from issue #792
This commit is contained in:
parent
e530f03b94
commit
2bd9b3aed0
|
@ -20,6 +20,7 @@
|
|||
package org.elasticsearch.common;
|
||||
|
||||
import gnu.trove.map.hash.*;
|
||||
import gnu.trove.set.hash.THashSet;
|
||||
import org.elasticsearch.common.trove.ExtTDoubleObjectHashMap;
|
||||
import org.elasticsearch.common.trove.ExtTHashMap;
|
||||
import org.elasticsearch.common.trove.ExtTLongObjectHashMap;
|
||||
|
@ -33,6 +34,7 @@ public class CacheRecycler {
|
|||
|
||||
public static void clear() {
|
||||
hashMap.clear();
|
||||
hashSet.clear();
|
||||
doubleObjectHashMap.clear();
|
||||
longObjectHashMap.clear();
|
||||
longLongHashMap.clear();
|
||||
|
@ -91,6 +93,32 @@ public class CacheRecycler {
|
|||
ref.add(map);
|
||||
}
|
||||
|
||||
// ----- THashSet -----
|
||||
|
||||
private static SoftWrapper<Queue<THashSet>> hashSet = new SoftWrapper<Queue<THashSet>>();
|
||||
|
||||
public static <T> THashSet<T> popHashSet() {
|
||||
Queue<THashSet> ref = hashSet.get();
|
||||
if (ref == null) {
|
||||
return new THashSet<T>();
|
||||
}
|
||||
THashSet set = ref.poll();
|
||||
if (set == null) {
|
||||
return new THashSet<T>();
|
||||
}
|
||||
return set;
|
||||
}
|
||||
|
||||
public static void pushHashSet(THashSet map) {
|
||||
Queue<THashSet> ref = hashSet.get();
|
||||
if (ref == null) {
|
||||
ref = ConcurrentCollections.newQueue();
|
||||
hashSet.set(ref);
|
||||
}
|
||||
map.clear();
|
||||
ref.add(map);
|
||||
}
|
||||
|
||||
// ------ ExtTDoubleObjectHashMap -----
|
||||
|
||||
private static SoftWrapper<Queue<ExtTDoubleObjectHashMap>> doubleObjectHashMap = new SoftWrapper<Queue<ExtTDoubleObjectHashMap>>();
|
||||
|
|
|
@ -26,7 +26,22 @@ import org.elasticsearch.common.bytes.HashedBytesArray;
|
|||
*/
|
||||
public interface IdReaderTypeCache {
|
||||
|
||||
/**
|
||||
* @param docId The Lucene docId of the child document to return the parent _uid for.
|
||||
* @return The parent _uid for the specified docId (which is a child document)
|
||||
*/
|
||||
HashedBytesArray parentIdByDoc(int docId);
|
||||
|
||||
int docById(HashedBytesArray id);
|
||||
/**
|
||||
* @param uid The uid of the document to return the lucene docId for
|
||||
* @return The lucene docId for the specified uid
|
||||
*/
|
||||
int docById(HashedBytesArray uid);
|
||||
|
||||
/**
|
||||
* @param docId The lucene docId of the document to return _uid for
|
||||
* @return The _uid of the specified docId
|
||||
*/
|
||||
HashedBytesArray idByDoc(int docId);
|
||||
|
||||
}
|
||||
|
|
|
@ -20,6 +20,7 @@
|
|||
package org.elasticsearch.index.cache.id.simple;
|
||||
|
||||
import gnu.trove.impl.Constants;
|
||||
import gnu.trove.map.hash.TIntObjectHashMap;
|
||||
import org.apache.lucene.index.*;
|
||||
import org.apache.lucene.util.StringHelper;
|
||||
import org.elasticsearch.ElasticSearchException;
|
||||
|
@ -138,6 +139,7 @@ public class SimpleIdCache extends AbstractIndexComponent implements IdCache, Se
|
|||
// when traversing, make sure to ignore deleted docs, so the key->docId will be correct
|
||||
if (!reader.isDeleted(termDocs.doc())) {
|
||||
typeBuilder.idToDoc.put(idAsBytes, termDocs.doc());
|
||||
typeBuilder.docToId[termDocs.doc()] = idAsBytes;
|
||||
}
|
||||
}
|
||||
} while (termEnum.next());
|
||||
|
@ -205,6 +207,7 @@ public class SimpleIdCache extends AbstractIndexComponent implements IdCache, Se
|
|||
for (Map.Entry<String, TypeBuilder> typeBuilderEntry : entry.getValue().entrySet()) {
|
||||
types.put(typeBuilderEntry.getKey(), new SimpleIdReaderTypeCache(typeBuilderEntry.getKey(),
|
||||
typeBuilderEntry.getValue().idToDoc,
|
||||
typeBuilderEntry.getValue().docToId,
|
||||
typeBuilderEntry.getValue().parentIdsValues.toArray(new HashedBytesArray[typeBuilderEntry.getValue().parentIdsValues.size()]),
|
||||
typeBuilderEntry.getValue().parentIdsOrdinals));
|
||||
}
|
||||
|
@ -246,6 +249,7 @@ public class SimpleIdCache extends AbstractIndexComponent implements IdCache, Se
|
|||
|
||||
static class TypeBuilder {
|
||||
final ExtTObjectIntHasMap<HashedBytesArray> idToDoc = new ExtTObjectIntHasMap<HashedBytesArray>(Constants.DEFAULT_CAPACITY, Constants.DEFAULT_LOAD_FACTOR, -1);
|
||||
final HashedBytesArray[] docToId;
|
||||
final ArrayList<HashedBytesArray> parentIdsValues = new ArrayList<HashedBytesArray>();
|
||||
final int[] parentIdsOrdinals;
|
||||
int t = 1; // current term number (0 indicated null value)
|
||||
|
@ -254,6 +258,7 @@ public class SimpleIdCache extends AbstractIndexComponent implements IdCache, Se
|
|||
parentIdsOrdinals = new int[reader.maxDoc()];
|
||||
// the first one indicates null value
|
||||
parentIdsValues.add(null);
|
||||
docToId = new HashedBytesArray[reader.maxDoc()];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -32,14 +32,17 @@ public class SimpleIdReaderTypeCache implements IdReaderTypeCache {
|
|||
|
||||
private final ExtTObjectIntHasMap<HashedBytesArray> idToDoc;
|
||||
|
||||
private final HashedBytesArray[] docIdToId;
|
||||
|
||||
private final HashedBytesArray[] parentIdsValues;
|
||||
|
||||
private final int[] parentIdsOrdinals;
|
||||
|
||||
public SimpleIdReaderTypeCache(String type, ExtTObjectIntHasMap<HashedBytesArray> idToDoc,
|
||||
public SimpleIdReaderTypeCache(String type, ExtTObjectIntHasMap<HashedBytesArray> idToDoc, HashedBytesArray[] docIdToId,
|
||||
HashedBytesArray[] parentIdsValues, int[] parentIdsOrdinals) {
|
||||
this.type = type;
|
||||
this.idToDoc = idToDoc;
|
||||
this.docIdToId = docIdToId;
|
||||
this.idToDoc.trimToSize();
|
||||
this.parentIdsValues = parentIdsValues;
|
||||
this.parentIdsOrdinals = parentIdsOrdinals;
|
||||
|
@ -53,8 +56,12 @@ public class SimpleIdReaderTypeCache implements IdReaderTypeCache {
|
|||
return parentIdsValues[parentIdsOrdinals[docId]];
|
||||
}
|
||||
|
||||
public int docById(HashedBytesArray id) {
|
||||
return idToDoc.get(id);
|
||||
public int docById(HashedBytesArray uid) {
|
||||
return idToDoc.get(uid);
|
||||
}
|
||||
|
||||
public HashedBytesArray idByDoc(int docId) {
|
||||
return docIdToId[docId];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -378,6 +378,10 @@ public abstract class FilterBuilders {
|
|||
return new HasChildFilterBuilder(type, query);
|
||||
}
|
||||
|
||||
public static HasParentFilterBuilder hasParentFilter(String parentType, QueryBuilder query) {
|
||||
return new HasParentFilterBuilder(parentType, query);
|
||||
}
|
||||
|
||||
public static BoolFilterBuilder boolFilter() {
|
||||
return new BoolFilterBuilder();
|
||||
}
|
||||
|
|
|
@ -0,0 +1,84 @@
|
|||
/*
|
||||
* Licensed to ElasticSearch and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. ElasticSearch licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.index.query;
|
||||
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Builder for the 'has_parent' filter.
|
||||
*/
|
||||
public class HasParentFilterBuilder extends BaseFilterBuilder {
|
||||
|
||||
private final QueryBuilder queryBuilder;
|
||||
private final String parentType;
|
||||
private String scope;
|
||||
private String filterName;
|
||||
private String executionType;
|
||||
|
||||
/**
|
||||
* @param parentType The parent type
|
||||
* @param parentQuery The query that will be matched with parent documents
|
||||
*/
|
||||
public HasParentFilterBuilder(String parentType, QueryBuilder parentQuery) {
|
||||
this.parentType = parentType;
|
||||
this.queryBuilder = parentQuery;
|
||||
}
|
||||
|
||||
public HasParentFilterBuilder scope(String scope) {
|
||||
this.scope = scope;
|
||||
return this;
|
||||
}
|
||||
|
||||
public HasParentFilterBuilder filterName(String filterName) {
|
||||
this.filterName = filterName;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Expert: Sets the low level parent to child filtering implementation. Can be: 'indirect' or 'uid'
|
||||
*
|
||||
* This option is experimental and will be removed.
|
||||
*/
|
||||
public HasParentFilterBuilder executionType(String executionType) {
|
||||
this.executionType = executionType;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doXContent(XContentBuilder builder, Params params) throws IOException {
|
||||
builder.startObject(HasParentFilterParser.NAME);
|
||||
builder.field("query");
|
||||
queryBuilder.toXContent(builder, params);
|
||||
builder.field("parent_type", parentType);
|
||||
if (scope != null) {
|
||||
builder.field("_scope", scope);
|
||||
}
|
||||
if (filterName != null) {
|
||||
builder.field("_name", filterName);
|
||||
}
|
||||
if (executionType != null) {
|
||||
builder.field("execution_type", executionType);
|
||||
}
|
||||
builder.endObject();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
/*
|
||||
* Licensed to ElasticSearch and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. ElasticSearch licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.index.query;
|
||||
|
||||
import org.apache.lucene.search.Filter;
|
||||
import org.apache.lucene.search.FilteredQuery;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.elasticsearch.common.Strings;
|
||||
import org.elasticsearch.common.inject.Inject;
|
||||
import org.elasticsearch.common.xcontent.XContentParser;
|
||||
import org.elasticsearch.index.mapper.DocumentMapper;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
|
||||
import org.elasticsearch.index.search.child.HasParentFilter;
|
||||
import org.elasticsearch.search.internal.SearchContext;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
public class HasParentFilterParser implements FilterParser {
|
||||
|
||||
public static final String NAME = "has_parent";
|
||||
|
||||
@Inject
|
||||
public HasParentFilterParser() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public String[] names() {
|
||||
return new String[]{NAME, Strings.toCamelCase(NAME)};
|
||||
}
|
||||
|
||||
@Override
|
||||
public Filter parse(QueryParseContext parseContext) throws IOException, QueryParsingException {
|
||||
XContentParser parser = parseContext.parser();
|
||||
|
||||
Query query = null;
|
||||
boolean queryFound = false;
|
||||
String parentType = null;
|
||||
String executionType = "uid";
|
||||
String scope = null;
|
||||
|
||||
String filterName = null;
|
||||
String currentFieldName = null;
|
||||
XContentParser.Token token;
|
||||
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
|
||||
if (token == XContentParser.Token.FIELD_NAME) {
|
||||
currentFieldName = parser.currentName();
|
||||
} else if (token == XContentParser.Token.START_OBJECT) {
|
||||
if ("query".equals(currentFieldName)) {
|
||||
// TODO handle `query` element before `type` element...
|
||||
String[] origTypes = QueryParseContext.setTypesWithPrevious(parentType == null ? null : new String[]{parentType});
|
||||
try {
|
||||
query = parseContext.parseInnerQuery();
|
||||
queryFound = true;
|
||||
} finally {
|
||||
QueryParseContext.setTypes(origTypes);
|
||||
}
|
||||
} else {
|
||||
throw new QueryParsingException(parseContext.index(), "[has_parent] filter does not support [" + currentFieldName + "]");
|
||||
}
|
||||
} else if (token.isValue()) {
|
||||
if ("type".equals(currentFieldName) || "parent_type".equals(currentFieldName) || "parentType".equals(currentFieldName)) {
|
||||
parentType = parser.text();
|
||||
} else if ("_scope".equals(currentFieldName)) {
|
||||
scope = parser.text();
|
||||
} else if ("_name".equals(currentFieldName)) {
|
||||
filterName = parser.text();
|
||||
// TODO: change to execution_type
|
||||
} else if ("execution_type".equals(currentFieldName) || "executionType".equals(currentFieldName)) { // This option is experimental and will most likely be removed.
|
||||
executionType = parser.text();
|
||||
} else {
|
||||
throw new QueryParsingException(parseContext.index(), "[has_parent] filter does not support [" + currentFieldName + "]");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!queryFound) {
|
||||
throw new QueryParsingException(parseContext.index(), "[parent] filter requires 'query' field");
|
||||
}
|
||||
if (query == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (parentType == null) {
|
||||
throw new QueryParsingException(parseContext.index(), "[parent] filter requires 'parent_type' field");
|
||||
}
|
||||
|
||||
DocumentMapper parentDocMapper = parseContext.mapperService().documentMapper(parentType);
|
||||
if (parentDocMapper == null) {
|
||||
throw new QueryParsingException(parseContext.index(), "[parent] filter configured 'parent_type' [" + parentType + "] is not a valid type");
|
||||
}
|
||||
|
||||
// wrap the query with type query
|
||||
query = new FilteredQuery(query, parseContext.cacheFilter(parentDocMapper.typeFilter(), null));
|
||||
|
||||
SearchContext searchContext = SearchContext.current();
|
||||
|
||||
HasParentFilter parentFilter = HasParentFilter.create(executionType, query, scope, parentType, searchContext);
|
||||
searchContext.addScopePhase(parentFilter);
|
||||
|
||||
if (filterName != null) {
|
||||
parseContext.addNamedFilter(filterName, parentFilter);
|
||||
}
|
||||
return parentFilter;
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,241 @@
|
|||
/*
|
||||
* Licensed to ElasticSearch and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. ElasticSearch licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.index.search.child;
|
||||
|
||||
import gnu.trove.set.hash.THashSet;
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.search.*;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.elasticsearch.ElasticSearchIllegalStateException;
|
||||
import org.elasticsearch.common.CacheRecycler;
|
||||
import org.elasticsearch.common.bytes.HashedBytesArray;
|
||||
import org.elasticsearch.common.collect.Tuple;
|
||||
import org.elasticsearch.common.lucene.docset.GetDocSet;
|
||||
import org.elasticsearch.common.lucene.search.NoopCollector;
|
||||
import org.elasticsearch.common.trove.ExtTHashMap;
|
||||
import org.elasticsearch.index.cache.id.IdReaderTypeCache;
|
||||
import org.elasticsearch.search.internal.ScopePhase;
|
||||
import org.elasticsearch.search.internal.SearchContext;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
|
||||
import static com.google.common.collect.Maps.newHashMap;
|
||||
|
||||
/**
|
||||
* A filter that only return child documents that are linked to the parent documents that matched with the inner query.
|
||||
*/
|
||||
public abstract class HasParentFilter extends Filter implements ScopePhase.CollectorPhase {
|
||||
|
||||
final Query query;
|
||||
final String scope;
|
||||
final String parentType;
|
||||
final SearchContext context;
|
||||
|
||||
HasParentFilter(Query query, String scope, String parentType, SearchContext context) {
|
||||
this.query = query;
|
||||
this.scope = scope;
|
||||
this.parentType = parentType;
|
||||
this.context = context;
|
||||
}
|
||||
|
||||
public String scope() {
|
||||
return scope;
|
||||
}
|
||||
|
||||
public Query query() {
|
||||
return query;
|
||||
}
|
||||
|
||||
public static HasParentFilter create(String executionType, Query query, String scope, String parentType, SearchContext context) {
|
||||
// This mechanism is experimental and will most likely be removed.
|
||||
if ("indirect".equals(executionType)) {
|
||||
return new InDirect(query, scope, parentType, context);
|
||||
} else if ("uid".equals(executionType)) {
|
||||
return new UidParentFilter(query, scope, parentType, context);
|
||||
}
|
||||
throw new ElasticSearchIllegalStateException("Illegal has_parent execution type: " + executionType);
|
||||
}
|
||||
|
||||
static class UidParentFilter extends HasParentFilter {
|
||||
|
||||
THashSet<HashedBytesArray> parents;
|
||||
|
||||
UidParentFilter(Query query, String scope, String parentType, SearchContext context) {
|
||||
super(query, scope, parentType, context);
|
||||
}
|
||||
|
||||
public boolean requiresProcessing() {
|
||||
return parents == null;
|
||||
}
|
||||
|
||||
public Collector collector() {
|
||||
parents = CacheRecycler.popHashSet();
|
||||
return new ParentUidsCollector(parents, context, parentType);
|
||||
}
|
||||
|
||||
public void processCollector(Collector collector) {
|
||||
parents = ((ParentUidsCollector) collector).collectedUids;
|
||||
}
|
||||
|
||||
public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
|
||||
IdReaderTypeCache idReaderTypeCache = context.idCache().reader(reader).type(parentType);
|
||||
return new ChildrenDocSet(reader, parents, idReaderTypeCache);
|
||||
}
|
||||
|
||||
public void clear() {
|
||||
CacheRecycler.pushHashSet(parents);
|
||||
parents = null;
|
||||
}
|
||||
|
||||
static class ChildrenDocSet extends GetDocSet {
|
||||
|
||||
final IndexReader reader;
|
||||
final THashSet<HashedBytesArray> parents;
|
||||
final IdReaderTypeCache idReaderTypeCache;
|
||||
|
||||
ChildrenDocSet(IndexReader reader, THashSet<HashedBytesArray> parents, IdReaderTypeCache idReaderTypeCache) {
|
||||
super(reader.maxDoc());
|
||||
this.reader = reader;
|
||||
this.parents = parents;
|
||||
this.idReaderTypeCache = idReaderTypeCache;
|
||||
}
|
||||
|
||||
public boolean get(int doc) {
|
||||
return !reader.isDeleted(doc) && parents.contains(idReaderTypeCache.parentIdByDoc(doc));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static class ParentUidsCollector extends NoopCollector {
|
||||
|
||||
final THashSet<HashedBytesArray> collectedUids;
|
||||
final SearchContext context;
|
||||
final String parentType;
|
||||
|
||||
IdReaderTypeCache typeCache;
|
||||
|
||||
ParentUidsCollector(THashSet<HashedBytesArray> collectedUids, SearchContext context, String parentType) {
|
||||
this.collectedUids = collectedUids;
|
||||
this.context = context;
|
||||
this.parentType = parentType;
|
||||
}
|
||||
|
||||
public void collect(int doc) throws IOException {
|
||||
collectedUids.add(typeCache.idByDoc(doc));
|
||||
}
|
||||
|
||||
public void setNextReader(IndexReader reader, int docBase) throws IOException {
|
||||
typeCache = context.idCache().reader(reader).type(parentType);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static class InDirect extends HasParentFilter {
|
||||
|
||||
Map<Object, FixedBitSet> parentDocs;
|
||||
|
||||
InDirect(Query query, String scope, String parentType, SearchContext context) {
|
||||
super(query, scope, parentType, context);
|
||||
}
|
||||
|
||||
public boolean requiresProcessing() {
|
||||
return parentDocs == null;
|
||||
}
|
||||
|
||||
public Collector collector() {
|
||||
return new ParentDocsCollector();
|
||||
}
|
||||
|
||||
public void processCollector(Collector collector) {
|
||||
parentDocs = ((ParentDocsCollector) collector).segmentResults;
|
||||
}
|
||||
|
||||
public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
|
||||
return new ChildrenDocSet(reader, parentDocs, context, parentType);
|
||||
}
|
||||
|
||||
public void clear() {
|
||||
parentDocs = null;
|
||||
}
|
||||
|
||||
static class ChildrenDocSet extends GetDocSet {
|
||||
|
||||
final IdReaderTypeCache currentTypeCache;
|
||||
final IndexReader currentReader;
|
||||
final Tuple<IndexReader, IdReaderTypeCache>[] readersToTypeCache;
|
||||
final Map<Object, FixedBitSet> parentDocs;
|
||||
|
||||
ChildrenDocSet(IndexReader currentReader, Map<Object, FixedBitSet> parentDocs,
|
||||
SearchContext context, String parentType) {
|
||||
super(currentReader.maxDoc());
|
||||
this.currentTypeCache = context.idCache().reader(currentReader).type(parentType);
|
||||
this.currentReader = currentReader;
|
||||
this.parentDocs = parentDocs;
|
||||
this.readersToTypeCache = new Tuple[context.searcher().subReaders().length];
|
||||
for (int i = 0; i < readersToTypeCache.length; i++) {
|
||||
IndexReader reader = context.searcher().subReaders()[i];
|
||||
readersToTypeCache[i] = new Tuple<IndexReader, IdReaderTypeCache>(reader, context.idCache().reader(reader).type(parentType));
|
||||
}
|
||||
}
|
||||
|
||||
public boolean get(int doc) {
|
||||
if (currentReader.isDeleted(doc) || doc == -1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
HashedBytesArray parentId = currentTypeCache.parentIdByDoc(doc);
|
||||
if (parentId == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
for (Tuple<IndexReader, IdReaderTypeCache> readerTypeCacheTuple : readersToTypeCache) {
|
||||
int parentDocId = readerTypeCacheTuple.v2().docById(parentId);
|
||||
if (parentDocId == -1) {
|
||||
continue;
|
||||
}
|
||||
|
||||
FixedBitSet currentParentDocs = parentDocs.get(readerTypeCacheTuple.v1().getCoreCacheKey());
|
||||
if (currentParentDocs.get(parentDocId)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static class ParentDocsCollector extends NoopCollector {
|
||||
|
||||
final Map<Object, FixedBitSet> segmentResults = newHashMap();
|
||||
FixedBitSet current;
|
||||
|
||||
public void collect(int doc) throws IOException {
|
||||
current.set(doc);
|
||||
}
|
||||
|
||||
public void setNextReader(IndexReader reader, int docBase) throws IOException {
|
||||
segmentResults.put(reader.getCoreCacheKey(), current = new FixedBitSet(reader.maxDoc()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -82,6 +82,7 @@ public class IndicesQueriesRegistry {
|
|||
|
||||
Map<String, FilterParser> filterParsers = Maps.newHashMap();
|
||||
addFilterParser(filterParsers, new HasChildFilterParser());
|
||||
addFilterParser(filterParsers, new HasParentFilterParser());
|
||||
addFilterParser(filterParsers, new NestedFilterParser());
|
||||
addFilterParser(filterParsers, new TypeFilterParser());
|
||||
addFilterParser(filterParsers, new IdsFilterParser());
|
||||
|
|
|
@ -30,6 +30,8 @@ import org.elasticsearch.common.settings.Settings;
|
|||
import org.elasticsearch.common.unit.SizeValue;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.index.query.FilterBuilders;
|
||||
import org.elasticsearch.index.query.QueryBuilders;
|
||||
import org.elasticsearch.node.Node;
|
||||
|
||||
import java.io.IOException;
|
||||
|
@ -40,6 +42,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF
|
|||
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
|
||||
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
|
||||
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
|
||||
import static org.elasticsearch.index.query.FilterBuilders.hasParentFilter;
|
||||
import static org.elasticsearch.index.query.QueryBuilders.*;
|
||||
import static org.elasticsearch.node.NodeBuilder.nodeBuilder;
|
||||
|
||||
|
@ -52,16 +55,12 @@ public class ChildSearchBenchmark {
|
|||
Settings settings = settingsBuilder()
|
||||
.put("index.engine.robin.refreshInterval", "-1")
|
||||
.put("gateway.type", "local")
|
||||
.put(SETTING_NUMBER_OF_SHARDS, 2)
|
||||
.put(SETTING_NUMBER_OF_REPLICAS, 1)
|
||||
.put(SETTING_NUMBER_OF_SHARDS, 1)
|
||||
.put(SETTING_NUMBER_OF_REPLICAS, 0)
|
||||
.build();
|
||||
|
||||
Node node1 = nodeBuilder().settings(settingsBuilder().put(settings).put("name", "node1")).node();
|
||||
Node node2 = nodeBuilder().settings(settingsBuilder().put(settings).put("name", "node2")).node();
|
||||
|
||||
Node clientNode = nodeBuilder().settings(settingsBuilder().put(settings).put("name", "client")).client(true).node();
|
||||
|
||||
Client client = clientNode.client();
|
||||
Client client = node1.client();
|
||||
|
||||
long COUNT = SizeValue.parseSizeValue("1m").singles();
|
||||
int CHILD_COUNT = 5;
|
||||
|
@ -162,6 +161,59 @@ public class ChildSearchBenchmark {
|
|||
}
|
||||
System.out.println("--> has_child Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");
|
||||
|
||||
String[] executionTypes = new String[]{"uid", "indirect"}; // either uid (faster, in general a bit more memory) or indirect (slower, but in general a bit less memory)
|
||||
for (String executionType : executionTypes) {
|
||||
System.out.println("--> Running has_parent filter with " + executionType + " execution type");
|
||||
// run parent child constant query
|
||||
for (int j = 0; j < QUERY_WARMUP; j++) {
|
||||
SearchResponse searchResponse = client.prepareSearch()
|
||||
.setQuery(constantScoreQuery(
|
||||
hasParentFilter("parent", termQuery("name", "test1")).executionType(executionType)
|
||||
))
|
||||
.execute().actionGet();
|
||||
if (searchResponse.failedShards() > 0) {
|
||||
System.err.println("Search Failures " + Arrays.toString(searchResponse.shardFailures()));
|
||||
}
|
||||
if (searchResponse.hits().totalHits() != CHILD_COUNT) {
|
||||
System.err.println("--> mismatch on hits [" + j + "], got [" + searchResponse.hits().totalHits() + "], expected [" + CHILD_COUNT + "]");
|
||||
}
|
||||
}
|
||||
|
||||
totalQueryTime = 0;
|
||||
for (int j = 1; j <= QUERY_COUNT; j++) {
|
||||
SearchResponse searchResponse = client.prepareSearch()
|
||||
.setQuery(constantScoreQuery(
|
||||
hasParentFilter("parent", termQuery("name", "test1")).executionType(executionType)
|
||||
))
|
||||
.execute().actionGet();
|
||||
if (searchResponse.failedShards() > 0) {
|
||||
System.err.println("Search Failures " + Arrays.toString(searchResponse.shardFailures()));
|
||||
}
|
||||
if (searchResponse.hits().totalHits() != CHILD_COUNT) {
|
||||
System.err.println("--> mismatch on hits [" + j + "], got [" + searchResponse.hits().totalHits() + "], expected [" + CHILD_COUNT + "]");
|
||||
}
|
||||
totalQueryTime += searchResponse.tookInMillis();
|
||||
}
|
||||
System.out.println("--> has_parent[" + executionType + "] Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");
|
||||
|
||||
System.out.println("--> Running has_parent[" + executionType + "] filter with match_all query as parent query");
|
||||
totalQueryTime = 0;
|
||||
for (int j = 1; j <= QUERY_COUNT; j++) {
|
||||
SearchResponse searchResponse = client.prepareSearch()
|
||||
.setQuery(constantScoreQuery(
|
||||
hasParentFilter("parent", matchAllQuery()).executionType(executionType)
|
||||
))
|
||||
.execute().actionGet();
|
||||
if (searchResponse.failedShards() > 0) {
|
||||
System.err.println("Search Failures " + Arrays.toString(searchResponse.shardFailures()));
|
||||
}
|
||||
if (searchResponse.hits().totalHits() != 5000000) {
|
||||
System.err.println("--> mismatch on hits [" + j + "], got [" + searchResponse.hits().totalHits() + "], expected [" + 5000000 + "]");
|
||||
}
|
||||
totalQueryTime += searchResponse.tookInMillis();
|
||||
}
|
||||
System.out.println("--> has_parent[" + executionType + "] with match_all query as parent query Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");
|
||||
}
|
||||
System.out.println("--> Running top_children query");
|
||||
// run parent child score query
|
||||
for (int j = 0; j < QUERY_WARMUP; j++) {
|
||||
|
@ -183,10 +235,8 @@ public class ChildSearchBenchmark {
|
|||
}
|
||||
System.out.println("--> top_children Query Avg: " + (totalQueryTime / QUERY_COUNT) + "ms");
|
||||
|
||||
clientNode.close();
|
||||
|
||||
client.close();
|
||||
node1.close();
|
||||
node2.close();
|
||||
}
|
||||
|
||||
private static XContentBuilder parentSource(String id, String nameValue) throws IOException {
|
||||
|
|
|
@ -19,6 +19,7 @@
|
|||
|
||||
package org.elasticsearch.test.integration.search.child;
|
||||
|
||||
import com.beust.jcommander.internal.Maps;
|
||||
import org.elasticsearch.action.search.SearchResponse;
|
||||
import org.elasticsearch.action.search.SearchType;
|
||||
import org.elasticsearch.action.search.ShardSearchFailure;
|
||||
|
@ -31,9 +32,12 @@ import org.testng.annotations.AfterClass;
|
|||
import org.testng.annotations.BeforeClass;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.*;
|
||||
|
||||
import static com.google.common.collect.Lists.newArrayList;
|
||||
import static com.google.common.collect.Maps.newHashMap;
|
||||
import static org.elasticsearch.index.query.FilterBuilders.hasChildFilter;
|
||||
import static org.elasticsearch.index.query.FilterBuilders.hasParentFilter;
|
||||
import static org.elasticsearch.index.query.QueryBuilders.*;
|
||||
import static org.elasticsearch.search.facet.FacetBuilders.termsFacet;
|
||||
import static org.hamcrest.MatcherAssert.assertThat;
|
||||
|
@ -237,6 +241,77 @@ public class SimpleChildQuerySearchTests extends AbstractNodesTests {
|
|||
assertThat(searchResponse.hits().totalHits(), equalTo(2l));
|
||||
assertThat(searchResponse.hits().getAt(0).id(), anyOf(equalTo("p2"), equalTo("p1")));
|
||||
assertThat(searchResponse.hits().getAt(1).id(), anyOf(equalTo("p2"), equalTo("p1")));
|
||||
|
||||
// HAS PARENT FILTER
|
||||
searchResponse = client.prepareSearch("test").setQuery(constantScoreQuery(hasParentFilter("parent", termQuery("p_field", "p_value2")))).execute().actionGet();
|
||||
assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0));
|
||||
assertThat(searchResponse.failedShards(), equalTo(0));
|
||||
assertThat(searchResponse.hits().totalHits(), equalTo(2l));
|
||||
assertThat(searchResponse.hits().getAt(0).id(), equalTo("c3"));
|
||||
assertThat(searchResponse.hits().getAt(1).id(), equalTo("c4"));
|
||||
|
||||
searchResponse = client.prepareSearch("test").setQuery(constantScoreQuery(hasParentFilter("parent", termQuery("p_field", "p_value1")))).execute().actionGet();
|
||||
assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0));
|
||||
assertThat(searchResponse.failedShards(), equalTo(0));
|
||||
assertThat(searchResponse.hits().totalHits(), equalTo(2l));
|
||||
assertThat(searchResponse.hits().getAt(0).id(), equalTo("c1"));
|
||||
assertThat(searchResponse.hits().getAt(1).id(), equalTo("c2"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHasParentFilter() throws Exception {
|
||||
client.admin().indices().prepareDelete().execute().actionGet();
|
||||
client.admin().indices().prepareCreate("test").execute().actionGet();
|
||||
client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
|
||||
client.admin().indices().preparePutMapping("test").setType("child").setSource(XContentFactory.jsonBuilder().startObject().startObject("type")
|
||||
.startObject("_parent").field("type", "parent").endObject()
|
||||
.endObject().endObject()).execute().actionGet();
|
||||
|
||||
Map<String, List<String>> parentToChildren = newHashMap();
|
||||
// Childless parent
|
||||
client.prepareIndex("test", "parent", "p0").setSource("p_field", "p0").execute().actionGet();
|
||||
parentToChildren.put("p0", new ArrayList<String>());
|
||||
|
||||
String previousParentId = null;
|
||||
int numChildDocs = 32;
|
||||
int numChildDocsPerParent = 0;
|
||||
for (int i = 1; i <= numChildDocs; i++) {
|
||||
if (previousParentId == null || i % numChildDocsPerParent == 0) {
|
||||
previousParentId = "p" + i;
|
||||
client.prepareIndex("test", "parent", previousParentId).setSource("p_field", previousParentId).execute().actionGet();
|
||||
client.admin().indices().prepareFlush("test").execute().actionGet();
|
||||
numChildDocsPerParent++;
|
||||
}
|
||||
|
||||
String childId = "c" + i;
|
||||
client.prepareIndex("test", "child", childId)
|
||||
.setSource("c_field", childId)
|
||||
.setParent(previousParentId)
|
||||
.execute().actionGet();
|
||||
|
||||
if (!parentToChildren.containsKey(previousParentId)) {
|
||||
parentToChildren.put(previousParentId, new ArrayList<String>());
|
||||
}
|
||||
parentToChildren.get(previousParentId).add(childId);
|
||||
}
|
||||
client.admin().indices().prepareRefresh().execute().actionGet();
|
||||
|
||||
assertThat(parentToChildren.isEmpty(), equalTo(false));
|
||||
for (Map.Entry<String, List<String>> parentToChildrenEntry : parentToChildren.entrySet()) {
|
||||
SearchResponse searchResponse = client.prepareSearch("test")
|
||||
.setQuery(constantScoreQuery(hasParentFilter("parent", termQuery("p_field", parentToChildrenEntry.getKey()))))
|
||||
.setSize(numChildDocsPerParent)
|
||||
.execute().actionGet();
|
||||
|
||||
assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0));
|
||||
assertThat(searchResponse.failedShards(), equalTo(0));
|
||||
List<String> childIds = parentToChildrenEntry.getValue();
|
||||
assertThat(searchResponse.hits().totalHits(), equalTo((long) childIds.size()));
|
||||
int counter = 0;
|
||||
for (String childId : childIds) {
|
||||
assertThat(searchResponse.hits().getAt(counter++).id(), equalTo(childId));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
Loading…
Reference in New Issue