INDEX_CODEC_SETTING = new Setting<>("index.codec", "default", s -> {
+ switch (s) {
case "default":
case "best_compression":
case "lucene_default":
return s;
default:
if (Codec.availableCodecs().contains(s) == false) { // we don't error message the not officially supported ones
- throw new IllegalArgumentException("unknown value for [index.codec] must be one of [default, best_compression] but was: " + s);
+ throw new IllegalArgumentException(
+ "unknown value for [index.codec] must be one of [default, best_compression] but was: " + s);
}
return s;
}
- }, Property.IndexScope);
+ }, Property.IndexScope, Property.NodeScope);
/** if set to true the engine will start even if the translog id in the commit point can not be found */
public static final String INDEX_FORCE_NEW_TRANSLOG = "index.engine.force_new_translog";
@@ -97,7 +102,8 @@ public final class EngineConfig {
IndexSettings indexSettings, Engine.Warmer warmer, Store store, SnapshotDeletionPolicy deletionPolicy,
MergePolicy mergePolicy,Analyzer analyzer,
Similarity similarity, CodecService codecService, Engine.EventListener eventListener,
- TranslogRecoveryPerformer translogRecoveryPerformer, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, TranslogConfig translogConfig, TimeValue flushMergesAfter) {
+ TranslogRecoveryPerformer translogRecoveryPerformer, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy,
+ TranslogConfig translogConfig, TimeValue flushMergesAfter) {
this.shardId = shardId;
final Settings settings = indexSettings.getSettings();
this.indexSettings = indexSettings;
@@ -138,7 +144,8 @@ public final class EngineConfig {
}
/**
- * Returns the initial index buffer size. This setting is only read on startup and otherwise controlled by {@link IndexingMemoryController}
+ * Returns the initial index buffer size. This setting is only read on startup and otherwise controlled
+ * by {@link IndexingMemoryController}
*/
public ByteSizeValue getIndexingBufferSize() {
return indexingBufferSize;
@@ -146,11 +153,12 @@ public final class EngineConfig {
/**
* Returns true iff delete garbage collection in the engine should be enabled. This setting is updateable
- * in realtime and forces a volatile read. Consumers can safely read this value directly go fetch it's latest value. The default is true
+ * in realtime and forces a volatile read. Consumers can safely read this value directly go fetch it's latest value.
+ * The default is true
*
* Engine GC deletion if enabled collects deleted documents from in-memory realtime data structures after a certain amount of
- * time ({@link IndexSettings#getGcDeletesInMillis()} if enabled. Before deletes are GCed they will cause re-adding the document that was deleted
- * to fail.
+ * time ({@link IndexSettings#getGcDeletesInMillis()} if enabled. Before deletes are GCed they will cause re-adding the document
+ * that was deleted to fail.
*
*/
public boolean isEnableGcDeletes() {
@@ -168,7 +176,8 @@ public final class EngineConfig {
}
/**
- * Returns a thread-pool mainly used to get estimated time stamps from {@link org.elasticsearch.threadpool.ThreadPool#estimatedTimeInMillis()} and to schedule
+ * Returns a thread-pool mainly used to get estimated time stamps from
+ * {@link org.elasticsearch.threadpool.ThreadPool#estimatedTimeInMillis()} and to schedule
* async force merge calls on the {@link org.elasticsearch.threadpool.ThreadPool.Names#FORCE_MERGE} thread-pool
*/
public ThreadPool getThreadPool() {
@@ -183,8 +192,9 @@ public final class EngineConfig {
}
/**
- * Returns the {@link org.elasticsearch.index.store.Store} instance that provides access to the {@link org.apache.lucene.store.Directory}
- * used for the engines {@link org.apache.lucene.index.IndexWriter} to write it's index files to.
+ * Returns the {@link org.elasticsearch.index.store.Store} instance that provides access to the
+ * {@link org.apache.lucene.store.Directory} used for the engines {@link org.apache.lucene.index.IndexWriter} to write it's index files
+ * to.
*
* Note: In order to use this instance the consumer needs to increment the stores reference before it's used the first time and hold
* it's reference until it's not needed anymore.
diff --git a/core/src/main/java/org/elasticsearch/index/fieldstats/FieldStatsProvider.java b/core/src/main/java/org/elasticsearch/index/fieldstats/FieldStatsProvider.java
new file mode 100644
index 00000000000..066d1207e3f
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/fieldstats/FieldStatsProvider.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fieldstats;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.action.fieldstats.FieldStats;
+import org.elasticsearch.action.fieldstats.IndexConstraint;
+import org.elasticsearch.action.fieldstats.IndexConstraint.Comparison;
+import org.elasticsearch.action.fieldstats.IndexConstraint.Property;
+import org.elasticsearch.common.joda.DateMathParser;
+import org.elasticsearch.index.engine.Engine.Searcher;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.core.DateFieldMapper.DateFieldType;
+import org.elasticsearch.index.mapper.ip.IpFieldMapper.IpFieldType;
+import org.joda.time.DateTimeZone;
+
+import java.io.IOException;
+
+/**
+ * Provides a service for gettings the {@link FieldStats} for a given field from
+ * the index.
+ */
+public class FieldStatsProvider {
+
+ private final Searcher searcher;
+ private final MapperService mapperService;
+
+ /**
+ * @param searcher
+ * the {@link Searcher}to use when retrieving the
+ * {@link FieldStats}
+ * @param mapperService
+ * the {@link MapperService}
+ */
+ public FieldStatsProvider(Searcher searcher, MapperService mapperService) {
+ this.searcher = searcher;
+ this.mapperService = mapperService;
+ }
+
+ /**
+ * @param field
+ * the name of the field to return {@link FieldStats} for.
+ * @return a {@link FieldStats} object for the given field
+ * @throws IOException
+ * if the field statistics cannot be read
+ */
+ public > FieldStats get(String field) throws IOException {
+ MappedFieldType mappedFieldType = mapperService.fullName(field);
+ if (mappedFieldType != null) {
+ IndexReader reader = searcher.reader();
+ Terms terms = MultiFields.getTerms(reader, field);
+ if (terms != null) {
+ return mappedFieldType.stats(terms, reader.maxDoc());
+ }
+ }
+ return null;
+ }
+
+ /**
+ * @param fieldName
+ * the fieldName to check
+ * @param from
+ * the minimum value for the query
+ * @param to
+ * the maximum value for the query
+ * @param includeLower
+ * whether the from value is inclusive
+ * @param includeUpper
+ * whether the to value is inclusive
+ * @param timeZone
+ * the timeZone to use for date fields
+ * @param dateMathParser
+ * the {@link DateMathParser} to use for date fields
+ * @return A {@link Relation} indicating the overlap of the range of terms
+ * for the field with the query range. This method will return:
+ *
+ * - {@link Relation#WITHIN} if the range of terms for the field
+ * in the shard is completely within the query range
+ * - {@link Relation#DISJOINT} if the range of terms for the field
+ * in the shard is completely outside the query range
+ * - {@link Relation#INTERSECTS} if the range of terms for the
+ * field in the shard intersects with the query range
+ *
+ * @throws IOException
+ * if the index cannot be read
+ */
+ public Relation isFieldWithinQuery(String fieldName, Object from, Object to, boolean includeLower, boolean includeUpper,
+ DateTimeZone timeZone, DateMathParser dateMathParser) throws IOException {
+ MappedFieldType mappedFieldType = mapperService.fullName(fieldName);
+ FieldStats fieldStats = get(fieldName);
+ if (fieldStats == null) {
+ // No fieldStats for the field so the field doesn't exist on
+ // this shard, so relation is DISJOINT
+ return Relation.DISJOINT;
+ } else {
+ // Convert the from and to values to Strings so they can be used
+ // in the IndexConstraints. Since DateTime is represented as a
+ // Long field in Lucene we need to use the millisecond value of
+ // the DateTime in that case
+ String fromString = null;
+ if (from != null) {
+ if (mappedFieldType instanceof DateFieldType) {
+ long millis = ((DateFieldType) mappedFieldType).parseToMilliseconds(from, !includeLower, timeZone, dateMathParser);
+ fromString = fieldStats.stringValueOf(millis, null);
+ } else if (mappedFieldType instanceof IpFieldType) {
+ if (from instanceof BytesRef) {
+ from = ((BytesRef) from).utf8ToString();
+ }
+ long ipAsLong = ((IpFieldType) mappedFieldType).value(from);
+ fromString = fieldStats.stringValueOf(ipAsLong, null);
+ } else {
+ fromString = fieldStats.stringValueOf(from, null);
+ }
+ }
+ String toString = null;
+ if (to != null) {
+ if (mappedFieldType instanceof DateFieldType) {
+ long millis = ((DateFieldType) mappedFieldType).parseToMilliseconds(to, includeUpper, timeZone, dateMathParser);
+ toString = fieldStats.stringValueOf(millis, null);
+ } else if (mappedFieldType instanceof IpFieldType) {
+ if (to instanceof BytesRef) {
+ to = ((BytesRef) to).utf8ToString();
+ }
+ long ipAsLong = ((IpFieldType) mappedFieldType).value(to);
+ toString = fieldStats.stringValueOf(ipAsLong, null);
+ } else {
+ toString = fieldStats.stringValueOf(to, null);
+ }
+ }
+ if ((from == null || fieldStats
+ .match(new IndexConstraint(fieldName, Property.MIN, includeLower ? Comparison.GTE : Comparison.GT, fromString)))
+ && (to == null || fieldStats.match(
+ new IndexConstraint(fieldName, Property.MAX, includeUpper ? Comparison.LTE : Comparison.LT, toString)))) {
+ // If the min and max terms for the field are both within
+ // the query range then all documents will match so relation is
+ // WITHIN
+ return Relation.WITHIN;
+ } else if ((to != null && fieldStats
+ .match(new IndexConstraint(fieldName, Property.MIN, includeUpper ? Comparison.GT : Comparison.GTE, toString)))
+ || (from != null && fieldStats.match(
+ new IndexConstraint(fieldName, Property.MAX, includeLower ? Comparison.LT : Comparison.LTE, fromString)))) {
+ // If the min and max terms are both outside the query range
+ // then no document will match so relation is DISJOINT (N.B.
+ // since from <= to we only need
+ // to check one bould for each side of the query range)
+ return Relation.DISJOINT;
+ }
+ }
+ // Range of terms doesn't match any of the constraints so must INTERSECT
+ return Relation.INTERSECTS;
+ }
+
+ /**
+ * An enum used to describe the relation between the range of terms in a
+ * shard when compared with a query range
+ */
+ public static enum Relation {
+ WITHIN, INTERSECTS, DISJOINT;
+ }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index 414ea0f7e9c..eaf897e7fbd 100755
--- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -33,12 +33,12 @@ import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.mapper.Mapper.BuilderContext;
import org.elasticsearch.index.mapper.object.ObjectMapper;
+import org.elasticsearch.index.percolator.PercolatorFieldMapper;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.indices.InvalidTypeNameException;
import org.elasticsearch.indices.TypeMissingException;
import org.elasticsearch.indices.mapper.MapperRegistry;
-import org.elasticsearch.percolator.PercolatorService;
import org.elasticsearch.script.ScriptService;
import java.io.Closeable;
@@ -333,7 +333,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
}
private boolean typeNameStartsWithIllegalDot(DocumentMapper mapper) {
- return mapper.type().startsWith(".") && !PercolatorService.TYPE_NAME.equals(mapper.type());
+ return mapper.type().startsWith(".") && !PercolatorFieldMapper.TYPE_NAME.equals(mapper.type());
}
private boolean assertSerialization(DocumentMapper mapper) {
@@ -405,7 +405,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable {
public DocumentMapper parse(String mappingType, CompressedXContent mappingSource, boolean applyDefault) throws MapperParsingException {
String defaultMappingSource;
- if (PercolatorService.TYPE_NAME.equals(mappingType)) {
+ if (PercolatorFieldMapper.TYPE_NAME.equals(mappingType)) {
defaultMappingSource = this.defaultPercolatorMappingSource;
} else {
defaultMappingSource = this.defaultMappingSource;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
index 048c58297cb..724c37fcfcd 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
@@ -417,10 +417,15 @@ public class DateFieldMapper extends NumberFieldMapper {
}
public long parseToMilliseconds(Object value, boolean inclusive, @Nullable DateTimeZone zone, @Nullable DateMathParser forcedDateParser) {
+ if (value instanceof Long) {
+ return ((Long) value).longValue();
+ }
+
DateMathParser dateParser = dateMathParser();
if (forcedDateParser != null) {
dateParser = forcedDateParser;
}
+
String strValue;
if (value instanceof BytesRef) {
strValue = ((BytesRef) value).utf8ToString();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
index 9a4cf70782b..2ffb5d4ecf5 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
@@ -22,12 +22,14 @@ package org.elasticsearch.index.mapper.ip;
import org.apache.lucene.analysis.LegacyNumericTokenStream;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.Terms;
import org.apache.lucene.search.LegacyNumericRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.LegacyNumericUtils;
import org.elasticsearch.Version;
+import org.elasticsearch.action.fieldstats.FieldStats;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Numbers;
@@ -262,6 +264,13 @@ public class IpFieldMapper extends NumberFieldMapper {
iValue + iSim,
true, true);
}
+
+ @Override
+ public FieldStats stats(Terms terms, int maxDoc) throws IOException {
+ long minValue = LegacyNumericUtils.getMinLong(terms);
+ long maxValue = LegacyNumericUtils.getMaxLong(terms);
+ return new FieldStats.Ip(maxDoc, terms.getDocCount(), terms.getSumDocFreq(), terms.getSumTotalTermFreq(), minValue, maxValue);
+ }
}
protected IpFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolateStats.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolateStats.java
deleted file mode 100644
index f4c899dff9a..00000000000
--- a/core/src/main/java/org/elasticsearch/index/percolator/PercolateStats.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.index.percolator;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentBuilderString;
-
-import java.io.IOException;
-
-/**
- * Exposes percolator related statistics.
- */
-public class PercolateStats implements Streamable, ToXContent {
-
- private long percolateCount;
- private long percolateTimeInMillis;
- private long current;
- private long memorySizeInBytes = -1;
- private long numQueries;
-
- /**
- * Noop constructor for serialization purposes.
- */
- public PercolateStats() {
- }
-
- PercolateStats(long percolateCount, long percolateTimeInMillis, long current, long memorySizeInBytes, long numQueries) {
- this.percolateCount = percolateCount;
- this.percolateTimeInMillis = percolateTimeInMillis;
- this.current = current;
- this.memorySizeInBytes = memorySizeInBytes;
- this.numQueries = numQueries;
- }
-
- /**
- * @return The number of times the percolate api has been invoked.
- */
- public long getCount() {
- return percolateCount;
- }
-
- /**
- * @return The total amount of time spend in the percolate api
- */
- public long getTimeInMillis() {
- return percolateTimeInMillis;
- }
-
- /**
- * @return The total amount of time spend in the percolate api
- */
- public TimeValue getTime() {
- return new TimeValue(getTimeInMillis());
- }
-
- /**
- * @return The total amount of active percolate api invocations.
- */
- public long getCurrent() {
- return current;
- }
-
- /**
- * @return The total number of loaded percolate queries.
- */
- public long getNumQueries() {
- return numQueries;
- }
-
- /**
- * @return Temporarily returns -1, but this used to return the total size the loaded queries take in
- * memory, but this is disabled now because the size estimation was too expensive cpu wise. This will be enabled
- * again when a cheaper size estimation can be found.
- */
- public long getMemorySizeInBytes() {
- return memorySizeInBytes;
- }
-
- /**
- * @return The total size the loaded queries take in memory.
- */
- public ByteSizeValue getMemorySize() {
- return new ByteSizeValue(memorySizeInBytes);
- }
-
- @Override
- public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
- builder.startObject(Fields.PERCOLATE);
- builder.field(Fields.TOTAL, percolateCount);
- builder.timeValueField(Fields.TIME_IN_MILLIS, Fields.TIME, percolateTimeInMillis);
- builder.field(Fields.CURRENT, current);
- builder.field(Fields.MEMORY_SIZE_IN_BYTES, memorySizeInBytes);
- builder.field(Fields.MEMORY_SIZE, getMemorySize());
- builder.field(Fields.QUERIES, getNumQueries());
- builder.endObject();
- return builder;
- }
-
- public void add(PercolateStats percolate) {
- if (percolate == null) {
- return;
- }
-
- percolateCount += percolate.getCount();
- percolateTimeInMillis += percolate.getTimeInMillis();
- current += percolate.getCurrent();
- numQueries += percolate.getNumQueries();
- }
-
- static final class Fields {
- static final XContentBuilderString PERCOLATE = new XContentBuilderString("percolate");
- static final XContentBuilderString TOTAL = new XContentBuilderString("total");
- static final XContentBuilderString TIME = new XContentBuilderString("time");
- static final XContentBuilderString TIME_IN_MILLIS = new XContentBuilderString("time_in_millis");
- static final XContentBuilderString CURRENT = new XContentBuilderString("current");
- static final XContentBuilderString MEMORY_SIZE_IN_BYTES = new XContentBuilderString("memory_size_in_bytes");
- static final XContentBuilderString MEMORY_SIZE = new XContentBuilderString("memory_size");
- static final XContentBuilderString QUERIES = new XContentBuilderString("queries");
- }
-
- public static PercolateStats readPercolateStats(StreamInput in) throws IOException {
- PercolateStats stats = new PercolateStats();
- stats.readFrom(in);
- return stats;
- }
-
- @Override
- public void readFrom(StreamInput in) throws IOException {
- percolateCount = in.readVLong();
- percolateTimeInMillis = in.readVLong();
- current = in.readVLong();
- numQueries = in.readVLong();
- }
-
- @Override
- public void writeTo(StreamOutput out) throws IOException {
- out.writeVLong(percolateCount);
- out.writeVLong(percolateTimeInMillis);
- out.writeVLong(current);
- out.writeVLong(numQueries);
- }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
index f44d454655e..338de5c333d 100644
--- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
@@ -19,34 +19,50 @@
package org.elasticsearch.index.percolator;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.search.Query;
+import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.core.BinaryFieldMapper;
import org.elasticsearch.index.mapper.core.KeywordFieldMapper;
+import org.elasticsearch.index.mapper.core.StringFieldMapper;
+import org.elasticsearch.index.query.ParsedQuery;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
public class PercolatorFieldMapper extends FieldMapper {
+ public static final String TYPE_NAME = ".percolator";
public static final String NAME = "query";
public static final String CONTENT_TYPE = "percolator";
public static final PercolatorFieldType FIELD_TYPE = new PercolatorFieldType();
private static final String EXTRACTED_TERMS_FIELD_NAME = "extracted_terms";
private static final String UNKNOWN_QUERY_FIELD_NAME = "unknown_query";
+ private static final String QUERY_BUILDER_FIELD_NAME = "query_builder_field";
+
public static final String EXTRACTED_TERMS_FULL_FIELD_NAME = NAME + "." + EXTRACTED_TERMS_FIELD_NAME;
public static final String UNKNOWN_QUERY_FULL_FIELD_NAME = NAME + "." + UNKNOWN_QUERY_FIELD_NAME;
+ public static final String QUERY_BUILDER_FULL_FIELD_NAME = NAME + "." + QUERY_BUILDER_FIELD_NAME;
public static class Builder extends FieldMapper.Builder {
@@ -60,19 +76,29 @@ public class PercolatorFieldMapper extends FieldMapper {
@Override
public PercolatorFieldMapper build(BuilderContext context) {
context.path().add(name);
- KeywordFieldMapper extractedTermsField = createStringFieldBuilder(EXTRACTED_TERMS_FIELD_NAME).build(context);
- KeywordFieldMapper unknownQueryField = createStringFieldBuilder(UNKNOWN_QUERY_FIELD_NAME).build(context);
+ KeywordFieldMapper extractedTermsField = createExtractQueryFieldBuilder(EXTRACTED_TERMS_FIELD_NAME).build(context);
+ KeywordFieldMapper unknownQueryField = createExtractQueryFieldBuilder(UNKNOWN_QUERY_FIELD_NAME).build(context);
+ BinaryFieldMapper queryBuilderField = createQueryBuilderFieldBuilder().build(context);
context.path().remove();
- return new PercolatorFieldMapper(name(), fieldType, defaultFieldType, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo, queryShardContext, extractedTermsField, unknownQueryField);
+ return new PercolatorFieldMapper(name(), fieldType, defaultFieldType, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo, queryShardContext, extractedTermsField, unknownQueryField, queryBuilderField);
}
- static KeywordFieldMapper.Builder createStringFieldBuilder(String name) {
+ static KeywordFieldMapper.Builder createExtractQueryFieldBuilder(String name) {
KeywordFieldMapper.Builder queryMetaDataFieldBuilder = new KeywordFieldMapper.Builder(name);
queryMetaDataFieldBuilder.docValues(false);
queryMetaDataFieldBuilder.store(false);
queryMetaDataFieldBuilder.indexOptions(IndexOptions.DOCS);
return queryMetaDataFieldBuilder;
}
+
+ static BinaryFieldMapper.Builder createQueryBuilderFieldBuilder() {
+ BinaryFieldMapper.Builder builder = new BinaryFieldMapper.Builder(QUERY_BUILDER_FIELD_NAME);
+ builder.docValues(true);
+ builder.indexOptions(IndexOptions.NONE);
+ builder.store(false);
+ builder.fieldType().setDocValuesType(DocValuesType.BINARY);
+ return builder;
+ }
}
public static class TypeParser implements FieldMapper.TypeParser {
@@ -111,26 +137,81 @@ public class PercolatorFieldMapper extends FieldMapper {
private final QueryShardContext queryShardContext;
private final KeywordFieldMapper queryTermsField;
private final KeywordFieldMapper unknownQueryField;
+ private final BinaryFieldMapper queryBuilderField;
- public PercolatorFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Settings indexSettings, MultiFields multiFields, CopyTo copyTo, QueryShardContext queryShardContext, KeywordFieldMapper queryTermsField, KeywordFieldMapper unknownQueryField) {
+ public PercolatorFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
+ Settings indexSettings, MultiFields multiFields, CopyTo copyTo, QueryShardContext queryShardContext,
+ KeywordFieldMapper queryTermsField, KeywordFieldMapper unknownQueryField,
+ BinaryFieldMapper queryBuilderField) {
super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
this.queryShardContext = queryShardContext;
this.queryTermsField = queryTermsField;
this.unknownQueryField = unknownQueryField;
- this.mapUnmappedFieldAsString = PercolatorQueriesRegistry.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING.get(indexSettings);
+ this.queryBuilderField = queryBuilderField;
+ this.mapUnmappedFieldAsString = PercolatorQueryCache.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING.get(indexSettings);
}
@Override
public Mapper parse(ParseContext context) throws IOException {
QueryShardContext queryShardContext = new QueryShardContext(this.queryShardContext);
- Query query = PercolatorQueriesRegistry.parseQuery(queryShardContext, mapUnmappedFieldAsString, context.parser());
+ QueryBuilder queryBuilder = parseQueryBuilder(queryShardContext.parseContext(), context.parser());
+ // Fetching of terms, shapes and indexed scripts happen during this rewrite:
+ queryBuilder = queryBuilder.rewrite(queryShardContext);
+
+ try (XContentBuilder builder = XContentFactory.contentBuilder(PercolatorQueryCache.QUERY_BUILDER_CONTENT_TYPE)) {
+ queryBuilder.toXContent(builder, new MapParams(Collections.emptyMap()));
+ builder.flush();
+ byte[] queryBuilderAsBytes = builder.bytes().toBytes();
+ context.doc().add(new Field(queryBuilderField.name(), queryBuilderAsBytes, queryBuilderField.fieldType()));
+ }
+
+ Query query = toQuery(queryShardContext, mapUnmappedFieldAsString, queryBuilder);
ExtractQueryTermsService.extractQueryTerms(query, context.doc(), queryTermsField.name(), unknownQueryField.name(), queryTermsField.fieldType());
return null;
}
+ public static Query parseQuery(QueryShardContext context, boolean mapUnmappedFieldsAsString, XContentParser parser) throws IOException {
+ return toQuery(context, mapUnmappedFieldsAsString, parseQueryBuilder(context.parseContext(), parser));
+ }
+
+ static Query toQuery(QueryShardContext context, boolean mapUnmappedFieldsAsString, QueryBuilder queryBuilder) throws IOException {
+ context.reset();
+ // This means that fields in the query need to exist in the mapping prior to registering this query
+ // The reason that this is required, is that if a field doesn't exist then the query assumes defaults, which may be undesired.
+ //
+ // Even worse when fields mentioned in percolator queries do go added to map after the queries have been registered
+ // then the percolator queries don't work as expected any more.
+ //
+ // Query parsing can't introduce new fields in mappings (which happens when registering a percolator query),
+ // because field type can't be inferred from queries (like document do) so the best option here is to disallow
+ // the usage of unmapped fields in percolator queries to avoid unexpected behaviour
+ //
+ // if index.percolator.map_unmapped_fields_as_string is set to true, query can contain unmapped fields which will be mapped
+ // as an analyzed string.
+ context.setAllowUnmappedFields(false);
+ context.setMapUnmappedFieldAsString(mapUnmappedFieldsAsString);
+ context.parseFieldMatcher(context.getIndexSettings().getParseFieldMatcher());
+ try {
+ return queryBuilder.toQuery(context);
+ } finally {
+ context.reset();
+ }
+ }
+
+ static QueryBuilder parseQueryBuilder(QueryParseContext context, XContentParser parser) {
+ context.reset(parser);
+ try {
+ return context.parseInnerQueryBuilder();
+ } catch (IOException e) {
+ throw new ParsingException(parser.getTokenLocation(), "Failed to parse", e);
+ } finally {
+ context.reset(null);
+ }
+ }
+
@Override
public Iterator iterator() {
- return Arrays.asList(queryTermsField, unknownQueryField).iterator();
+ return Arrays.asList(queryTermsField, unknownQueryField, queryBuilderField).iterator();
}
@Override
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhase.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhase.java
new file mode 100644
index 00000000000..c1f9720b53b
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhase.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.percolator;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.ReaderUtil;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.BoostQuery;
+import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.index.percolator.PercolatorFieldMapper;
+import org.elasticsearch.index.percolator.PercolatorQueryCache;
+import org.elasticsearch.index.query.ParsedQuery;
+import org.elasticsearch.index.query.PercolatorQuery;
+import org.elasticsearch.search.SearchParseElement;
+import org.elasticsearch.search.fetch.FetchSubPhase;
+import org.elasticsearch.search.highlight.HighlightPhase;
+import org.elasticsearch.search.highlight.SearchContextHighlight;
+import org.elasticsearch.search.internal.InternalSearchHit;
+import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.internal.SubSearchContext;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+// Highlighting in the case of the percolator query is a bit different, because the PercolatorQuery itself doesn't get highlighted,
+// but the source of the PercolatorQuery gets highlighted by each hit with type '.percolator' (percolator queries).
+public class PercolatorHighlightSubFetchPhase implements FetchSubPhase {
+
+ private final HighlightPhase highlightPhase;
+
+ @Inject
+ public PercolatorHighlightSubFetchPhase(HighlightPhase highlightPhase) {
+ this.highlightPhase = highlightPhase;
+ }
+
+ @Override
+ public boolean hitsExecutionNeeded(SearchContext context) {
+ return context.highlight() != null && locatePercolatorQuery(context.query()) != null;
+ }
+
+ @Override
+ public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {
+ PercolatorQuery percolatorQuery = locatePercolatorQuery(context.query());
+ if (percolatorQuery == null) {
+ // shouldn't happen as we checked for the existence of a percolator query in hitsExecutionNeeded(...)
+ throw new IllegalStateException("couldn't locate percolator query");
+ }
+
+ List ctxs = context.searcher().getIndexReader().leaves();
+ PercolatorQueryCache queriesRegistry = context.percolatorQueryCache();
+ IndexSearcher percolatorIndexSearcher = percolatorQuery.getPercolatorIndexSearcher();
+
+ LeafReaderContext percolatorLeafReaderContext = percolatorIndexSearcher.getIndexReader().leaves().get(0);
+ FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext();
+ SubSearchContext subSearchContext =
+ createSubSearchContext(context, percolatorLeafReaderContext, percolatorQuery.getDocumentSource());
+
+ for (InternalSearchHit hit : hits) {
+ if (PercolatorFieldMapper.TYPE_NAME.equals(hit.getType())) {
+ LeafReaderContext ctx = ctxs.get(ReaderUtil.subIndex(hit.docId(), ctxs));
+ Query query = queriesRegistry.getQueries(ctx).getQuery(hit.docId() - ctx.docBase);
+ subSearchContext.parsedQuery(new ParsedQuery(query));
+ hitContext.reset(
+ new InternalSearchHit(0, "unknown", new Text(percolatorQuery.getDocumentType()), Collections.emptyMap()),
+ percolatorLeafReaderContext, 0, percolatorIndexSearcher
+ );
+ hitContext.cache().clear();
+ highlightPhase.hitExecute(subSearchContext, hitContext);
+ hit.highlightFields().putAll(hitContext.hit().getHighlightFields());
+ }
+ }
+
+ }
+
+ @Override
+ public Map parseElements() {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public boolean hitExecutionNeeded(SearchContext context) {
+ return false;
+ }
+
+ @Override
+ public void hitExecute(SearchContext context, HitContext hitContext) {
+ }
+
+ static PercolatorQuery locatePercolatorQuery(Query query) {
+ if (query instanceof PercolatorQuery) {
+ return (PercolatorQuery) query;
+ } else if (query instanceof BooleanQuery) {
+ for (BooleanClause clause : ((BooleanQuery) query).clauses()) {
+ PercolatorQuery result = locatePercolatorQuery(clause.getQuery());
+ if (result != null) {
+ return result;
+ }
+ }
+ } else if (query instanceof ConstantScoreQuery) {
+ return locatePercolatorQuery(((ConstantScoreQuery) query).getQuery());
+ } else if (query instanceof BoostQuery) {
+ return locatePercolatorQuery(((BoostQuery) query).getQuery());
+ }
+
+ return null;
+ }
+
+ private SubSearchContext createSubSearchContext(SearchContext context, LeafReaderContext leafReaderContext, BytesReference source) {
+ SubSearchContext subSearchContext = new SubSearchContext(context);
+ subSearchContext.highlight(new SearchContextHighlight(context.highlight().fields()));
+ // Enforce highlighting by source, because MemoryIndex doesn't support stored fields.
+ subSearchContext.highlight().globalForceSource(true);
+ subSearchContext.lookup().source().setSegmentAndDocument(leafReaderContext, 0);
+ subSearchContext.lookup().source().setSource(source);
+ return subSearchContext;
+ }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java
deleted file mode 100644
index 0a0cb9e96d9..00000000000
--- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueriesRegistry.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.percolator;
-
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.metrics.CounterMetric;
-import org.elasticsearch.common.metrics.MeanMetric;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.mapper.Uid;
-import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
-import org.elasticsearch.index.mapper.internal.UidFieldMapper;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.shard.AbstractIndexShardComponent;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.percolator.PercolatorService;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Map;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Each shard will have a percolator registry even if there isn't a {@link PercolatorService#TYPE_NAME} document type in the index.
- * For shards with indices that have no {@link PercolatorService#TYPE_NAME} document type, this will hold no percolate queries.
- *
- * Once a document type has been created, the real-time percolator will start to listen to write events and update the
- * this registry with queries in real time.
- */
-public final class PercolatorQueriesRegistry extends AbstractIndexShardComponent implements Closeable {
-
- public final static Setting INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING =
- Setting.boolSetting("index.percolator.map_unmapped_fields_as_string", false, Property.IndexScope);
-
- private final ConcurrentMap percolateQueries = ConcurrentCollections.newConcurrentMapWithAggressiveConcurrency();
- private final QueryShardContext queryShardContext;
- private boolean mapUnmappedFieldsAsString;
- private final MeanMetric percolateMetric = new MeanMetric();
- private final CounterMetric currentMetric = new CounterMetric();
- private final CounterMetric numberOfQueries = new CounterMetric();
-
- public PercolatorQueriesRegistry(ShardId shardId, IndexSettings indexSettings, QueryShardContext queryShardContext) {
- super(shardId, indexSettings);
- this.queryShardContext = queryShardContext;
- this.mapUnmappedFieldsAsString = indexSettings.getValue(INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING);
- }
-
- public ConcurrentMap getPercolateQueries() {
- return percolateQueries;
- }
-
- @Override
- public void close() {
- clear();
- }
-
- public void clear() {
- percolateQueries.clear();
- }
-
-
- public void addPercolateQuery(String idAsString, BytesReference source) {
- Query newquery = parsePercolatorDocument(idAsString, source);
- BytesRef id = new BytesRef(idAsString);
- percolateQueries.put(id, newquery);
- numberOfQueries.inc();
-
- }
-
- public void removePercolateQuery(String idAsString) {
- BytesRef id = new BytesRef(idAsString);
- Query query = percolateQueries.remove(id);
- if (query != null) {
- numberOfQueries.dec();
- }
- }
-
- public Query parsePercolatorDocument(String id, BytesReference source) {
- try (XContentParser sourceParser = XContentHelper.createParser(source)) {
- String currentFieldName = null;
- XContentParser.Token token = sourceParser.nextToken(); // move the START_OBJECT
- if (token != XContentParser.Token.START_OBJECT) {
- throw new ElasticsearchException("failed to parse query [" + id + "], not starting with OBJECT");
- }
- while ((token = sourceParser.nextToken()) != XContentParser.Token.END_OBJECT) {
- if (token == XContentParser.Token.FIELD_NAME) {
- currentFieldName = sourceParser.currentName();
- } else if (token == XContentParser.Token.START_OBJECT) {
- if ("query".equals(currentFieldName)) {
- return parseQuery(queryShardContext, mapUnmappedFieldsAsString, sourceParser);
- } else {
- sourceParser.skipChildren();
- }
- } else if (token == XContentParser.Token.START_ARRAY) {
- sourceParser.skipChildren();
- }
- }
- } catch (Exception e) {
- throw new PercolatorException(shardId().getIndex(), "failed to parse query [" + id + "]", e);
- }
- return null;
- }
-
- public static Query parseQuery(QueryShardContext queryShardContext, boolean mapUnmappedFieldsAsString, XContentParser parser) {
- QueryShardContext context = new QueryShardContext(queryShardContext);
- try {
- context.reset(parser);
- // This means that fields in the query need to exist in the mapping prior to registering this query
- // The reason that this is required, is that if a field doesn't exist then the query assumes defaults, which may be undesired.
- //
- // Even worse when fields mentioned in percolator queries do go added to map after the queries have been registered
- // then the percolator queries don't work as expected any more.
- //
- // Query parsing can't introduce new fields in mappings (which happens when registering a percolator query),
- // because field type can't be inferred from queries (like document do) so the best option here is to disallow
- // the usage of unmapped fields in percolator queries to avoid unexpected behaviour
- //
- // if index.percolator.map_unmapped_fields_as_string is set to true, query can contain unmapped fields which will be mapped
- // as an analyzed string.
- context.setAllowUnmappedFields(false);
- context.setMapUnmappedFieldAsString(mapUnmappedFieldsAsString);
- return context.parseInnerQuery();
- } catch (IOException e) {
- throw new ParsingException(parser.getTokenLocation(), "Failed to parse", e);
- } finally {
- context.reset(null);
- }
- }
-
- public void loadQueries(IndexReader reader) {
- logger.trace("loading percolator queries...");
- final int loadedQueries;
- try {
- Query query = new TermQuery(new Term(TypeFieldMapper.NAME, PercolatorService.TYPE_NAME));
- QueriesLoaderCollector queryCollector = new QueriesLoaderCollector(PercolatorQueriesRegistry.this, logger);
- IndexSearcher indexSearcher = new IndexSearcher(reader);
- indexSearcher.setQueryCache(null);
- indexSearcher.search(query, queryCollector);
- Map queries = queryCollector.queries();
- for (Map.Entry entry : queries.entrySet()) {
- percolateQueries.put(entry.getKey(), entry.getValue());
- numberOfQueries.inc();
- }
- loadedQueries = queries.size();
- } catch (Exception e) {
- throw new PercolatorException(shardId.getIndex(), "failed to load queries from percolator index", e);
- }
- logger.debug("done loading [{}] percolator queries", loadedQueries);
- }
-
- public boolean isPercolatorQuery(Engine.Index operation) {
- if (PercolatorService.TYPE_NAME.equals(operation.type())) {
- parsePercolatorDocument(operation.id(), operation.source());
- return true;
- }
- return false;
- }
-
- public boolean isPercolatorQuery(Engine.Delete operation) {
- return PercolatorService.TYPE_NAME.equals(operation.type());
- }
-
- public synchronized void updatePercolateQuery(Engine engine, String id) {
- // this can be called out of order as long as for every change to a percolator document it's invoked. This will always
- // fetch the latest change but might fetch the same change twice if updates / deletes happen concurrently.
- try (Engine.GetResult getResult = engine.get(new Engine.Get(true, new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(PercolatorService.TYPE_NAME, id))))) {
- if (getResult.exists()) {
- addPercolateQuery(id, getResult.source().source);
- } else {
- removePercolateQuery(id);
- }
- }
- }
-
- public void prePercolate() {
- currentMetric.inc();
- }
-
- public void postPercolate(long tookInNanos) {
- currentMetric.dec();
- percolateMetric.inc(tookInNanos);
- }
-
- /**
- * @return The current metrics
- */
- public PercolateStats stats() {
- return new PercolateStats(percolateMetric.count(), TimeUnit.NANOSECONDS.toMillis(percolateMetric.sum()), currentMetric.count(), -1, numberOfQueries.count());
- }
-
- // Enable when a more efficient manner is found for estimating the size of a Lucene query.
- /*private static long computeSizeInMemory(HashedBytesRef id, Query query) {
- long size = (3 * RamUsageEstimator.NUM_BYTES_INT) + RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + id.bytes.bytes.length;
- size += RamEstimator.sizeOf(query);
- return size;
- }
-
- private static final class RamEstimator {
- // we move this into it's own class to exclude it from the forbidden API checks
- // it's fine to use here!
- static long sizeOf(Query query) {
- return RamUsageEstimator.sizeOf(query);
- }
- }*/
-}
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java
new file mode 100644
index 00000000000..7c9602b4909
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java
@@ -0,0 +1,266 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.percolator;
+
+import com.carrotsearch.hppc.IntObjectHashMap;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.cache.Cache;
+import org.elasticsearch.common.cache.CacheBuilder;
+import org.elasticsearch.common.lucene.index.ElasticsearchLeafReader;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.AbstractIndexComponent;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexWarmer;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
+import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
+import org.elasticsearch.index.query.PercolatorQuery;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.shard.ShardUtils;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+import java.util.function.Supplier;
+
+public final class PercolatorQueryCache extends AbstractIndexComponent
+ implements Closeable, LeafReader.CoreClosedListener, PercolatorQuery.QueryRegistry {
+
+ public final static Setting INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING =
+ Setting.boolSetting("index.percolator.map_unmapped_fields_as_string", false, Setting.Property.IndexScope);
+
+ public final static XContentType QUERY_BUILDER_CONTENT_TYPE = XContentType.SMILE;
+
+ private final Supplier queryShardContextSupplier;
+ private final Cache cache;
+ private final boolean mapUnmappedFieldsAsString;
+
+ public PercolatorQueryCache(IndexSettings indexSettings, Supplier queryShardContextSupplier) {
+ super(indexSettings);
+ this.queryShardContextSupplier = queryShardContextSupplier;
+ cache = CacheBuilder.builder().build();
+ this.mapUnmappedFieldsAsString = indexSettings.getValue(INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING);
+ }
+
+ @Override
+ public Leaf getQueries(LeafReaderContext ctx) {
+ QueriesLeaf percolatorQueries = cache.get(ctx.reader().getCoreCacheKey());
+ if (percolatorQueries == null) {
+ throw new IllegalStateException("queries not loaded, queries should be have been preloaded during index warming...");
+ }
+ return percolatorQueries;
+ }
+
+ public IndexWarmer.Listener createListener(ThreadPool threadPool) {
+ return new IndexWarmer.Listener() {
+
+ final Executor executor = threadPool.executor(ThreadPool.Names.WARMER);
+
+ @Override
+ public IndexWarmer.TerminationHandle warmNewReaders(IndexShard indexShard, Engine.Searcher searcher) {
+ final CountDownLatch latch = new CountDownLatch(searcher.reader().leaves().size());
+ for (final LeafReaderContext ctx : searcher.reader().leaves()) {
+ executor.execute(() -> {
+ try {
+ final long start = System.nanoTime();
+ QueriesLeaf queries = loadQueries(ctx, indexShard.indexSettings().getIndexVersionCreated());
+ cache.put(ctx.reader().getCoreCacheKey(), queries);
+ if (indexShard.warmerService().logger().isTraceEnabled()) {
+ indexShard.warmerService().logger().trace(
+ "loading percolator queries took [{}]",
+ TimeValue.timeValueNanos(System.nanoTime() - start)
+ );
+ }
+ } catch (Throwable t) {
+ indexShard.warmerService().logger().warn("failed to load percolator queries", t);
+ } finally {
+ latch.countDown();
+ }
+ });
+ }
+ return () -> latch.await();
+ }
+
+ @Override
+ public IndexWarmer.TerminationHandle warmTopReader(IndexShard indexShard, Engine.Searcher searcher) {
+ return IndexWarmer.TerminationHandle.NO_WAIT;
+ }
+ };
+ }
+
+ QueriesLeaf loadQueries(LeafReaderContext context, Version indexVersionCreated) throws IOException {
+ LeafReader leafReader = context.reader();
+ ShardId shardId = ShardUtils.extractShardId(leafReader);
+ if (shardId == null) {
+ throw new IllegalStateException("can't resolve shard id");
+ }
+ if (indexSettings.getIndex().equals(shardId.getIndex()) == false) {
+ // percolator cache insanity
+ String message = "Trying to load queries for index " + shardId.getIndex() + " with cache of index " + indexSettings.getIndex();
+ throw new IllegalStateException(message);
+ }
+
+ IntObjectHashMap queries = new IntObjectHashMap<>();
+ boolean legacyLoading = indexVersionCreated.before(Version.V_5_0_0);
+ PostingsEnum postings = leafReader.postings(new Term(TypeFieldMapper.NAME, PercolatorFieldMapper.TYPE_NAME), PostingsEnum.NONE);
+ if (postings != null) {
+ if (legacyLoading) {
+ LegacyQueryFieldVisitor visitor = new LegacyQueryFieldVisitor();
+ for (int docId = postings.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = postings.nextDoc()) {
+ leafReader.document(docId, visitor);
+ queries.put(docId, parseLegacyPercolatorDocument(docId, visitor.source));
+ visitor.source = null; // reset
+ }
+ } else {
+ BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues(PercolatorFieldMapper.QUERY_BUILDER_FULL_FIELD_NAME);
+ if (binaryDocValues != null) {
+ for (int docId = postings.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = postings.nextDoc()) {
+ BytesRef queryBuilder = binaryDocValues.get(docId);
+ if (queryBuilder.length > 0) {
+ queries.put(docId, parseQueryBuilder(docId, queryBuilder));
+ }
+ }
+ }
+ }
+ }
+ leafReader.addCoreClosedListener(this);
+ return new QueriesLeaf(shardId, queries);
+ }
+
+ private Query parseQueryBuilder(int docId, BytesRef queryBuilder) {
+ XContent xContent = QUERY_BUILDER_CONTENT_TYPE.xContent();
+ try (XContentParser sourceParser = xContent.createParser(queryBuilder.bytes, queryBuilder.offset, queryBuilder.length)) {
+ QueryShardContext context = queryShardContextSupplier.get();
+ return PercolatorFieldMapper.parseQuery(context, mapUnmappedFieldsAsString, sourceParser);
+ } catch (IOException e) {
+ throw new PercolatorException(index(), "failed to parse query builder for document [" + docId + "]", e);
+ }
+ }
+
+ private Query parseLegacyPercolatorDocument(int docId, BytesReference source) {
+ try (XContentParser sourceParser = XContentHelper.createParser(source)) {
+ String currentFieldName = null;
+ XContentParser.Token token = sourceParser.nextToken(); // move the START_OBJECT
+ if (token != XContentParser.Token.START_OBJECT) {
+ throw new ElasticsearchException("failed to parse query [" + docId + "], not starting with OBJECT");
+ }
+ while ((token = sourceParser.nextToken()) != XContentParser.Token.END_OBJECT) {
+ if (token == XContentParser.Token.FIELD_NAME) {
+ currentFieldName = sourceParser.currentName();
+ } else if (token == XContentParser.Token.START_OBJECT) {
+ if ("query".equals(currentFieldName)) {
+ QueryShardContext context = queryShardContextSupplier.get();
+ return PercolatorFieldMapper.parseQuery(context, mapUnmappedFieldsAsString, sourceParser);
+ } else {
+ sourceParser.skipChildren();
+ }
+ } else if (token == XContentParser.Token.START_ARRAY) {
+ sourceParser.skipChildren();
+ }
+ }
+ } catch (Exception e) {
+ throw new PercolatorException(index(), "failed to parse query [" + docId + "]", e);
+ }
+ return null;
+ }
+
+ public PercolatorQueryCacheStats getStats(ShardId shardId) {
+ int numberOfQueries = 0;
+ for (QueriesLeaf queries : cache.values()) {
+ if (shardId.equals(queries.shardId)) {
+ numberOfQueries += queries.queries.size();
+ }
+ }
+ return new PercolatorQueryCacheStats(numberOfQueries);
+ }
+
+ @Override
+ public void onClose(Object cacheKey) throws IOException {
+ cache.invalidate(cacheKey);
+ }
+
+ @Override
+ public void close() throws IOException {
+ cache.invalidateAll();
+ }
+
+ final static class LegacyQueryFieldVisitor extends StoredFieldVisitor {
+
+ private BytesArray source;
+
+ @Override
+ public void binaryField(FieldInfo fieldInfo, byte[] bytes) throws IOException {
+ source = new BytesArray(bytes);
+ }
+
+ @Override
+ public Status needsField(FieldInfo fieldInfo) throws IOException {
+ if (source != null) {
+ return Status.STOP;
+ }
+ if (SourceFieldMapper.NAME.equals(fieldInfo.name)) {
+ return Status.YES;
+ } else {
+ return Status.NO;
+ }
+ }
+
+ }
+
+ final static class QueriesLeaf implements Leaf {
+
+ final ShardId shardId;
+ final IntObjectHashMap queries;
+
+ QueriesLeaf(ShardId shardId, IntObjectHashMap queries) {
+ this.shardId = shardId;
+ this.queries = queries;
+ }
+
+ @Override
+ public Query getQuery(int docId) {
+ return queries.get(docId);
+ }
+ }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCacheStats.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCacheStats.java
new file mode 100644
index 00000000000..a8e3b7f4799
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCacheStats.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.percolator;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentBuilderString;
+
+import java.io.IOException;
+
+/**
+ * Exposes percolator query cache statistics.
+ */
+public class PercolatorQueryCacheStats implements Streamable, ToXContent {
+
+ private long numQueries;
+
+ /**
+ * Noop constructor for serialization purposes.
+ */
+ public PercolatorQueryCacheStats() {
+ }
+
+ PercolatorQueryCacheStats(long numQueries) {
+ this.numQueries = numQueries;
+ }
+
+ /**
+ * @return The total number of loaded percolate queries.
+ */
+ public long getNumQueries() {
+ return numQueries;
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject(Fields.PERCOLATOR);
+ builder.field(Fields.QUERIES, getNumQueries());
+ builder.endObject();
+ return builder;
+ }
+
+ public void add(PercolatorQueryCacheStats percolate) {
+ if (percolate == null) {
+ return;
+ }
+
+ numQueries += percolate.getNumQueries();
+ }
+
+ static final class Fields {
+ static final XContentBuilderString PERCOLATOR = new XContentBuilderString("percolator");
+ static final XContentBuilderString QUERIES = new XContentBuilderString("num_queries");
+ }
+
+ public static PercolatorQueryCacheStats readPercolateStats(StreamInput in) throws IOException {
+ PercolatorQueryCacheStats stats = new PercolatorQueryCacheStats();
+ stats.readFrom(in);
+ return stats;
+ }
+
+ @Override
+ public void readFrom(StreamInput in) throws IOException {
+ numQueries = in.readVLong();
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ out.writeVLong(numQueries);
+ }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/QueriesLoaderCollector.java b/core/src/main/java/org/elasticsearch/index/percolator/QueriesLoaderCollector.java
deleted file mode 100644
index 1bea43e4ea1..00000000000
--- a/core/src/main/java/org/elasticsearch/index/percolator/QueriesLoaderCollector.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.index.percolator;
-
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.SimpleCollector;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.IndexFieldDataService;
-import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
-import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.Uid;
-import org.elasticsearch.index.mapper.internal.UidFieldMapper;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- */
-final class QueriesLoaderCollector extends SimpleCollector {
-
- private final Map queries = new HashMap<>();
- private final FieldsVisitor fieldsVisitor = new FieldsVisitor(true);
- private final PercolatorQueriesRegistry percolator;
- private final ESLogger logger;
-
- private LeafReader reader;
-
- QueriesLoaderCollector(PercolatorQueriesRegistry percolator, ESLogger logger) {
- this.percolator = percolator;
- this.logger = logger;
- }
-
- public Map queries() {
- return this.queries;
- }
-
- @Override
- public void collect(int doc) throws IOException {
- fieldsVisitor.reset();
- reader.document(doc, fieldsVisitor);
- final Uid uid = fieldsVisitor.uid();
-
- try {
- // id is only used for logging, if we fail we log the id in the catch statement
- final Query parseQuery = percolator.parsePercolatorDocument(null, fieldsVisitor.source());
- if (parseQuery != null) {
- queries.put(new BytesRef(uid.id()), parseQuery);
- } else {
- logger.warn("failed to add query [{}] - parser returned null", uid);
- }
-
- } catch (Exception e) {
- logger.warn("failed to add query [{}]", e, uid);
- }
- }
-
- @Override
- protected void doSetNextReader(LeafReaderContext context) throws IOException {
- reader = context.reader();
- }
-
- @Override
- public void setScorer(Scorer scorer) throws IOException {
- }
-
- @Override
- public boolean needsScores() {
- return false;
- }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
index 3c5ca1ce444..324552d7b54 100644
--- a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
@@ -189,7 +189,7 @@ public class HasChildQueryBuilder extends AbstractQueryBuilderhas_child queries.
@@ -104,12 +105,21 @@ public class HasChildQueryParser implements QueryParser {
return ScoreMode.Max;
} else if ("avg".equals(scoreModeString)) {
return ScoreMode.Avg;
- } else if ("total".equals(scoreModeString)) {
+ } else if ("sum".equals(scoreModeString)) {
return ScoreMode.Total;
}
throw new IllegalArgumentException("No score mode for child query [" + scoreModeString + "] found");
}
+ public static String scoreModeAsString(ScoreMode scoreMode) {
+ if (scoreMode == ScoreMode.Total) {
+ // Lucene uses 'total' but 'sum' is more consistent with other elasticsearch APIs
+ return "sum";
+ } else {
+ return scoreMode.name().toLowerCase(Locale.ROOT);
+ }
+ }
+
@Override
public HasChildQueryBuilder getBuilderPrototype() {
return HasChildQueryBuilder.PROTOTYPE;
diff --git a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
index 596c2499211..bd5f348db33 100644
--- a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
@@ -121,7 +121,7 @@ public class NestedQueryBuilder extends AbstractQueryBuilder
query.toXContent(builder, params);
builder.field(NestedQueryParser.PATH_FIELD.getPreferredName(), path);
if (scoreMode != null) {
- builder.field(NestedQueryParser.SCORE_MODE_FIELD.getPreferredName(), scoreMode.name().toLowerCase(Locale.ROOT));
+ builder.field(NestedQueryParser.SCORE_MODE_FIELD.getPreferredName(), HasChildQueryParser.scoreModeAsString(scoreMode));
}
printBoostAndQueryName(builder);
if (queryInnerHits != null) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java
index ba5d7c2447e..218919f7ed2 100644
--- a/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java
@@ -68,20 +68,7 @@ public class NestedQueryParser implements QueryParser {
} else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
boost = parser.floatValue();
} else if (parseContext.parseFieldMatcher().match(currentFieldName, SCORE_MODE_FIELD)) {
- String sScoreMode = parser.text();
- if ("avg".equals(sScoreMode)) {
- scoreMode = ScoreMode.Avg;
- } else if ("min".equals(sScoreMode)) {
- scoreMode = ScoreMode.Min;
- } else if ("max".equals(sScoreMode)) {
- scoreMode = ScoreMode.Max;
- } else if ("total".equals(sScoreMode) || "sum".equals(sScoreMode)) {
- scoreMode = ScoreMode.Total;
- } else if ("none".equals(sScoreMode)) {
- scoreMode = ScoreMode.None;
- } else {
- throw new ParsingException(parser.getTokenLocation(), "illegal score_mode for nested query [" + sScoreMode + "]");
- }
+ scoreMode = HasChildQueryParser.parseScoreMode(parser.text());
} else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
queryName = parser.text();
} else {
diff --git a/core/src/main/java/org/elasticsearch/index/query/ParentIdQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ParentIdQueryBuilder.java
index f9bd7623f35..4a2efa95c9a 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ParentIdQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ParentIdQueryBuilder.java
@@ -19,13 +19,18 @@
package org.elasticsearch.index.query;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.DocValuesTermsQuery;
import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
+import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
import java.io.IOException;
import java.util.Objects;
@@ -71,7 +76,12 @@ public final class ParentIdQueryBuilder extends AbstractQueryBuilder percolatorQueries;
- private Query percolateQuery;
private Query queriesMetaDataQuery;
private final Query percolateTypeQuery;
/**
- * @param percolatorIndexSearcher The index searcher on top of the in-memory index that holds the document being percolated
- * @param percolatorQueries All the registered percolator queries
- * @param percolateTypeQuery A query that identifies all document containing percolator queries
+ * @param docType The type of the document being percolated
+ * @param queryRegistry The registry holding all the percolator queries as Lucene queries.
+ * @param documentSource The source of the document being percolated
+ * @param percolatorIndexSearcher The index searcher on top of the in-memory index that holds the document being percolated
+ * @param percolateTypeQuery A query that identifies all document containing percolator queries
*/
- Builder(IndexSearcher percolatorIndexSearcher, Map percolatorQueries, Query percolateTypeQuery) {
- this.percolatorIndexSearcher = percolatorIndexSearcher;
- this.percolatorQueries = percolatorQueries;
- this.percolateTypeQuery = percolateTypeQuery;
- }
-
- /**
- * Optionally sets a query that reduces the number of queries to percolate based on custom metadata attached
- * on the percolator documents.
- */
- void setPercolateQuery(Query percolateQuery) {
- this.percolateQuery = percolateQuery;
+ public Builder(String docType, QueryRegistry queryRegistry, BytesReference documentSource, IndexSearcher percolatorIndexSearcher,
+ Query percolateTypeQuery) {
+ this.docType = Objects.requireNonNull(docType);
+ this.documentSource = Objects.requireNonNull(documentSource);
+ this.percolatorIndexSearcher = Objects.requireNonNull(percolatorIndexSearcher);
+ this.queryRegistry = Objects.requireNonNull(queryRegistry);
+ this.percolateTypeQuery = Objects.requireNonNull(percolateTypeQuery);
}
/**
@@ -85,39 +81,43 @@ final class PercolatorQuery extends Query {
* @param extractedTermsFieldName The name of the field to get the extracted terms from
* @param unknownQueryFieldname The field used to mark documents whose queries couldn't all get extracted
*/
- void extractQueryTermsQuery(String extractedTermsFieldName, String unknownQueryFieldname) throws IOException {
- this.queriesMetaDataQuery = ExtractQueryTermsService.createQueryTermsQuery(percolatorIndexSearcher.getIndexReader(), extractedTermsFieldName, unknownQueryFieldname);
+ public void extractQueryTermsQuery(String extractedTermsFieldName, String unknownQueryFieldname) throws IOException {
+ this.queriesMetaDataQuery = ExtractQueryTermsService.createQueryTermsQuery(
+ percolatorIndexSearcher.getIndexReader(), extractedTermsFieldName, unknownQueryFieldname
+ );
}
- PercolatorQuery build() {
+ public PercolatorQuery build() {
BooleanQuery.Builder builder = new BooleanQuery.Builder();
builder.add(percolateTypeQuery, FILTER);
if (queriesMetaDataQuery != null) {
builder.add(queriesMetaDataQuery, FILTER);
}
- if (percolateQuery != null){
- builder.add(percolateQuery, MUST);
- }
- return new PercolatorQuery(builder.build(), percolatorIndexSearcher, percolatorQueries);
+ return new PercolatorQuery(docType, queryRegistry, documentSource, builder.build(), percolatorIndexSearcher);
}
}
+ private final String documentType;
+ private final QueryRegistry queryRegistry;
+ private final BytesReference documentSource;
private final Query percolatorQueriesQuery;
private final IndexSearcher percolatorIndexSearcher;
- private final Map percolatorQueries;
- private PercolatorQuery(Query percolatorQueriesQuery, IndexSearcher percolatorIndexSearcher, Map percolatorQueries) {
+ private PercolatorQuery(String documentType, QueryRegistry queryRegistry, BytesReference documentSource,
+ Query percolatorQueriesQuery, IndexSearcher percolatorIndexSearcher) {
+ this.documentType = documentType;
+ this.documentSource = documentSource;
this.percolatorQueriesQuery = percolatorQueriesQuery;
+ this.queryRegistry = queryRegistry;
this.percolatorIndexSearcher = percolatorIndexSearcher;
- this.percolatorQueries = percolatorQueries;
}
@Override
public Query rewrite(IndexReader reader) throws IOException {
Query rewritten = percolatorQueriesQuery.rewrite(reader);
if (rewritten != percolatorQueriesQuery) {
- return new PercolatorQuery(rewritten, percolatorIndexSearcher, percolatorQueries);
+ return new PercolatorQuery(documentType, queryRegistry, documentSource, rewritten, percolatorIndexSearcher);
} else {
return this;
}
@@ -160,7 +160,7 @@ final class PercolatorQuery extends Query {
return null;
}
- final LeafReader leafReader = leafReaderContext.reader();
+ final QueryRegistry.Leaf percolatorQueries = queryRegistry.getQueries(leafReaderContext);
return new Scorer(this) {
@Override
@@ -173,7 +173,7 @@ final class PercolatorQuery extends Query {
return new TwoPhaseIterator(approximation.iterator()) {
@Override
public boolean matches() throws IOException {
- return matchDocId(approximation.docID(), leafReader);
+ return matchDocId(approximation.docID());
}
@Override
@@ -198,27 +198,30 @@ final class PercolatorQuery extends Query {
return approximation.docID();
}
- boolean matchDocId(int docId, LeafReader leafReader) throws IOException {
- SingleFieldsVisitor singleFieldsVisitor = new SingleFieldsVisitor(UidFieldMapper.NAME);
- leafReader.document(docId, singleFieldsVisitor);
- BytesRef percolatorQueryId = new BytesRef(singleFieldsVisitor.uid().id());
- return matchQuery(percolatorQueryId);
+ boolean matchDocId(int docId) throws IOException {
+ Query query = percolatorQueries.getQuery(docId);
+ if (query != null) {
+ return Lucene.exists(percolatorIndexSearcher, query);
+ } else {
+ return false;
+ }
}
};
}
};
}
- boolean matchQuery(BytesRef percolatorQueryId) throws IOException {
- Query percolatorQuery = percolatorQueries.get(percolatorQueryId);
- if (percolatorQuery != null) {
- return Lucene.exists(percolatorIndexSearcher, percolatorQuery);
- } else {
- return false;
- }
+ public IndexSearcher getPercolatorIndexSearcher() {
+ return percolatorIndexSearcher;
}
- private final Object instance = new Object();
+ public String getDocumentType() {
+ return documentType;
+ }
+
+ public BytesReference getDocumentSource() {
+ return documentSource;
+ }
@Override
public boolean equals(Object o) {
@@ -228,19 +231,46 @@ final class PercolatorQuery extends Query {
PercolatorQuery that = (PercolatorQuery) o;
- return instance.equals(that.instance);
+ if (!documentType.equals(that.documentType)) return false;
+ return documentSource.equals(that.documentSource);
}
@Override
public int hashCode() {
int result = super.hashCode();
- result = 31 * result + instance.hashCode();
+ result = 31 * result + documentType.hashCode();
+ result = 31 * result + documentSource.hashCode();
return result;
}
@Override
public String toString(String s) {
- return "PercolatorQuery{inner={" + percolatorQueriesQuery.toString(s) + "}}";
+ return "PercolatorQuery{document_type={" + documentType + "},document_source={" + documentSource.toUtf8() +
+ "},inner={" + percolatorQueriesQuery.toString(s) + "}}";
}
+
+ @Override
+ public long ramBytesUsed() {
+ long sizeInBytes = 0;
+ if (documentSource.hasArray()) {
+ sizeInBytes += documentSource.array().length;
+ } else {
+ sizeInBytes += documentSource.length();
+ }
+ return sizeInBytes;
+ }
+
+ public interface QueryRegistry {
+
+ Leaf getQueries(LeafReaderContext ctx);
+
+ interface Leaf {
+
+ Query getQuery(int docId);
+
+ }
+
+ }
+
}
diff --git a/core/src/main/java/org/elasticsearch/index/query/PercolatorQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/PercolatorQueryBuilder.java
new file mode 100644
index 00000000000..5cb1e54d203
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/query/PercolatorQueryBuilder.java
@@ -0,0 +1,375 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.query;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.MultiReader;
+import org.apache.lucene.index.SlowCompositeReaderWrapper;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.memory.MemoryIndex;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.Weight;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.DocumentMapperForType;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
+import org.elasticsearch.index.mapper.internal.UidFieldMapper;
+import org.elasticsearch.index.percolator.PercolatorFieldMapper;
+import org.elasticsearch.index.percolator.PercolatorQueryCache;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.index.mapper.SourceToParse.source;
+
+public class PercolatorQueryBuilder extends AbstractQueryBuilder {
+
+ public static final String NAME = "percolator";
+ static final PercolatorQueryBuilder PROTO = new PercolatorQueryBuilder(null, null, null, null, null, null, null, null);
+
+ private final String documentType;
+ private final BytesReference document;
+
+ private final String indexedDocumentIndex;
+ private final String indexedDocumentType;
+ private final String indexedDocumentId;
+ private final String indexedDocumentRouting;
+ private final String indexedDocumentPreference;
+ private final Long indexedDocumentVersion;
+
+ public PercolatorQueryBuilder(String documentType, BytesReference document) {
+ if (documentType == null) {
+ throw new IllegalArgumentException("[document_type] is a required argument");
+ }
+ if (document == null) {
+ throw new IllegalArgumentException("[document] is a required argument");
+ }
+ this.documentType = documentType;
+ this.document = document;
+ indexedDocumentIndex = null;
+ indexedDocumentType = null;
+ indexedDocumentId = null;
+ indexedDocumentRouting = null;
+ indexedDocumentPreference = null;
+ indexedDocumentVersion = null;
+ }
+
+ public PercolatorQueryBuilder(String documentType, String indexedDocumentIndex, String indexedDocumentType,
+ String indexedDocumentId, String indexedDocumentRouting, String indexedDocumentPreference,
+ Long indexedDocumentVersion) {
+ if (documentType == null) {
+ throw new IllegalArgumentException("[document_type] is a required argument");
+ }
+ if (indexedDocumentIndex == null) {
+ throw new IllegalArgumentException("[index] is a required argument");
+ }
+ if (indexedDocumentType == null) {
+ throw new IllegalArgumentException("[type] is a required argument");
+ }
+ if (indexedDocumentId == null) {
+ throw new IllegalArgumentException("[id] is a required argument");
+ }
+ this.documentType = documentType;
+ this.indexedDocumentIndex = indexedDocumentIndex;
+ this.indexedDocumentType = indexedDocumentType;
+ this.indexedDocumentId = indexedDocumentId;
+ this.indexedDocumentRouting = indexedDocumentRouting;
+ this.indexedDocumentPreference = indexedDocumentPreference;
+ this.indexedDocumentVersion = indexedDocumentVersion;
+ this.document = null;
+ }
+
+ private PercolatorQueryBuilder(String documentType, BytesReference document, String indexedDocumentIndex, String indexedDocumentType,
+ String indexedDocumentId, String indexedDocumentRouting, String indexedDocumentPreference,
+ Long indexedDocumentVersion) {
+ this.documentType = documentType;
+ this.document = document;
+ this.indexedDocumentIndex = indexedDocumentIndex;
+ this.indexedDocumentType = indexedDocumentType;
+ this.indexedDocumentId = indexedDocumentId;
+ this.indexedDocumentRouting = indexedDocumentRouting;
+ this.indexedDocumentPreference = indexedDocumentPreference;
+ this.indexedDocumentVersion = indexedDocumentVersion;
+ }
+
+ @Override
+ protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject(NAME);
+ builder.field(PercolatorQueryParser.DOCUMENT_TYPE_FIELD.getPreferredName(), documentType);
+ if (document != null) {
+ XContentType contentType = XContentFactory.xContentType(document);
+ if (contentType == builder.contentType()) {
+ builder.rawField(PercolatorQueryParser.DOCUMENT_FIELD.getPreferredName(), document);
+ } else {
+ XContentParser parser = XContentFactory.xContent(contentType).createParser(document);
+ parser.nextToken();
+ builder.field(PercolatorQueryParser.DOCUMENT_FIELD.getPreferredName());
+ builder.copyCurrentStructure(parser);
+ }
+ }
+ if (indexedDocumentIndex != null || indexedDocumentType != null || indexedDocumentId != null) {
+ if (indexedDocumentIndex != null) {
+ builder.field(PercolatorQueryParser.INDEXED_DOCUMENT_FIELD_INDEX.getPreferredName(), indexedDocumentIndex);
+ }
+ if (indexedDocumentType != null) {
+ builder.field(PercolatorQueryParser.INDEXED_DOCUMENT_FIELD_TYPE.getPreferredName(), indexedDocumentType);
+ }
+ if (indexedDocumentId != null) {
+ builder.field(PercolatorQueryParser.INDEXED_DOCUMENT_FIELD_ID.getPreferredName(), indexedDocumentId);
+ }
+ if (indexedDocumentRouting != null) {
+ builder.field(PercolatorQueryParser.INDEXED_DOCUMENT_FIELD_ROUTING.getPreferredName(), indexedDocumentRouting);
+ }
+ if (indexedDocumentPreference != null) {
+ builder.field(PercolatorQueryParser.INDEXED_DOCUMENT_FIELD_PREFERENCE.getPreferredName(), indexedDocumentPreference);
+ }
+ if (indexedDocumentVersion != null) {
+ builder.field(PercolatorQueryParser.INDEXED_DOCUMENT_FIELD_VERSION.getPreferredName(), indexedDocumentVersion);
+ }
+ }
+ printBoostAndQueryName(builder);
+ builder.endObject();
+ }
+
+ @Override
+ protected PercolatorQueryBuilder doReadFrom(StreamInput in) throws IOException {
+ String docType = in.readString();
+ String documentIndex = in.readOptionalString();
+ String documentType = in.readOptionalString();
+ String documentId = in.readOptionalString();
+ String documentRouting = in.readOptionalString();
+ String documentPreference = in.readOptionalString();
+ Long documentVersion = null;
+ if (in.readBoolean()) {
+ documentVersion = in.readVLong();
+ }
+ BytesReference documentSource = null;
+ if (in.readBoolean()) {
+ documentSource = in.readBytesReference();
+ }
+ return new PercolatorQueryBuilder(docType, documentSource, documentIndex, documentType, documentId,
+ documentRouting, documentPreference, documentVersion);
+ }
+
+ @Override
+ protected void doWriteTo(StreamOutput out) throws IOException {
+ out.writeString(documentType);
+ out.writeOptionalString(indexedDocumentIndex);
+ out.writeOptionalString(indexedDocumentType);
+ out.writeOptionalString(indexedDocumentId);
+ out.writeOptionalString(indexedDocumentRouting);
+ out.writeOptionalString(indexedDocumentPreference);
+ if (indexedDocumentVersion != null) {
+ out.writeBoolean(true);
+ out.writeVLong(indexedDocumentVersion);
+ } else {
+ out.writeBoolean(false);
+ }
+ if (document != null) {
+ out.writeBoolean(true);
+ out.writeBytesReference(document);
+ } else {
+ out.writeBoolean(false);
+ }
+ }
+
+ @Override
+ protected boolean doEquals(PercolatorQueryBuilder other) {
+ return Objects.equals(documentType, other.documentType)
+ && Objects.equals(document, other.document)
+ && Objects.equals(indexedDocumentIndex, other.indexedDocumentIndex)
+ && Objects.equals(indexedDocumentType, other.indexedDocumentType)
+ && Objects.equals(indexedDocumentId, other.indexedDocumentId);
+ }
+
+ @Override
+ protected int doHashCode() {
+ return Objects.hash(documentType, document, indexedDocumentIndex, indexedDocumentType, indexedDocumentId);
+ }
+
+ @Override
+ public String getWriteableName() {
+ return NAME;
+ }
+
+ @Override
+ protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException {
+ if (document != null) {
+ return this;
+ }
+
+ GetRequest getRequest = new GetRequest(indexedDocumentIndex, indexedDocumentType, indexedDocumentId);
+ getRequest.preference("_local");
+ getRequest.routing(indexedDocumentRouting);
+ getRequest.preference(indexedDocumentPreference);
+ if (indexedDocumentVersion != null) {
+ getRequest.version(indexedDocumentVersion);
+ }
+ GetResponse getResponse = queryShardContext.getClient().get(getRequest).actionGet();
+ if (getResponse.isExists() == false) {
+ throw new ResourceNotFoundException(
+ "indexed document [{}/{}/{}] couldn't be found", indexedDocumentIndex, indexedDocumentType, indexedDocumentId
+ );
+ }
+ return new PercolatorQueryBuilder(documentType, getResponse.getSourceAsBytesRef());
+ }
+
+ @Override
+ protected Query doToQuery(QueryShardContext context) throws IOException {
+ if (indexedDocumentIndex != null || indexedDocumentType != null || indexedDocumentId != null) {
+ throw new IllegalStateException("query builder must be rewritten first");
+ }
+
+ if (document == null) {
+ throw new IllegalStateException("nothing to percolator");
+ }
+
+ MapperService mapperService = context.getMapperService();
+ DocumentMapperForType docMapperForType = mapperService.documentMapperWithAutoCreate(documentType);
+ DocumentMapper docMapper = docMapperForType.getDocumentMapper();
+
+ ParsedDocument doc = docMapper.parse(source(document)
+ .index(context.index().getName())
+ .id("_temp_id")
+ .type(documentType));
+
+ Analyzer defaultAnalyzer = context.getAnalysisService().defaultIndexAnalyzer();
+ final IndexSearcher docSearcher;
+ if (doc.docs().size() > 1) {
+ assert docMapper.hasNestedObjects();
+ docSearcher = createMultiDocumentSearcher(docMapper, defaultAnalyzer, doc);
+ } else {
+ // TODO: we may want to bring to MemoryIndex thread local cache back...
+ // but I'm unsure about the real benefits.
+ MemoryIndex memoryIndex = new MemoryIndex(true);
+ indexDoc(docMapper, defaultAnalyzer, doc.rootDoc(), memoryIndex);
+ docSearcher = memoryIndex.createSearcher();
+ docSearcher.setQueryCache(null);
+ }
+
+ PercolatorQueryCache registry = context.getPercolatorQueryCache();
+ if (registry == null) {
+ throw new QueryShardException(context, "no percolator query registry");
+ }
+
+ Query percolateTypeQuery = new TermQuery(new Term(TypeFieldMapper.NAME, PercolatorFieldMapper.TYPE_NAME));
+ PercolatorQuery.Builder builder = new PercolatorQuery.Builder(
+ documentType, registry, document, docSearcher, percolateTypeQuery
+ );
+ Settings indexSettings = registry.getIndexSettings().getSettings();
+ if (indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null).onOrAfter(Version.V_5_0_0)) {
+ builder.extractQueryTermsQuery(
+ PercolatorFieldMapper.EXTRACTED_TERMS_FULL_FIELD_NAME, PercolatorFieldMapper.UNKNOWN_QUERY_FULL_FIELD_NAME
+ );
+ }
+ return builder.build();
+ }
+
+ public String getDocumentType() {
+ return documentType;
+ }
+
+ public BytesReference getDocument() {
+ return document;
+ }
+
+ private IndexSearcher createMultiDocumentSearcher(DocumentMapper docMapper, Analyzer defaultAnalyzer, ParsedDocument doc) {
+ IndexReader[] memoryIndices = new IndexReader[doc.docs().size()];
+ List docs = doc.docs();
+ int rootDocIndex = docs.size() - 1;
+ assert rootDocIndex > 0;
+ for (int i = 0; i < docs.size(); i++) {
+ ParseContext.Document d = docs.get(i);
+ MemoryIndex memoryIndex = new MemoryIndex(true);
+ indexDoc(docMapper, defaultAnalyzer, d, memoryIndex);
+ memoryIndices[i] = memoryIndex.createSearcher().getIndexReader();
+ }
+ try {
+ MultiReader mReader = new MultiReader(memoryIndices, true);
+ LeafReader slowReader = SlowCompositeReaderWrapper.wrap(mReader);
+ final IndexSearcher slowSearcher = new IndexSearcher(slowReader) {
+
+ @Override
+ public Weight createNormalizedWeight(Query query, boolean needsScores) throws IOException {
+ BooleanQuery.Builder bq = new BooleanQuery.Builder();
+ bq.add(query, BooleanClause.Occur.MUST);
+ bq.add(Queries.newNestedFilter(), BooleanClause.Occur.MUST_NOT);
+ return super.createNormalizedWeight(bq.build(), needsScores);
+ }
+
+ };
+ slowSearcher.setQueryCache(null);
+ return slowSearcher;
+ } catch (IOException e) {
+ throw new ElasticsearchException("Failed to create index for percolator with nested document ", e);
+ }
+ }
+
+ private void indexDoc(DocumentMapper documentMapper, Analyzer defaultAnalyzer, ParseContext.Document document,
+ MemoryIndex memoryIndex) {
+ for (IndexableField field : document.getFields()) {
+ if (field.fieldType().indexOptions() == IndexOptions.NONE && field.name().equals(UidFieldMapper.NAME)) {
+ continue;
+ }
+
+ Analyzer analyzer = defaultAnalyzer;
+ if (documentMapper != null && documentMapper.mappers().getMapper(field.name()) != null) {
+ analyzer = documentMapper.mappers().indexAnalyzer();
+ }
+ try {
+ try (TokenStream tokenStream = field.tokenStream(analyzer, null)) {
+ if (tokenStream != null) {
+ memoryIndex.addField(field.name(), tokenStream, field.boost());
+ }
+ }
+ } catch (IOException e) {
+ throw new ElasticsearchException("Failed to create token stream", e);
+ }
+ }
+ }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/index/query/PercolatorQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/PercolatorQueryParser.java
new file mode 100644
index 00000000000..a559db59927
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/query/PercolatorQueryParser.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.query;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+
+public class PercolatorQueryParser implements QueryParser {
+
+ public static final ParseField DOCUMENT_FIELD = new ParseField("document");
+ public static final ParseField DOCUMENT_TYPE_FIELD = new ParseField("document_type");
+ public static final ParseField INDEXED_DOCUMENT_FIELD_INDEX = new ParseField("index");
+ public static final ParseField INDEXED_DOCUMENT_FIELD_TYPE = new ParseField("type");
+ public static final ParseField INDEXED_DOCUMENT_FIELD_ID = new ParseField("id");
+ public static final ParseField INDEXED_DOCUMENT_FIELD_ROUTING = new ParseField("routing");
+ public static final ParseField INDEXED_DOCUMENT_FIELD_PREFERENCE = new ParseField("preference");
+ public static final ParseField INDEXED_DOCUMENT_FIELD_VERSION = new ParseField("version");
+
+ @Override
+ public String[] names() {
+ return new String[]{PercolatorQueryBuilder.NAME};
+ }
+
+ @Override
+ public PercolatorQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+ XContentParser parser = parseContext.parser();
+ float boost = AbstractQueryBuilder.DEFAULT_BOOST;
+
+ String documentType = null;
+
+ String indexedDocumentIndex = null;
+ String indexedDocumentType = null;
+ String indexedDocumentId = null;
+ String indexedDocumentRouting = null;
+ String indexedDocumentPreference = null;
+ Long indexedDocumentVersion = null;
+
+ BytesReference source = null;
+
+ String queryName = null;
+ String currentFieldName = null;
+
+ XContentParser.Token token;
+ while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+ if (token == XContentParser.Token.FIELD_NAME) {
+ currentFieldName = parser.currentName();
+ } else if (token == XContentParser.Token.START_OBJECT) {
+ if (parseContext.parseFieldMatcher().match(currentFieldName, DOCUMENT_FIELD)) {
+ try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
+ builder.copyCurrentStructure(parser);
+ builder.flush();
+ source = builder.bytes();
+ }
+ } else {
+ throw new ParsingException(parser.getTokenLocation(), "[" + PercolatorQueryBuilder.NAME +
+ "] query does not support [" + token + "]");
+ }
+ } else if (token.isValue()) {
+ if (parseContext.parseFieldMatcher().match(currentFieldName, DOCUMENT_TYPE_FIELD)) {
+ documentType = parser.text();
+ } else if (parseContext.parseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_INDEX)) {
+ indexedDocumentIndex = parser.text();
+ } else if (parseContext.parseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_TYPE)) {
+ indexedDocumentType = parser.text();
+ } else if (parseContext.parseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_ID)) {
+ indexedDocumentId = parser.text();
+ } else if (parseContext.parseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_ROUTING)) {
+ indexedDocumentRouting = parser.text();
+ } else if (parseContext.parseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_PREFERENCE)) {
+ indexedDocumentPreference = parser.text();
+ } else if (parseContext.parseFieldMatcher().match(currentFieldName, INDEXED_DOCUMENT_FIELD_VERSION)) {
+ indexedDocumentVersion = parser.longValue();
+ } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) {
+ boost = parser.floatValue();
+ } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) {
+ queryName = parser.text();
+ } else {
+ throw new ParsingException(parser.getTokenLocation(), "[" + PercolatorQueryBuilder.NAME +
+ "] query does not support [" + currentFieldName + "]");
+ }
+ } else {
+ throw new ParsingException(parser.getTokenLocation(), "[" + PercolatorQueryBuilder.NAME +
+ "] query does not support [" + token + "]");
+ }
+ }
+
+ if (documentType == null) {
+ throw new IllegalArgumentException("[" + PercolatorQueryBuilder.NAME + "] query is missing required [" +
+ DOCUMENT_TYPE_FIELD.getPreferredName() + "] parameter");
+ }
+
+ PercolatorQueryBuilder queryBuilder;
+ if (source != null) {
+ queryBuilder = new PercolatorQueryBuilder(documentType, source);
+ } else if (indexedDocumentId != null) {
+ queryBuilder = new PercolatorQueryBuilder(documentType, indexedDocumentIndex, indexedDocumentType,
+ indexedDocumentId, indexedDocumentRouting, indexedDocumentPreference, indexedDocumentVersion);
+ } else {
+ throw new IllegalArgumentException("[" + PercolatorQueryBuilder.NAME + "] query, nothing to percolate");
+ }
+ queryBuilder.queryName(queryName);
+ queryBuilder.boost(boost);
+ return queryBuilder;
+ }
+
+ @Override
+ public PercolatorQueryBuilder getBuilderPrototype() {
+ return PercolatorQueryBuilder.PROTO;
+ }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java b/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
index 21c1f3ff695..f04f03fcbcd 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
@@ -832,6 +832,18 @@ public abstract class QueryBuilders {
return new ExistsQueryBuilder(name);
}
+ public static PercolatorQueryBuilder percolatorQuery(String documentType, BytesReference document) {
+ return new PercolatorQueryBuilder(documentType, document);
+ }
+
+ public static PercolatorQueryBuilder percolatorQuery(String documentType, String indexedDocumentIndex,
+ String indexedDocumentType, String indexedDocumentId,
+ String indexedDocumentRouting, String indexedDocumentPreference,
+ Long indexedDocumentVersion) {
+ return new PercolatorQueryBuilder(documentType, indexedDocumentIndex, indexedDocumentType, indexedDocumentId,
+ indexedDocumentRouting, indexedDocumentPreference, indexedDocumentVersion);
+ }
+
private QueryBuilders() {
}
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java
index e057aff06b1..11164659b3f 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java
@@ -20,6 +20,7 @@ package org.elasticsearch.index.query;
import org.elasticsearch.client.Client;
import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.fieldstats.FieldStatsProvider;
import org.elasticsearch.indices.query.IndicesQueriesRegistry;
import org.elasticsearch.script.ScriptService;
@@ -31,6 +32,7 @@ public class QueryRewriteContext {
protected final IndexSettings indexSettings;
protected final IndicesQueriesRegistry indicesQueriesRegistry;
protected final QueryParseContext parseContext;
+ protected FieldStatsProvider fieldStatsProvider;
public QueryRewriteContext(IndexSettings indexSettings, ScriptService scriptService, IndicesQueriesRegistry indicesQueriesRegistry) {
this.scriptService = scriptService;
@@ -39,6 +41,14 @@ public class QueryRewriteContext {
this.parseContext = new QueryParseContext(indicesQueriesRegistry);
}
+ public void setFieldStatsProvider(FieldStatsProvider fieldStatsProvider) {
+ this.fieldStatsProvider = fieldStatsProvider;
+ }
+
+ public FieldStatsProvider getFieldStatsProvider() {
+ return fieldStatsProvider;
+ }
+
/**
* Returns a clients to fetch resources from local or remove nodes.
*/
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
index 6acd5272f89..a21b53cdf51 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
@@ -44,9 +44,9 @@ import org.elasticsearch.index.mapper.ContentPath;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.core.StringFieldMapper;
import org.elasticsearch.index.mapper.core.TextFieldMapper;
import org.elasticsearch.index.mapper.object.ObjectMapper;
+import org.elasticsearch.index.percolator.PercolatorQueryCache;
import org.elasticsearch.index.query.support.InnerHitsQueryParserHelper;
import org.elasticsearch.index.query.support.NestedScope;
import org.elasticsearch.index.similarity.SimilarityService;
@@ -87,13 +87,15 @@ public class QueryShardContext extends QueryRewriteContext {
private final Map namedQueries = new HashMap<>();
private final MapperQueryParser queryParser = new MapperQueryParser(this);
+ private final IndicesQueriesRegistry indicesQueriesRegistry;
+ private final PercolatorQueryCache percolatorQueryCache;
private boolean allowUnmappedFields;
private boolean mapUnmappedFieldAsString;
private NestedScope nestedScope;
boolean isFilter; // pkg private for testing
public QueryShardContext(IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache, IndexFieldDataService indexFieldDataService, MapperService mapperService, SimilarityService similarityService, ScriptService scriptService,
- final IndicesQueriesRegistry indicesQueriesRegistry) {
+ final IndicesQueriesRegistry indicesQueriesRegistry, PercolatorQueryCache percolatorQueryCache) {
super(indexSettings, scriptService, indicesQueriesRegistry);
this.indexSettings = indexSettings;
this.similarityService = similarityService;
@@ -101,17 +103,18 @@ public class QueryShardContext extends QueryRewriteContext {
this.bitsetFilterCache = bitsetFilterCache;
this.indexFieldDataService = indexFieldDataService;
this.allowUnmappedFields = indexSettings.isDefaultAllowUnmappedFields();
-
+ this.indicesQueriesRegistry = indicesQueriesRegistry;
+ this.percolatorQueryCache = percolatorQueryCache;
}
public QueryShardContext(QueryShardContext source) {
- this(source.indexSettings, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService, source.similarityService, source.scriptService, source.indicesQueriesRegistry);
+ this(source.indexSettings, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService, source.similarityService, source.scriptService, source.indicesQueriesRegistry, source.percolatorQueryCache);
this.types = source.getTypes();
}
public QueryShardContext clone() {
- return new QueryShardContext(indexSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, scriptService, indicesQueriesRegistry);
+ return new QueryShardContext(indexSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, scriptService, indicesQueriesRegistry, percolatorQueryCache);
}
public void parseFieldMatcher(ParseFieldMatcher parseFieldMatcher) {
@@ -148,6 +151,10 @@ public class QueryShardContext extends QueryRewriteContext {
return mapperService;
}
+ public PercolatorQueryCache getPercolatorQueryCache() {
+ return percolatorQueryCache;
+ }
+
public Similarity getSearchSimilarity() {
return similarityService != null ? similarityService.similarity(mapperService) : null;
}
diff --git a/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
index cd99bec0f74..c3953a51170 100644
--- a/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
@@ -22,6 +22,10 @@ package org.elasticsearch.index.query;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.action.fieldstats.FieldStats;
+import org.elasticsearch.action.fieldstats.IndexConstraint;
+import org.elasticsearch.action.fieldstats.IndexConstraint.Comparison;
+import org.elasticsearch.action.fieldstats.IndexConstraint.Property;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
@@ -30,6 +34,7 @@ import org.elasticsearch.common.joda.FormatDateTimeFormatter;
import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.fieldstats.FieldStatsProvider;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.core.DateFieldMapper;
import org.joda.time.DateTimeZone;
@@ -253,6 +258,43 @@ public class RangeQueryBuilder extends AbstractQueryBuilder i
return NAME;
}
+ @Override
+ protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException {
+ FieldStatsProvider fieldStatsProvider = queryShardContext.getFieldStatsProvider();
+ // If the fieldStatsProvider is null we are not on the shard and cannot
+ // rewrite so just return without rewriting
+ if (fieldStatsProvider != null) {
+ DateMathParser dateMathParser = format == null ? null : new DateMathParser(format);
+ FieldStatsProvider.Relation relation = fieldStatsProvider.isFieldWithinQuery(fieldName, from, to, includeLower, includeUpper,
+ timeZone, dateMathParser);
+ switch (relation) {
+ case DISJOINT:
+ return new MatchNoneQueryBuilder();
+ case WITHIN:
+ FieldStats fieldStats = fieldStatsProvider.get(fieldName);
+ if (!(fieldStats.getMinValue().equals(from) && fieldStats.getMaxValue().equals(to) && includeUpper && includeLower)) {
+ // Rebuild the range query with the bounds for this shard.
+ // The includeLower/Upper values are preserved only if the
+ // bound has not been changed by the rewrite
+ RangeQueryBuilder newRangeQuery = new RangeQueryBuilder(fieldName);
+ String dateFormatString = format == null ? null : format.format();
+ newRangeQuery.from(fieldStats.getMinValue(), includeLower || fieldStats.match(
+ new IndexConstraint(fieldName, Property.MIN, Comparison.GT, fieldStats.stringValueOf(from, dateFormatString))));
+ newRangeQuery.to(fieldStats.getMaxValue(), includeUpper || fieldStats.match(
+ new IndexConstraint(fieldName, Property.MAX, Comparison.LT, fieldStats.stringValueOf(to, dateFormatString))));
+ newRangeQuery.format = format;
+ newRangeQuery.timeZone = timeZone;
+ return newRangeQuery;
+ } else {
+ return this;
+ }
+ case INTERSECTS:
+ break;
+ }
+ }
+ return this;
+ }
+
@Override
protected Query doToQuery(QueryShardContext context) throws IOException {
Query query = null;
diff --git a/core/src/main/java/org/elasticsearch/index/query/support/NestedInnerQueryParseSupport.java b/core/src/main/java/org/elasticsearch/index/query/support/NestedInnerQueryParseSupport.java
index 86983026b19..5c65a57d532 100644
--- a/core/src/main/java/org/elasticsearch/index/query/support/NestedInnerQueryParseSupport.java
+++ b/core/src/main/java/org/elasticsearch/index/query/support/NestedInnerQueryParseSupport.java
@@ -20,9 +20,7 @@
package org.elasticsearch.index.query.support;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.join.BitSetProducer;
import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
@@ -45,46 +43,18 @@ public class NestedInnerQueryParseSupport {
protected final QueryParseContext parseContext;
private BytesReference source;
- private Query innerQuery;
private Query innerFilter;
protected String path;
private boolean filterParsed = false;
- private boolean queryParsed = false;
- protected boolean queryFound = false;
protected boolean filterFound = false;
- protected BitSetProducer parentFilter;
- protected Query childFilter;
-
protected ObjectMapper nestedObjectMapper;
- private ObjectMapper parentObjectMapper;
public NestedInnerQueryParseSupport(XContentParser parser, QueryShardContext context) {
shardContext = context;
parseContext = shardContext.parseContext();
shardContext.reset(parser);
-
- }
-
- public NestedInnerQueryParseSupport(QueryShardContext context) {
- this.parseContext = context.parseContext();
- this.shardContext = context;
- }
-
- public void query() throws IOException {
- if (path != null) {
- setPathLevel();
- try {
- innerQuery = parseContext.parseInnerQueryBuilder().toQuery(this.shardContext);
- } finally {
- resetPathLevel();
- }
- queryParsed = true;
- } else {
- source = XContentFactory.smileBuilder().copyCurrentStructure(parseContext.parser()).bytes();
- }
- queryFound = true;
}
public void filter() throws IOException {
@@ -103,35 +73,6 @@ public class NestedInnerQueryParseSupport {
filterFound = true;
}
- public Query getInnerQuery() throws IOException {
- if (queryParsed) {
- return innerQuery;
- } else {
- if (path == null) {
- throw new QueryShardException(shardContext, "[nested] requires 'path' field");
- }
- if (!queryFound) {
- throw new QueryShardException(shardContext, "[nested] requires either 'query' or 'filter' field");
- }
-
- XContentParser old = parseContext.parser();
- try {
- XContentParser innerParser = XContentHelper.createParser(source);
- parseContext.parser(innerParser);
- setPathLevel();
- try {
- innerQuery = parseContext.parseInnerQueryBuilder().toQuery(this.shardContext);
- } finally {
- resetPathLevel();
- }
- queryParsed = true;
- return innerQuery;
- } finally {
- parseContext.parser(old);
- }
- }
- }
-
public Query getInnerFilter() throws IOException {
if (filterParsed) {
return innerFilter;
@@ -178,27 +119,12 @@ public class NestedInnerQueryParseSupport {
return nestedObjectMapper;
}
- public boolean queryFound() {
- return queryFound;
- }
-
public boolean filterFound() {
return filterFound;
}
- public ObjectMapper getParentObjectMapper() {
- return parentObjectMapper;
- }
-
private void setPathLevel() {
- ObjectMapper objectMapper = shardContext.nestedScope().getObjectMapper();
- if (objectMapper == null) {
- parentFilter = shardContext.bitsetFilter(Queries.newNonNestedFilter());
- } else {
- parentFilter = shardContext.bitsetFilter(objectMapper.nestedTypeFilter());
- }
- childFilter = nestedObjectMapper.nestedTypeFilter();
- parentObjectMapper = shardContext.nestedScope().nextLevel(nestedObjectMapper);
+ shardContext.nestedScope().nextLevel(nestedObjectMapper);
}
private void resetPathLevel() {
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 5e2df030001..7b19a12bbaa 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -46,14 +46,12 @@ import org.elasticsearch.common.metrics.MeanMetric;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.common.util.Callback;
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.common.util.concurrent.FutureUtils;
import org.elasticsearch.common.util.concurrent.SuspendableRefContainer;
-import org.elasticsearch.gateway.MetaDataStateFormat;
import org.elasticsearch.index.IndexModule;
import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.NodeServicesProvider;
import org.elasticsearch.index.SearchSlowLog;
import org.elasticsearch.index.VersionType;
import org.elasticsearch.index.cache.IndexCache;
@@ -83,9 +81,7 @@ import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.SourceToParse;
import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.index.merge.MergeStats;
-import org.elasticsearch.index.percolator.PercolateStats;
-import org.elasticsearch.index.percolator.PercolatorQueriesRegistry;
-import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.percolator.PercolatorFieldMapper;
import org.elasticsearch.index.recovery.RecoveryStats;
import org.elasticsearch.index.refresh.RefreshStats;
import org.elasticsearch.index.search.stats.SearchStats;
@@ -106,7 +102,6 @@ import org.elasticsearch.index.warmer.WarmerStats;
import org.elasticsearch.indices.IndexingMemoryController;
import org.elasticsearch.indices.recovery.RecoveryFailedException;
import org.elasticsearch.indices.recovery.RecoveryState;
-import org.elasticsearch.percolator.PercolatorService;
import org.elasticsearch.search.suggest.completion.CompletionFieldStats;
import org.elasticsearch.search.suggest.completion.CompletionStats;
import org.elasticsearch.threadpool.ThreadPool;
@@ -122,7 +117,6 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
@@ -140,7 +134,6 @@ public class IndexShard extends AbstractIndexShardComponent {
private final ShardIndexWarmerService shardWarmerService;
private final ShardRequestCache shardQueryCache;
private final ShardFieldData shardFieldData;
- private final PercolatorQueriesRegistry percolatorQueriesRegistry;
private final IndexFieldDataService indexFieldDataService;
private final ShardSuggestMetric shardSuggestMetric = new ShardSuggestMetric();
private final ShardBitsetFilterCache shardBitsetFilterCache;
@@ -159,7 +152,6 @@ public class IndexShard extends AbstractIndexShardComponent {
* being indexed/deleted. */
private final AtomicLong writingBytes = new AtomicLong();
- private volatile ScheduledFuture refreshScheduledFuture;
protected volatile ShardRouting shardRouting;
protected volatile IndexShardState state;
protected final AtomicReference currentEngineReference = new AtomicReference<>();
@@ -201,7 +193,8 @@ public class IndexShard extends AbstractIndexShardComponent {
public IndexShard(ShardId shardId, IndexSettings indexSettings, ShardPath path, Store store, IndexCache indexCache,
MapperService mapperService, SimilarityService similarityService, IndexFieldDataService indexFieldDataService,
@Nullable EngineFactory engineFactory,
- IndexEventListener indexEventListener, IndexSearcherWrapper indexSearcherWrapper, NodeServicesProvider provider, SearchSlowLog slowLog, Engine.Warmer warmer, IndexingOperationListener... listeners) {
+ IndexEventListener indexEventListener, IndexSearcherWrapper indexSearcherWrapper, ThreadPool threadPool, BigArrays bigArrays,
+ SearchSlowLog slowLog, Engine.Warmer warmer, IndexingOperationListener... listeners) {
super(shardId, indexSettings);
final Settings settings = indexSettings.getSettings();
this.codecService = new CodecService(mapperService, logger);
@@ -212,7 +205,7 @@ public class IndexShard extends AbstractIndexShardComponent {
this.engineFactory = engineFactory == null ? new InternalEngineFactory() : engineFactory;
this.store = store;
this.indexEventListener = indexEventListener;
- this.threadPool = provider.getThreadPool();
+ this.threadPool = threadPool;
this.mapperService = mapperService;
this.indexCache = indexCache;
this.internalIndexingStats = new InternalIndexingStats();
@@ -233,7 +226,7 @@ public class IndexShard extends AbstractIndexShardComponent {
this.checkIndexOnStartup = indexSettings.getValue(IndexSettings.INDEX_CHECK_ON_STARTUP);
this.translogConfig = new TranslogConfig(shardId, shardPath().resolveTranslog(), indexSettings,
- provider.getBigArrays());
+ bigArrays);
final QueryCachingPolicy cachingPolicy;
// the query cache is a node-level thing, however we want the most popular filters
// to be computed on a per-shard basis
@@ -246,8 +239,6 @@ public class IndexShard extends AbstractIndexShardComponent {
this.engineConfig = newEngineConfig(translogConfig, cachingPolicy);
this.suspendableRefContainer = new SuspendableRefContainer();
this.searcherWrapper = indexSearcherWrapper;
- QueryShardContext queryShardContext = new QueryShardContext(indexSettings, indexCache.bitsetFilterCache(), indexFieldDataService, mapperService, similarityService, provider.getScriptService(), provider.getIndicesQueriesRegistry());
- this.percolatorQueriesRegistry = new PercolatorQueriesRegistry(shardId, indexSettings, queryShardContext);
}
public Store store() {
@@ -476,12 +467,8 @@ public class IndexShard extends AbstractIndexShardComponent {
if (logger.isTraceEnabled()) {
logger.trace("index [{}][{}]{}", index.type(), index.id(), index.docs());
}
- final boolean isPercolatorQuery = percolatorQueriesRegistry.isPercolatorQuery(index);
Engine engine = getEngine();
created = engine.index(index);
- if (isPercolatorQuery) {
- percolatorQueriesRegistry.updatePercolateQuery(engine, index.id());
- }
index.endTime(System.nanoTime());
} catch (Throwable ex) {
indexingOperationListeners.postIndex(index, ex);
@@ -519,12 +506,8 @@ public class IndexShard extends AbstractIndexShardComponent {
if (logger.isTraceEnabled()) {
logger.trace("delete [{}]", delete.uid().text());
}
- final boolean isPercolatorQuery = percolatorQueriesRegistry.isPercolatorQuery(delete);
Engine engine = getEngine();
engine.delete(delete);
- if (isPercolatorQuery) {
- percolatorQueriesRegistry.updatePercolateQuery(engine, delete.id());
- }
delete.endTime(System.nanoTime());
} catch (Throwable ex) {
indexingOperationListeners.postDelete(delete, ex);
@@ -644,10 +627,6 @@ public class IndexShard extends AbstractIndexShardComponent {
return shardFieldData.stats(fields);
}
- public PercolatorQueriesRegistry percolateRegistry() {
- return percolatorQueriesRegistry;
- }
-
public TranslogStats translogStats() {
return getEngine().getTranslog().stats();
}
@@ -784,10 +763,6 @@ public class IndexShard extends AbstractIndexShardComponent {
public void close(String reason, boolean flushEngine) throws IOException {
synchronized (mutex) {
try {
- if (state != IndexShardState.CLOSED) {
- FutureUtils.cancel(refreshScheduledFuture);
- refreshScheduledFuture = null;
- }
changeState(IndexShardState.CLOSED, reason);
} finally {
final Engine engine = this.currentEngineReference.getAndSet(null);
@@ -796,18 +771,15 @@ public class IndexShard extends AbstractIndexShardComponent {
engine.flushAndClose();
}
} finally { // playing safe here and close the engine even if the above succeeds - close can be called multiple times
- IOUtils.close(engine, percolatorQueriesRegistry);
+ IOUtils.close(engine);
}
}
}
}
public IndexShard postRecovery(String reason) throws IndexShardStartedException, IndexShardRelocatedException, IndexShardClosedException {
- if (mapperService.hasMapping(PercolatorService.TYPE_NAME)) {
+ if (mapperService.hasMapping(PercolatorFieldMapper.TYPE_NAME)) {
refresh("percolator_load_queries");
- try (Engine.Searcher searcher = getEngine().acquireSearcher("percolator_load_queries")) {
- this.percolatorQueriesRegistry.loadQueries(searcher.reader());
- }
}
synchronized (mutex) {
if (state == IndexShardState.CLOSED) {
@@ -1104,10 +1076,6 @@ public class IndexShard extends AbstractIndexShardComponent {
return getEngine().getTranslog();
}
- public PercolateStats percolateStats() {
- return percolatorQueriesRegistry.stats();
- }
-
public IndexEventListener getIndexEventListener() {
return indexEventListener;
}
diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
index 5518d1b1273..774052b3a5f 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
@@ -20,8 +20,8 @@ package org.elasticsearch.index.shard;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.NodeServicesProvider;
import org.elasticsearch.index.cache.IndexCache;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.engine.EngineConfig;
@@ -33,6 +33,7 @@ import org.elasticsearch.index.SearchSlowLog;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.translog.TranslogStats;
+import org.elasticsearch.threadpool.ThreadPool;
import java.io.IOException;
@@ -44,9 +45,13 @@ import java.io.IOException;
*/
public final class ShadowIndexShard extends IndexShard {
- public ShadowIndexShard(ShardId shardId, IndexSettings indexSettings, ShardPath path, Store store, IndexCache indexCache, MapperService mapperService, SimilarityService similarityService, IndexFieldDataService indexFieldDataService, @Nullable EngineFactory engineFactory,
- IndexEventListener indexEventListener, IndexSearcherWrapper wrapper, NodeServicesProvider provider, SearchSlowLog searchSlowLog, Engine.Warmer engineWarmer) throws IOException {
- super(shardId, indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldDataService, engineFactory, indexEventListener, wrapper, provider, searchSlowLog, engineWarmer);
+ public ShadowIndexShard(ShardId shardId, IndexSettings indexSettings, ShardPath path, Store store, IndexCache indexCache,
+ MapperService mapperService, SimilarityService similarityService, IndexFieldDataService indexFieldDataService,
+ @Nullable EngineFactory engineFactory, IndexEventListener indexEventListener, IndexSearcherWrapper wrapper,
+ ThreadPool threadPool, BigArrays bigArrays, SearchSlowLog searchSlowLog, Engine.Warmer engineWarmer)
+ throws IOException {
+ super(shardId, indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldDataService, engineFactory,
+ indexEventListener, wrapper, threadPool, bigArrays, searchSlowLog, engineWarmer);
}
/**
diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShardStateMetaData.java b/core/src/main/java/org/elasticsearch/index/shard/ShardStateMetaData.java
index 315371c7286..407f271fd65 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/ShardStateMetaData.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/ShardStateMetaData.java
@@ -102,7 +102,7 @@ public final class ShardStateMetaData {
return "version [" + legacyVersion + "], primary [" + primary + "], allocation [" + allocationId + "]";
}
- public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(XContentType.JSON, SHARD_STATE_FILE_PREFIX) {
+ public static final MetaDataStateFormat FORMAT = new MetaDataStateFormat(XContentType.SMILE, SHARD_STATE_FILE_PREFIX) {
@Override
protected XContentBuilder newXContentBuilder(XContentType type, OutputStream stream) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
index 330787a68a3..c15d2cfcdbe 100644
--- a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
+++ b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
@@ -32,9 +32,9 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.metadata.SnapshotId;
import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.blobstore.BlobContainer;
import org.elasticsearch.common.blobstore.BlobMetaData;
diff --git a/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java b/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
index 933fd784588..584b98cff33 100644
--- a/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
+++ b/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
@@ -61,8 +61,9 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim
return SimpleFSLockFactory.INSTANCE;
default:
throw new IllegalArgumentException("unrecognized [index.store.fs.fs_lock] \"" + s + "\": must be native or simple");
- }
- }, Property.IndexScope);
+ } // can we set on both - node and index level, some nodes might be running on NFS so they might need simple rather than native
+ }, Property.IndexScope, Property.NodeScope);
+
private final CounterMetric rateLimitingTimeInNanos = new CounterMetric();
private final ShardPath path;
@@ -108,7 +109,8 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim
protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
- final String storeType = indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.DEFAULT.getSettingsKey());
+ final String storeType = indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(),
+ IndexModule.Type.DEFAULT.getSettingsKey());
if (IndexModule.Type.FS.match(storeType) || IndexModule.Type.DEFAULT.match(storeType)) {
final FSDirectory open = FSDirectory.open(location, lockFactory); // use lucene defaults
if (open instanceof MMapDirectory && Constants.WINDOWS == false) {
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java b/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
index 9129a3b1360..4b4aa4e8df2 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
@@ -70,7 +70,7 @@ public final class IndicesRequestCache extends AbstractComponent implements Remo
* since we are checking on the cluster state IndexMetaData always.
*/
public static final Setting INDEX_CACHE_REQUEST_ENABLED_SETTING =
- Setting.boolSetting("index.requests.cache.enable", false, Property.Dynamic, Property.IndexScope);
+ Setting.boolSetting("index.requests.cache.enable", true, Property.Dynamic, Property.IndexScope);
public static final Setting INDICES_CACHE_QUERY_SIZE =
Setting.byteSizeSetting("indices.requests.cache.size", "1%", Property.NodeScope);
public static final Setting INDICES_CACHE_QUERY_EXPIRE =
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index b0e1bbdbd2b..a0dba7089a9 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -19,6 +19,7 @@
package org.elasticsearch.indices;
+import com.carrotsearch.hppc.cursors.ObjectCursor;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.CollectionUtil;
@@ -30,10 +31,11 @@ import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
import org.elasticsearch.action.admin.indices.stats.ShardStats;
import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.breaker.CircuitBreaker;
import org.elasticsearch.common.bytes.BytesReference;
@@ -66,6 +68,7 @@ import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
import org.elasticsearch.index.flush.FlushStats;
import org.elasticsearch.index.get.GetStats;
+import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.merge.MergeStats;
import org.elasticsearch.index.recovery.RecoveryStats;
import org.elasticsearch.index.refresh.RefreshStats;
@@ -74,6 +77,7 @@ import org.elasticsearch.index.shard.IllegalIndexShardStateException;
import org.elasticsearch.index.shard.IndexEventListener;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.IndexShardState;
+import org.elasticsearch.index.shard.IndexingOperationListener;
import org.elasticsearch.index.shard.IndexingStats;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.store.IndexStoreConfig;
@@ -88,12 +92,13 @@ import org.elasticsearch.search.query.QueryPhase;
import org.elasticsearch.search.query.QuerySearchResult;
import org.elasticsearch.threadpool.ThreadPool;
+import java.io.Closeable;
import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -265,7 +270,7 @@ public class IndicesService extends AbstractLifecycleComponent i
if (indexShard.routingEntry() == null) {
continue;
}
- IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesQueryCache, indexShard, flags), indexShard.commitStats()) });
+ IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesQueryCache, indexService.cache().getPercolatorQueryCache(), indexShard, flags), indexShard.commitStats()) });
if (!statsByShard.containsKey(indexService.index())) {
statsByShard.put(indexService.index(), arrayAsArrayList(indexShardStats));
} else {
@@ -325,6 +330,7 @@ public class IndicesService extends AbstractLifecycleComponent i
* @throws IndexAlreadyExistsException if the index already exists.
*/
public synchronized IndexService createIndex(final NodeServicesProvider nodeServicesProvider, IndexMetaData indexMetaData, List builtInListeners) throws IOException {
+
if (!lifecycle.started()) {
throw new IllegalStateException("Can't create an index [" + indexMetaData.getIndex() + "], node is closed");
}
@@ -332,37 +338,22 @@ public class IndicesService extends AbstractLifecycleComponent i
throw new IllegalArgumentException("index must have a real UUID found value: [" + indexMetaData.getIndexUUID() + "]");
}
final Index index = indexMetaData.getIndex();
- final Predicate indexNameMatcher = (indexExpression) -> indexNameExpressionResolver.matchesIndex(index.getName(), indexExpression, clusterService.state());
- final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexNameMatcher, indexScopeSetting);
if (hasIndex(index)) {
throw new IndexAlreadyExistsException(index);
}
- logger.debug("creating Index [{}], shards [{}]/[{}{}]",
- indexMetaData.getIndex(),
- idxSettings.getNumberOfShards(),
- idxSettings.getNumberOfReplicas(),
- idxSettings.isShadowReplicaIndex() ? "s" : "");
-
- final IndexModule indexModule = new IndexModule(idxSettings, indexStoreConfig, analysisRegistry);
- pluginsService.onIndexModule(indexModule);
- for (IndexEventListener listener : builtInListeners) {
- indexModule.addIndexEventListener(listener);
- }
+ List finalListeners = new ArrayList<>(builtInListeners);
final IndexEventListener onStoreClose = new IndexEventListener() {
@Override
public void onStoreClosed(ShardId shardId) {
indicesQueryCache.onClose(shardId);
}
};
- indexModule.addIndexEventListener(onStoreClose);
- indexModule.addIndexEventListener(oldShardsStats);
- final IndexEventListener listener = indexModule.freeze();
- listener.beforeIndexCreated(index, idxSettings.getSettings());
- final IndexService indexService = indexModule.newIndexService(nodeEnv, this, nodeServicesProvider, indicesQueryCache, mapperRegistry, indicesFieldDataCache, indexingMemoryController);
+ finalListeners.add(onStoreClose);
+ finalListeners.add(oldShardsStats);
+ final IndexService indexService = createIndexService("create index", nodeServicesProvider, indexMetaData, indicesQueryCache, indicesFieldDataCache, finalListeners, indexingMemoryController);
boolean success = false;
try {
- assert indexService.getIndexEventListener() == listener;
- listener.afterIndexCreated(indexService);
+ indexService.getIndexEventListener().afterIndexCreated(indexService);
indices = newMapBuilder(indices).put(index.getUUID(), indexService).immutableMap();
success = true;
return indexService;
@@ -371,7 +362,54 @@ public class IndicesService extends AbstractLifecycleComponent i
indexService.close("plugins_failed", true);
}
}
+ }
+ /**
+ * This creates a new IndexService without registering it
+ */
+ private synchronized IndexService createIndexService(final String reason, final NodeServicesProvider nodeServicesProvider, IndexMetaData indexMetaData, IndicesQueryCache indicesQueryCache, IndicesFieldDataCache indicesFieldDataCache, List builtInListeners, IndexingOperationListener... indexingOperationListeners) throws IOException {
+ final Index index = indexMetaData.getIndex();
+ final Predicate indexNameMatcher = (indexExpression) -> indexNameExpressionResolver.matchesIndex(index.getName(), indexExpression, clusterService.state());
+ final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexNameMatcher, indexScopeSetting);
+ logger.debug("creating Index [{}], shards [{}]/[{}{}] - reason [{}]",
+ indexMetaData.getIndex(),
+ idxSettings.getNumberOfShards(),
+ idxSettings.getNumberOfReplicas(),
+ idxSettings.isShadowReplicaIndex() ? "s" : "", reason);
+
+ final IndexModule indexModule = new IndexModule(idxSettings, indexStoreConfig, analysisRegistry);
+ pluginsService.onIndexModule(indexModule);
+ for (IndexEventListener listener : builtInListeners) {
+ indexModule.addIndexEventListener(listener);
+ }
+ final IndexEventListener listener = indexModule.freeze();
+ listener.beforeIndexCreated(index, idxSettings.getSettings());
+ return indexModule.newIndexService(nodeEnv, this, nodeServicesProvider, indicesQueryCache, mapperRegistry, indicesFieldDataCache, indexingOperationListeners);
+ }
+
+ /**
+ * This method verifies that the given {@link IndexMetaData} holds sane values to create an {@link IndexService}. This method will throw an
+ * exception if the creation fails. The created {@link IndexService} will not be registered and will be closed immediately.
+ */
+ public synchronized void verifyIndexMetadata(final NodeServicesProvider nodeServicesProvider, IndexMetaData metaData) throws IOException {
+ final List closeables = new ArrayList<>();
+ try {
+ IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {});
+ closeables.add(indicesFieldDataCache);
+ IndicesQueryCache indicesQueryCache = new IndicesQueryCache(settings);
+ closeables.add(indicesQueryCache);
+ // this will also fail if some plugin fails etc. which is nice since we can verify that early
+ final IndexService service = createIndexService("metadata verification", nodeServicesProvider,
+ metaData, indicesQueryCache, indicesFieldDataCache, Collections.emptyList());
+ for (ObjectCursor typeMapping : metaData.getMappings().values()) {
+ // don't apply the default mapping, it has been applied when the mapping was created
+ service.mapperService().merge(typeMapping.value.type(), typeMapping.value.source(),
+ MapperService.MergeReason.MAPPING_RECOVERY, true);
+ }
+ closeables.add(() -> service.close("metadata verification", false));
+ } finally {
+ IOUtils.close(closeables);
+ }
}
/**
@@ -542,6 +580,7 @@ public class IndicesService extends AbstractLifecycleComponent i
* @param indexSettings the shards index settings.
* @throws IOException if an IOException occurs
*/
+ @Override
public void deleteShardStore(String reason, ShardLock lock, IndexSettings indexSettings) throws IOException {
ShardId shardId = lock.getShardId();
logger.trace("{} deleting shard reason [{}]", shardId, reason);
@@ -654,6 +693,7 @@ public class IndicesService extends AbstractLifecycleComponent i
/**
* Adds a pending delete for the given index shard.
*/
+ @Override
public void addPendingDelete(ShardId shardId, IndexSettings settings) {
if (shardId == null) {
throw new IllegalArgumentException("shardId must not be null");
diff --git a/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java b/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java
index da8e617759d..fce4e8411db 100644
--- a/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java
+++ b/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java
@@ -38,7 +38,7 @@ import org.elasticsearch.index.flush.FlushStats;
import org.elasticsearch.index.get.GetStats;
import org.elasticsearch.index.shard.IndexingStats;
import org.elasticsearch.index.merge.MergeStats;
-import org.elasticsearch.index.percolator.PercolateStats;
+import org.elasticsearch.index.percolator.PercolatorQueryCacheStats;
import org.elasticsearch.index.recovery.RecoveryStats;
import org.elasticsearch.index.refresh.RefreshStats;
import org.elasticsearch.index.search.stats.SearchStats;
@@ -105,8 +105,8 @@ public class NodeIndicesStats implements Streamable, ToXContent {
}
@Nullable
- public PercolateStats getPercolate() {
- return stats.getPercolate();
+ public PercolatorQueryCacheStats getPercolate() {
+ return stats.getPercolatorCache();
}
@Nullable
diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
index e73396fcd7f..20a1d341cf9 100644
--- a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
+++ b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
@@ -160,15 +160,21 @@ public final class AnalysisModule extends AbstractModule {
@Override
protected void configure() {
try {
- HunspellService service = new HunspellService(environment.settings(), environment, knownDictionaries);
- AnalysisRegistry registry = new AnalysisRegistry(service, environment, charFilters, tokenFilters, tokenizers, analyzers);
- bind(HunspellService.class).toInstance(service);
+ AnalysisRegistry registry = buildRegistry();
+ bind(HunspellService.class).toInstance(registry.getHunspellService());
bind(AnalysisRegistry.class).toInstance(registry);
} catch (IOException e) {
throw new ElasticsearchException("failed to load hunspell service", e);
}
}
+ /**
+ * Builds an {@link AnalysisRegistry} from the current configuration.
+ */
+ public AnalysisRegistry buildRegistry() throws IOException {
+ return new AnalysisRegistry(new HunspellService(environment.settings(), environment, knownDictionaries), environment, charFilters, tokenFilters, tokenizers, analyzers);
+ }
+
/**
* AnalysisProvider is the basic factory interface for registering analysis components like:
*
diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
index 5d501b65686..46ead3fbf36 100644
--- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
+++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
@@ -22,7 +22,6 @@ package org.elasticsearch.indices.cluster;
import com.carrotsearch.hppc.IntHashSet;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.cluster.action.index.NodeIndexDeletedAction;
@@ -38,6 +37,7 @@ import org.elasticsearch.cluster.routing.RestoreSource;
import org.elasticsearch.cluster.routing.RoutingNodes;
import org.elasticsearch.cluster.routing.RoutingTable;
import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.compress.CompressedXContent;
diff --git a/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java b/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
index 6943ecd7504..b1d7af7ff9c 100644
--- a/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
+++ b/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
@@ -23,7 +23,6 @@ import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.flush.FlushRequest;
import org.elasticsearch.action.admin.indices.flush.SyncedFlushResponse;
import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -31,6 +30,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.IndexRoutingTable;
import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java
index 934730c7c93..aaf351f6056 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySource.java
@@ -20,9 +20,9 @@
package org.elasticsearch.indices.recovery;
import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.routing.RoutingNode;
import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetService.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetService.java
index ab8c87cd636..d57cfbb98c8 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetService.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetService.java
@@ -24,10 +24,10 @@ import org.apache.lucene.store.RateLimiter;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateObserver;
import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
diff --git a/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
index 5dc8af41e80..d2db41a7a0c 100644
--- a/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
+++ b/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
@@ -21,7 +21,6 @@ package org.elasticsearch.indices.store;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.cluster.ClusterStateObserver;
@@ -31,6 +30,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.IndexRoutingTable;
import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
diff --git a/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java b/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
index 0422c0944e2..35a34ebea1b 100644
--- a/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
+++ b/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
@@ -29,11 +29,11 @@ import org.elasticsearch.action.support.nodes.BaseNodesRequest;
import org.elasticsearch.action.support.nodes.BaseNodesResponse;
import org.elasticsearch.action.support.nodes.TransportNodesAction;
import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
diff --git a/core/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java b/core/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java
index 422a08042ce..a30ae49ff7b 100644
--- a/core/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java
+++ b/core/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java
@@ -30,9 +30,9 @@ import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.bulk.TransportBulkAction;
import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.component.AbstractLifecycleComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.uid.Versions;
diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
index ac2df419f55..7e0dc1b4ffa 100644
--- a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
+++ b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
@@ -28,11 +28,11 @@ import org.elasticsearch.action.ingest.PutPipelineRequest;
import org.elasticsearch.action.ingest.WritePipelineResponse;
import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
@@ -40,7 +40,6 @@ import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.ingest.core.IngestInfo;
import org.elasticsearch.ingest.core.Pipeline;
import org.elasticsearch.ingest.core.Processor;
-import org.elasticsearch.ingest.core.ProcessorInfo;
import org.elasticsearch.ingest.core.TemplateService;
import org.elasticsearch.script.ScriptService;
@@ -51,7 +50,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
public class PipelineStore extends AbstractComponent implements Closeable, ClusterStateListener {
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/ForEachProcessor.java b/core/src/main/java/org/elasticsearch/ingest/processor/ForEachProcessor.java
index 5b101fbfb32..b6f6a85d219 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/ForEachProcessor.java
+++ b/core/src/main/java/org/elasticsearch/ingest/processor/ForEachProcessor.java
@@ -59,11 +59,8 @@ public final class ForEachProcessor extends AbstractProcessor {
List