diff --git a/core/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index 024e0439ac5..1838b60050e 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -20,10 +20,14 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.hppc.ObjectArrayList; + import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.ByteArrayDataOutput; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; @@ -126,6 +130,15 @@ public class BinaryFieldMapper extends FieldMapper { return new BytesBinaryDVIndexFieldData.Builder(); } + @Override + public Query existsQuery(QueryShardContext context) { + if (hasDocValues()) { + return new DocValuesFieldExistsQuery(name()); + } else { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + } + @Override public Query termQuery(Object value, QueryShardContext context) { throw new QueryShardException(context, "Binary fields do not support searching"); @@ -165,6 +178,11 @@ public class BinaryFieldMapper extends FieldMapper { } else { field.add(value); } + } else { + // Only add an entry to the field names field if the field is stored + // but has no doc values so exists query will work on a field with + // no doc values + createFieldNamesField(context, fields); } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 6fe8a37a46e..45cd9e17ad1 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -23,7 +23,10 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; @@ -136,6 +139,15 @@ public class BooleanFieldMapper extends FieldMapper { return CONTENT_TYPE; } + @Override + public Query existsQuery(QueryShardContext context) { + if (hasDocValues()) { + return new DocValuesFieldExistsQuery(name()); + } else { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + } + @Override public Boolean nullValue() { return (Boolean)super.nullValue(); @@ -253,6 +265,8 @@ public class BooleanFieldMapper extends FieldMapper { } if (fieldType().hasDocValues()) { fields.add(new SortedNumericDocValuesField(fieldType().name(), value ? 1 : 0)); + } else { + createFieldNamesField(context, fields); } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 1ab84eda639..1c92150676c 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -21,6 +21,8 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.suggest.document.Completion50PostingsFormat; import org.apache.lucene.search.suggest.document.CompletionAnalyzer; import org.apache.lucene.search.suggest.document.CompletionQuery; @@ -40,11 +42,13 @@ import org.elasticsearch.common.xcontent.XContentParser.NumberType; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.suggest.completion.CompletionSuggester; import org.elasticsearch.search.suggest.completion.context.ContextMapping; import org.elasticsearch.search.suggest.completion.context.ContextMappings; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -257,6 +261,11 @@ public class CompletionFieldMapper extends FieldMapper implements ArrayValueMapp return postingsFormat; } + @Override + public Query existsQuery(QueryShardContext context) { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + /** * Completion prefix query */ @@ -456,6 +465,11 @@ public class CompletionFieldMapper extends FieldMapper implements ArrayValueMapp context.doc().add(new SuggestField(fieldType().name(), input, metaData.weight)); } } + List fields = new ArrayList<>(1); + createFieldNamesField(context, fields); + for (IndexableField field : fields) { + context.doc().add(field); + } multiFields.parse(this, context); return null; } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 6de6e860a8f..36e7a73aa9a 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -26,9 +26,12 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.Term; import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; @@ -245,6 +248,15 @@ public class DateFieldMapper extends FieldMapper { return dateTimeFormatter().parser().parseMillis(value); } + @Override + public Query existsQuery(QueryShardContext context) { + if (hasDocValues()) { + return new DocValuesFieldExistsQuery(name()); + } else { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + } + @Override public Query termQuery(Object value, @Nullable QueryShardContext context) { Query query = rangeQuery(value, value, true, true, ShapeRelation.INTERSECTS, null, null, context); @@ -451,6 +463,8 @@ public class DateFieldMapper extends FieldMapper { } if (fieldType().hasDocValues()) { fields.add(new SortedNumericDocValuesField(fieldType().name(), timestamp)); + } else if (fieldType().stored() || fieldType().indexOptions() != IndexOptions.NONE) { + createFieldNamesField(context, fields); } if (fieldType().stored()) { fields.add(new StoredField(fieldType().name(), timestamp)); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 589670fcccd..c6e0dd9c00b 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -22,6 +22,7 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; @@ -33,6 +34,7 @@ import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.mapper.FieldNamesFieldMapper.FieldNamesFieldType; import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.index.similarity.SimilarityService; @@ -285,6 +287,16 @@ public abstract class FieldMapper extends Mapper implements Cloneable { */ protected abstract void parseCreateField(ParseContext context, List fields) throws IOException; + protected void createFieldNamesField(ParseContext context, List fields) { + FieldNamesFieldType fieldNamesFieldType = (FieldNamesFieldMapper.FieldNamesFieldType) context.docMapper() + .metadataMapper(FieldNamesFieldMapper.class).fieldType(); + if (fieldNamesFieldType != null && fieldNamesFieldType.isEnabled()) { + for (String fieldName : FieldNamesFieldMapper.extractFieldNames(fieldType().name())) { + fields.add(new Field(FieldNamesFieldMapper.NAME, fieldName, fieldNamesFieldType)); + } + } + } + @Override public Iterator iterator() { return multiFields.iterator(); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java index c2923be4c74..8482a94cfc7 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java @@ -23,6 +23,10 @@ import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -44,6 +48,9 @@ import java.util.Objects; */ public class FieldNamesFieldMapper extends MetadataFieldMapper { + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger( + ESLoggerFactory.getLogger(FieldNamesFieldMapper.class)); + public static final String NAME = "_field_names"; public static final String CONTENT_TYPE = "_field_names"; @@ -178,11 +185,18 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper { return enabled; } + @Override + public Query existsQuery(QueryShardContext context) { + throw new UnsupportedOperationException("Cannot run exists query on _field_names"); + } + @Override public Query termQuery(Object value, QueryShardContext context) { if (isEnabled() == false) { throw new IllegalStateException("Cannot run [exists] queries if the [_field_names] field is disabled"); } + DEPRECATION_LOGGER.deprecated( + "terms query on the _field_names field is deprecated and will be removed, use exists query instead"); return super.termQuery(value, context); } } @@ -206,12 +220,14 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper { @Override public void postParse(ParseContext context) throws IOException { - super.parse(context); + if (context.indexSettings().getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).before(Version.V_6_1_0)) { + super.parse(context); + } } @Override public Mapper parse(ParseContext context) throws IOException { - // we parse in post parse + // Adding values to the _field_names field is handled by the mappers for each field type return null; } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index 700b44a204e..45237eb572d 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -23,7 +23,10 @@ import org.apache.lucene.document.LatLonPoint; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.geo.GeoPoint; @@ -37,6 +40,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; import java.io.IOException; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -180,6 +184,15 @@ public class GeoPointFieldMapper extends FieldMapper implements ArrayValueMapper return new AbstractLatLonPointDVIndexFieldData.Builder(); } + @Override + public Query existsQuery(QueryShardContext context) { + if (hasDocValues()) { + return new DocValuesFieldExistsQuery(name()); + } else { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + } + @Override public Query termQuery(Object value, QueryShardContext context) { throw new QueryShardException(context, "Geo fields do not support exact searching, use dedicated geo queries instead: [" @@ -207,6 +220,12 @@ public class GeoPointFieldMapper extends FieldMapper implements ArrayValueMapper } if (fieldType.hasDocValues()) { context.doc().add(new LatLonDocValuesField(fieldType().name(), point.lat(), point.lon())); + } else if (fieldType().stored() || fieldType().indexOptions() != IndexOptions.NONE) { + List fields = new ArrayList<>(1); + createFieldNamesField(context, fields); + for (IndexableField field : fields) { + context.doc().add(field); + } } // if the mapping contains multifields then use the geohash string if (multiFields.iterator().hasNext()) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 72bb35668bd..c605b8d0936 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -18,10 +18,12 @@ */ package org.elasticsearch.index.mapper; -import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.spatial.prefix.PrefixTreeStrategy; import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; import org.apache.lucene.spatial.prefix.TermQueryPrefixTreeStrategy; @@ -29,6 +31,7 @@ import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree; import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree; import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; +import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.geo.SpatialStrategy; @@ -44,6 +47,8 @@ import org.locationtech.spatial4j.shape.Shape; import org.locationtech.spatial4j.shape.jts.JtsGeometry; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -125,6 +130,11 @@ public class GeoShapeFieldMapper extends FieldMapper { return builder; } + @Override + protected boolean defaultDocValues(Version indexCreated) { + return false; + } + protected Explicit coerce(BuilderContext context) { if (coerce != null) { return new Explicit<>(coerce, true); @@ -406,6 +416,11 @@ public class GeoShapeFieldMapper extends FieldMapper { throw new IllegalArgumentException("Unknown prefix tree strategy [" + strategyName + "]"); } + @Override + public Query existsQuery(QueryShardContext context) { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + @Override public Query termQuery(Object value, QueryShardContext context) { throw new QueryShardException(context, "Geo fields do not support exact searching, use dedicated geo queries instead"); @@ -440,11 +455,9 @@ public class GeoShapeFieldMapper extends FieldMapper { throw new MapperParsingException("[{" + fieldType().name() + "}] is configured for points only but a " + ((shape instanceof JtsGeometry) ? ((JtsGeometry)shape).getGeom().getGeometryType() : shape.getClass()) + " was found"); } - Field[] fields = fieldType().defaultStrategy().createIndexableFields(shape); - if (fields == null || fields.length == 0) { - return null; - } - for (Field field : fields) { + List fields = new ArrayList<>(Arrays.asList(fieldType().defaultStrategy().createIndexableFields(shape))); + createFieldNamesField(context, fields); + for (IndexableField field : fields) { context.doc().add(field); } } catch (Exception e) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index 55898e5f96c..41256d3a5bb 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -23,6 +23,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermInSetQuery; @@ -36,10 +37,10 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.AtomicFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.indices.breaker.CircuitBreakerService; @@ -126,6 +127,11 @@ public class IdFieldMapper extends MetadataFieldMapper { return termsQuery(Arrays.asList(value), context); } + @Override + public Query existsQuery(QueryShardContext context) { + return new MatchAllDocsQuery(); + } + @Override public Query termsQuery(List values, QueryShardContext context) { if (indexOptions() != IndexOptions.NONE) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index 7f3f934419a..0010211a955 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -21,9 +21,9 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.Queries; @@ -111,6 +111,11 @@ public class IndexFieldMapper extends MetadataFieldMapper { return true; } + @Override + public Query existsQuery(QueryShardContext context) { + return new MatchAllDocsQuery(); + } + /** * This termQuery impl looks at the context to determine the index that * is being queried and then returns a MATCH_ALL_QUERY or MATCH_NO_QUERY diff --git a/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index faa486dd972..bc811d041e3 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -25,8 +25,11 @@ import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Explicit; @@ -153,6 +156,15 @@ public class IpFieldMapper extends FieldMapper { } } + @Override + public Query existsQuery(QueryShardContext context) { + if (hasDocValues()) { + return new DocValuesFieldExistsQuery(name()); + } else { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + } + @Override public Query termQuery(Object value, @Nullable QueryShardContext context) { failIfNotIndexed(); @@ -369,6 +381,8 @@ public class IpFieldMapper extends FieldMapper { } if (fieldType().hasDocValues()) { fields.add(new SortedSetDocValuesField(fieldType().name(), new BytesRef(InetAddressPoint.encode(address)))); + } else if (fieldType().stored() || fieldType().indexOptions() != IndexOptions.NONE) { + createFieldNamesField(context, fields); } if (fieldType().stored()) { fields.add(new StoredField(fieldType().name(), new BytesRef(InetAddressPoint.encode(address)))); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 32fe4c95c43..cb2c4b6b6fd 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -25,7 +25,10 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; @@ -35,6 +38,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; +import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; import java.util.Iterator; @@ -210,6 +214,15 @@ public final class KeywordFieldMapper extends FieldMapper { this.normalizer = normalizer; } + @Override + public Query existsQuery(QueryShardContext context) { + if (hasDocValues()) { + return new DocValuesFieldExistsQuery(name()); + } else { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + } + @Override public Query nullValueQuery() { if (nullValue() == null) { @@ -328,6 +341,8 @@ public final class KeywordFieldMapper extends FieldMapper { } if (fieldType().hasDocValues()) { fields.add(new SortedSetDocValuesField(fieldType().name(), binaryValue)); + } else if (fieldType().stored() || fieldType().indexOptions() != IndexOptions.NONE) { + createFieldNamesField(context, fields); } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 2796a5342c1..6eab9087534 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -379,6 +379,8 @@ public abstract class MappedFieldType extends FieldType { return new ConstantScoreQuery(termQuery(nullValue, null)); } + public abstract Query existsQuery(QueryShardContext context); + /** * An enum used to describe the relation between the range of terms in a * shard when compared with a query range diff --git a/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 80dc01a965f..a44611d6406 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -29,10 +29,13 @@ import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.Explicit; @@ -867,6 +870,15 @@ public class NumberFieldMapper extends FieldMapper { return type.name; } + @Override + public Query existsQuery(QueryShardContext context) { + if (hasDocValues()) { + return new DocValuesFieldExistsQuery(name()); + } else { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + } + @Override public Query termQuery(Object value, QueryShardContext context) { failIfNotIndexed(); @@ -1001,6 +1013,9 @@ public class NumberFieldMapper extends FieldMapper { boolean docValued = fieldType().hasDocValues(); boolean stored = fieldType().stored(); fields.addAll(fieldType().type.createFields(fieldType().name(), numericValue, indexed, docValued, stored)); + if (docValued == false && (stored || indexed)) { + createFieldNamesField(context, fields); + } } @Override diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java index df6612cb222..73109a3ecd8 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java @@ -25,6 +25,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.DocValuesTermsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; @@ -178,6 +179,11 @@ public class ParentFieldMapper extends MetadataFieldMapper { return CONTENT_TYPE; } + @Override + public Query existsQuery(QueryShardContext context) { + return new DocValuesFieldExistsQuery(name()); + } + @Override public Query termQuery(Object value, @Nullable QueryShardContext context) { return termsQuery(Collections.singletonList(value), context); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java index 88679d910b1..a4b009f9f1f 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java @@ -22,9 +22,13 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; import java.util.Collections; @@ -121,6 +125,11 @@ public class RoutingFieldMapper extends MetadataFieldMapper { public String typeName() { return CONTENT_TYPE; } + + @Override + public Query existsQuery(QueryShardContext context) { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } } private boolean required; @@ -165,6 +174,7 @@ public class RoutingFieldMapper extends MetadataFieldMapper { if (routing != null) { if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { fields.add(new Field(fieldType().name(), routing, fieldType())); + createFieldNamesField(context, fields); } } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java index bcf901388f1..7d74f9e52aa 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java @@ -24,6 +24,7 @@ import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; @@ -162,6 +163,11 @@ public class SeqNoFieldMapper extends MetadataFieldMapper { return Long.parseLong(value.toString()); } + @Override + public Query existsQuery(QueryShardContext context) { + return new DocValuesFieldExistsQuery(name()); + } + @Override public Query termQuery(Object value, @Nullable QueryShardContext context) { long v = parse(value); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index fc177e45d5c..47d5e64438e 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -24,7 +24,6 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.Version; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; @@ -162,6 +161,11 @@ public class SourceFieldMapper extends MetadataFieldMapper { return CONTENT_TYPE; } + @Override + public Query existsQuery(QueryShardContext context) { + throw new QueryShardException(context, "The _source field is not searchable"); + } + @Override public Query termQuery(Object value, QueryShardContext context) { throw new QueryShardException(context, "The _source field is not searchable"); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 82223ae2de7..24c3443658a 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -22,13 +22,17 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; +import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; import java.util.Iterator; @@ -274,6 +278,15 @@ public class TextFieldMapper extends FieldMapper { return CONTENT_TYPE; } + @Override + public Query existsQuery(QueryShardContext context) { + if (hasDocValues()) { + return new DocValuesFieldExistsQuery(name()); + } else { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + } + @Override public Query nullValueQuery() { if (nullValue() == null) { @@ -332,6 +345,7 @@ public class TextFieldMapper extends FieldMapper { if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { Field field = new Field(fieldType().name(), value, fieldType()); fields.add(field); + createFieldNamesField(context, fields); } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java index 09ef33f0795..d0e30e77c9e 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java @@ -132,6 +132,11 @@ public class TypeFieldMapper extends MetadataFieldMapper { return true; } + @Override + public Query existsQuery(QueryShardContext context) { + return new MatchAllDocsQuery(); + } + @Override public Query termQuery(Object value, QueryShardContext context) { return termsQuery(Arrays.asList(value), context); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/UidFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/UidFieldMapper.java index f981fc94b2d..95dc40bca63 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/UidFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/UidFieldMapper.java @@ -22,6 +22,7 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermInSetQuery; @@ -133,6 +134,11 @@ public class UidFieldMapper extends MetadataFieldMapper { } } + @Override + public Query existsQuery(QueryShardContext context) { + return new MatchAllDocsQuery(); + } + @Override public Query termQuery(Object value, @Nullable QueryShardContext context) { return termsQuery(Arrays.asList(value), context); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java index 1d2e997acba..90ea85024c1 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java @@ -24,6 +24,7 @@ import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -87,6 +88,11 @@ public class VersionFieldMapper extends MetadataFieldMapper { return CONTENT_TYPE; } + @Override + public Query existsQuery(QueryShardContext context) { + return new DocValuesFieldExistsQuery(name()); + } + @Override public Query termQuery(Object value, QueryShardContext context) { throw new QueryShardException(context, "The _version field is not searchable"); diff --git a/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java index 799998e2c9f..97378e01236 100644 --- a/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java @@ -19,10 +19,14 @@ package org.elasticsearch.index.query; +import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; @@ -32,6 +36,7 @@ import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import java.io.IOException; import java.util.Collection; @@ -126,8 +131,9 @@ public class ExistsQueryBuilder extends AbstractQueryBuilder } public static Query newFilter(QueryShardContext context, String fieldPattern) { - final FieldNamesFieldMapper.FieldNamesFieldType fieldNamesFieldType = - (FieldNamesFieldMapper.FieldNamesFieldType) context.getMapperService().fullName(FieldNamesFieldMapper.NAME); + + final FieldNamesFieldMapper.FieldNamesFieldType fieldNamesFieldType = (FieldNamesFieldMapper.FieldNamesFieldType) context + .getMapperService().fullName(FieldNamesFieldMapper.NAME); if (fieldNamesFieldType == null) { // can only happen when no types exist, so no docs exist either return Queries.newMatchNoDocsQuery("Missing types in \"" + NAME + "\" query."); @@ -142,19 +148,62 @@ public class ExistsQueryBuilder extends AbstractQueryBuilder fields = context.simpleMatchToIndexNames(fieldPattern); } + if (context.indexVersionCreated().before(Version.V_6_1_0)) { + return newLegacyExistsQuery(fields); + } + if (fields.size() == 1) { - Query filter = fieldNamesFieldType.termQuery(fields.iterator().next(), context); + String field = fields.iterator().next(); + return newFieldExistsQuery(context, field); + } + + BooleanQuery.Builder boolFilterBuilder = new BooleanQuery.Builder(); + for (String field : fields) { + boolFilterBuilder.add(newFieldExistsQuery(context, field), BooleanClause.Occur.SHOULD); + } + return new ConstantScoreQuery(boolFilterBuilder.build()); + } + + private static Query newLegacyExistsQuery(Collection fields) { + // We create TermsQuery directly here rather than using FieldNamesFieldType.termsQuery() + // so we don't end up with deprecation warnings + if (fields.size() == 1) { + Query filter = new TermQuery(new Term(FieldNamesFieldMapper.NAME, fields.iterator().next())); return new ConstantScoreQuery(filter); } BooleanQuery.Builder boolFilterBuilder = new BooleanQuery.Builder(); for (String field : fields) { - Query filter = fieldNamesFieldType.termQuery(field, context); + Query filter = new TermQuery(new Term(FieldNamesFieldMapper.NAME, field)); boolFilterBuilder.add(filter, BooleanClause.Occur.SHOULD); } return new ConstantScoreQuery(boolFilterBuilder.build()); } + private static Query newFieldExistsQuery(QueryShardContext context, String field) { + MappedFieldType fieldType = context.getMapperService().fullName(field); + if (fieldType == null) { + // The field does not exist as a leaf but could be an object so + // check for an object mapper + if (context.getObjectMapper(field) != null) { + return newObjectFieldExistsQuery(context, field); + } + return Queries.newMatchNoDocsQuery("No field \"" + field + "\" exists in mappings."); + } + Query filter = fieldType.existsQuery(context); + return new ConstantScoreQuery(filter); + } + + private static Query newObjectFieldExistsQuery(QueryShardContext context, String objField) { + BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder(); + Collection fields = context.simpleMatchToIndexNames(objField + ".*"); + for (String field : fields) { + Query existsQuery = context.getMapperService().fullName(field).existsQuery(context); + booleanQuery.add(existsQuery, Occur.SHOULD); + } + return new ConstantScoreQuery(booleanQuery.build()); + } + @Override protected int doHashCode() { return Objects.hash(fieldName); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java index 398708d75f9..4e79a68c50e 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java @@ -23,14 +23,18 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; import java.io.StringReader; @@ -88,6 +92,15 @@ public class DocumentFieldMapperTests extends LuceneTestCase { return "fake"; } + @Override + public Query existsQuery(QueryShardContext context) { + if (hasDocValues()) { + return new DocValuesFieldExistsQuery(name()); + } else { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + } + } static class FakeFieldMapper extends FieldMapper { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/core/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java index 4a2c36d829f..33e3bc20183 100755 --- a/core/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java @@ -20,12 +20,17 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.index.IndexableField; -import org.locationtech.spatial4j.shape.Point; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.builders.ShapeBuilders; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.QueryShardContext; +import org.locationtech.spatial4j.shape.Point; import java.io.IOException; import java.nio.charset.Charset; @@ -128,6 +133,15 @@ public class ExternalMapper extends FieldMapper { public String typeName() { return "faketype"; } + + @Override + public Query existsQuery(QueryShardContext context) { + if (hasDocValues()) { + return new DocValuesFieldExistsQuery(name()); + } else { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + } } private final String generatedValue; diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java b/core/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java index 642282c9d5c..464b0d9f840 100755 --- a/core/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java @@ -23,16 +23,14 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.index.mapper.StringFieldType; +import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; import java.util.List; @@ -114,6 +112,15 @@ public class FakeStringFieldMapper extends FieldMapper { } return termQuery(nullValue(), null); } + + @Override + public Query existsQuery(QueryShardContext context) { + if (hasDocValues()) { + return new DocValuesFieldExistsQuery(name()); + } else { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + } } protected FakeStringFieldMapper(String simpleName, FakeStringFieldType fieldType, MappedFieldType defaultFieldType, diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java index 9d25e2b70b8..70022dc6326 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldMapperTests.java @@ -19,29 +19,16 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.index.IndexableField; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.test.ESSingleNodeTestCase; -import java.io.IOException; import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; -import java.util.function.Supplier; public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { @@ -100,12 +87,13 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { .bytes(), XContentType.JSON)); - assertFieldNames(set("a", "a.keyword", "b", "b.c", "_id", "_version", "_seq_no", "_primary_term", "_source"), doc); + assertFieldNames(set("a"), doc); } public void testExplicitEnabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_field_names").field("enabled", true).endObject() + .startObject("properties").startObject("field").field("type", "keyword").field("doc_values", false).endObject().endObject() .endObject().endObject().string(); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); FieldNamesFieldMapper fieldNamesMapper = docMapper.metadataMapper(FieldNamesFieldMapper.class); @@ -118,27 +106,7 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { .bytes(), XContentType.JSON)); - assertFieldNames(set("field", "field.keyword", "_id", "_version", "_seq_no", "_primary_term", "_source"), doc); - } - - public void testDedup() throws Exception { - String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_field_names").field("enabled", true).endObject() - .endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); - FieldNamesFieldMapper fieldNamesMapper = docMapper.metadataMapper(FieldNamesFieldMapper.class); - assertTrue(fieldNamesMapper.fieldType().isEnabled()); - - ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder() - .startObject() - .field("field", 3) // will create 2 lucene fields under the hood: index and doc values - .endObject() - .bytes(), - XContentType.JSON)); - - Set fields = set("field", "_id", "_version", "_seq_no", "_primary_term", "_source"); - assertFieldNames(fields, doc); - assertEquals(fields.size(), doc.rootDoc().getValues("_field_names").length); + assertFieldNames(set("field"), doc); } public void testDisabled() throws Exception { @@ -175,110 +143,4 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { mapperEnabled = mapperService.merge("type", new CompressedXContent(enabledMapping), MapperService.MergeReason.MAPPING_UPDATE, false); assertTrue(mapperEnabled.metadataMapper(FieldNamesFieldMapper.class).fieldType().isEnabled()); } - - private static class DummyMetadataFieldMapper extends MetadataFieldMapper { - - public static class TypeParser implements MetadataFieldMapper.TypeParser { - - @Override - public Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - return new MetadataFieldMapper.Builder("_dummy", FIELD_TYPE, FIELD_TYPE) { - @Override - public DummyMetadataFieldMapper build(BuilderContext context) { - return new DummyMetadataFieldMapper(context.indexSettings()); - } - }; - } - - @Override - public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) { - final Settings indexSettings = context.mapperService().getIndexSettings().getSettings(); - return new DummyMetadataFieldMapper(indexSettings); - } - - } - - private static class DummyFieldType extends TermBasedFieldType { - - DummyFieldType() { - super(); - } - - private DummyFieldType(MappedFieldType other) { - super(other); - } - - @Override - public MappedFieldType clone() { - return new DummyFieldType(this); - } - - @Override - public String typeName() { - return "_dummy"; - } - - } - - private static final MappedFieldType FIELD_TYPE = new DummyFieldType(); - static { - FIELD_TYPE.setTokenized(false); - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); - FIELD_TYPE.setName("_dummy"); - FIELD_TYPE.freeze(); - } - - protected DummyMetadataFieldMapper(Settings indexSettings) { - super("_dummy", FIELD_TYPE, FIELD_TYPE, indexSettings); - } - - @Override - public void preParse(ParseContext context) throws IOException { - } - - @Override - public void postParse(ParseContext context) throws IOException { - context.doc().add(new Field("_dummy", "dummy", FIELD_TYPE)); - } - - @Override - protected void parseCreateField(ParseContext context, List fields) throws IOException { - } - - @Override - protected String contentType() { - return "_dummy"; - } - - } - - public void testSeesFieldsFromPlugins() throws IOException { - IndexService indexService = createIndex("test"); - IndicesModule indicesModule = newTestIndicesModule( - Collections.emptyMap(), - Collections.singletonMap("_dummy", new DummyMetadataFieldMapper.TypeParser()) - ); - final MapperRegistry mapperRegistry = indicesModule.getMapperRegistry(); - Supplier queryShardContext = () -> { - return indexService.newQueryShardContext(0, null, () -> { throw new UnsupportedOperationException(); }, null); - }; - MapperService mapperService = new MapperService(indexService.getIndexSettings(), indexService.getIndexAnalyzers(), - indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, queryShardContext); - DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), mapperService, - indexService.getIndexAnalyzers(), indexService.xContentRegistry(), indexService.similarityService(), mapperRegistry, - queryShardContext); - String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); - DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); - ParsedDocument parsedDocument = mapper.parse(SourceToParse.source("index", "type", "id", new BytesArray("{}"), - XContentType.JSON)); - IndexableField[] fields = parsedDocument.rootDoc().getFields(FieldNamesFieldMapper.NAME); - boolean found = false; - for (IndexableField f : fields) { - if ("_dummy".equals(f.stringValue())) { - found = true; - break; - } - } - assertTrue("Could not find the dummy field among " + Arrays.toString(fields), found); - } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java index b3c9da806fa..945407fc394 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java @@ -21,10 +21,18 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.elasticsearch.index.mapper.FieldNamesFieldMapper; -import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.query.QueryShardContext; import org.junit.Before; +import java.util.Collections; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public class FieldNamesFieldTypeTests extends FieldTypeTestCase { @Override protected MappedFieldType createDefaultFieldType() { @@ -43,13 +51,28 @@ public class FieldNamesFieldTypeTests extends FieldTypeTestCase { } public void testTermQuery() { - FieldNamesFieldMapper.FieldNamesFieldType type = new FieldNamesFieldMapper.FieldNamesFieldType(); - type.setName(FieldNamesFieldMapper.CONTENT_TYPE); - type.setEnabled(true); - Query termQuery = type.termQuery("field_name", null); + + FieldNamesFieldMapper.FieldNamesFieldType fieldNamesFieldType = new FieldNamesFieldMapper.FieldNamesFieldType(); + fieldNamesFieldType.setName(FieldNamesFieldMapper.CONTENT_TYPE); + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType(); + fieldType.setName("field_name"); + + Settings settings = settings(Version.CURRENT).build(); + IndexSettings indexSettings = new IndexSettings( + new IndexMetaData.Builder("foo").settings(settings).numberOfShards(1).numberOfReplicas(0).build(), settings); + MapperService mapperService = mock(MapperService.class); + when(mapperService.fullName("_field_names")).thenReturn(fieldNamesFieldType); + when(mapperService.fullName("field_name")).thenReturn(fieldType); + when(mapperService.simpleMatchToIndexNames("field_name")).thenReturn(Collections.singletonList("field_name")); + + QueryShardContext queryShardContext = new QueryShardContext(0, + indexSettings, null, null, mapperService, null, null, null, null, null, null, () -> 0L, null); + fieldNamesFieldType.setEnabled(true); + Query termQuery = fieldNamesFieldType.termQuery("field_name", queryShardContext); assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.CONTENT_TYPE, "field_name")), termQuery); - type.setEnabled(false); - IllegalStateException e = expectThrows(IllegalStateException.class, () -> type.termQuery("field_name", null)); + assertWarnings("terms query on the _field_names field is deprecated and will be removed, use exists query instead"); + fieldNamesFieldType.setEnabled(false); + IllegalStateException e = expectThrows(IllegalStateException.class, () -> fieldNamesFieldType.termQuery("field_name", null)); assertEquals("Cannot run [exists] queries if the [_field_names] field is disabled", e.getMessage()); } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java index 4ae9b004413..fe885a46b87 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java @@ -19,6 +19,11 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.test.ESTestCase; import java.util.Arrays; @@ -223,5 +228,14 @@ public class FieldTypeLookupTests extends ESTestCase { public String typeName() { return "otherfaketype"; } + + @Override + public Query existsQuery(QueryShardContext context) { + if (hasDocValues()) { + return new DocValuesFieldExistsQuery(name()); + } else { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + } } } diff --git a/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java index cfc2d789420..40d3f90dd4e 100644 --- a/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java @@ -22,15 +22,20 @@ package org.elasticsearch.index.query; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.test.AbstractQueryTestCase; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; @@ -60,23 +65,66 @@ public class ExistsQueryBuilderTests extends AbstractQueryTestCase fields = context.getQueryShardContext().simpleMatchToIndexNames(fieldPattern); + Collection mappedFields = fields.stream().filter((field) -> context.getQueryShardContext().getObjectMapper(field) != null + || context.getQueryShardContext().getMapperService().fullName(field) != null).collect(Collectors.toList()); if (getCurrentTypes().length == 0) { assertThat(query, instanceOf(MatchNoDocsQuery.class)); MatchNoDocsQuery matchNoDocsQuery = (MatchNoDocsQuery) query; assertThat(matchNoDocsQuery.toString(null), containsString("Missing types in \"exists\" query.")); + } else if (context.mapperService().getIndexSettings().getIndexVersionCreated().before(Version.V_6_1_0)) { + if (fields.size() == 1) { + assertThat(query, instanceOf(ConstantScoreQuery.class)); + ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) query; + String field = fields.iterator().next(); + assertThat(constantScoreQuery.getQuery(), instanceOf(TermQuery.class)); + TermQuery termQuery = (TermQuery) constantScoreQuery.getQuery(); + assertEquals(field, termQuery.getTerm().text()); + } else { + assertThat(query, instanceOf(ConstantScoreQuery.class)); + ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) query; + assertThat(constantScoreQuery.getQuery(), instanceOf(BooleanQuery.class)); + BooleanQuery booleanQuery = (BooleanQuery) constantScoreQuery.getQuery(); + assertThat(booleanQuery.clauses().size(), equalTo(mappedFields.size())); + for (int i = 0; i < mappedFields.size(); i++) { + BooleanClause booleanClause = booleanQuery.clauses().get(i); + assertThat(booleanClause.getOccur(), equalTo(BooleanClause.Occur.SHOULD)); + } + } + } else if (fields.size() == 1 && mappedFields.size() == 0) { + assertThat(query, instanceOf(MatchNoDocsQuery.class)); + MatchNoDocsQuery matchNoDocsQuery = (MatchNoDocsQuery) query; + assertThat(matchNoDocsQuery.toString(null), + containsString("No field \"" + fields.iterator().next() + "\" exists in mappings.")); } else if (fields.size() == 1) { assertThat(query, instanceOf(ConstantScoreQuery.class)); ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) query; - assertThat(constantScoreQuery.getQuery(), instanceOf(TermQuery.class)); - TermQuery termQuery = (TermQuery) constantScoreQuery.getQuery(); - assertEquals(fields.iterator().next(), termQuery.getTerm().text()); + String field = fields.iterator().next(); + if (context.getQueryShardContext().getObjectMapper(field) != null) { + assertThat(constantScoreQuery.getQuery(), instanceOf(BooleanQuery.class)); + BooleanQuery booleanQuery = (BooleanQuery) constantScoreQuery.getQuery(); + List childFields = new ArrayList<>(); + context.getQueryShardContext().getObjectMapper(field).forEach(mapper -> childFields.add(mapper.name())); + assertThat(booleanQuery.clauses().size(), equalTo(childFields.size())); + for (int i = 0; i < childFields.size(); i++) { + BooleanClause booleanClause = booleanQuery.clauses().get(i); + assertThat(booleanClause.getOccur(), equalTo(BooleanClause.Occur.SHOULD)); + } + } else if (context.getQueryShardContext().getMapperService().fullName(field).hasDocValues()) { + assertThat(constantScoreQuery.getQuery(), instanceOf(DocValuesFieldExistsQuery.class)); + DocValuesFieldExistsQuery dvExistsQuery = (DocValuesFieldExistsQuery) constantScoreQuery.getQuery(); + assertEquals(field, dvExistsQuery.getField()); + } else { + assertThat(constantScoreQuery.getQuery(), instanceOf(TermQuery.class)); + TermQuery termQuery = (TermQuery) constantScoreQuery.getQuery(); + assertEquals(field, termQuery.getTerm().text()); + } } else { assertThat(query, instanceOf(ConstantScoreQuery.class)); ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) query; assertThat(constantScoreQuery.getQuery(), instanceOf(BooleanQuery.class)); BooleanQuery booleanQuery = (BooleanQuery) constantScoreQuery.getQuery(); - assertThat(booleanQuery.clauses().size(), equalTo(fields.size())); - for (int i = 0; i < fields.size(); i++) { + assertThat(booleanQuery.clauses().size(), equalTo(mappedFields.size())); + for (int i = 0; i < mappedFields.size(); i++) { BooleanClause booleanClause = booleanQuery.clauses().get(i); assertThat(booleanClause.getOccur(), equalTo(BooleanClause.Occur.SHOULD)); } diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java index 94b55fba618..cf71bc0872a 100644 --- a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java @@ -801,11 +801,11 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase 0) { - expected = new ConstantScoreQuery(new TermQuery(new Term("_field_names", "foo"))); + expected = new ConstantScoreQuery(new TermQuery(new Term("_field_names", STRING_FIELD_NAME))); } else { expected = new MatchNoDocsQuery(); } diff --git a/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java index 4c0e2192cec..ad3946a6755 100644 --- a/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java @@ -23,6 +23,7 @@ import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; import org.apache.lucene.index.Term; import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.PointRangeQuery; @@ -30,6 +31,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermRangeQuery; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.lucene.BytesRefs; @@ -124,7 +126,12 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase 0) { - expectedQuery = new ConstantScoreQuery(new TermQuery(new Term(FieldNamesFieldMapper.NAME, queryBuilder.fieldName()))); + if (context.mapperService().getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_1_0) + && context.mapperService().fullName(queryBuilder.fieldName()).hasDocValues()) { + expectedQuery = new ConstantScoreQuery(new DocValuesFieldExistsQuery(queryBuilder.fieldName())); + } else { + expectedQuery = new ConstantScoreQuery(new TermQuery(new Term(FieldNamesFieldMapper.NAME, queryBuilder.fieldName()))); + } } else { expectedQuery = new MatchNoDocsQuery("no mappings yet"); } @@ -385,7 +392,7 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase 0) { - expectedQuery = new ConstantScoreQuery(new TermQuery(new Term(FieldNamesFieldMapper.NAME, query.fieldName()))); + if (queryShardContext.getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_1_0) + && queryShardContext.fieldMapper(query.fieldName()).hasDocValues()) { + expectedQuery = new ConstantScoreQuery(new DocValuesFieldExistsQuery(query.fieldName())); + } else { + expectedQuery = new ConstantScoreQuery(new TermQuery(new Term(FieldNamesFieldMapper.NAME, query.fieldName()))); + } } else { expectedQuery = new MatchNoDocsQuery("no mappings yet"); } @@ -416,7 +428,7 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase fields = new ArrayList<>(1); + createFieldNamesField(context, fields); + for (IndexableField field : fields) { + context.doc().add(field); + } } static Query parseQuery(QueryShardContext context, boolean mapUnmappedFieldsAsString, XContentParser parser) throws IOException { diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index ea966c5fc3a..f927f920f90 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -23,13 +23,17 @@ import com.ibm.icu.text.Collator; import com.ibm.icu.text.RawCollationKey; import com.ibm.icu.text.RuleBasedCollator; import com.ibm.icu.util.ULocale; + import org.apache.lucene.document.Field; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamOutput; @@ -122,6 +126,15 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper { this.collator = collator.isFrozen() ? collator : collator.freeze(); } + @Override + public Query existsQuery(QueryShardContext context) { + if (hasDocValues()) { + return new DocValuesFieldExistsQuery(name()); + } else { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + } + @Override public Query nullValueQuery() { if (nullValue() == null) { @@ -750,6 +763,8 @@ public class ICUCollationKeywordFieldMapper extends FieldMapper { if (fieldType().hasDocValues()) { fields.add(getDVField.apply(fieldType().name(), binaryValue)); + } else if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { + createFieldNamesField(context, fields); } } } diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index 6824c8bf202..a6dc27b1f8a 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -19,14 +19,11 @@ package org.elasticsearch.index.mapper.murmur3; -import java.io.IOException; -import java.util.List; -import java.util.Map; - import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; @@ -44,6 +41,10 @@ import org.elasticsearch.index.mapper.TypeParsers; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; +import java.io.IOException; +import java.util.List; +import java.util.Map; + public class Murmur3FieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "murmur3"; @@ -127,6 +128,11 @@ public class Murmur3FieldMapper extends FieldMapper { return new DocValuesIndexFieldData.Builder().numericType(NumericType.LONG); } + @Override + public Query existsQuery(QueryShardContext context) { + return new DocValuesFieldExistsQuery(name()); + } + @Override public Query termQuery(Object value, QueryShardContext context) { throw new QueryShardException(context, "Murmur3 fields are not searchable: [" + name() + "]"); diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/160_exists_query.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/160_exists_query.yml new file mode 100644 index 00000000000..a34b110e910 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/160_exists_query.yml @@ -0,0 +1,1289 @@ +setup: + - skip: + features: ["headers"] + + - do: + indices.create: + index: test + body: + mappings: + test: + dynamic: false + properties: + binary: + type: binary + doc_values: true + boolean: + type: boolean + date: + type: date + geo_point: + type: geo_point + geo_shape: + type: geo_shape + ip: + type: ip + keyword: + type: keyword + byte: + type: byte + double: + type: double + float: + type: float + half_float: + type: half_float + integer: + type: integer + long: + type: long + short: + type: short + object: + type: object + properties: + inner1: + type: keyword + inner2: + type: keyword + text: + type: text + + - do: + headers: + Content-Type: application/json + index: + index: "test" + type: "test" + id: 1 + body: + binary: "YWJjZGUxMjM0" + boolean: true + date: "2017-01-01" + geo_point: [0.0, 20.0] + geo_shape: + type: "point" + coordinates: [0.0, 20.0] + ip: "192.168.0.1" + keyword: "foo" + byte: 1 + double: 1.0 + float: 1.0 + half_float: 1.0 + integer: 1 + long: 1 + short: 1 + object: + inner1: "foo" + inner2: "bar" + text: "foo bar" + + - do: + headers: + Content-Type: application/json + index: + index: "test" + type: "test" + id: 2 + body: + binary: "YWJjZGUxMjM0" + boolean: false + date: "2017-01-01" + geo_point: [0.0, 20.0] + geo_shape: + type: "point" + coordinates: [0.0, 20.0] + ip: "192.168.0.1" + keyword: "foo" + byte: 1 + double: 1.0 + float: 1.0 + half_float: 1.0 + integer: 1 + long: 1 + short: 1 + object: + inner1: "foo" + text: "foo bar" + + - do: + headers: + Content-Type: application/json + index: + index: "test" + type: "test" + id: 3 + routing: "route_me" + body: + binary: "YWJjZGUxMjM0" + boolean: true + date: "2017-01-01" + geo_point: [0.0, 20.0] + geo_shape: + type: "point" + coordinates: [0.0, 20.0] + ip: "192.168.0.1" + keyword: "foo" + byte: 1 + double: 1.0 + float: 1.0 + half_float: 1.0 + integer: 1 + long: 1 + short: 1 + object: + inner2: "bar" + text: "foo bar" + + - do: + index: + index: "test" + type: "test" + id: 4 + body: {} + + - do: + indices.create: + index: test-no-dv + body: + mappings: + test: + dynamic: false + properties: + binary: + type: binary + doc_values: false + store: true + boolean: + type: boolean + doc_values: false + date: + type: date + doc_values: false + geo_point: + type: geo_point + doc_values: false + geo_shape: + type: geo_shape + ip: + type: ip + doc_values: false + keyword: + type: keyword + doc_values: false + byte: + type: byte + doc_values: false + double: + type: double + doc_values: false + float: + type: float + doc_values: false + half_float: + type: half_float + doc_values: false + integer: + type: integer + doc_values: false + long: + type: long + doc_values: false + short: + type: short + doc_values: false + object: + type: object + properties: + inner1: + type: keyword + doc_values: false + inner2: + type: keyword + doc_values: false + text: + type: text + doc_values: false + + - do: + headers: + Content-Type: application/json + index: + index: "test-no-dv" + type: "test" + id: 1 + body: + binary: "YWJjZGUxMjM0" + boolean: true + date: "2017-01-01" + geo_point: [0.0, 20.0] + geo_shape: + type: "point" + coordinates: [0.0, 20.0] + ip: "192.168.0.1" + keyword: "foo" + byte: 1 + double: 1.0 + float: 1.0 + half_float: 1.0 + integer: 1 + long: 1 + short: 1 + object: + inner1: "foo" + inner2: "bar" + text: "foo bar" + + - do: + headers: + Content-Type: application/json + index: + index: "test-no-dv" + type: "test" + id: 2 + body: + binary: "YWJjZGUxMjM0" + boolean: false + date: "2017-01-01" + geo_point: [0.0, 20.0] + geo_shape: + type: "point" + coordinates: [0.0, 20.0] + ip: "192.168.0.1" + keyword: "foo" + byte: 1 + double: 1.0 + float: 1.0 + half_float: 1.0 + integer: 1 + long: 1 + short: 1 + object: + inner1: "foo" + text: "foo bar" + + - do: + headers: + Content-Type: application/json + index: + index: "test-no-dv" + type: "test" + id: 3 + routing: "route_me" + body: + binary: "YWJjZGUxMjM0" + boolean: true + date: "2017-01-01" + geo_point: [0.0, 20.0] + geo_shape: + type: "point" + coordinates: [0.0, 20.0] + ip: "192.168.0.1" + keyword: "foo" + byte: 1 + double: 1.0 + float: 1.0 + half_float: 1.0 + integer: 1 + long: 1 + short: 1 + object: + inner2: "bar" + text: "foo bar" + + - do: + index: + index: "test-no-dv" + type: "test" + id: 4 + body: {} + + - do: + indices.create: + index: test-unmapped + body: + mappings: + test: + dynamic: false + properties: + unrelated: + type: keyword + + - do: + index: + index: "test-unmapped" + type: "test" + id: 1 + body: + unrelated: "foo" + + - do: + indices.create: + index: test-empty + body: + mappings: + test: + dynamic: false + properties: + binary: + type: binary + date: + type: date + geo_point: + type: geo_point + geo_shape: + type: geo_shape + ip: + type: ip + keyword: + type: keyword + byte: + type: byte + double: + type: double + float: + type: float + half_float: + type: half_float + integer: + type: integer + long: + type: long + short: + type: short + object: + type: object + properties: + inner1: + type: keyword + inner2: + type: keyword + text: + type: text + + - do: + indices.refresh: + index: [test, test-unmapped, test-empty, test-no-dv] + +--- +"Test exists query on mapped binary field": + - do: + search: + index: test + body: + query: + exists: + field: binary + + - match: {hits.total: 3} + +--- +"Test exists query on mapped boolean field": + - do: + search: + index: test + body: + query: + exists: + field: boolean + + - match: {hits.total: 3} + +--- +"Test exists query on mapped date field": + - do: + search: + index: test + body: + query: + exists: + field: date + + - match: {hits.total: 3} + +--- +"Test exists query on mapped geo_point field": + - do: + search: + index: test + body: + query: + exists: + field: geo_point + + - match: {hits.total: 3} + +--- +"Test exists query on mapped geo_shape field": + - do: + search: + index: test + body: + query: + exists: + field: geo_shape + + - match: {hits.total: 3} + +--- +"Test exists query on mapped ip field": + - do: + search: + index: test + body: + query: + exists: + field: ip + + - match: {hits.total: 3} + +--- +"Test exists query on mapped keyword field": + - do: + search: + index: test + body: + query: + exists: + field: keyword + + - match: {hits.total: 3} + +--- +"Test exists query on mapped byte field": + - do: + search: + index: test + body: + query: + exists: + field: byte + + - match: {hits.total: 3} + +--- +"Test exists query on mapped double field": + - do: + search: + index: test + body: + query: + exists: + field: double + + - match: {hits.total: 3} + +--- +"Test exists query on mapped float field": + - do: + search: + index: test + body: + query: + exists: + field: float + + - match: {hits.total: 3} + +--- +"Test exists query on mapped half_float field": + - do: + search: + index: test + body: + query: + exists: + field: half_float + + - match: {hits.total: 3} + +--- +"Test exists query on mapped integer field": + - do: + search: + index: test + body: + query: + exists: + field: integer + + - match: {hits.total: 3} + +--- +"Test exists query on mapped long field": + - do: + search: + index: test + body: + query: + exists: + field: long + + - match: {hits.total: 3} + +--- +"Test exists query on mapped short field": + - do: + search: + index: test + body: + query: + exists: + field: short + + - match: {hits.total: 3} + +--- +"Test exists query on mapped object field": + - do: + search: + index: test + body: + query: + exists: + field: object + + - match: {hits.total: 3} + +--- +"Test exists query on mapped object inner field": + - do: + search: + index: test + body: + query: + exists: + field: object.inner1 + + - match: {hits.total: 2} + +--- +"Test exists query on mapped text field": + - do: + search: + index: test + body: + query: + exists: + field: text + + - match: {hits.total: 3} + +--- +"Test exists query on _id field": + - do: + search: + index: test + body: + query: + exists: + field: _id + + - match: {hits.total: 4} + +--- +"Test exists query on _uid field": + - skip: + version: " - 6.1.0" + reason: exists on _uid not supported prior to 6.1.0 + - do: + search: + index: test + body: + query: + exists: + field: _uid + + - match: {hits.total: 4} + +--- +"Test exists query on _index field": + - skip: + version: " - 6.1.0" + reason: exists on _index not supported prior to 6.1.0 + - do: + search: + index: test + body: + query: + exists: + field: _index + + - match: {hits.total: 4} + +--- +"Test exists query on _type field": + - skip: + version: " - 6.1.0" + reason: exists on _type not supported prior to 6.1.0 + - do: + search: + index: test + body: + query: + exists: + field: _type + + - match: {hits.total: 4} + +--- +"Test exists query on _routing field": + - do: + search: + index: test + body: + query: + exists: + field: _routing + + - match: {hits.total: 1} + +--- +"Test exists query on _seq_no field": + - do: + search: + index: test + body: + query: + exists: + field: _seq_no + + - match: {hits.total: 4} + +--- +"Test exists query on _source field": + - skip: + version: " - 6.1.0" + reason: exists on _source not supported prior to 6.1.0 + - do: + catch: /query_shard_exception/ + search: + index: test + body: + query: + exists: + field: _source + +--- +"Test exists query on _version field": + - do: + search: + index: test + body: + query: + exists: + field: _version + + - match: {hits.total: 4} + +--- +"Test exists query on unmapped binary field": + - do: + search: + index: test-unmapped + body: + query: + exists: + field: binary + + - match: {hits.total: 0} + +--- +"Test exists query on unmapped boolean field": + - do: + search: + index: test-unmapped + body: + query: + exists: + field: boolean + + - match: {hits.total: 0} + +--- +"Test exists query on unmapped date field": + - do: + search: + index: test-unmapped + body: + query: + exists: + field: date + + - match: {hits.total: 0} + +--- +"Test exists query on unmapped geo_point field": + - do: + search: + index: test-unmapped + body: + query: + exists: + field: geo_point + + - match: {hits.total: 0} + +--- +"Test exists query on unmapped geo_shape field": + - do: + search: + index: test-unmapped + body: + query: + exists: + field: geo_shape + + - match: {hits.total: 0} + +--- +"Test exists query on unmapped ip field": + - do: + search: + index: test-unmapped + body: + query: + exists: + field: ip + + - match: {hits.total: 0} + +--- +"Test exists query on unmapped keyword field": + - do: + search: + index: test-unmapped + body: + query: + exists: + field: keyword + + - match: {hits.total: 0} + +--- +"Test exists query on unmapped byte field": + - do: + search: + index: test-unmapped + body: + query: + exists: + field: byte + + - match: {hits.total: 0} + +--- +"Test exists query on unmapped double field": + - do: + search: + index: test-unmapped + body: + query: + exists: + field: double + + - match: {hits.total: 0} + +--- +"Test exists query on unmapped float field": + - do: + search: + index: test-unmapped + body: + query: + exists: + field: float + + - match: {hits.total: 0} + +--- +"Test exists query on unmapped half_float field": + - do: + search: + index: test-unmapped + body: + query: + exists: + field: half_float + + - match: {hits.total: 0} + +--- +"Test exists query on unmapped integer field": + - do: + search: + index: test-unmapped + body: + query: + exists: + field: integer + + - match: {hits.total: 0} + +--- +"Test exists query on unmapped long field": + - do: + search: + index: test-unmapped + body: + query: + exists: + field: long + + - match: {hits.total: 0} + +--- +"Test exists query on unmapped short field": + - do: + search: + index: test-unmapped + body: + query: + exists: + field: short + + - match: {hits.total: 0} + +--- +"Test exists query on unmapped object field": + - do: + search: + index: test-unmapped + body: + query: + exists: + field: object + + - match: {hits.total: 0} + +--- +"Test exists query on unmapped object inner field": + - do: + search: + index: test-unmapped + body: + query: + exists: + field: object.inner1 + + - match: {hits.total: 0} + +--- +"Test exists query on unmapped text field": + - do: + search: + index: test-unmapped + body: + query: + exists: + field: text + + - match: {hits.total: 0} + +--- +"Test exists query on binary field in empty index": + - do: + search: + index: test-empty + body: + query: + exists: + field: binary + + - match: {hits.total: 0} + +--- +"Test exists query on boolean field in empty index": + - do: + search: + index: test-empty + body: + query: + exists: + field: boolean + + - match: {hits.total: 0} + +--- +"Test exists query on date field in empty index": + - do: + search: + index: test-empty + body: + query: + exists: + field: date + + - match: {hits.total: 0} + +--- +"Test exists query on geo_point field in empty index": + - do: + search: + index: test-empty + body: + query: + exists: + field: geo_point + + - match: {hits.total: 0} + +--- +"Test exists query on geo_shape field in empty index": + - do: + search: + index: test-empty + body: + query: + exists: + field: geo_shape + + - match: {hits.total: 0} + +--- +"Test exists query on ip field in empty index": + - do: + search: + index: test-empty + body: + query: + exists: + field: ip + + - match: {hits.total: 0} + +--- +"Test exists query on keyword field in empty index": + - do: + search: + index: test-empty + body: + query: + exists: + field: keyword + + - match: {hits.total: 0} + +--- +"Test exists query on byte field in empty index": + - do: + search: + index: test-empty + body: + query: + exists: + field: byte + + - match: {hits.total: 0} + +--- +"Test exists query on double field in empty index": + - do: + search: + index: test-empty + body: + query: + exists: + field: double + + - match: {hits.total: 0} + +--- +"Test exists query on float field in empty index": + - do: + search: + index: test-empty + body: + query: + exists: + field: float + + - match: {hits.total: 0} + +--- +"Test exists query on half_float field in empty index": + - do: + search: + index: test-empty + body: + query: + exists: + field: half_float + + - match: {hits.total: 0} + +--- +"Test exists query on integer field in empty index": + - do: + search: + index: test-empty + body: + query: + exists: + field: integer + + - match: {hits.total: 0} + +--- +"Test exists query on long field in empty index": + - do: + search: + index: test-empty + body: + query: + exists: + field: long + + - match: {hits.total: 0} + +--- +"Test exists query on short field in empty index": + - do: + search: + index: test-empty + body: + query: + exists: + field: short + + - match: {hits.total: 0} + +--- +"Test exists query on object field in empty index": + - do: + search: + index: test-empty + body: + query: + exists: + field: object + + - match: {hits.total: 0} + +--- +"Test exists query on object inner field in empty index": + - do: + search: + index: test-empty + body: + query: + exists: + field: object.inner1 + + - match: {hits.total: 0} + +--- +"Test exists query on text field in empty index": + - do: + search: + index: test-empty + body: + query: + exists: + field: text + + - match: {hits.total: 0} + +--- +"Test exists query on mapped binary field with no doc values": + - do: + search: + index: test-no-dv + body: + query: + exists: + field: binary + + - match: {hits.total: 3} + +--- +"Test exists query on mapped boolean field with no doc values": + - do: + search: + index: test-no-dv + body: + query: + exists: + field: boolean + + - match: {hits.total: 3} + +--- +"Test exists query on mapped date field with no doc values": + - do: + search: + index: test-no-dv + body: + query: + exists: + field: date + + - match: {hits.total: 3} + +--- +"Test exists query on mapped geo_point field with no doc values": + - do: + search: + index: test-no-dv + body: + query: + exists: + field: geo_point + + - match: {hits.total: 3} + +--- +"Test exists query on mapped geo_shape field with no doc values": + - do: + search: + index: test-no-dv + body: + query: + exists: + field: geo_shape + + - match: {hits.total: 3} + +--- +"Test exists query on mapped ip field with no doc values": + - do: + search: + index: test-no-dv + body: + query: + exists: + field: ip + + - match: {hits.total: 3} + +--- +"Test exists query on mapped keyword field with no doc values": + - do: + search: + index: test-no-dv + body: + query: + exists: + field: keyword + + - match: {hits.total: 3} + +--- +"Test exists query on mapped byte field with no doc values": + - do: + search: + index: test-no-dv + body: + query: + exists: + field: byte + + - match: {hits.total: 3} + +--- +"Test exists query on mapped double field with no doc values": + - do: + search: + index: test-no-dv + body: + query: + exists: + field: double + + - match: {hits.total: 3} + +--- +"Test exists query on mapped float field with no doc values": + - do: + search: + index: test-no-dv + body: + query: + exists: + field: float + + - match: {hits.total: 3} + +--- +"Test exists query on mapped half_float field with no doc values": + - do: + search: + index: test-no-dv + body: + query: + exists: + field: half_float + + - match: {hits.total: 3} + +--- +"Test exists query on mapped integer field with no doc values": + - do: + search: + index: test-no-dv + body: + query: + exists: + field: integer + + - match: {hits.total: 3} + +--- +"Test exists query on mapped long field with no doc values": + - do: + search: + index: test-no-dv + body: + query: + exists: + field: long + + - match: {hits.total: 3} + +--- +"Test exists query on mapped short field with no doc values": + - do: + search: + index: test-no-dv + body: + query: + exists: + field: short + + - match: {hits.total: 3} + +--- +"Test exists query on mapped object field with no doc values": + - do: + search: + index: test-no-dv + body: + query: + exists: + field: object + + - match: {hits.total: 3} + +--- +"Test exists query on mapped object inner field with no doc values": + - do: + search: + index: test-no-dv + body: + query: + exists: + field: object.inner1 + + - match: {hits.total: 2} + +--- +"Test exists query on mapped text field with no doc values": + - do: + search: + index: test-no-dv + body: + query: + exists: + field: text + + - match: {hits.total: 3} diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java index ae91a791535..2b6f4c38a90 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java @@ -19,11 +19,13 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.search.Query; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.similarity.BM25SimilarityProvider; import org.elasticsearch.test.ESTestCase; @@ -285,6 +287,8 @@ public abstract class FieldTypeTestCase extends ESTestCase { public MappedFieldType clone() {return null;} @Override public String typeName() { return fieldType.typeName();} + @Override + public Query existsQuery(QueryShardContext context) { return null; } }; try { fieldType.checkCompatibility(bogus, conflicts, random().nextBoolean()); @@ -299,6 +303,8 @@ public abstract class FieldTypeTestCase extends ESTestCase { public MappedFieldType clone() {return null;} @Override public String typeName() { return "othertype";} + @Override + public Query existsQuery(QueryShardContext context) { return null; } }; try { fieldType.checkCompatibility(other, conflicts, random().nextBoolean()); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index d8c9c8d797b..b374a6b4034 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -19,12 +19,17 @@ package org.elasticsearch.index.mapper; -import java.io.IOException; -import java.util.List; - +import org.apache.lucene.index.Term; +import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryShardContext; + +import java.io.IOException; +import java.util.List; // this sucks how much must be overridden just do get a dummy field mapper... public class MockFieldMapper extends FieldMapper { @@ -66,6 +71,15 @@ public class MockFieldMapper extends FieldMapper { public String typeName() { return "faketype"; } + + @Override + public Query existsQuery(QueryShardContext context) { + if (hasDocValues()) { + return new DocValuesFieldExistsQuery(name()); + } else { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + } } @Override