Index field names of documents.

The `exists` and `missing` filters need to merge postings lists of all existing
terms, which can be very costly, especially on high-cardinality fields. This
commit indexes the field names of a document under `_field_names` and reuses it
to speed up the `exists` and `missing` filters.

This is only enabled for indices that are created on or after Elasticsearch
1.3.0.

Close #5659

docs/reference/mapping/fields.asciidoc

@@ -21,6 +21,8 @@ include::fields/boost-field.asciidoc[]
 
 include::fields/parent-field.asciidoc[]
 
+include::fields/field-names-field.asciidoc[]
+
 include::fields/routing-field.asciidoc[]
 
 include::fields/index-field.asciidoc[]

docs/reference/mapping/fields/field-names-field.asciidoc

@@ -0,0 +1,11 @@
+[[mapping-field-names-field]]
+=== `_field_names`
+
+coming[1.3.0]
+
+The `_field_names` field indexes the field names of a document, which can later
+be used to search for documents based on the fields that they contain typically
+using the `exists` and `missing` filters.
+
+`_field_names` is indexed by default for indices that have been created after
+Elasticsearch 1.3.0.

src/main/java/org/elasticsearch/Version.java

@@ -19,12 +19,14 @@
 
 package org.elasticsearch;
 
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 
 import java.io.IOException;
@@ -344,6 +346,15 @@ public class Version implements Serializable {
         }
     }
 
+    /**
+     * Return the {@link Version} of Elasticsearch that has been used to create an index given its settings.
+     */
+    public static Version indexCreated(Settings indexSettings) {
+        assert indexSettings.get(IndexMetaData.SETTING_UUID) == null // if the UUDI is there the index has actually been created otherwise this might be a test
+                || indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null) != null : IndexMetaData.SETTING_VERSION_CREATED + " not set in IndexSettings";
+        return indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
+    }
+
     public static void writeVersion(Version version, StreamOutput out) throws IOException {
         out.writeVInt(version.id);
     }

src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java

@@ -180,6 +180,8 @@ public class DocumentMapper implements ToXContent {
             this.rootMappers.put(TTLFieldMapper.class, new TTLFieldMapper());
             this.rootMappers.put(VersionFieldMapper.class, new VersionFieldMapper());
             this.rootMappers.put(ParentFieldMapper.class, new ParentFieldMapper());
+            // _field_names last so that it can see all other fields
+            this.rootMappers.put(FieldNamesFieldMapper.class, new FieldNamesFieldMapper(indexSettings));
         }
 
         public Builder meta(ImmutableMap<String, Object> meta) {

src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java

@@ -21,9 +21,7 @@ package org.elasticsearch.index.mapper;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
-import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.MapBuilder;
@@ -51,7 +49,6 @@ import org.elasticsearch.index.similarity.SimilarityLookupService;
 
 import java.util.Iterator;
 import java.util.Map;
-import java.util.Set;
 
 import static org.elasticsearch.index.mapper.MapperBuilders.doc;
 
@@ -122,10 +119,9 @@ public class DocumentMapperParser extends AbstractIndexComponent {
                 .put(UidFieldMapper.NAME, new UidFieldMapper.TypeParser())
                 .put(VersionFieldMapper.NAME, new VersionFieldMapper.TypeParser())
                 .put(IdFieldMapper.NAME, new IdFieldMapper.TypeParser())
+                .put(FieldNamesFieldMapper.NAME, new FieldNamesFieldMapper.TypeParser())
                 .immutableMap();
-        assert indexSettings.get(IndexMetaData.SETTING_UUID) == null // if the UUDI is there the index has actually been created otherwise this might be a test
-                || indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null) != null : IndexMetaData.SETTING_VERSION_CREATED + " not set in IndexSettings";
-        indexVersionCreated = indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
+        indexVersionCreated = Version.indexCreated(indexSettings);
     }
 
     public void putTypeParser(String type, Mapper.TypeParser typeParser) {

src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java

@@ -74,6 +74,10 @@ public final class MapperBuilders {
         return new TypeFieldMapper.Builder();
     }
 
+    public static FieldNamesFieldMapper.Builder fieldNames() {
+        return new FieldNamesFieldMapper.Builder();
+    }
+
     public static IndexFieldMapper.Builder index() {
         return new IndexFieldMapper.Builder();
     }

src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java

@@ -0,0 +1,248 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper.internal;
+
+import com.google.common.collect.UnmodifiableIterator;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.XStringField;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.codec.docvaluesformat.DocValuesFormatProvider;
+import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
+import org.elasticsearch.index.fielddata.FieldDataType;
+import org.elasticsearch.index.mapper.*;
+import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.index.mapper.MapperBuilders.fieldNames;
+import static org.elasticsearch.index.mapper.core.TypeParsers.parseField;
+
+/**
+ * A mapper that indexes the field names of a document under <code>_field_names</code>. This mapper is typically useful in order
+ * to have fast <code>exists</code> and <code>missing</code> queries/filters.
+ *
+ * Added in Elasticsearch 1.3.
+ */
+public class FieldNamesFieldMapper extends AbstractFieldMapper<String> implements InternalMapper, RootMapper {
+
+    public static final String NAME = "_field_names";
+
+    public static final String CONTENT_TYPE = "_field_names";
+
+    public static class Defaults extends AbstractFieldMapper.Defaults {
+        public static final String NAME = FieldNamesFieldMapper.NAME;
+        public static final String INDEX_NAME = FieldNamesFieldMapper.NAME;
+
+        public static final FieldType FIELD_TYPE = new FieldType(AbstractFieldMapper.Defaults.FIELD_TYPE);
+        public static final FieldType FIELD_TYPE_PRE_1_3_0;
+
+        static {
+            FIELD_TYPE.setIndexed(true);
+            FIELD_TYPE.setTokenized(false);
+            FIELD_TYPE.setStored(false);
+            FIELD_TYPE.setOmitNorms(true);
+            FIELD_TYPE.setIndexOptions(IndexOptions.DOCS_ONLY);
+            FIELD_TYPE.freeze();
+            FIELD_TYPE_PRE_1_3_0 = new FieldType(FIELD_TYPE);
+            FIELD_TYPE_PRE_1_3_0.setIndexed(false);
+            FIELD_TYPE_PRE_1_3_0.freeze();
+        }
+    }
+
+    public static class Builder extends AbstractFieldMapper.Builder<Builder, FieldNamesFieldMapper> {
+
+        private boolean indexIsExplicit;
+
+        public Builder() {
+            super(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE));
+            indexName = Defaults.INDEX_NAME;
+        }
+
+        @Override
+        public Builder index(boolean index) {
+            indexIsExplicit = true;
+            return super.index(index);
+        }
+
+        @Override
+        public FieldNamesFieldMapper build(BuilderContext context) {
+            if ((context.indexCreatedVersion() == null || context.indexCreatedVersion().before(Version.V_1_3_0)) && !indexIsExplicit) {
+                fieldType.setIndexed(false);
+            }
+            return new FieldNamesFieldMapper(name, indexName, boost, fieldType, postingsProvider, docValuesProvider, fieldDataSettings, context.indexSettings());
+        }
+    }
+
+    public static class TypeParser implements Mapper.TypeParser {
+        @Override
+        public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+            FieldNamesFieldMapper.Builder builder = fieldNames();
+            parseField(builder, builder.name, node, parserContext);
+            return builder;
+        }
+    }
+
+    private final FieldType defaultFieldType;
+
+    private static FieldType defaultFieldType(Settings indexSettings) {
+        return indexSettings != null && Version.indexCreated(indexSettings).onOrAfter(Version.V_1_3_0) ? Defaults.FIELD_TYPE : Defaults.FIELD_TYPE_PRE_1_3_0;
+    }
+
+    public FieldNamesFieldMapper(Settings indexSettings) {
+        this(Defaults.NAME, Defaults.INDEX_NAME, indexSettings);
+    }
+
+    protected FieldNamesFieldMapper(String name, String indexName, Settings indexSettings) {
+        this(name, indexName, Defaults.BOOST, new FieldType(defaultFieldType(indexSettings)), null, null, null, indexSettings);
+    }
+
+    public FieldNamesFieldMapper(String name, String indexName, float boost, FieldType fieldType, PostingsFormatProvider postingsProvider,
+                           DocValuesFormatProvider docValuesProvider, @Nullable Settings fieldDataSettings, Settings indexSettings) {
+        super(new Names(name, indexName, indexName, name), boost, fieldType, null, Lucene.KEYWORD_ANALYZER,
+                Lucene.KEYWORD_ANALYZER, postingsProvider, docValuesProvider, null, null, fieldDataSettings, indexSettings);
+        this.defaultFieldType = defaultFieldType(indexSettings);
+    }
+
+    @Override
+    public FieldType defaultFieldType() {
+        return defaultFieldType;
+    }
+
+    @Override
+    public FieldDataType defaultFieldDataType() {
+        return new FieldDataType("string");
+    }
+
+    @Override
+    public String value(Object value) {
+        if (value == null) {
+            return null;
+        }
+        return value.toString();
+    }
+
+    @Override
+    public boolean useTermQueryWithQueryString() {
+        return true;
+    }
+
+    @Override
+    public void preParse(ParseContext context) throws IOException {
+    }
+
+    @Override
+    public void postParse(ParseContext context) throws IOException {
+        super.parse(context);
+    }
+
+    @Override
+    public void parse(ParseContext context) throws IOException {
+        // we parse in post parse
+    }
+
+    @Override
+    public boolean includeInObject() {
+        return false;
+    }
+
+    static Iterable<String> extractFieldNames(final String fullPath) {
+        return new Iterable<String>() {
+            @Override
+            public Iterator<String> iterator() {
+                return new UnmodifiableIterator<String>() {
+
+                    int endIndex = nextEndIndex(0);
+
+                    private int nextEndIndex(int index) {
+                        while (index < fullPath.length() && fullPath.charAt(index) != '.') {
+                            index += 1;
+                        }
+                        return index;
+                    }
+
+                    @Override
+                    public boolean hasNext() {
+                        return endIndex <= fullPath.length();
+                    }
+
+                    @Override
+                    public String next() {
+                        final String result = fullPath.substring(0, endIndex);
+                        endIndex = nextEndIndex(endIndex + 1);
+                        return result;
+                    }
+
+                };
+            }
+        };
+    }
+
+    @Override
+    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+        if (!fieldType.indexed() && !fieldType.stored() && !hasDocValues()) {
+            return;
+        }
+        for (ParseContext.Document document : context.docs()) {
+            final List<String> paths = new ArrayList<>();
+            for (IndexableField field : document.getFields()) {
+                paths.add(field.name());
+            }
+            for (String path : paths) {
+                for (String fieldName : extractFieldNames(path)) {
+                    if (fieldType.indexed() || fieldType.stored()) {
+                        document.add(new XStringField(names().indexName(), fieldName, fieldType));
+                    }
+                    if (hasDocValues()) {
+                        document.add(new SortedSetDocValuesField(names().indexName(), new BytesRef(fieldName)));
+                    }
+                }
+            }
+        }
+    }
+
+    @Override
+    protected String contentType() {
+        return CONTENT_TYPE;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        XContentBuilder json = XContentFactory.jsonBuilder();
+        super.toXContent(json, params);
+        if (json.string().equals("\"" + NAME + "\"{\"type\":\"" + CONTENT_TYPE + "\"}")) {
+            return builder;
+        }
+        return super.toXContent(builder, params);
+    }
+}

src/main/java/org/elasticsearch/index/query/ExistsFilterParser.java

@@ -27,7 +27,9 @@ import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.lucene.search.XBooleanFilter;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.cache.filter.support.CacheKeyFilter;
+import org.elasticsearch.index.mapper.FieldMappers;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.internal.FieldNamesFieldMapper;
 
 import java.io.IOException;
 import java.util.Set;
@@ -81,6 +83,8 @@ public class ExistsFilterParser implements FilterParser {
     }
 
     public static Filter newFilter(QueryParseContext parseContext, String fieldPattern, String filterName) {
+        final FieldMappers fieldNamesMapper = parseContext.mapperService().indexName(FieldNamesFieldMapper.CONTENT_TYPE);
+
         MapperService.SmartNameObjectMapper smartNameObjectMapper = parseContext.smartObjectMapper(fieldPattern);
         if (smartNameObjectMapper != null && smartNameObjectMapper.hasMapper()) {
             // automatic make the object mapper pattern
@@ -101,7 +105,17 @@ public class ExistsFilterParser implements FilterParser {
                 nonNullFieldMappers = smartNameFieldMappers;
             }
             Filter filter = null;
-            if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) {
+            if (fieldNamesMapper!= null && fieldNamesMapper.mapper().fieldType().indexed()) {
+                final String f;
+                if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) {
+                    f = smartNameFieldMappers.mapper().names().indexName();
+                } else {
+                    f = field;
+                }
+                filter = fieldNamesMapper.mapper().termFilter(f, parseContext);
+            }
+            // if _field_names are not indexed, we need to go the slow way
+            if (filter == null && smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) {
                 filter = smartNameFieldMappers.mapper().rangeFilter(null, null, true, true, parseContext);
             }
             if (filter == null) {

src/main/java/org/elasticsearch/index/query/MissingFilterParser.java

@@ -28,7 +28,9 @@ import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.lucene.search.XBooleanFilter;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.cache.filter.support.CacheKeyFilter;
+import org.elasticsearch.index.mapper.FieldMappers;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.internal.FieldNamesFieldMapper;
 
 import java.io.IOException;
 import java.util.Set;
@@ -94,6 +96,7 @@ public class MissingFilterParser implements FilterParser {
             throw new QueryParsingException(parseContext.index(), "missing must have either existence, or null_value, or both set to true");
         }
 
+        final FieldMappers fieldNamesMapper = parseContext.mapperService().indexName(FieldNamesFieldMapper.NAME);
         MapperService.SmartNameObjectMapper smartNameObjectMapper = parseContext.smartObjectMapper(fieldPattern);
         if (smartNameObjectMapper != null && smartNameObjectMapper.hasMapper()) {
             // automatic make the object mapper pattern
@@ -122,7 +125,17 @@ public class MissingFilterParser implements FilterParser {
                     nonNullFieldMappers = smartNameFieldMappers;
                 }
                 Filter filter = null;
-                if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) {
+                if (fieldNamesMapper != null && fieldNamesMapper.mapper().fieldType().indexed()) {
+                    final String f;
+                    if (smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) {
+                        f = smartNameFieldMappers.mapper().names().indexName();
+                    } else {
+                        f = field;
+                    }
+                    filter = fieldNamesMapper.mapper().termFilter(f, parseContext);
+                }
+                // if _field_names are not indexed, we need to go the slow way
+                if (filter == null && smartNameFieldMappers != null && smartNameFieldMappers.hasMapper()) {
                     filter = smartNameFieldMappers.mapper().rangeFilter(null, null, true, true, parseContext);
                 }
                 if (filter == null) {

src/test/java/org/elasticsearch/VersionTests.java

@@ -19,6 +19,8 @@
 
 package org.elasticsearch;
 
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.test.ElasticsearchTestCase;
 import org.junit.Test;
 
@@ -94,4 +96,12 @@ public class VersionTests extends ElasticsearchTestCase {
     public void testWrongVersionFromString() {
         Version.fromString("WRONG.VERSION");
     }
+
+    public void testVersion() {
+        // test scenario
+        assertEquals(Version.CURRENT, Version.indexCreated(ImmutableSettings.builder().build()));
+        // an actual index has a IndexMetaData.SETTING_UUID
+        final Version version = randomFrom(Version.V_0_18_0, Version.V_0_90_13, Version.V_1_3_0);
+        assertEquals(version, Version.indexCreated(ImmutableSettings.builder().put(IndexMetaData.SETTING_UUID, "foo").put(IndexMetaData.SETTING_VERSION_CREATED, version).build()));
+    }
 }

src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java

@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper.internal;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.lucene.index.IndexableField;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.MapperTestUtils;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.test.ElasticsearchTestCase;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+public class FieldNamesFieldMapperTests extends ElasticsearchTestCase {
+
+    private static Set<String> extract(String path) {
+        return ImmutableSet.<String>builder().addAll(FieldNamesFieldMapper.extractFieldNames(path)).build();
+    }
+
+    private static <T> Set<T> set(T... values) {
+        return new HashSet<T>(Arrays.asList(values));
+    }
+
+    public void testExtractFieldNames() {
+        assertEquals(set("abc"), extract("abc"));
+        assertEquals(set("a", "a.b"), extract("a.b"));
+        assertEquals(set("a", "a.b", "a.b.c"), extract("a.b.c"));
+        // and now corner cases
+        assertEquals(set("", ".a"), extract(".a"));
+        assertEquals(set("a", "a."), extract("a."));
+        assertEquals(set("", ".", ".."), extract(".."));
+    }
+
+    public void test() throws Exception {
+        DocumentMapper defaultMapper = MapperTestUtils.newParser().parse(XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string());
+
+        ParsedDocument doc = defaultMapper.parse("type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                    .field("a", "100")
+                    .startObject("b")
+                        .field("c", 42)
+                    .endObject()
+                .endObject()
+                .bytes());
+
+        final Set<String> fieldNames = new HashSet<>();
+        for (IndexableField field : doc.rootDoc().getFields()) {
+            if (FieldNamesFieldMapper.CONTENT_TYPE.equals(field.name())) {
+                fieldNames.add(field.stringValue());
+            }
+        }
+        assertEquals(new HashSet<>(Arrays.asList("a", "b", "b.c", "_uid", "_type", "_version", "_source", "_all")), fieldNames);
+    }
+}

src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsTests.java

@@ -22,6 +22,7 @@ import com.google.common.base.Strings;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.index.mapper.internal.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.internal.IndexFieldMapper;
 import org.elasticsearch.index.query.FilterBuilders;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
@@ -1265,5 +1266,14 @@ public class StringTermsTests extends ElasticsearchIntegrationTest {
             assertThat(bucket.getDocCount(), equalTo(i == 0 ? 5L : 2L));
             i++;
         }
+
+        response = client().prepareSearch("idx", "empty_bucket_idx").setTypes("type")
+                .addAggregation(terms("terms")
+                        .executionHint(randomExecutionHint())
+                        .field(FieldNamesFieldMapper.NAME)
+                ).execute().actionGet();
+        assertSearchResponse(response);
+        terms = response.getAggregations().get("terms");
+        assertEquals(5L, terms.getBucketByKey("i").getDocCount());
     }
 }

src/test/java/org/elasticsearch/search/query/ExistsMissingTests.java

@@ -0,0 +1,98 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.query;
+
+import com.google.common.collect.ImmutableMap;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.mapper.internal.FieldNamesFieldMapper;
+import org.elasticsearch.index.query.FilterBuilders;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.test.ElasticsearchIntegrationTest;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+
+
+public class ExistsMissingTests extends ElasticsearchIntegrationTest {
+
+    public void testExistsMissing() throws Exception {
+        assertAcked(client().admin().indices().prepareCreate("idx").addMapping("type", XContentBuilder.builder(JsonXContent.jsonXContent)
+                .startObject()
+                    .startObject("type")
+                        .startObject(FieldNamesFieldMapper.NAME)
+                            // by setting randomly index to no we also test the pre-1.3 behavior
+                            .field("index", randomFrom("no", "not_analyzed"))
+                            .field("store", randomFrom("no", "yes"))
+                        .endObject()
+                    .endObject()
+                .endObject()));
+            @SuppressWarnings("unchecked")
+            Map<String, Object>[] sources = new Map[] {
+                    // simple property
+                    ImmutableMap.of("foo", "bar"),
+                    // object fields
+                    ImmutableMap.of("bar", ImmutableMap.of("foo", "bar", "bar", ImmutableMap.of("bar", "foo"))),
+                    ImmutableMap.of("bar", ImmutableMap.of("baz", 42)),
+                    // empty doc
+                    ImmutableMap.of()
+            };
+            List<IndexRequestBuilder> reqs = new ArrayList<IndexRequestBuilder>();
+            for (Map<String, Object> source : sources) {
+                reqs.add(client().prepareIndex("idx", "type").setSource(source));
+            }
+            indexRandom(true, reqs);
+
+        final Map<String, Integer> expected = new LinkedHashMap<String, Integer>();
+        expected.put("foo", 1);
+        expected.put("f*", 2); // foo and bar.foo, that's how the expansion works
+        expected.put("bar", 2);
+        expected.put("bar.*", 2);
+        expected.put("bar.foo", 1);
+        expected.put("bar.bar", 1);
+        expected.put("bar.bar.bar", 1);
+        expected.put("baz", 1);
+        expected.put("foobar", 0);
+
+        final long numDocs = client().prepareSearch("idx").execute().actionGet().getHits().totalHits();
+
+        for (Map.Entry<String, Integer> entry : expected.entrySet()) {
+            final String fieldName = entry.getKey();
+            final int count = entry.getValue();
+            // exists
+            SearchResponse resp = client().prepareSearch("idx").setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), FilterBuilders.existsFilter(fieldName))).execute().actionGet();
+            assertSearchResponse(resp);
+            assertEquals(count, resp.getHits().totalHits());
+
+            // missing
+            resp = client().prepareSearch("idx").setQuery(QueryBuilders.filteredQuery(QueryBuilders.matchAllQuery(), FilterBuilders.missingFilter(fieldName))).execute().actionGet();
+            assertSearchResponse(resp);
+            assertEquals(numDocs - count, resp.getHits().totalHits());
+        }
+    }
+
+}

src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java

@@ -71,6 +71,7 @@ import org.elasticsearch.discovery.zen.elect.ElectMasterService;
 import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.FieldMapper.Loading;
+import org.elasticsearch.index.mapper.internal.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.internal.IdFieldMapper;
 import org.elasticsearch.index.merge.policy.*;
 import org.elasticsearch.index.merge.scheduler.ConcurrentMergeSchedulerProvider;
@@ -333,6 +334,11 @@ public abstract class ElasticsearchIntegrationTest extends ElasticsearchTestCase
                             .field("index", randomFrom("not_analyzed", "no"))
                         .endObject();
                 }
+                mappings.startObject(FieldNamesFieldMapper.NAME)
+                        .startObject("fielddata")
+                            .field(FieldDataType.FORMAT_KEY, randomFrom("paged_bytes", "fst", "doc_values"))
+                        .endObject()
+                    .endObject();
                 mappings.startArray("dynamic_templates")
                         .startObject()
                             .startObject("template-strings")