Query DSL: Query String _missing_ and _exists_ syntax, closes #446.

modules/elasticsearch/src/main/java/org/apache/lucene/queryParser/ExistsFieldQueryExtension.java

@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lucene.queryParser;
+
+import org.apache.lucene.search.DeletionAwareConstantScoreQuery;
+import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermRangeFilter;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.query.xcontent.QueryParseContext;
+
+import static org.elasticsearch.index.query.support.QueryParsers.*;
+
+/**
+ * @author kimchy (shay.banon)
+ */
+public class ExistsFieldQueryExtension implements FieldQueryExtension {
+
+    public static final String NAME = "_exists_";
+
+    @Override public Query query(QueryParseContext parseContext, String queryText) {
+        String fieldName = queryText;
+        Filter filter = null;
+        MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName);
+        if (smartNameFieldMappers != null) {
+            if (smartNameFieldMappers.hasMapper()) {
+                filter = smartNameFieldMappers.mapper().rangeFilter(null, null, true, true);
+            }
+        }
+        if (filter == null) {
+            filter = new TermRangeFilter(fieldName, null, null, true, true);
+        }
+
+        // we always cache this one, really does not change...
+        filter = parseContext.cacheFilter(filter);
+
+        filter = wrapSmartNameFilter(filter, smartNameFieldMappers, parseContext);
+
+        return new DeletionAwareConstantScoreQuery(filter);
+    }
+}

modules/elasticsearch/src/main/java/org/apache/lucene/queryParser/FieldQueryExtension.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lucene.queryParser;
+
+import org.apache.lucene.search.Query;
+import org.elasticsearch.index.query.xcontent.QueryParseContext;
+
+/**
+ * @author kimchy (shay.banon)
+ */
+public interface FieldQueryExtension {
+
+    Query query(QueryParseContext parseContext, String queryText);
+}

modules/elasticsearch/src/main/java/org/apache/lucene/queryParser/MapperQueryParser.java

@@ -23,6 +23,7 @@ import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.Query;
+import org.elasticsearch.common.collect.ImmutableMap;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.index.mapper.FieldMapper;
@@ -46,6 +47,15 @@ import static org.elasticsearch.index.query.support.QueryParsers.*;
  */
 public class MapperQueryParser extends QueryParser {
 
+    public static final ImmutableMap<String, FieldQueryExtension> fieldQueryExtensions;
+
+    static {
+        fieldQueryExtensions = ImmutableMap.<String, FieldQueryExtension>builder()
+                .put(ExistsFieldQueryExtension.NAME, new ExistsFieldQueryExtension())
+                .put(MissingFieldQueryExtension.NAME, new MissingFieldQueryExtension())
+                .build();
+    }
+
     private final QueryParseContext parseContext;
 
     private FieldMapper currentMapper;
@@ -89,6 +99,10 @@ public class MapperQueryParser extends QueryParser {
     }
 
     @Override public Query getFieldQuery(String field, String queryText) throws ParseException {
+        FieldQueryExtension fieldQueryExtension = fieldQueryExtensions.get(field);
+        if (fieldQueryExtension != null) {
+            return fieldQueryExtension.query(parseContext, queryText);
+        }
         currentMapper = null;
         if (parseContext.mapperService() != null) {
             MapperService.SmartNameFieldMappers fieldMappers = parseContext.mapperService().smartName(field);

modules/elasticsearch/src/main/java/org/apache/lucene/queryParser/MissingFieldQueryExtension.java

@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lucene.queryParser;
+
+import org.apache.lucene.search.DeletionAwareConstantScoreQuery;
+import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermRangeFilter;
+import org.elasticsearch.common.lucene.search.NotFilter;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.query.xcontent.QueryParseContext;
+
+import static org.elasticsearch.index.query.support.QueryParsers.*;
+
+/**
+ * @author kimchy (shay.banon)
+ */
+public class MissingFieldQueryExtension implements FieldQueryExtension {
+
+    public static final String NAME = "_missing_";
+
+    @Override public Query query(QueryParseContext parseContext, String queryText) {
+        String fieldName = queryText;
+
+        Filter filter = null;
+        MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName);
+        if (smartNameFieldMappers != null) {
+            if (smartNameFieldMappers.hasMapper()) {
+                filter = smartNameFieldMappers.mapper().rangeFilter(null, null, true, true);
+            }
+        }
+        if (filter == null) {
+            filter = new TermRangeFilter(fieldName, null, null, true, true);
+        }
+
+        filter = new NotFilter(filter);
+        // we always cache this one, really does not change...
+        filter = parseContext.cacheFilter(filter);
+
+        filter = wrapSmartNameFilter(filter, smartNameFieldMappers, parseContext);
+
+        return new DeletionAwareConstantScoreQuery(filter, true);
+    }
+}

modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/NotDocSet.java

@@ -19,22 +19,18 @@
 
 package org.elasticsearch.common.lucene.docset;
 
-import org.apache.lucene.search.DocIdSetIterator;
-
 import java.io.IOException;
 
 /**
  * @author kimchy (shay.banon)
  */
-public class NotDocSet extends DocSet {
+public class NotDocSet extends GetDocSet {
 
     private final DocSet set;
 
-    private final int max;
-
     public NotDocSet(DocSet set, int max) {
+        super(max);
         this.set = set;
-        this.max = max;
     }
 
     @Override public boolean isCacheable() {
@@ -44,65 +40,4 @@ public class NotDocSet extends DocSet {
     @Override public boolean get(int doc) throws IOException {
         return !set.get(doc);
     }
-
-    @Override public DocIdSetIterator iterator() throws IOException {
-        return new NotDocIdSetIterator();
-    }
-
-    class NotDocIdSetIterator extends DocIdSetIterator {
-        int lastReturn = -1;
-        private DocIdSetIterator it1 = null;
-        private int innerDocid = -1;
-
-        NotDocIdSetIterator() throws IOException {
-            initialize();
-        }
-
-        private void initialize() throws IOException {
-            it1 = set.iterator();
-
-            try {
-                if ((innerDocid = it1.nextDoc()) == DocIdSetIterator.NO_MORE_DOCS) it1 = null;
-            } catch (IOException e) {
-                e.printStackTrace();
-            }
-        }
-
-        @Override
-        public int docID() {
-            return lastReturn;
-        }
-
-        @Override
-        public int nextDoc() throws IOException {
-            return advance(0);
-        }
-
-        @Override
-        public int advance(int target) throws IOException {
-
-            if (lastReturn == DocIdSetIterator.NO_MORE_DOCS) {
-                return DocIdSetIterator.NO_MORE_DOCS;
-            }
-
-            if (target <= lastReturn) target = lastReturn + 1;
-
-            if (it1 != null && innerDocid < target) {
-                if ((innerDocid = it1.advance(target)) == DocIdSetIterator.NO_MORE_DOCS) {
-                    it1 = null;
-                }
-            }
-
-            while (it1 != null && innerDocid == target) {
-                target++;
-                if (target >= max) {
-                    return (lastReturn = DocIdSetIterator.NO_MORE_DOCS);
-                }
-                if ((innerDocid = it1.advance(target)) == DocIdSetIterator.NO_MORE_DOCS) {
-                    it1 = null;
-                }
-            }
-            return (lastReturn = target);
-        }
-    }
 }

modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/OpenBitDocSet.java

@@ -44,6 +44,10 @@ public class OpenBitDocSet extends DocSet {
         this.set = new OpenBitSetDISI(disi, numBits);
     }
 
+    @Override public boolean isCacheable() {
+        return true;
+    }
+
     public OpenBitSet set() {
         return set;
     }

modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/ExistsFilterParser.java

@@ -19,10 +19,7 @@
 
 package org.elasticsearch.index.query.xcontent;
 
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.search.DocIdSet;
 import org.apache.lucene.search.Filter;
-import org.apache.lucene.search.MultiTermQueryWrapperFilter;
 import org.apache.lucene.search.TermRangeFilter;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -87,7 +84,6 @@ public class ExistsFilterParser extends AbstractIndexComponent implements XConte
             filter = new TermRangeFilter(fieldName, null, null, true, true);
         }
 
-        filter = new ExistsFilter((MultiTermQueryWrapperFilter) filter);
         // we always cache this one, really does not change...
         filter = parseContext.cacheFilter(filter);
 
@@ -97,32 +93,4 @@ public class ExistsFilterParser extends AbstractIndexComponent implements XConte
         }
         return filter;
     }
-
-    public static final class ExistsFilter extends Filter {
-
-        private final MultiTermQueryWrapperFilter filter;
-
-        public ExistsFilter(MultiTermQueryWrapperFilter filter) {
-            this.filter = filter;
-        }
-
-        @Override public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
-            return filter.getDocIdSet(reader);
-        }
-
-        @Override public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-
-            ExistsFilter that = (ExistsFilter) o;
-
-            if (filter != null ? !filter.equals(that.filter) : that.filter != null) return false;
-
-            return true;
-        }
-
-        @Override public int hashCode() {
-            return filter != null ? filter.hashCode() : 0;
-        }
-    }
 }

modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/MissingFilterParser.java

@@ -20,7 +20,6 @@
 package org.elasticsearch.index.query.xcontent;
 
 import org.apache.lucene.search.Filter;
-import org.apache.lucene.search.MultiTermQueryWrapperFilter;
 import org.apache.lucene.search.TermRangeFilter;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lucene.search.NotFilter;
@@ -86,7 +85,7 @@ public class MissingFilterParser extends AbstractIndexComponent implements XCont
             filter = new TermRangeFilter(fieldName, null, null, true, true);
         }
 
-        filter = new NotFilter(new ExistsFilterParser.ExistsFilter((MultiTermQueryWrapperFilter) filter));
+        filter = new NotFilter(filter);
         // we always cache this one, really does not change...
         filter = parseContext.cacheFilter(filter);
 

modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/query/SimpleQueryTests.java

@@ -21,6 +21,7 @@ package org.elasticsearch.test.integration.search.query;
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.test.integration.AbstractNodesTests;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
@@ -59,6 +60,8 @@ public class SimpleQueryTests extends AbstractNodesTests {
             // ignore
         }
 
+        client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder().put("number_of_shards", 1)).execute().actionGet();
+
         client.prepareIndex("test", "type1", "1").setSource("field1", "value1_1", "field2", "value2_1").execute().actionGet();
         client.prepareIndex("test", "type1", "2").setSource("field1", "value1_2").execute().actionGet();
         client.prepareIndex("test", "type1", "3").setSource("field2", "value2_3").execute().actionGet();
@@ -71,6 +74,16 @@ public class SimpleQueryTests extends AbstractNodesTests {
         assertThat(searchResponse.hits().getAt(0).id(), anyOf(equalTo("1"), equalTo("2")));
         assertThat(searchResponse.hits().getAt(1).id(), anyOf(equalTo("1"), equalTo("2")));
 
+        searchResponse = client.prepareSearch().setQuery(constantScoreQuery(exists("field1"))).execute().actionGet();
+        assertThat(searchResponse.hits().totalHits(), equalTo(2l));
+        assertThat(searchResponse.hits().getAt(0).id(), anyOf(equalTo("1"), equalTo("2")));
+        assertThat(searchResponse.hits().getAt(1).id(), anyOf(equalTo("1"), equalTo("2")));
+
+        searchResponse = client.prepareSearch().setQuery(queryString("_exists_:field1")).execute().actionGet();
+        assertThat(searchResponse.hits().totalHits(), equalTo(2l));
+        assertThat(searchResponse.hits().getAt(0).id(), anyOf(equalTo("1"), equalTo("2")));
+        assertThat(searchResponse.hits().getAt(1).id(), anyOf(equalTo("1"), equalTo("2")));
+
         searchResponse = client.prepareSearch().setQuery(filtered(matchAllQuery(), exists("field2"))).execute().actionGet();
         assertThat(searchResponse.hits().totalHits(), equalTo(2l));
         assertThat(searchResponse.hits().getAt(0).id(), anyOf(equalTo("1"), equalTo("3")));
@@ -84,5 +97,21 @@ public class SimpleQueryTests extends AbstractNodesTests {
         assertThat(searchResponse.hits().totalHits(), equalTo(2l));
         assertThat(searchResponse.hits().getAt(0).id(), anyOf(equalTo("3"), equalTo("4")));
         assertThat(searchResponse.hits().getAt(1).id(), anyOf(equalTo("3"), equalTo("4")));
+
+        // double check for cache
+        searchResponse = client.prepareSearch().setQuery(filtered(matchAllQuery(), missing("field1"))).execute().actionGet();
+        assertThat(searchResponse.hits().totalHits(), equalTo(2l));
+        assertThat(searchResponse.hits().getAt(0).id(), anyOf(equalTo("3"), equalTo("4")));
+        assertThat(searchResponse.hits().getAt(1).id(), anyOf(equalTo("3"), equalTo("4")));
+
+        searchResponse = client.prepareSearch().setQuery(constantScoreQuery(missing("field1"))).execute().actionGet();
+        assertThat(searchResponse.hits().totalHits(), equalTo(2l));
+        assertThat(searchResponse.hits().getAt(0).id(), anyOf(equalTo("3"), equalTo("4")));
+        assertThat(searchResponse.hits().getAt(1).id(), anyOf(equalTo("3"), equalTo("4")));
+
+        searchResponse = client.prepareSearch().setQuery(queryString("_missing_:field1")).execute().actionGet();
+        assertThat(searchResponse.hits().totalHits(), equalTo(2l));
+        assertThat(searchResponse.hits().getAt(0).id(), anyOf(equalTo("3"), equalTo("4")));
+        assertThat(searchResponse.hits().getAt(1).id(), anyOf(equalTo("3"), equalTo("4")));
     }
 }