Query DSL: Add exists and missing filters to filter documents where a field either has a value or not in them, closes #445.

modules/elasticsearch/src/main/java/org/elasticsearch/action/index/IndexRequest.java

@@ -26,6 +26,7 @@ import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.WriteConsistencyLevel;
 import org.elasticsearch.action.support.replication.ReplicationType;
 import org.elasticsearch.action.support.replication.ShardReplicationOperationRequest;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.Required;
 import org.elasticsearch.common.Unicode;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -116,6 +117,8 @@ public class IndexRequest extends ShardReplicationOperationRequest {
 
     private boolean refresh = false;
 
+    private XContentType contentType = Requests.INDEX_CONTENT_TYPE;
+
     public IndexRequest() {
     }
 
@@ -173,6 +176,14 @@ public class IndexRequest extends ShardReplicationOperationRequest {
         return this;
     }
 
+    /**
+     * Sets the content type that will be used when generating a document from user provided objects (like Map).
+     */
+    public IndexRequest contentType(XContentType contentType) {
+        this.contentType = contentType;
+        return this;
+    }
+
     /**
      * Should the listener be called on a separate thread if needed.
      */
@@ -221,7 +232,7 @@ public class IndexRequest extends ShardReplicationOperationRequest {
     }
 
     /**
-     * The source of the JSON document to index.
+     * The source of the document to index.
      */
     public byte[] source() {
         if (sourceUnsafe || sourceOffset > 0) {
@@ -233,12 +244,12 @@ public class IndexRequest extends ShardReplicationOperationRequest {
     }
 
     /**
-     * Index the Map as a JSON.
+     * Index the Map as a {@link org.elasticsearch.client.Requests#INDEX_CONTENT_TYPE}.
      *
      * @param source The map to index
      */
     @Required public IndexRequest source(Map source) throws ElasticSearchGenerationException {
-        return source(source, XContentType.JSON);
+        return source(source, contentType);
     }
 
     /**
@@ -287,6 +298,46 @@ public class IndexRequest extends ShardReplicationOperationRequest {
         return this;
     }
 
+    @Required public IndexRequest source(String field1, Object value1) {
+        try {
+            XContentBuilder builder = XContentFactory.contentBuilder(contentType);
+            builder.startObject().field(field1, value1).endObject();
+            return source(builder);
+        } catch (IOException e) {
+            throw new ElasticSearchGenerationException("Failed to generate", e);
+        }
+    }
+
+    @Required public IndexRequest source(String field1, Object value1, String field2, Object value2) {
+        try {
+            XContentBuilder builder = XContentFactory.contentBuilder(contentType);
+            builder.startObject().field(field1, value1).field(field2, value2).endObject();
+            return source(builder);
+        } catch (IOException e) {
+            throw new ElasticSearchGenerationException("Failed to generate", e);
+        }
+    }
+
+    @Required public IndexRequest source(String field1, Object value1, String field2, Object value2, String field3, Object value3) {
+        try {
+            XContentBuilder builder = XContentFactory.contentBuilder(contentType);
+            builder.startObject().field(field1, value1).field(field2, value2).field(field3, value3).endObject();
+            return source(builder);
+        } catch (IOException e) {
+            throw new ElasticSearchGenerationException("Failed to generate", e);
+        }
+    }
+
+    @Required public IndexRequest source(String field1, Object value1, String field2, Object value2, String field3, Object value3, String field4, Object value4) {
+        try {
+            XContentBuilder builder = XContentFactory.contentBuilder(contentType);
+            builder.startObject().field(field1, value1).field(field2, value2).field(field3, value3).field(field4, value4).endObject();
+            return source(builder);
+        } catch (IOException e) {
+            throw new ElasticSearchGenerationException("Failed to generate", e);
+        }
+    }
+
     /**
      * Sets the document to index in bytes form.
      */

modules/elasticsearch/src/main/java/org/elasticsearch/client/Requests.java

@@ -63,6 +63,11 @@ public class Requests {
      */
     public static XContentType CONTENT_TYPE = XContentType.SMILE;
 
+    /**
+     * The default content type to use to generate source documents when indexing.
+     */
+    public static XContentType INDEX_CONTENT_TYPE = XContentType.JSON;
+
     public static IndexRequest indexRequest() {
         return new IndexRequest();
     }

modules/elasticsearch/src/main/java/org/elasticsearch/client/action/index/IndexRequestBuilder.java

@@ -142,6 +142,46 @@ public class IndexRequestBuilder extends BaseRequestBuilder<IndexRequest, IndexR
         return this;
     }
 
+    /**
+     * Constructs a simple document with a field and a value.
+     */
+    public IndexRequestBuilder setSource(String field1, Object value1) {
+        request.source(field1, value1);
+        return this;
+    }
+
+    /**
+     * Constructs a simple document with a field and value pairs.
+     */
+    public IndexRequestBuilder setSource(String field1, Object value1, String field2, Object value2) {
+        request.source(field1, value1, field2, value2);
+        return this;
+    }
+
+    /**
+     * Constructs a simple document with a field and value pairs.
+     */
+    public IndexRequestBuilder setSource(String field1, Object value1, String field2, Object value2, String field3, Object value3) {
+        request.source(field1, value1, field2, value2, field3, value3);
+        return this;
+    }
+
+    /**
+     * Constructs a simple document with a field and value pairs.
+     */
+    public IndexRequestBuilder setSource(String field1, Object value1, String field2, Object value2, String field3, Object value3, String field4, Object value4) {
+        request.source(field1, value1, field2, value2, field3, value3, field4, value4);
+        return this;
+    }
+
+    /**
+     * The content type that will be used to generate a document from user provided objects (like Map).
+     */
+    public IndexRequestBuilder setContentType(XContentType contentType) {
+        request.contentType(contentType);
+        return this;
+    }
+
     /**
      * A timeout to wait if the index operation can't be performed immediately. Defaults to <tt>1m</tt>.
      */

modules/elasticsearch/src/main/java/org/elasticsearch/index/query/IndexQueryParserModule.java

@@ -262,6 +262,8 @@ public class IndexQueryParserModule extends AbstractModule {
             bindings.processXContentQueryFilter(OrFilterParser.NAME, OrFilterParser.class);
             bindings.processXContentQueryFilter(NotFilterParser.NAME, NotFilterParser.class);
             bindings.processXContentQueryFilter(MatchAllFilterParser.NAME, MatchAllFilterParser.class);
+            bindings.processXContentQueryFilter(ExistsFilterParser.NAME, ExistsFilterParser.class);
+            bindings.processXContentQueryFilter(MissingFilterParser.NAME, MissingFilterParser.class);
         }
     }
 }

modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/ExistsFilterBuilder.java

@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.query.xcontent;
+
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+/**
+ * Constructs a filter that only match on documents that the field has a value in them.
+ *
+ * @author kimchy (shay.banon)
+ */
+public class ExistsFilterBuilder extends BaseFilterBuilder {
+
+    private String name;
+
+    private String filterName;
+
+    public ExistsFilterBuilder(String name) {
+        this.name = name;
+    }
+
+    /**
+     * Sets the filter name for the filter that can be used when searching for matched_filters per hit.
+     */
+    public ExistsFilterBuilder filterName(String filterName) {
+        this.filterName = filterName;
+        return this;
+    }
+
+
+    @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(ExistsFilterParser.NAME);
+        builder.field("field", name);
+        if (filterName != null) {
+            builder.field("_name", filterName);
+        }
+        builder.endObject();
+    }
+}

modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/ExistsFilterParser.java

@@ -0,0 +1,128 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.query.xcontent;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.DocIdSet;
+import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.MultiTermQueryWrapperFilter;
+import org.apache.lucene.search.TermRangeFilter;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.AbstractIndexComponent;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.query.QueryParsingException;
+import org.elasticsearch.index.settings.IndexSettings;
+
+import java.io.IOException;
+
+import static org.elasticsearch.index.query.support.QueryParsers.*;
+
+/**
+ * @author kimchy (shay.banon)
+ */
+public class ExistsFilterParser extends AbstractIndexComponent implements XContentFilterParser {
+
+    public static final String NAME = "exists";
+
+    @Inject public ExistsFilterParser(Index index, @IndexSettings Settings settings) {
+        super(index, settings);
+    }
+
+    @Override public String[] names() {
+        return new String[]{NAME};
+    }
+
+    @Override public Filter parse(QueryParseContext parseContext) throws IOException, QueryParsingException {
+        XContentParser parser = parseContext.parser();
+
+        String fieldName = null;
+        String filterName = null;
+
+        XContentParser.Token token;
+        String currentFieldName = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if ("field".equals(currentFieldName)) {
+                    fieldName = parser.text();
+                } else if ("_name".equals(currentFieldName)) {
+                    filterName = parser.text();
+                }
+            }
+        }
+
+        if (fieldName == null) {
+            throw new QueryParsingException(index, "exists must be provided with a [field]");
+        }
+
+        Filter filter = null;
+        MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName);
+        if (smartNameFieldMappers != null) {
+            if (smartNameFieldMappers.hasMapper()) {
+                filter = smartNameFieldMappers.mapper().rangeFilter(null, null, true, true);
+            }
+        }
+        if (filter == null) {
+            filter = new TermRangeFilter(fieldName, null, null, true, true);
+        }
+
+        filter = new ExistsFilter((MultiTermQueryWrapperFilter) filter);
+        // we always cache this one, really does not change...
+        filter = parseContext.cacheFilter(filter);
+
+        filter = wrapSmartNameFilter(filter, smartNameFieldMappers, parseContext);
+        if (filterName != null) {
+            parseContext.addNamedFilter(filterName, filter);
+        }
+        return filter;
+    }
+
+    public static final class ExistsFilter extends Filter {
+
+        private final MultiTermQueryWrapperFilter filter;
+
+        public ExistsFilter(MultiTermQueryWrapperFilter filter) {
+            this.filter = filter;
+        }
+
+        @Override public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
+            return filter.getDocIdSet(reader);
+        }
+
+        @Override public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+
+            ExistsFilter that = (ExistsFilter) o;
+
+            if (filter != null ? !filter.equals(that.filter) : that.filter != null) return false;
+
+            return true;
+        }
+
+        @Override public int hashCode() {
+            return filter != null ? filter.hashCode() : 0;
+        }
+    }
+}

modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/FilterBuilders.java

@@ -228,6 +228,24 @@ public abstract class FilterBuilders {
         return new GeoPolygonFilterBuilder(name);
     }
 
+    /**
+     * A filter to filter only documents where a field exists in them.
+     *
+     * @param name The name of the field
+     */
+    public static ExistsFilterBuilder exists(String name) {
+        return new ExistsFilterBuilder(name);
+    }
+
+    /**
+     * A filter to filter only documents where a field does not exists in them.
+     *
+     * @param name The name of the field
+     */
+    public static MissingFilterBuilder missing(String name) {
+        return new MissingFilterBuilder(name);
+    }
+
     public static BoolFilterBuilder boolFilter() {
         return new BoolFilterBuilder();
     }

modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/MissingFilterBuilder.java

@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.query.xcontent;
+
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+/**
+ * Constructs a filter that only match on documents that the field has a value in them.
+ *
+ * @author kimchy (shay.banon)
+ */
+public class MissingFilterBuilder extends BaseFilterBuilder {
+
+    private String name;
+
+    private String filterName;
+
+    public MissingFilterBuilder(String name) {
+        this.name = name;
+    }
+
+    /**
+     * Sets the filter name for the filter that can be used when searching for matched_filters per hit.
+     */
+    public MissingFilterBuilder filterName(String filterName) {
+        this.filterName = filterName;
+        return this;
+    }
+
+
+    @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(MissingFilterParser.NAME);
+        builder.field("field", name);
+        if (filterName != null) {
+            builder.field("_name", filterName);
+        }
+        builder.endObject();
+    }
+}

modules/elasticsearch/src/main/java/org/elasticsearch/index/query/xcontent/MissingFilterParser.java

@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.query.xcontent;
+
+import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.MultiTermQueryWrapperFilter;
+import org.apache.lucene.search.TermRangeFilter;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.lucene.search.NotFilter;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.AbstractIndexComponent;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.query.QueryParsingException;
+import org.elasticsearch.index.settings.IndexSettings;
+
+import java.io.IOException;
+
+import static org.elasticsearch.index.query.support.QueryParsers.*;
+
+/**
+ * @author kimchy (shay.banon)
+ */
+public class MissingFilterParser extends AbstractIndexComponent implements XContentFilterParser {
+
+    public static final String NAME = "missing";
+
+    @Inject public MissingFilterParser(Index index, @IndexSettings Settings settings) {
+        super(index, settings);
+    }
+
+    @Override public String[] names() {
+        return new String[]{NAME};
+    }
+
+    @Override public Filter parse(QueryParseContext parseContext) throws IOException, QueryParsingException {
+        XContentParser parser = parseContext.parser();
+
+        String fieldName = null;
+        String filterName = null;
+
+        XContentParser.Token token;
+        String currentFieldName = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if ("field".equals(currentFieldName)) {
+                    fieldName = parser.text();
+                } else if ("_name".equals(currentFieldName)) {
+                    filterName = parser.text();
+                }
+            }
+        }
+
+        if (fieldName == null) {
+            throw new QueryParsingException(index, "exists must be provided with a [field]");
+        }
+
+        Filter filter = null;
+        MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName);
+        if (smartNameFieldMappers != null) {
+            if (smartNameFieldMappers.hasMapper()) {
+                filter = smartNameFieldMappers.mapper().rangeFilter(null, null, true, true);
+            }
+        }
+        if (filter == null) {
+            filter = new TermRangeFilter(fieldName, null, null, true, true);
+        }
+
+        filter = new NotFilter(new ExistsFilterParser.ExistsFilter((MultiTermQueryWrapperFilter) filter));
+        // we always cache this one, really does not change...
+        filter = parseContext.cacheFilter(filter);
+
+        filter = wrapSmartNameFilter(filter, smartNameFieldMappers, parseContext);
+        if (filterName != null) {
+            parseContext.addNamedFilter(filterName, filter);
+        }
+        return filter;
+    }
+}

modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/query/SimpleQueryTests.java

@@ -0,0 +1,88 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.integration.search.query;
+
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.test.integration.AbstractNodesTests;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.elasticsearch.index.query.xcontent.FilterBuilders.*;
+import static org.elasticsearch.index.query.xcontent.QueryBuilders.*;
+import static org.hamcrest.MatcherAssert.*;
+import static org.hamcrest.Matchers.*;
+
+/**
+ * @author kimchy (shay.banon)
+ */
+public class SimpleQueryTests extends AbstractNodesTests {
+
+    private Client client;
+
+    @BeforeClass public void createNodes() throws Exception {
+        startNode("node1");
+        client = getClient();
+    }
+
+    @AfterClass public void closeNodes() {
+        client.close();
+        closeAllNodes();
+    }
+
+    protected Client getClient() {
+        return client("node1");
+    }
+
+    @Test public void filterExistsMissingTests() throws Exception {
+        try {
+            client.admin().indices().prepareDelete("test").execute().actionGet();
+        } catch (Exception e) {
+            // ignore
+        }
+
+        client.prepareIndex("test", "type1", "1").setSource("field1", "value1_1", "field2", "value2_1").execute().actionGet();
+        client.prepareIndex("test", "type1", "2").setSource("field1", "value1_2").execute().actionGet();
+        client.prepareIndex("test", "type1", "3").setSource("field2", "value2_3").execute().actionGet();
+        client.prepareIndex("test", "type1", "4").setSource("field3", "value3_4").execute().actionGet();
+
+        client.admin().indices().prepareRefresh().execute().actionGet();
+
+        SearchResponse searchResponse = client.prepareSearch().setQuery(filtered(matchAllQuery(), exists("field1"))).execute().actionGet();
+        assertThat(searchResponse.hits().totalHits(), equalTo(2l));
+        assertThat(searchResponse.hits().getAt(0).id(), anyOf(equalTo("1"), equalTo("2")));
+        assertThat(searchResponse.hits().getAt(1).id(), anyOf(equalTo("1"), equalTo("2")));
+
+        searchResponse = client.prepareSearch().setQuery(filtered(matchAllQuery(), exists("field2"))).execute().actionGet();
+        assertThat(searchResponse.hits().totalHits(), equalTo(2l));
+        assertThat(searchResponse.hits().getAt(0).id(), anyOf(equalTo("1"), equalTo("3")));
+        assertThat(searchResponse.hits().getAt(1).id(), anyOf(equalTo("1"), equalTo("3")));
+
+        searchResponse = client.prepareSearch().setQuery(filtered(matchAllQuery(), exists("field3"))).execute().actionGet();
+        assertThat(searchResponse.hits().totalHits(), equalTo(1l));
+        assertThat(searchResponse.hits().getAt(0).id(), equalTo("4"));
+
+        searchResponse = client.prepareSearch().setQuery(filtered(matchAllQuery(), missing("field1"))).execute().actionGet();
+        assertThat(searchResponse.hits().totalHits(), equalTo(2l));
+        assertThat(searchResponse.hits().getAt(0).id(), anyOf(equalTo("3"), equalTo("4")));
+        assertThat(searchResponse.hits().getAt(1).id(), anyOf(equalTo("3"), equalTo("4")));
+    }
+}