Search: Support partial fields that can returns partial view of the _source, closes #1570.

src/main/java/org/elasticsearch/client/action/search/SearchRequestBuilder.java

@@ -27,6 +27,7 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.action.support.BaseRequestBuilder;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.FilterBuilder;
@@ -392,6 +393,32 @@ public class SearchRequestBuilder extends BaseRequestBuilder<SearchRequest, Sear
         return this;
     }
 
+    /**
+     * Adds a partial field based on _source, with an "include" and/or "exclude" set which can include simple wildcard
+     * elements.
+     *
+     * @param name    The name of the field
+     * @param include An optional include (optionally wildcarded) pattern from _source
+     * @param exclude An optional exclude (optionally wildcarded) pattern from _source
+     */
+    public SearchRequestBuilder addPartialField(String name, @Nullable String include, @Nullable String exclude) {
+        sourceBuilder().partialField(name, include, exclude);
+        return this;
+    }
+
+    /**
+     * Adds a partial field based on _source, with an "includes" and/or "excludes set which can include simple wildcard
+     * elements.
+     *
+     * @param name     The name of the field
+     * @param includes An optional list of includes (optionally wildcarded) patterns from _source
+     * @param excludes An optional list of excludes (optionally wildcarded) patterns from _source
+     */
+    public SearchRequestBuilder addPartialField(String name, @Nullable String[] includes, @Nullable String[] excludes) {
+        sourceBuilder().partialField(name, includes, excludes);
+        return this;
+    }
+
     /**
      * Adds a script based field to load and return. The field does not have to be stored,
      * but its recommended to use non analyzed or numeric fields.

src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java

@@ -146,7 +146,9 @@ public class XContentMapValues {
             if (includes.length > 0) {
                 boolean atLeastOnOneIncludeMatched = false;
                 for (String include : includes) {
-                    if (Regex.simpleMatch(include, path)) {
+                    // check for prefix as well, something like: obj1.arr1.*
+                    // note, this does not work well with middle matches, like obj1.*.obj3
+                    if (include.startsWith(path) || Regex.simpleMatch(include, path)) {
                         atLeastOnOneIncludeMatched = true;
                         break;
                     }

src/main/java/org/elasticsearch/search/SearchHitField.java

@@ -26,7 +26,6 @@ import java.util.List;
 /**
  * A single field name and values part of a {@link SearchHit}.
  *
- *
  * @see SearchHit
  */
 public interface SearchHitField extends Streamable, Iterable<Object> {
@@ -44,12 +43,12 @@ public interface SearchHitField extends Streamable, Iterable<Object> {
     /**
      * The first value of the hit.
      */
-    Object value();
+    <V> V value();
 
     /**
      * The first value of the hit.
      */
-    Object getValue();
+    <V> V getValue();
 
     /**
      * The field values.

src/main/java/org/elasticsearch/search/SearchModule.java

@@ -30,6 +30,7 @@ import org.elasticsearch.search.facet.FacetModule;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.explain.ExplainFetchSubPhase;
 import org.elasticsearch.search.fetch.matchedfilters.MatchedFiltersFetchSubPhase;
+import org.elasticsearch.search.fetch.partial.PartialFieldsFetchSubPhase;
 import org.elasticsearch.search.fetch.script.ScriptFieldsFetchSubPhase;
 import org.elasticsearch.search.fetch.version.VersionFetchSubPhase;
 import org.elasticsearch.search.highlight.HighlightPhase;
@@ -55,6 +56,7 @@ public class SearchModule extends AbstractModule implements SpawnModules {
         bind(FetchPhase.class).asEagerSingleton();
         bind(ExplainFetchSubPhase.class).asEagerSingleton();
         bind(ScriptFieldsFetchSubPhase.class).asEagerSingleton();
+        bind(PartialFieldsFetchSubPhase.class).asEagerSingleton();
         bind(VersionFetchSubPhase.class).asEagerSingleton();
         bind(MatchedFiltersFetchSubPhase.class).asEagerSingleton();
         bind(HighlightPhase.class).asEagerSingleton();

src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java

@@ -25,6 +25,7 @@ import gnu.trove.iterator.TObjectFloatIterator;
 import gnu.trove.map.hash.TObjectFloatHashMap;
 import org.elasticsearch.ElasticSearchGenerationException;
 import org.elasticsearch.client.Requests;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Unicode;
 import org.elasticsearch.common.io.BytesStream;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -48,7 +49,6 @@ import java.util.Map;
  * A search source builder allowing to easily build search source. Simple construction
  * using {@link org.elasticsearch.search.builder.SearchSourceBuilder#searchSource()}.
  *
- *
  * @see org.elasticsearch.action.search.SearchRequest#source(SearchSourceBuilder)
  */
 public class SearchSourceBuilder implements ToXContent {
@@ -94,8 +94,8 @@ public class SearchSourceBuilder implements ToXContent {
     private Float minScore;
 
     private List<String> fieldNames;
-
     private List<ScriptField> scriptFields;
+    private List<PartialField> partialFields;
 
     private List<AbstractFacetBuilder> facets;
 
@@ -463,6 +463,38 @@ public class SearchSourceBuilder implements ToXContent {
         return this;
     }
 
+    /**
+     * Adds a partial field based on _source, with an "include" and/or "exclude" set which can include simple wildcard
+     * elements.
+     *
+     * @param name    The name of the field
+     * @param include An optional include (optionally wildcarded) pattern from _source
+     * @param exclude An optional exclude (optionally wildcarded) pattern from _source
+     */
+    public SearchSourceBuilder partialField(String name, @Nullable String include, @Nullable String exclude) {
+        if (partialFields == null) {
+            partialFields = Lists.newArrayList();
+        }
+        partialFields.add(new PartialField(name, include, exclude));
+        return this;
+    }
+
+    /**
+     * Adds a partial field based on _source, with an "includes" and/or "excludes set which can include simple wildcard
+     * elements.
+     *
+     * @param name     The name of the field
+     * @param includes An optional list of includes (optionally wildcarded) patterns from _source
+     * @param excludes An optional list of excludes (optionally wildcarded) patterns from _source
+     */
+    public SearchSourceBuilder partialField(String name, @Nullable String[] includes, @Nullable String[] excludes) {
+        if (partialFields == null) {
+            partialFields = Lists.newArrayList();
+        }
+        partialFields.add(new PartialField(name, includes, excludes));
+        return this;
+    }
+
     /**
      * Sets the boost a specific index will receive when the query is executeed against it.
      *
@@ -582,6 +614,29 @@ public class SearchSourceBuilder implements ToXContent {
             }
         }
 
+        if (partialFields != null) {
+            builder.startObject("partial_fields");
+            for (PartialField partialField : partialFields) {
+                builder.startObject(partialField.name());
+                if (partialField.includes() != null) {
+                    if (partialField.includes().length == 1) {
+                        builder.field("include", partialField.includes()[0]);
+                    } else {
+                        builder.field("include", partialField.includes());
+                    }
+                }
+                if (partialField.excludes() != null) {
+                    if (partialField.excludes().length == 1) {
+                        builder.field("exclude", partialField.excludes()[0]);
+                    } else {
+                        builder.field("exclude", partialField.excludes());
+                    }
+                }
+                builder.endObject();
+            }
+            builder.endObject();
+        }
+
         if (scriptFields != null) {
             builder.startObject("script_fields");
             for (ScriptField scriptField : scriptFields) {
@@ -683,4 +738,34 @@ public class SearchSourceBuilder implements ToXContent {
             return params;
         }
     }
+
+    private static class PartialField {
+        private final String name;
+        private final String[] includes;
+        private final String[] excludes;
+
+        private PartialField(String name, String[] includes, String[] excludes) {
+            this.name = name;
+            this.includes = includes;
+            this.excludes = excludes;
+        }
+
+        private PartialField(String name, String include, String exclude) {
+            this.name = name;
+            this.includes = include == null ? null : new String[]{include};
+            this.excludes = exclude == null ? null : new String[]{exclude};
+        }
+
+        public String name() {
+            return name;
+        }
+
+        public String[] includes() {
+            return includes;
+        }
+
+        public String[] excludes() {
+            return excludes;
+        }
+    }
 }

src/main/java/org/elasticsearch/search/fetch/FetchPhase.java

@@ -44,6 +44,7 @@ import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.SearchPhase;
 import org.elasticsearch.search.fetch.explain.ExplainFetchSubPhase;
 import org.elasticsearch.search.fetch.matchedfilters.MatchedFiltersFetchSubPhase;
+import org.elasticsearch.search.fetch.partial.PartialFieldsFetchSubPhase;
 import org.elasticsearch.search.fetch.script.ScriptFieldsFetchSubPhase;
 import org.elasticsearch.search.fetch.version.VersionFetchSubPhase;
 import org.elasticsearch.search.highlight.HighlightPhase;
@@ -66,9 +67,9 @@ public class FetchPhase implements SearchPhase {
     private final FetchSubPhase[] fetchSubPhases;
 
     @Inject
-    public FetchPhase(HighlightPhase highlightPhase, ScriptFieldsFetchSubPhase scriptFieldsPhase,
+    public FetchPhase(HighlightPhase highlightPhase, ScriptFieldsFetchSubPhase scriptFieldsPhase, PartialFieldsFetchSubPhase partialFieldsPhase,
                       MatchedFiltersFetchSubPhase matchFiltersPhase, ExplainFetchSubPhase explainPhase, VersionFetchSubPhase versionPhase) {
-        this.fetchSubPhases = new FetchSubPhase[]{scriptFieldsPhase, matchFiltersPhase, explainPhase, highlightPhase, versionPhase};
+        this.fetchSubPhases = new FetchSubPhase[]{scriptFieldsPhase, partialFieldsPhase, matchFiltersPhase, explainPhase, highlightPhase, versionPhase};
     }
 
     @Override
@@ -89,13 +90,19 @@ public class FetchPhase implements SearchPhase {
         ResetFieldSelector fieldSelector;
         List<String> extractFieldNames = null;
         boolean sourceRequested = false;
-        if (context.hasScriptFields() && !context.hasFieldNames()) {
+        if (!context.hasFieldNames()) {
-            // we ask for script fields, and no field names, don't load the source
+            if (context.hasPartialFields()) {
-            fieldSelector = UidFieldSelector.INSTANCE;
+                // partial fields need the source, so fetch it, but don't return it
-            sourceRequested = false;
+                fieldSelector = new UidAndSourceFieldSelector();
-        } else if (!context.hasFieldNames()) {
+                sourceRequested = false;
-            fieldSelector = new UidAndSourceFieldSelector();
+            } else if (context.hasScriptFields()) {
-            sourceRequested = true;
+                // we ask for script fields, and no field names, don't load the source
+                fieldSelector = UidFieldSelector.INSTANCE;
+                sourceRequested = false;
+            } else {
+                fieldSelector = new UidAndSourceFieldSelector();
+                sourceRequested = true;
+            }
         } else if (context.fieldNames().isEmpty()) {
             fieldSelector = UidFieldSelector.INSTANCE;
             sourceRequested = false;

src/main/java/org/elasticsearch/search/fetch/partial/PartialFieldsContext.java

@@ -0,0 +1,67 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.fetch.partial;
+
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+/**
+ */
+public class PartialFieldsContext {
+
+    public static class PartialField {
+        private final String name;
+        private final String[] includes;
+        private final String[] excludes;
+
+        public PartialField(String name, String[] includes, String[] excludes) {
+            this.name = name;
+            this.includes = includes;
+            this.excludes = excludes;
+        }
+
+        public String name() {
+            return this.name;
+        }
+
+        public String[] includes() {
+            return this.includes;
+        }
+
+        public String[] excludes() {
+            return this.excludes;
+        }
+    }
+
+    private final List<PartialField> fields = Lists.newArrayList();
+
+    public PartialFieldsContext() {
+
+    }
+
+    public void add(PartialField field) {
+        fields.add(field);
+    }
+
+    public List<PartialField> fields() {
+        return this.fields;
+    }
+}

src/main/java/org/elasticsearch/search/fetch/partial/PartialFieldsFetchSubPhase.java

@@ -0,0 +1,84 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.fetch.partial;
+
+import com.google.common.collect.ImmutableMap;
+import org.elasticsearch.ElasticSearchException;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.search.SearchHitField;
+import org.elasticsearch.search.SearchParseElement;
+import org.elasticsearch.search.fetch.FetchSubPhase;
+import org.elasticsearch.search.internal.InternalSearchHit;
+import org.elasticsearch.search.internal.InternalSearchHitField;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ */
+public class PartialFieldsFetchSubPhase implements FetchSubPhase {
+
+    @Inject
+    public PartialFieldsFetchSubPhase() {
+
+    }
+
+    @Override
+    public Map<String, ? extends SearchParseElement> parseElements() {
+        ImmutableMap.Builder<String, SearchParseElement> parseElements = ImmutableMap.builder();
+        parseElements.put("partial_fields", new PartialFieldsParseElement())
+                .put("partialFields", new PartialFieldsParseElement());
+        return parseElements.build();
+    }
+
+    @Override
+    public boolean hitsExecutionNeeded(SearchContext context) {
+        return false;
+    }
+
+    @Override
+    public void hitsExecute(SearchContext context, InternalSearchHit[] hits) throws ElasticSearchException {
+    }
+
+    @Override
+    public boolean hitExecutionNeeded(SearchContext context) {
+        return context.hasPartialFields();
+    }
+
+    @Override
+    public void hitExecute(SearchContext context, HitContext hitContext) throws ElasticSearchException {
+        for (PartialFieldsContext.PartialField field : context.partialFields().fields()) {
+            Object value = context.lookup().source().filter(field.includes(), field.excludes());
+
+            if (hitContext.hit().fieldsOrNull() == null) {
+                hitContext.hit().fields(new HashMap<String, SearchHitField>(2));
+            }
+
+            SearchHitField hitField = hitContext.hit().fields().get(field.name());
+            if (hitField == null) {
+                hitField = new InternalSearchHitField(field.name(), new ArrayList<Object>(2));
+                hitContext.hit().fields().put(field.name(), hitField);
+            }
+            hitField.values().add(value);
+        }
+    }
+}

src/main/java/org/elasticsearch/search/fetch/partial/PartialFieldsParseElement.java

@@ -0,0 +1,95 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.fetch.partial;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.SearchParseElement;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * <pre>
+ * "partial_fields" : {
+ *  "test1" : {
+ *      "includes" : "doc['field_name'].value"
+ *  },
+ *  "test2" : {
+ *      "excludes" : "..."
+ *  }
+ * }
+ * </pre>
+ */
+public class PartialFieldsParseElement implements SearchParseElement {
+
+    @Override
+    public void parse(XContentParser parser, SearchContext context) throws Exception {
+        XContentParser.Token token;
+        String currentFieldName = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                String fieldName = currentFieldName;
+                List<String> includes = null;
+                List<String> excludes = null;
+                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                    if (token == XContentParser.Token.FIELD_NAME) {
+                        currentFieldName = parser.currentName();
+                    } else if (token == XContentParser.Token.START_ARRAY) {
+                        if ("includes".equals(currentFieldName) || "include".equals(currentFieldName)) {
+                            if (includes == null) {
+                                includes = new ArrayList<String>(2);
+                            }
+                            while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                                includes.add(parser.text());
+                            }
+                        } else if ("excludes".equals(currentFieldName) || "exclude".equals(currentFieldName)) {
+                            if (excludes == null) {
+                                excludes = new ArrayList<String>(2);
+                            }
+                            while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                                excludes.add(parser.text());
+                            }
+                        }
+                    } else if (token.isValue()) {
+                        if ("include".equals(currentFieldName)) {
+                            if (includes == null) {
+                                includes = new ArrayList<String>(2);
+                            }
+                            includes.add(parser.text());
+                        } else if ("exclude".equals(currentFieldName)) {
+                            if (excludes == null) {
+                                excludes = new ArrayList<String>(2);
+                            }
+                            excludes.add(parser.text());
+                        }
+                    }
+                }
+                PartialFieldsContext.PartialField field = new PartialFieldsContext.PartialField(fieldName,
+                        includes == null ? Strings.EMPTY_ARRAY : includes.toArray(new String[includes.size()]),
+                        excludes == null ? Strings.EMPTY_ARRAY : excludes.toArray(new String[excludes.size()]));
+                context.partialFields().add(field);
+            }
+        }
+    }
+}

src/main/java/org/elasticsearch/search/internal/SearchContext.java

@@ -50,6 +50,7 @@ import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.dfs.DfsSearchResult;
 import org.elasticsearch.search.facet.SearchContextFacets;
 import org.elasticsearch.search.fetch.FetchSearchResult;
+import org.elasticsearch.search.fetch.partial.PartialFieldsContext;
 import org.elasticsearch.search.fetch.script.ScriptFieldsContext;
 import org.elasticsearch.search.highlight.SearchContextHighlight;
 import org.elasticsearch.search.lookup.SearchLookup;
@@ -117,6 +118,8 @@ public class SearchContext implements Releasable {
     private boolean version = false; // by default, we don't return versions
 
     private List<String> fieldNames;
+    private ScriptFieldsContext scriptFields;
+    private PartialFieldsContext partialFields;
 
     private int from = -1;
 
@@ -146,8 +149,6 @@ public class SearchContext implements Releasable {
 
     private SearchContextHighlight highlight;
 
-    private ScriptFieldsContext scriptFields;
-
     private SearchLookup searchLookup;
 
     private boolean queryRewritten;
@@ -278,6 +279,17 @@ public class SearchContext implements Releasable {
         return this.scriptFields;
     }
 
+    public boolean hasPartialFields() {
+        return partialFields != null;
+    }
+
+    public PartialFieldsContext partialFields() {
+        if (partialFields == null) {
+            partialFields = new PartialFieldsContext();
+        }
+        return this.partialFields;
+    }
+
     public ContextIndexSearcher searcher() {
         return this.searcher;
     }

src/main/java/org/elasticsearch/search/lookup/SourceLookup.java

@@ -116,6 +116,10 @@ public class SourceLookup implements Map {
         return XContentMapValues.extractRawValues(path, loadSourceIfNeeded());
     }
 
+    public Object filter(String[] includes, String[] excludes) {
+        return XContentMapValues.filter(loadSourceIfNeeded(), includes, excludes);
+    }
+
     public Object extractValue(String path) {
         return XContentMapValues.extractValue(path, loadSourceIfNeeded());
     }

src/test/java/org/elasticsearch/test/integration/search/fields/SearchFieldsTests.java

@@ -21,12 +21,20 @@ package org.elasticsearch.test.integration.search.fields;
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.integration.AbstractNodesTests;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.client.Requests.refreshRequest;
+import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -41,8 +49,8 @@ public class SearchFieldsTests extends AbstractNodesTests {
 
     @BeforeClass
     public void createNodes() throws Exception {
-        startNode("server1");
+        startNode("node1", settingsBuilder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0));
-        startNode("server2");
+        startNode("client1", settingsBuilder().put("node.client", true).build());
         client = getClient();
     }
 
@@ -53,13 +61,14 @@ public class SearchFieldsTests extends AbstractNodesTests {
     }
 
     protected Client getClient() {
-        return client("server1");
+        return client("client1");
     }
 
     @Test
     public void testStoredFields() throws Exception {
+        client.admin().indices().prepareDelete().execute().actionGet();
         client.admin().indices().prepareCreate("test").execute().actionGet();
-        client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
+        client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
 
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
                 .startObject("field1").field("type", "string").field("store", "yes").endObject()
@@ -112,4 +121,160 @@ public class SearchFieldsTests extends AbstractNodesTests {
         assertThat(searchResponse.hits().getAt(0).fields().get("field1").value().toString(), equalTo("value1"));
         assertThat(searchResponse.hits().getAt(0).fields().get("field3").value().toString(), equalTo("value3"));
     }
+
+    @Test
+    public void testScriptDocAndFields() throws Exception {
+        client.admin().indices().prepareDelete().execute().actionGet();
+        client.admin().indices().prepareCreate("test").execute().actionGet();
+        client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
+
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
+                .startObject("num1").field("type", "double").field("store", "yes").endObject()
+                .endObject().endObject().endObject().string();
+
+        client.admin().indices().preparePutMapping().setType("type1").setSource(mapping).execute().actionGet();
+
+        client.prepareIndex("test", "type1", "1")
+                .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).field("date", "1970-01-01T00:00:00").endObject())
+                .execute().actionGet();
+        client.admin().indices().prepareFlush().execute().actionGet();
+        client.prepareIndex("test", "type1", "2")
+                .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).field("date", "1970-01-01T00:00:25").endObject())
+                .execute().actionGet();
+        client.admin().indices().prepareFlush().execute().actionGet();
+        client.prepareIndex("test", "type1", "3")
+                .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).field("date", "1970-01-01T00:02:00").endObject())
+                .execute().actionGet();
+        client.admin().indices().refresh(refreshRequest()).actionGet();
+
+        logger.info("running doc['num1'].value");
+        SearchResponse response = client.prepareSearch()
+                .setQuery(matchAllQuery())
+                .addSort("num1", SortOrder.ASC)
+                .addScriptField("sNum1", "doc['num1'].value")
+                .addScriptField("sNum1_field", "_fields['num1'].value")
+                .addScriptField("date1", "doc['date'].date.millis")
+                .execute().actionGet();
+
+        assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
+
+        assertThat(response.hits().totalHits(), equalTo(3l));
+        assertThat(response.hits().getAt(0).isSourceEmpty(), equalTo(true));
+        assertThat(response.hits().getAt(0).id(), equalTo("1"));
+        assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(1.0));
+        assertThat((Double) response.hits().getAt(0).fields().get("sNum1_field").values().get(0), equalTo(1.0));
+        assertThat((Long) response.hits().getAt(0).fields().get("date1").values().get(0), equalTo(0l));
+        assertThat(response.hits().getAt(1).id(), equalTo("2"));
+        assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(2.0));
+        assertThat((Double) response.hits().getAt(1).fields().get("sNum1_field").values().get(0), equalTo(2.0));
+        assertThat((Long) response.hits().getAt(1).fields().get("date1").values().get(0), equalTo(25000l));
+        assertThat(response.hits().getAt(2).id(), equalTo("3"));
+        assertThat((Double) response.hits().getAt(2).fields().get("sNum1").values().get(0), equalTo(3.0));
+        assertThat((Double) response.hits().getAt(2).fields().get("sNum1_field").values().get(0), equalTo(3.0));
+        assertThat((Long) response.hits().getAt(2).fields().get("date1").values().get(0), equalTo(120000l));
+
+        logger.info("running doc['num1'].value * factor");
+        Map<String, Object> params = MapBuilder.<String, Object>newMapBuilder().put("factor", 2.0).map();
+        response = client.prepareSearch()
+                .setQuery(matchAllQuery())
+                .addSort("num1", SortOrder.ASC)
+                .addScriptField("sNum1", "doc['num1'].value * factor", params)
+                .execute().actionGet();
+
+        assertThat(response.hits().totalHits(), equalTo(3l));
+        assertThat(response.hits().getAt(0).id(), equalTo("1"));
+        assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(2.0));
+        assertThat(response.hits().getAt(1).id(), equalTo("2"));
+        assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(4.0));
+        assertThat(response.hits().getAt(2).id(), equalTo("3"));
+        assertThat((Double) response.hits().getAt(2).fields().get("sNum1").values().get(0), equalTo(6.0));
+    }
+
+    @Test
+    public void testScriptFieldUsingSource() throws Exception {
+        client.admin().indices().prepareDelete().execute().actionGet();
+        client.admin().indices().prepareCreate("test").execute().actionGet();
+        client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
+
+        client.prepareIndex("test", "type1", "1")
+                .setSource(jsonBuilder().startObject()
+                        .startObject("obj1").field("test", "something").endObject()
+                        .startObject("obj2").startArray("arr2").value("arr_value1").value("arr_value2").endArray().endObject()
+                        .startArray("arr3").startObject().field("arr3_field1", "arr3_value1").endObject().endArray()
+                        .endObject())
+                .execute().actionGet();
+        client.admin().indices().refresh(refreshRequest()).actionGet();
+
+        SearchResponse response = client.prepareSearch()
+                .setQuery(matchAllQuery())
+                .addField("_source.obj1") // we also automatically detect _source in fields
+                .addScriptField("s_obj1", "_source.obj1")
+                .addScriptField("s_obj1_test", "_source.obj1.test")
+                .addScriptField("s_obj2", "_source.obj2")
+                .addScriptField("s_obj2_arr2", "_source.obj2.arr2")
+                .addScriptField("s_arr3", "_source.arr3")
+                .execute().actionGet();
+
+        assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
+
+        Map<String, Object> sObj1 = response.hits().getAt(0).field("_source.obj1").value();
+        assertThat(sObj1.get("test").toString(), equalTo("something"));
+        assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something"));
+
+        sObj1 = response.hits().getAt(0).field("s_obj1").value();
+        assertThat(sObj1.get("test").toString(), equalTo("something"));
+        assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something"));
+
+        Map<String, Object> sObj2 = response.hits().getAt(0).field("s_obj2").value();
+        List sObj2Arr2 = (List) sObj2.get("arr2");
+        assertThat(sObj2Arr2.size(), equalTo(2));
+        assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1"));
+        assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2"));
+
+        sObj2Arr2 = (List) response.hits().getAt(0).field("s_obj2_arr2").value();
+        assertThat(sObj2Arr2.size(), equalTo(2));
+        assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1"));
+        assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2"));
+
+        List sObj2Arr3 = (List) response.hits().getAt(0).field("s_arr3").value();
+        assertThat(((Map) sObj2Arr3.get(0)).get("arr3_field1").toString(), equalTo("arr3_value1"));
+    }
+
+    @Test
+    public void testPartialFields() throws Exception {
+        client.admin().indices().prepareDelete().execute().actionGet();
+        client.admin().indices().prepareCreate("test").execute().actionGet();
+        client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
+
+
+        client.prepareIndex("test", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject()
+                .field("field1", "value1")
+                .startObject("obj1")
+                .startArray("arr1")
+                .startObject().startObject("obj2").field("field2", "value21").endObject().endObject()
+                .startObject().startObject("obj2").field("field2", "value22").endObject().endObject()
+                .endArray()
+                .endObject()
+                .endObject())
+                .execute().actionGet();
+
+        client.admin().indices().prepareRefresh().execute().actionGet();
+
+        SearchResponse response = client.prepareSearch("test")
+                .addPartialField("partial1", "obj1.arr1.*", null)
+                .addPartialField("partial2", null, "obj1.*")
+                .execute().actionGet();
+        assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
+
+        Map<String, Object> partial1 = response.hits().getAt(0).field("partial1").value();
+        assertThat(partial1, notNullValue());
+        assertThat(partial1.containsKey("field1"), equalTo(false));
+        assertThat(partial1.containsKey("obj1"), equalTo(true));
+        assertThat(((Map) partial1.get("obj1")).get("arr1"), instanceOf(List.class));
+
+        Map<String, Object> partial2 = response.hits().getAt(0).field("partial2").value();
+        assertThat(partial2, notNullValue());
+        assertThat(partial2.containsKey("obj1"), equalTo(false));
+        assertThat(partial2.containsKey("field1"), equalTo(true));
+    }
 }

src/test/java/org/elasticsearch/test/integration/search/scriptfield/ScriptFieldSearchTests.java

@@ -1,192 +0,0 @@
-/*
- * Licensed to ElasticSearch and Shay Banon under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. ElasticSearch licenses this
- * file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.test.integration.search.scriptfield;
-
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.common.collect.MapBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.search.sort.SortOrder;
-import org.elasticsearch.test.integration.AbstractNodesTests;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import static org.elasticsearch.client.Requests.refreshRequest;
-import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.equalTo;
-
-/**
- *
- */
-@Test
-public class ScriptFieldSearchTests extends AbstractNodesTests {
-
-    private Client client;
-
-    @BeforeClass
-    public void createNodes() throws Exception {
-        startNode("server1", settingsBuilder().put("number_of_shards", 1).put("number_of_replicas", 0));
-        startNode("client1", settingsBuilder().put("node.client", true).build());
-        client = getClient();
-    }
-
-    @AfterClass
-    public void closeNodes() {
-        client.close();
-        closeNode("client1");
-        closeAllNodes();
-    }
-
-    protected Client getClient() {
-        return client("client1");
-    }
-
-    @Test
-    public void testDocAndFields() throws Exception {
-        try {
-            client.admin().indices().prepareDelete("test").execute().actionGet();
-        } catch (Exception e) {
-            // its ok
-        }
-        client.admin().indices().prepareCreate("test").execute().actionGet();
-
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
-                .startObject("num1").field("type", "double").field("store", "yes").endObject()
-                .endObject().endObject().endObject().string();
-
-        client.admin().indices().preparePutMapping().setType("type1").setSource(mapping).execute().actionGet();
-
-        client.prepareIndex("test", "type1", "1")
-                .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).field("date", "1970-01-01T00:00:00").endObject())
-                .execute().actionGet();
-        client.admin().indices().prepareFlush().execute().actionGet();
-        client.prepareIndex("test", "type1", "2")
-                .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).field("date", "1970-01-01T00:00:25").endObject())
-                .execute().actionGet();
-        client.admin().indices().prepareFlush().execute().actionGet();
-        client.prepareIndex("test", "type1", "3")
-                .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).field("date", "1970-01-01T00:02:00").endObject())
-                .execute().actionGet();
-        client.admin().indices().refresh(refreshRequest()).actionGet();
-
-        logger.info("running doc['num1'].value");
-        SearchResponse response = client.prepareSearch()
-                .setQuery(matchAllQuery())
-                .addSort("num1", SortOrder.ASC)
-                .addScriptField("sNum1", "doc['num1'].value")
-                .addScriptField("sNum1_field", "_fields['num1'].value")
-                .addScriptField("date1", "doc['date'].date.millis")
-                .execute().actionGet();
-
-        assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
-
-        assertThat(response.hits().totalHits(), equalTo(3l));
-        assertThat(response.hits().getAt(0).isSourceEmpty(), equalTo(true));
-        assertThat(response.hits().getAt(0).id(), equalTo("1"));
-        assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(1.0));
-        assertThat((Double) response.hits().getAt(0).fields().get("sNum1_field").values().get(0), equalTo(1.0));
-        assertThat((Long) response.hits().getAt(0).fields().get("date1").values().get(0), equalTo(0l));
-        assertThat(response.hits().getAt(1).id(), equalTo("2"));
-        assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(2.0));
-        assertThat((Double) response.hits().getAt(1).fields().get("sNum1_field").values().get(0), equalTo(2.0));
-        assertThat((Long) response.hits().getAt(1).fields().get("date1").values().get(0), equalTo(25000l));
-        assertThat(response.hits().getAt(2).id(), equalTo("3"));
-        assertThat((Double) response.hits().getAt(2).fields().get("sNum1").values().get(0), equalTo(3.0));
-        assertThat((Double) response.hits().getAt(2).fields().get("sNum1_field").values().get(0), equalTo(3.0));
-        assertThat((Long) response.hits().getAt(2).fields().get("date1").values().get(0), equalTo(120000l));
-
-        logger.info("running doc['num1'].value * factor");
-        Map<String, Object> params = MapBuilder.<String, Object>newMapBuilder().put("factor", 2.0).map();
-        response = client.prepareSearch()
-                .setQuery(matchAllQuery())
-                .addSort("num1", SortOrder.ASC)
-                .addScriptField("sNum1", "doc['num1'].value * factor", params)
-                .execute().actionGet();
-
-        assertThat(response.hits().totalHits(), equalTo(3l));
-        assertThat(response.hits().getAt(0).id(), equalTo("1"));
-        assertThat((Double) response.hits().getAt(0).fields().get("sNum1").values().get(0), equalTo(2.0));
-        assertThat(response.hits().getAt(1).id(), equalTo("2"));
-        assertThat((Double) response.hits().getAt(1).fields().get("sNum1").values().get(0), equalTo(4.0));
-        assertThat(response.hits().getAt(2).id(), equalTo("3"));
-        assertThat((Double) response.hits().getAt(2).fields().get("sNum1").values().get(0), equalTo(6.0));
-    }
-
-    @Test
-    public void testScriptFieldUsingSource() throws Exception {
-        try {
-            client.admin().indices().prepareDelete("test").execute().actionGet();
-        } catch (Exception e) {
-            // its ok
-        }
-        client.admin().indices().prepareCreate("test").execute().actionGet();
-        client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet();
-        client.prepareIndex("test", "type1", "1")
-                .setSource(jsonBuilder().startObject()
-                        .startObject("obj1").field("test", "something").endObject()
-                        .startObject("obj2").startArray("arr2").value("arr_value1").value("arr_value2").endArray().endObject()
-                        .startArray("arr3").startObject().field("arr3_field1", "arr3_value1").endObject().endArray()
-                        .endObject())
-                .execute().actionGet();
-        client.admin().indices().refresh(refreshRequest()).actionGet();
-
-        SearchResponse response = client.prepareSearch()
-                .setQuery(matchAllQuery())
-                .addField("_source.obj1") // we also automatically detect _source in fields
-                .addScriptField("s_obj1", "_source.obj1")
-                .addScriptField("s_obj1_test", "_source.obj1.test")
-                .addScriptField("s_obj2", "_source.obj2")
-                .addScriptField("s_obj2_arr2", "_source.obj2.arr2")
-                .addScriptField("s_arr3", "_source.arr3")
-                .execute().actionGet();
-
-        assertThat("Failures " + Arrays.toString(response.shardFailures()), response.shardFailures().length, equalTo(0));
-
-        Map<String, Object> sObj1 = (Map<String, Object>) response.hits().getAt(0).field("_source.obj1").value();
-        assertThat(sObj1.get("test").toString(), equalTo("something"));
-        assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something"));
-
-        sObj1 = (Map<String, Object>) response.hits().getAt(0).field("s_obj1").value();
-        assertThat(sObj1.get("test").toString(), equalTo("something"));
-        assertThat(response.hits().getAt(0).field("s_obj1_test").value().toString(), equalTo("something"));
-
-        Map<String, Object> sObj2 = (Map<String, Object>) response.hits().getAt(0).field("s_obj2").value();
-        List sObj2Arr2 = (List) sObj2.get("arr2");
-        assertThat(sObj2Arr2.size(), equalTo(2));
-        assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1"));
-        assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2"));
-
-        sObj2Arr2 = (List) response.hits().getAt(0).field("s_obj2_arr2").value();
-        assertThat(sObj2Arr2.size(), equalTo(2));
-        assertThat(sObj2Arr2.get(0).toString(), equalTo("arr_value1"));
-        assertThat(sObj2Arr2.get(1).toString(), equalTo("arr_value2"));
-
-        List sObj2Arr3 = (List) response.hits().getAt(0).field("s_arr3").value();
-        assertThat(((Map) sObj2Arr3.get(0)).get("arr3_field1").toString(), equalTo("arr3_value1"));
-    }
-}

src/test/java/org/elasticsearch/test/unit/common/xcontent/support/XContentMapValuesTests.java

@@ -80,6 +80,10 @@ public class XContentMapValuesTests {
         filter = XContentMapValues.filter(source, new String[]{"test1*"}, Strings.EMPTY_ARRAY);
         assertThat(filter.get("test1"), equalTo(source.get("test1")));
         assertThat(filter.containsKey("path1"), equalTo(false));
+
+        filter = XContentMapValues.filter(source, new String[]{"path1.path2.*"}, Strings.EMPTY_ARRAY);
+        assertThat(filter.get("path1"), equalTo(source.get("path1")));
+        assertThat(filter.containsKey("test1"), equalTo(false));
     }
 
     @SuppressWarnings({"unchecked"})