Ensure source filtering automatons are only compiled once (#20857)

This change adds a overloaded `XContentMapValues#filter` method that returns
a function enclosing the compiled automatons that can be reused across filter
calls. This for instance prevents compiling automatons over and over again when
hits are filtered or in the SourceFieldMapper for each document.

Closes #20839

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -880,7 +880,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]IndicesStoreIntegrationIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]IndicesStoreTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]template[/\\]SimpleIndexTemplateIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]mget[/\\]SimpleMgetIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]JvmGcMonitorServiceSettingsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]PluginInfoTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]PluginsServiceTests.java" checks="LineLength" />

core/src/main/java/org/elasticsearch/action/explain/ExplainRequestBuilder.java

@@ -99,12 +99,9 @@ public class ExplainRequestBuilder extends SingleShardOperationRequestBuilder<Ex
      * Indicates whether the response should contain the stored _source
      */
     public ExplainRequestBuilder setFetchSource(boolean fetch) {
-        FetchSourceContext context = request.fetchSourceContext();
+        FetchSourceContext fetchSourceContext = request.fetchSourceContext() != null ? request.fetchSourceContext()
-        if (context == null) {
+            : FetchSourceContext.FETCH_SOURCE;
-            request.fetchSourceContext(new FetchSourceContext(fetch));
+        request.fetchSourceContext(new FetchSourceContext(fetch, fetchSourceContext.includes(), fetchSourceContext.excludes()));
-        } else {
-            context.fetchSource(fetch);
-        }
         return this;
     }
 
@@ -129,14 +126,9 @@ public class ExplainRequestBuilder extends SingleShardOperationRequestBuilder<Ex
      * @param excludes An optional list of exclude (optionally wildcarded) pattern to filter the returned _source
      */
     public ExplainRequestBuilder setFetchSource(@Nullable String[] includes, @Nullable String[] excludes) {
-        FetchSourceContext context = request.fetchSourceContext();
+        FetchSourceContext fetchSourceContext = request.fetchSourceContext() != null ? request.fetchSourceContext()
-        if (context == null) {
+            : FetchSourceContext.FETCH_SOURCE;
-            request.fetchSourceContext(new FetchSourceContext(includes, excludes));
+        request.fetchSourceContext(new FetchSourceContext(fetchSourceContext.fetchSource(), includes, excludes));
-        } else {
-            context.fetchSource(true);
-            context.includes(includes);
-            context.excludes(excludes);
-        }
         return this;
     }
 }

core/src/main/java/org/elasticsearch/action/get/GetRequestBuilder.java

@@ -99,12 +99,8 @@ public class GetRequestBuilder extends SingleShardOperationRequestBuilder<GetReq
      * @return this for chaining
      */
     public GetRequestBuilder setFetchSource(boolean fetch) {
-        FetchSourceContext context = request.fetchSourceContext();
+        FetchSourceContext context = request.fetchSourceContext() == null ? FetchSourceContext.FETCH_SOURCE : request.fetchSourceContext();
-        if (context == null) {
+        request.fetchSourceContext(new FetchSourceContext(fetch, context.includes(), context.excludes()));
-            request.fetchSourceContext(new FetchSourceContext(fetch));
-        } else {
-            context.fetchSource(fetch);
-        }
         return this;
     }
 
@@ -129,14 +125,8 @@ public class GetRequestBuilder extends SingleShardOperationRequestBuilder<GetReq
      * @param excludes An optional list of exclude (optionally wildcarded) pattern to filter the returned _source
      */
     public GetRequestBuilder setFetchSource(@Nullable String[] includes, @Nullable String[] excludes) {
-        FetchSourceContext context = request.fetchSourceContext();
+        FetchSourceContext context = request.fetchSourceContext() == null ? FetchSourceContext.FETCH_SOURCE : request.fetchSourceContext();
-        if (context == null) {
+        request.fetchSourceContext(new FetchSourceContext(context.fetchSource(), includes, excludes));
-            request.fetchSourceContext(new FetchSourceContext(includes, excludes));
-        } else {
-            context.fetchSource(true);
-            context.includes(includes);
-            context.excludes(excludes);
-        }
         return this;
     }
 

core/src/main/java/org/elasticsearch/action/get/MultiGetRequest.java

@@ -370,7 +370,7 @@ public class MultiGetRequest extends ActionRequest<MultiGetRequest> implements I
             long version = Versions.MATCH_ANY;
             VersionType versionType = VersionType.INTERNAL;
 
-            FetchSourceContext fetchSourceContext = null;
+            FetchSourceContext fetchSourceContext = FetchSourceContext.FETCH_SOURCE;
 
             while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                 if (token == XContentParser.Token.FIELD_NAME) {
@@ -401,9 +401,11 @@ public class MultiGetRequest extends ActionRequest<MultiGetRequest> implements I
                         versionType = VersionType.fromString(parser.text());
                     } else if ("_source".equals(currentFieldName)) {
                         if (parser.isBooleanValue()) {
-                            fetchSourceContext = new FetchSourceContext(parser.booleanValue());
+                            fetchSourceContext = new FetchSourceContext(parser.booleanValue(), fetchSourceContext.includes(),
+                                fetchSourceContext.excludes());
                         } else if (token == XContentParser.Token.VALUE_STRING) {
-                            fetchSourceContext = new FetchSourceContext(new String[]{parser.text()});
+                            fetchSourceContext = new FetchSourceContext(fetchSourceContext.fetchSource(),
+                                new String[]{parser.text()}, fetchSourceContext.excludes());
                         } else {
                             throw new ElasticsearchParseException("illegal type for _source: [{}]", token);
                         }
@@ -422,7 +424,8 @@ public class MultiGetRequest extends ActionRequest<MultiGetRequest> implements I
                         while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                             includes.add(parser.text());
                         }
-                        fetchSourceContext = new FetchSourceContext(includes.toArray(Strings.EMPTY_ARRAY));
+                        fetchSourceContext = new FetchSourceContext(fetchSourceContext.fetchSource(), includes.toArray(Strings.EMPTY_ARRAY)
+                            , fetchSourceContext.excludes());
                     }
 
                 } else if (token == XContentParser.Token.START_OBJECT) {
@@ -450,7 +453,7 @@ public class MultiGetRequest extends ActionRequest<MultiGetRequest> implements I
                             }
                         }
 
-                        fetchSourceContext = new FetchSourceContext(
+                        fetchSourceContext = new FetchSourceContext(fetchSourceContext.fetchSource(),
                                 includes == null ? Strings.EMPTY_ARRAY : includes.toArray(new String[includes.size()]),
                                 excludes == null ? Strings.EMPTY_ARRAY : excludes.toArray(new String[excludes.size()]));
                     }
@@ -463,7 +466,7 @@ public class MultiGetRequest extends ActionRequest<MultiGetRequest> implements I
                 aFields = defaultFields;
             }
             items.add(new Item(index, type, id).routing(routing).storedFields(aFields).parent(parent).version(version).versionType(versionType)
-                    .fetchSourceContext(fetchSourceContext == null ? defaultFetchSource : fetchSourceContext));
+                    .fetchSourceContext(fetchSourceContext == FetchSourceContext.FETCH_SOURCE ? defaultFetchSource : fetchSourceContext));
         }
     }
 

core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java

@@ -307,7 +307,7 @@ public class UpdateHelper extends AbstractComponent {
         if (request.fetchSource() != null && request.fetchSource().fetchSource()) {
             sourceRequested = true;
             if (request.fetchSource().includes().length > 0 || request.fetchSource().excludes().length > 0) {
-                Object value = sourceLookup.filter(request.fetchSource().includes(), request.fetchSource().excludes());
+                Object value = sourceLookup.filter(request.fetchSource());
                 try {
                     final int initialCapacity = Math.min(1024, sourceAsBytes.length());
                     BytesStreamOutput streamOutput = new BytesStreamOutput(initialCapacity);

core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java

@@ -400,7 +400,8 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
      *            the returned _source
      */
     public UpdateRequest fetchSource(@Nullable String include, @Nullable String exclude) {
-        this.fetchSourceContext = new FetchSourceContext(include, exclude);
+        FetchSourceContext context = this.fetchSourceContext == null ? FetchSourceContext.FETCH_SOURCE : this.fetchSourceContext;
+        this.fetchSourceContext = new FetchSourceContext(context.fetchSource(), new String[] {include}, new String[]{exclude});
         return this;
     }
 
@@ -417,7 +418,8 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
      *            filter the returned _source
      */
     public UpdateRequest fetchSource(@Nullable String[] includes, @Nullable String[] excludes) {
-        this.fetchSourceContext = new FetchSourceContext(includes, excludes);
+        FetchSourceContext context = this.fetchSourceContext == null ? FetchSourceContext.FETCH_SOURCE : this.fetchSourceContext;
+        this.fetchSourceContext = new FetchSourceContext(context.fetchSource(), includes, excludes);
         return this;
     }
 
@@ -425,7 +427,8 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
      * Indicates whether the response should contain the updated _source.
      */
     public UpdateRequest fetchSource(boolean fetchSource) {
-        this.fetchSourceContext = new FetchSourceContext(fetchSource);
+        FetchSourceContext context = this.fetchSourceContext == null ? FetchSourceContext.FETCH_SOURCE : this.fetchSourceContext;
+        this.fetchSourceContext = new FetchSourceContext(fetchSource, context.includes(), context.excludes());
         return this;
     }
 

core/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java

@@ -33,6 +33,7 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
 /**
  *
@@ -155,6 +156,14 @@ public class XContentMapValues {
      * then {@code a.b} will be kept in the filtered map.
      */
     public static Map<String, Object> filter(Map<String, ?> map, String[] includes, String[] excludes) {
+        return filter(includes, excludes).apply(map);
+    }
+
+    /**
+     * Returns a function that filters a document map based on the given include and exclude rules.
+     * @see #filter(Map, String[], String[]) for details
+     */
+    public static Function<Map<String, ?>, Map<String, Object>> filter(String[] includes, String[] excludes) {
         CharacterRunAutomaton matchAllAutomaton = new CharacterRunAutomaton(Automata.makeAnyString());
 
         CharacterRunAutomaton include;
@@ -178,10 +187,10 @@ public class XContentMapValues {
         // NOTE: We cannot use Operations.minus because of the special case that
         // we want all sub properties to match as soon as an object matches
 
-        return filter(map,
+        return (map) -> filter(map,
-                include, include.getInitialState(),
+            include, include.getInitialState(),
-                exclude, exclude.getInitialState(),
+            exclude, exclude.getInitialState(),
-                matchAllAutomaton);
+            matchAllAutomaton);
     }
 
     /** Make matches on objects also match dots in field names.

core/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java

@@ -45,6 +45,7 @@ import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
 
@@ -56,6 +57,7 @@ public class SourceFieldMapper extends MetadataFieldMapper {
     public static final String NAME = "_source";
 
     public static final String CONTENT_TYPE = "_source";
+    private final Function<Map<String, ?>, Map<String, Object>> filter;
 
     public static class Defaults {
         public static final String NAME = SourceFieldMapper.NAME;
@@ -190,6 +192,8 @@ public class SourceFieldMapper extends MetadataFieldMapper {
         this.enabled = enabled;
         this.includes = includes;
         this.excludes = excludes;
+        final boolean filtered = (includes != null && includes.length > 0) || (excludes != null && excludes.length > 0);
+        this.filter = enabled && filtered && fieldType().stored() ? XContentMapValues.filter(includes, excludes) : null;
         this.complete = enabled && includes == null && excludes == null;
     }
 
@@ -239,12 +243,10 @@ public class SourceFieldMapper extends MetadataFieldMapper {
             return;
         }
 
-        boolean filtered = (includes != null && includes.length > 0) || (excludes != null && excludes.length > 0);
+        if (filter != null) {
-        if (filtered) {
             // we don't update the context source if we filter, we want to keep it as is...
-
             Tuple<XContentType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(source, true);
-            Map<String, Object> filteredSource = XContentMapValues.filter(mapTuple.v2(), includes, excludes);
+            Map<String, Object> filteredSource = filter.apply(mapTuple.v2());
             BytesStreamOutput bStream = new BytesStreamOutput();
             XContentType contentType = mapTuple.v1();
             XContentBuilder builder = XContentFactory.contentBuilder(contentType, bStream).map(filteredSource);

core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java

@@ -283,11 +283,9 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
      * every hit
      */
     public TopHitsAggregationBuilder fetchSource(boolean fetch) {
-        if (this.fetchSourceContext == null) {
+        FetchSourceContext fetchSourceContext = this.fetchSourceContext != null ? this.fetchSourceContext
-            this.fetchSourceContext = new FetchSourceContext(fetch);
+            : FetchSourceContext.FETCH_SOURCE;
-        } else {
+        this.fetchSourceContext = new FetchSourceContext(fetch, fetchSourceContext.includes(), fetchSourceContext.excludes());
-            this.fetchSourceContext.fetchSource(fetch);
-        }
         return this;
     }
 
@@ -322,7 +320,9 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
      *            pattern to filter the returned _source
      */
     public TopHitsAggregationBuilder fetchSource(@Nullable String[] includes, @Nullable String[] excludes) {
-        fetchSourceContext = new FetchSourceContext(includes, excludes);
+        FetchSourceContext fetchSourceContext = this.fetchSourceContext != null ? this.fetchSourceContext
+            : FetchSourceContext.FETCH_SOURCE;
+        this.fetchSourceContext = new FetchSourceContext(fetchSourceContext.fetchSource(), includes, excludes);
         return this;
     }
 

core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java

@@ -637,11 +637,9 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
      * every hit
      */
     public SearchSourceBuilder fetchSource(boolean fetch) {
-        if (this.fetchSourceContext == null) {
+        FetchSourceContext fetchSourceContext = this.fetchSourceContext != null ? this.fetchSourceContext
-            this.fetchSourceContext = new FetchSourceContext(fetch);
+            : FetchSourceContext.FETCH_SOURCE;
-        } else {
+        this.fetchSourceContext = new FetchSourceContext(fetch, fetchSourceContext.includes(), fetchSourceContext.excludes());
-            this.fetchSourceContext.fetchSource(fetch);
-        }
         return this;
     }
 
@@ -675,7 +673,9 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
      *            filter the returned _source
      */
     public SearchSourceBuilder fetchSource(@Nullable String[] includes, @Nullable String[] excludes) {
-        fetchSourceContext = new FetchSourceContext(includes, excludes);
+        FetchSourceContext fetchSourceContext = this.fetchSourceContext != null ? this.fetchSourceContext
+            : FetchSourceContext.FETCH_SOURCE;
+        this.fetchSourceContext = new FetchSourceContext(fetchSourceContext.fetchSource(), includes, excludes);
         return this;
     }
 

core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java

@@ -99,11 +99,9 @@ public class FetchPhase implements SearchPhase {
         } else {
             for (String fieldName : context.storedFieldsContext().fieldNames()) {
                 if (fieldName.equals(SourceFieldMapper.NAME)) {
-                    if (context.hasFetchSourceContext()) {
+                    FetchSourceContext fetchSourceContext = context.hasFetchSourceContext() ? context.fetchSourceContext()
-                        context.fetchSourceContext().fetchSource(true);
+                        : FetchSourceContext.FETCH_SOURCE;
-                    } else {
+                    context.fetchSourceContext(new FetchSourceContext(true, fetchSourceContext.includes(), fetchSourceContext.excludes()));
-                        context.fetchSourceContext(new FetchSourceContext(true));
-                    }
                     continue;
                 }
                 if (Regex.isSimpleMatchPattern(fieldName)) {

core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceContext.java

@@ -30,12 +30,15 @@ import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.rest.RestRequest;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
 
 /**
  * Context used to fetch the {@code _source}.
@@ -47,39 +50,13 @@ public class FetchSourceContext implements Writeable, ToXContent {
 
     public static final FetchSourceContext FETCH_SOURCE = new FetchSourceContext(true);
     public static final FetchSourceContext DO_NOT_FETCH_SOURCE = new FetchSourceContext(false);
-    private boolean fetchSource;
+    private final boolean fetchSource;
-    private String[] includes;
+    private final String[] includes;
-    private String[] excludes;
+    private final String[] excludes;
+    private Function<Map<String, ?>, Map<String, Object>> filter;
 
     public static FetchSourceContext parse(XContentParser parser) throws IOException {
-        FetchSourceContext fetchSourceContext = new FetchSourceContext();
+        return fromXContent(parser, ParseFieldMatcher.STRICT);
-        fetchSourceContext.fromXContent(parser, ParseFieldMatcher.STRICT);
-        return fetchSourceContext;
-    }
-
-    public FetchSourceContext() {
-    }
-
-    public FetchSourceContext(boolean fetchSource) {
-        this(fetchSource, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY);
-    }
-
-    public FetchSourceContext(String include) {
-        this(include, null);
-    }
-
-    public FetchSourceContext(String include, String exclude) {
-        this(true,
-                include == null ? Strings.EMPTY_ARRAY : new String[]{include},
-                exclude == null ? Strings.EMPTY_ARRAY : new String[]{exclude});
-    }
-
-    public FetchSourceContext(String[] includes) {
-        this(true, includes, Strings.EMPTY_ARRAY);
-    }
-
-    public FetchSourceContext(String[] includes, String[] excludes) {
-        this(true, includes, excludes);
     }
 
     public FetchSourceContext(boolean fetchSource, String[] includes, String[] excludes) {
@@ -88,6 +65,10 @@ public class FetchSourceContext implements Writeable, ToXContent {
         this.excludes = excludes == null ? Strings.EMPTY_ARRAY : excludes;
     }
 
+    public FetchSourceContext(boolean fetchSource) {
+        this(fetchSource, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY);
+    }
+
     public FetchSourceContext(StreamInput in) throws IOException {
         fetchSource = in.readBoolean();
         includes = in.readStringArray();
@@ -105,29 +86,14 @@ public class FetchSourceContext implements Writeable, ToXContent {
         return this.fetchSource;
     }
 
-    public FetchSourceContext fetchSource(boolean fetchSource) {
-        this.fetchSource = fetchSource;
-        return this;
-    }
-
     public String[] includes() {
         return this.includes;
     }
 
-    public FetchSourceContext includes(String[] includes) {
-        this.includes = includes;
-        return this;
-    }
-
     public String[] excludes() {
         return this.excludes;
     }
 
-    public FetchSourceContext excludes(String[] excludes) {
-        this.excludes = excludes;
-        return this;
-    }
-
     public static FetchSourceContext parseFromRestRequest(RestRequest request) {
         Boolean fetchSource = null;
         String[] source_excludes = null;
@@ -161,7 +127,7 @@ public class FetchSourceContext implements Writeable, ToXContent {
         return null;
     }
 
-    public void fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
+    public static FetchSourceContext fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
         XContentParser.Token token = parser.currentToken();
         boolean fetchSource = true;
         String[] includes = Strings.EMPTY_ARRAY;
@@ -226,9 +192,7 @@ public class FetchSourceContext implements Writeable, ToXContent {
             throw new ParsingException(parser.getTokenLocation(), "Expected one of [" + XContentParser.Token.VALUE_BOOLEAN + ", "
                     + XContentParser.Token.START_OBJECT + "] but found [" + token + "]", parser.getTokenLocation());
         }
-        this.fetchSource = fetchSource;
+        return new FetchSourceContext(fetchSource, includes, excludes);
-        this.includes = includes;
-        this.excludes = excludes;
     }
 
     @Override
@@ -265,4 +229,15 @@ public class FetchSourceContext implements Writeable, ToXContent {
         result = 31 * result + (excludes != null ? Arrays.hashCode(excludes) : 0);
         return result;
     }
+
+    /**
+     * Returns a filter function that expects the source map as an input and returns
+     * the filtered map.
+     */
+    public Function<Map<String, ?>, Map<String, Object>> getFilter() {
+        if (filter == null) {
+            filter = XContentMapValues.filter(includes, excludes);
+        }
+        return filter;
+    }
 }

core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhase.java

@@ -48,7 +48,7 @@ public final class FetchSourceSubPhase implements FetchSubPhase {
                     "for index [" + context.indexShard().shardId().getIndexName() + "]");
         }
 
-        Object value = source.filter(fetchSourceContext.includes(), fetchSourceContext.excludes());
+        final Object value = source.filter(fetchSourceContext);
         try {
             final int initialCapacity = Math.min(1024, source.internalSourceRef().length());
             BytesStreamOutput streamOutput = new BytesStreamOutput(initialCapacity);

core/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java

@@ -27,6 +27,7 @@ import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 
 import java.util.Collection;
 import java.util.List;
@@ -130,8 +131,8 @@ public class SourceLookup implements Map {
         return XContentMapValues.extractRawValues(path, loadSourceIfNeeded());
     }
 
-    public Object filter(String[] includes, String[] excludes) {
+    public Object filter(FetchSourceContext context) {
-        return XContentMapValues.filter(loadSourceIfNeeded(), includes, excludes);
+        return context.getFilter().apply(loadSourceIfNeeded());
     }
 
     public Object extractValue(String path) {

core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java

@@ -236,7 +236,7 @@ public class InnerHitBuilderTests extends ESTestCase {
         if (randomBoolean()) {
             randomFetchSourceContext = new FetchSourceContext(randomBoolean());
         } else {
-            randomFetchSourceContext = new FetchSourceContext(
+            randomFetchSourceContext = new FetchSourceContext(true,
                     generateRandomStringArray(12, 16, false),
                     generateRandomStringArray(12, 16, false)
             );
@@ -322,7 +322,7 @@ public class InnerHitBuilderTests extends ESTestCase {
                     if (randomBoolean()) {
                         randomFetchSourceContext = new FetchSourceContext(randomBoolean());
                     } else {
-                        randomFetchSourceContext = new FetchSourceContext(
+                        randomFetchSourceContext = new FetchSourceContext(true,
                                 generateRandomStringArray(12, 16, false),
                                 generateRandomStringArray(12, 16, false)
                         );

core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java

@@ -60,7 +60,8 @@ public class SimpleMgetIT extends ESIntegTestCase {
         assertThat(mgetResponse.getResponses()[1].getIndex(), is("nonExistingIndex"));
         assertThat(mgetResponse.getResponses()[1].isFailed(), is(true));
         assertThat(mgetResponse.getResponses()[1].getFailure().getMessage(), is("no such index"));
-        assertThat(((ElasticsearchException) mgetResponse.getResponses()[1].getFailure().getFailure()).getIndex().getName(), is("nonExistingIndex"));
+        assertThat(((ElasticsearchException) mgetResponse.getResponses()[1].getFailure().getFailure()).getIndex().getName(),
+            is("nonExistingIndex"));
 
 
         mgetResponse = client().prepareMultiGet()
@@ -70,7 +71,8 @@ public class SimpleMgetIT extends ESIntegTestCase {
         assertThat(mgetResponse.getResponses()[0].getIndex(), is("nonExistingIndex"));
         assertThat(mgetResponse.getResponses()[0].isFailed(), is(true));
         assertThat(mgetResponse.getResponses()[0].getFailure().getMessage(), is("no such index"));
-        assertThat(((ElasticsearchException) mgetResponse.getResponses()[0].getFailure().getFailure()).getIndex().getName(), is("nonExistingIndex"));
+        assertThat(((ElasticsearchException) mgetResponse.getResponses()[0].getFailure().getFailure()).getIndex().getName(),
+            is("nonExistingIndex"));
 
 
     }
@@ -119,9 +121,11 @@ public class SimpleMgetIT extends ESIntegTestCase {
         MultiGetRequestBuilder request = client().prepareMultiGet();
         for (int i = 0; i < 100; i++) {
             if (i % 2 == 0) {
-                request.add(new MultiGetRequest.Item(indexOrAlias(), "type", Integer.toString(i)).fetchSourceContext(new FetchSourceContext("included", "*.hidden_field")));
+                request.add(new MultiGetRequest.Item(indexOrAlias(), "type", Integer.toString(i))
+                    .fetchSourceContext(new FetchSourceContext(true, new String[] {"included"}, new String[] {"*.hidden_field"})));
             } else {
-                request.add(new MultiGetRequest.Item(indexOrAlias(), "type", Integer.toString(i)).fetchSourceContext(new FetchSourceContext(false)));
+                request.add(new MultiGetRequest.Item(indexOrAlias(), "type", Integer.toString(i))
+                    .fetchSourceContext(new FetchSourceContext(false)));
             }
         }
 

core/src/test/java/org/elasticsearch/search/AbstractSearchTestCase.java

@@ -153,19 +153,20 @@ public abstract class AbstractSearchTestCase extends ESTestCase {
                     fetchSourceContext = new FetchSourceContext(randomBoolean());
                     break;
                 case 1:
-                    fetchSourceContext = new FetchSourceContext(includes, excludes);
+                    fetchSourceContext = new FetchSourceContext(true, includes, excludes);
                     break;
                 case 2:
-                    fetchSourceContext = new FetchSourceContext(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20));
+                    fetchSourceContext = new FetchSourceContext(true, new String[]{randomAsciiOfLengthBetween(5, 20)},
+                        new String[]{randomAsciiOfLengthBetween(5, 20)});
                     break;
                 case 3:
                     fetchSourceContext = new FetchSourceContext(true, includes, excludes);
                     break;
                 case 4:
-                    fetchSourceContext = new FetchSourceContext(includes);
+                    fetchSourceContext = new FetchSourceContext(true, includes, null);
                     break;
                 case 5:
-                    fetchSourceContext = new FetchSourceContext(randomAsciiOfLengthBetween(5, 20));
+                    fetchSourceContext = new FetchSourceContext(true, new String[] {randomAsciiOfLengthBetween(5, 20)}, null);
                     break;
                 default:
                     throw new IllegalStateException();

core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java

@@ -107,26 +107,27 @@ public class TopHitsTests extends BaseAggregationTestCase<TopHitsAggregationBuil
                 excludes[i] = randomAsciiOfLengthBetween(5, 20);
             }
             switch (branch) {
-            case 0:
+                case 0:
-                fetchSourceContext = new FetchSourceContext(randomBoolean());
+                    fetchSourceContext = new FetchSourceContext(randomBoolean());
-                break;
+                    break;
-            case 1:
+                case 1:
-                fetchSourceContext = new FetchSourceContext(includes, excludes);
+                    fetchSourceContext = new FetchSourceContext(true, includes, excludes);
-                break;
+                    break;
-            case 2:
+                case 2:
-                fetchSourceContext = new FetchSourceContext(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20));
+                    fetchSourceContext = new FetchSourceContext(true, new String[]{randomAsciiOfLengthBetween(5, 20)},
-                break;
+                        new String[]{randomAsciiOfLengthBetween(5, 20)});
-            case 3:
+                    break;
-                fetchSourceContext = new FetchSourceContext(true, includes, excludes);
+                case 3:
-                break;
+                    fetchSourceContext = new FetchSourceContext(true, includes, excludes);
-            case 4:
+                    break;
-                fetchSourceContext = new FetchSourceContext(includes);
+                case 4:
-                break;
+                    fetchSourceContext = new FetchSourceContext(true, includes, null);
-            case 5:
+                    break;
-                fetchSourceContext = new FetchSourceContext(randomAsciiOfLengthBetween(5, 20));
+                case 5:
-                break;
+                    fetchSourceContext = new FetchSourceContext(true, new String[] {randomAsciiOfLengthBetween(5, 20)}, null);
-            default:
+                    break;
-                throw new IllegalStateException();
+                default:
+                    throw new IllegalStateException();
             }
             factory.fetchSource(fetchSourceContext);
         }

core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java

@@ -666,7 +666,7 @@ public class InnerHitsIT extends ESIntegTestCase {
                         .innerHit(new InnerHitBuilder())).get();
         assertNoFailures(response);
         assertHitCount(response, 1);
-        SearchHit hit = response.getHits().getAt(0); 
+        SearchHit hit = response.getHits().getAt(0);
         assertThat(hit.id(), equalTo("1"));
         SearchHits messages = hit.getInnerHits().get("comments.messages");
         assertThat(messages.getTotalHits(), equalTo(1L));
@@ -982,7 +982,8 @@ public class InnerHitsIT extends ESIntegTestCase {
         // other features (like in the query dsl or aggs) in order for consistency:
         SearchResponse response = client().prepareSearch()
                 .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.None)
-                .innerHit(new InnerHitBuilder().setFetchSourceContext(new FetchSourceContext("comments.message"))))
+                .innerHit(new InnerHitBuilder().setFetchSourceContext(new FetchSourceContext(true,
+                    new String[]{"comments.message"}, null))))
                 .get();
         assertNoFailures(response);
         assertHitCount(response, 1);