Filter mappings fields when field level security is configured (elastic/x-pack-elasticsearch#3173)

This PR uses a new extension point that's being added to Elasticsearch (see https://github.com/elastic/elasticsearch/pull/27603) so that the security plugin can filter the mappings fields returned by get index, get mappings, get field mappings and field capabilities API.

This effort aims at filtering information returned by API in the `indices/admin` category and field capabilities. It doesn't filter what the cluster state api returns as that is a cluster level operation.

One question is about backwards compatibility given that we would like to have this in 6.2. Shall we treat this as a bug as mappings should have been filtered before? Not sure if it's going to break existing integrations.

relates elastic/x-pack-elasticsearch#340

Original commit: elastic/x-pack-elasticsearch@d7e3fd3fa1

docs/en/release-notes/7.0.0-alpha1.asciidoc

@@ -9,6 +9,11 @@ Machine Learning::
 * The `max_running_jobs` node property is removed in this release. Use the
 `xpack.ml.max_open_jobs` setting instead. For more information, see <<ml-settings>>.
 
+Security::
+* The fields returned as part of the mappings section by get index, get
+mappings, get field mappings and field capabilities API are now only the
+ones that the user is authorized to access in case field level security is enabled.
+
 See also:
 
 * <<release-notes-7.0.0-alpha1,{es} 7.0.0-alpha1 Release Notes>>

docs/en/release-notes/xpack-breaking.asciidoc

@@ -26,6 +26,11 @@ Machine Learning::
 * The `max_running_jobs` node property is removed in this release. Use the
 `xpack.ml.max_open_jobs` setting instead. For more information, <<ml-settings>>.
 
+Security::
+* The fields returned as part of the mappings section by get index, get
+mappings, get field mappings and field capabilities API are now only the ones
+that the user is authorized to access in case field level security is enabled.
+
 See also:
 
 * <<breaking-changes-7.0,{es} Breaking Changes>>

plugin/src/main/java/org/elasticsearch/xpack/XPackPlugin.java

@@ -46,6 +46,7 @@ import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.ClusterPlugin;
 import org.elasticsearch.plugins.DiscoveryPlugin;
 import org.elasticsearch.plugins.IngestPlugin;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ScriptPlugin;
@@ -103,12 +104,15 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.BiConsumer;
+import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, IngestPlugin, NetworkPlugin, ClusterPlugin, DiscoveryPlugin {
+public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, IngestPlugin, NetworkPlugin, ClusterPlugin,
+        DiscoveryPlugin, MapperPlugin {
 
     public static final String NAME = "x-pack";
 
@@ -532,4 +536,9 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
     public BiConsumer<DiscoveryNode, ClusterState> getJoinValidator() {
         return security.getJoinValidator();
     }
+
+    @Override
+    public Function<String, Predicate<String>> getFieldFilter() {
+        return security.getFieldFilter();
+    }
 }

plugin/src/main/java/org/elasticsearch/xpack/ml/action/OpenJobAction.java

@@ -49,6 +49,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
@@ -537,7 +538,13 @@ public class OpenJobAction extends Action<OpenJobAction.Request, OpenJobAction.R
             String[] concreteIndices = aliasOrIndex.getIndices().stream().map(IndexMetaData::getIndex).map(Index::getName)
                     .toArray(String[]::new);
 
-            String[] indicesThatRequireAnUpdate = mappingRequiresUpdate(state, concreteIndices, Version.CURRENT, logger);
+            String[] indicesThatRequireAnUpdate;
+            try {
+                indicesThatRequireAnUpdate = mappingRequiresUpdate(state, concreteIndices, Version.CURRENT, logger);
+            } catch (IOException e) {
+                listener.onFailure(e);
+                return;
+            }
 
             if (indicesThatRequireAnUpdate.length > 0) {
                 try (XContentBuilder mapping = mappingSupplier.get()) {
@@ -742,7 +749,7 @@ public class OpenJobAction extends Action<OpenJobAction.Request, OpenJobAction.R
                 continue;
             }
 
-            if (nodeSupportsJobVersion(node.getVersion(), job.getJobVersion()) == false) {
+            if (nodeSupportsJobVersion(node.getVersion()) == false) {
                 String reason = "Not opening job [" + jobId + "] on node [" + node
                         + "], because this node does not support jobs of version [" + job.getJobVersion() + "]";
                 logger.trace(reason);
@@ -884,15 +891,16 @@ public class OpenJobAction extends Action<OpenJobAction.Request, OpenJobAction.R
         return unavailableIndices;
     }
 
-    static boolean nodeSupportsJobVersion(Version nodeVersion, Version jobVersion) {
+    private static boolean nodeSupportsJobVersion(Version nodeVersion) {
         return nodeVersion.onOrAfter(Version.V_5_5_0);
     }
 
-    static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndices, Version minVersion, Logger logger) {
+    static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndices, Version minVersion,
+                                          Logger logger) throws IOException {
         List<String> indicesToUpdate = new ArrayList<>();
 
         ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> currentMapping = state.metaData().findMappings(concreteIndices,
-                new String[] { ElasticsearchMappings.DOC_TYPE });
+                new String[] { ElasticsearchMappings.DOC_TYPE }, MapperPlugin.NOOP_FIELD_FILTER);
 
         for (String index : concreteIndices) {
             ImmutableOpenMap<String, MappingMetaData> innerMap = currentMapping.get(index);

plugin/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -61,6 +61,7 @@ import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.ClusterPlugin;
 import org.elasticsearch.plugins.DiscoveryPlugin;
 import org.elasticsearch.plugins.IngestPlugin;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
@@ -139,9 +140,11 @@ import org.elasticsearch.xpack.security.authc.support.mapper.expressiondsl.Expre
 import org.elasticsearch.xpack.security.authz.AuthorizationService;
 import org.elasticsearch.xpack.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.security.authz.SecuritySearchOperationListener;
+import org.elasticsearch.xpack.security.authz.accesscontrol.IndicesAccessControl;
 import org.elasticsearch.xpack.security.authz.accesscontrol.OptOutQueryCache;
 import org.elasticsearch.xpack.security.authz.accesscontrol.SecurityIndexSearcherWrapper;
 import org.elasticsearch.xpack.security.authz.accesscontrol.SetSecurityUserProcessor;
+import org.elasticsearch.xpack.security.authz.permission.FieldPermissions;
 import org.elasticsearch.xpack.security.authz.permission.FieldPermissionsCache;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
 import org.elasticsearch.xpack.security.authz.store.FileRolesStore;
@@ -180,7 +183,6 @@ import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.security.GeneralSecurityException;
 import java.time.Clock;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -194,6 +196,7 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.function.BiConsumer;
 import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
@@ -206,7 +209,7 @@ import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY
 import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_TEMPLATE_NAME;
 import static org.elasticsearch.xpack.security.support.IndexLifecycleManager.INTERNAL_INDEX_FORMAT;
 
-public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, ClusterPlugin, DiscoveryPlugin {
+public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, ClusterPlugin, DiscoveryPlugin, MapperPlugin {
 
     private static final Logger logger = Loggers.getLogger(XPackPlugin.class);
 
@@ -239,8 +242,7 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, Clus
     private final SetOnce<SecurityActionFilter> securityActionFilter = new SetOnce<>();
     private final List<BootstrapCheck> bootstrapChecks;
 
-    public Security(Settings settings, Environment env, XPackLicenseState licenseState, SSLService sslService)
+    public Security(Settings settings, Environment env, XPackLicenseState licenseState, SSLService sslService) {
-                    throws IOException, GeneralSecurityException {
         this.settings = settings;
         this.env = env;
         this.transportClientMode = XPackPlugin.transportClientMode(settings);
@@ -343,7 +345,7 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, Clus
             }
         }
         final AuditTrailService auditTrailService =
-                new AuditTrailService(settings,  auditTrails.stream().collect(Collectors.toList()), licenseState);
+                new AuditTrailService(settings, new ArrayList<>(auditTrails), licenseState);
         components.add(auditTrailService);
         this.auditTrailService.set(auditTrailService);
 
@@ -359,9 +361,8 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, Clus
         final AnonymousUser anonymousUser = new AnonymousUser(settings);
         final ReservedRealm reservedRealm = new ReservedRealm(env, settings, nativeUsersStore,
                 anonymousUser, securityLifecycleService, threadPool.getThreadContext());
-        Map<String, Realm.Factory> realmFactories = new HashMap<>();
+        Map<String, Realm.Factory> realmFactories = new HashMap<>(InternalRealms.getFactories(threadPool, resourceWatcherService,
-        realmFactories.putAll(InternalRealms.getFactories(threadPool, resourceWatcherService, sslService, nativeUsersStore,
+                sslService, nativeUsersStore, nativeRoleMappingStore, securityLifecycleService));
-                nativeRoleMappingStore, securityLifecycleService));
         for (XPackExtension extension : extensions) {
             Map<String, Realm.Factory> newRealms = extension.getRealms(resourceWatcherService);
             for (Map.Entry<String, Realm.Factory> entry : newRealms.entrySet()) {
@@ -529,11 +530,8 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, Clus
 
     
     public List<String> getSettingsFilter(@Nullable XPackExtensionsService extensionsService) {
-        ArrayList<String> settingsFilter = new ArrayList<>();
         List<String> asArray = settings.getAsList(setting("hide_settings"));
-        for (String pattern : asArray) {
+        ArrayList<String> settingsFilter = new ArrayList<>(asArray);
-            settingsFilter.add(pattern);
-        }
 
         final List<XPackExtension> extensions = extensionsService == null ? Collections.emptyList() : extensionsService.getExtensions();
         settingsFilter.addAll(RealmSettings.getSettingsFilter(extensions));
@@ -775,8 +773,7 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, Clus
         }
 
         String[] matches = Strings.commaDelimitedListToStringArray(value);
-        List<String> indices = new ArrayList<>();
+        List<String> indices = new ArrayList<>(SecurityLifecycleService.indexNames());
-        indices.addAll(SecurityLifecycleService.indexNames());
         if (indexAuditingEnabled) {
             DateTime now = new DateTime(DateTimeZone.UTC);
             // just use daily rollover
@@ -941,6 +938,31 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, Clus
         }
     }
 
+    @Override
+    public Function<String, Predicate<String>> getFieldFilter() {
+        if (enabled) {
+            return index -> {
+                if (licenseState.isDocumentAndFieldLevelSecurityAllowed() == false) {
+                    return MapperPlugin.NOOP_FIELD_PREDICATE;
+                }
+                IndicesAccessControl indicesAccessControl = threadContext.get().getTransient(AuthorizationService.INDICES_PERMISSIONS_KEY);
+                IndicesAccessControl.IndexAccessControl indexPermissions = indicesAccessControl.getIndexPermissions(index);
+                if (indexPermissions == null) {
+                    return MapperPlugin.NOOP_FIELD_PREDICATE;
+                }
+                if (indexPermissions.isGranted() == false) {
+                    throw new IllegalStateException("unexpected call to getFieldFilter for index [" + index + "] which is not granted");
+                }
+                FieldPermissions fieldPermissions = indexPermissions.getFieldPermissions();
+                if (fieldPermissions == null) {
+                    return MapperPlugin.NOOP_FIELD_PREDICATE;
+                }
+                return fieldPermissions::grantsAccessTo;
+            };
+        }
+        return MapperPlugin.super.getFieldFilter();
+    }
+
     @Override
     public BiConsumer<DiscoveryNode, ClusterState> getJoinValidator() {
         if (enabled) {

plugin/src/main/java/org/elasticsearch/xpack/security/authz/permission/FieldPermissions.java

@@ -115,7 +115,7 @@ public final class FieldPermissions implements Accountable {
         return a.getNumStates() * 5; // wild guess, better than 0
     }
 
-    static Automaton initializePermittedFieldsAutomaton(FieldPermissionsDefinition fieldPermissionsDefinition) {
+    public static Automaton initializePermittedFieldsAutomaton(FieldPermissionsDefinition fieldPermissionsDefinition) {
         Set<FieldGrantExcludeGroup> groups = fieldPermissionsDefinition.getFieldGrantExcludeGroups();
         assert groups.size() > 0 : "there must always be a single group for field inclusion/exclusion";
         List<Automaton> automatonList =

plugin/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java

@@ -5,24 +5,36 @@
  */
 package org.elasticsearch.integration;
 
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.action.fieldcaps.FieldCapabilities;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.test.SecurityIntegTestCase;
 import org.elasticsearch.xpack.XPackSettings;
 import org.elasticsearch.xpack.security.authc.support.Hasher;
-import org.elasticsearch.test.SecurityIntegTestCase;
 
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
 
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
-import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER;
-import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
+import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER;
+import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.hamcrest.Matchers.equalTo;
 
 public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
@@ -92,7 +104,7 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
                 .build();
     }
 
-    public void testSimpleQuery() throws Exception {
+    public void testSimpleQuery() {
         assertAcked(client().admin().indices().prepareCreate("test")
                         .addMapping("type1", "field1", "type=text", "field2", "type=text")
         );
@@ -130,7 +142,7 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
         assertThat(response.getHits().getAt(1).getSourceAsMap().get("field2").toString(), equalTo("value2"));
     }
 
-    public void testDLSIsAppliedBeforeFLS() throws Exception {
+    public void testDLSIsAppliedBeforeFLS() {
         assertAcked(client().admin().indices().prepareCreate("test")
                 .addMapping("type1", "field1", "type=text", "field2", "type=text")
         );
@@ -157,7 +169,7 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
         assertHitCount(response, 0);
     }
 
-    public void testQueryCache() throws Exception {
+    public void testQueryCache() {
         assertAcked(client().admin().indices().prepareCreate("test")
                         .setSettings(Settings.builder().put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true))
                         .addMapping("type1", "field1", "type=text", "field2", "type=text")
@@ -214,4 +226,216 @@ public class DocumentAndFieldLevelSecurityTests extends SecurityIntegTestCase {
         }
     }
 
+    public void testGetMappingsIsFiltered() {
+        assertAcked(client().admin().indices().prepareCreate("test")
+                .addMapping("type1", "field1", "type=text", "field2", "type=text")
+        );
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1")
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
+        client().prepareIndex("test", "type1", "2").setSource("field2", "value2")
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
+
+        {
+            GetMappingsResponse getMappingsResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+                    .admin().indices().prepareGetMappings("test").get();
+            assertExpectedFields(getMappingsResponse.getMappings(), "field1");
+        }
+
+        {
+            GetMappingsResponse getMappingsResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
+                    .admin().indices().prepareGetMappings("test").get();
+            assertExpectedFields(getMappingsResponse.getMappings(), "field2");
+        }
+
+        {
+            GetMappingsResponse getMappingsResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)))
+                    .admin().indices().prepareGetMappings("test").get();
+            assertExpectedFields(getMappingsResponse.getMappings(), "field1");
+        }
+
+        {
+            GetMappingsResponse getMappingsResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD)))
+                    .admin().indices().prepareGetMappings("test").get();
+            assertExpectedFields(getMappingsResponse.getMappings(), "field1", "field2");
+        }
+    }
+
+    public void testGetIndexMappingsIsFiltered() {
+        assertAcked(client().admin().indices().prepareCreate("test")
+                .addMapping("type1", "field1", "type=text", "field2", "type=text")
+        );
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1")
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
+        client().prepareIndex("test", "type1", "2").setSource("field2", "value2")
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
+
+        {
+            GetIndexResponse getIndexResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+                    .admin().indices().prepareGetIndex().setIndices("test").get();
+            assertExpectedFields(getIndexResponse.getMappings(), "field1");
+        }
+        {
+            GetIndexResponse getIndexResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
+                    .admin().indices().prepareGetIndex().setIndices("test").get();
+            assertExpectedFields(getIndexResponse.getMappings(), "field2");
+        }
+        {
+            GetIndexResponse getIndexResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)))
+                    .admin().indices().prepareGetIndex().setIndices("test").get();
+            assertExpectedFields(getIndexResponse.getMappings(), "field1");
+        }
+        {
+            GetIndexResponse getIndexResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD)))
+                    .admin().indices().prepareGetIndex().setIndices("test").get();
+            assertExpectedFields(getIndexResponse.getMappings(), "field1", "field2");
+        }
+    }
+
+    public void testGetFieldMappingsIsFiltered() {
+        assertAcked(client().admin().indices().prepareCreate("test")
+                .addMapping("type1", "field1", "type=text", "field2", "type=text")
+        );
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1")
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
+        client().prepareIndex("test", "type1", "2").setSource("field2", "value2")
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
+
+        {
+            GetFieldMappingsResponse getFieldMappingsResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+                    .admin().indices().prepareGetFieldMappings("test").setFields("*").get();
+
+            Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetaData>>> mappings =
+                    getFieldMappingsResponse.mappings();
+            assertEquals(1, mappings.size());
+            assertExpectedFields(mappings.get("test"), "field1");
+        }
+        {
+            GetFieldMappingsResponse getFieldMappingsResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
+                    .admin().indices().prepareGetFieldMappings("test").setFields("*").get();
+
+            Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetaData>>> mappings =
+                    getFieldMappingsResponse.mappings();
+            assertEquals(1, mappings.size());
+            assertExpectedFields(mappings.get("test"), "field2");
+        }
+        {
+            GetFieldMappingsResponse getFieldMappingsResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)))
+                    .admin().indices().prepareGetFieldMappings("test").setFields("*").get();
+
+            Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetaData>>> mappings =
+                    getFieldMappingsResponse.mappings();
+            assertEquals(1, mappings.size());
+            assertExpectedFields(mappings.get("test"), "field1");
+        }
+        {
+            GetFieldMappingsResponse getFieldMappingsResponse = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD)))
+                    .admin().indices().prepareGetFieldMappings("test").setFields("*").get();
+
+            Map<String, Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetaData>>> mappings =
+                    getFieldMappingsResponse.mappings();
+            assertEquals(1, mappings.size());
+            assertExpectedFields(mappings.get("test"), "field1", "field2");
+        }
+    }
+
+    public void testFieldCapabilitiesIsFiltered() {
+        assertAcked(client().admin().indices().prepareCreate("test")
+                .addMapping("type1", "field1", "type=text", "field2", "type=text")
+        );
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1")
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
+        client().prepareIndex("test", "type1", "2").setSource("field2", "value2")
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
+
+        {
+            FieldCapabilitiesRequest fieldCapabilitiesRequest = new FieldCapabilitiesRequest().fields("*").indices("test");
+            FieldCapabilitiesResponse response = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)))
+                    .fieldCaps(fieldCapabilitiesRequest).actionGet();
+            assertExpectedFields(response, "field1");
+        }
+        {
+            FieldCapabilitiesRequest fieldCapabilitiesRequest = new FieldCapabilitiesRequest().fields("*").indices("test");
+            FieldCapabilitiesResponse response = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)))
+                    .fieldCaps(fieldCapabilitiesRequest).actionGet();
+            assertExpectedFields(response, "field2");
+        }
+        {
+            FieldCapabilitiesRequest fieldCapabilitiesRequest = new FieldCapabilitiesRequest().fields("*").indices("test");
+            FieldCapabilitiesResponse response = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)))
+                    .fieldCaps(fieldCapabilitiesRequest).actionGet();
+            assertExpectedFields(response, "field1");
+        }
+        {
+            FieldCapabilitiesRequest fieldCapabilitiesRequest = new FieldCapabilitiesRequest().fields("*").indices("test");
+            FieldCapabilitiesResponse response = client().filterWithHeader(
+                    Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD)))
+                    .fieldCaps(fieldCapabilitiesRequest).actionGet();
+            assertExpectedFields(response, "field1", "field2");
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void assertExpectedFields(ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings,
+                                             String... fields) {
+        Map<String, Object> sourceAsMap = mappings.get("test").get("type1").getSourceAsMap();
+        assertEquals(1, sourceAsMap.size());
+        Map<String, Object> properties = (Map<String, Object>)sourceAsMap.get("properties");
+        assertEquals(fields.length, properties.size());
+        for (String field : fields) {
+            assertNotNull(properties.get(field));
+        }
+    }
+
+    private static void assertExpectedFields(FieldCapabilitiesResponse fieldCapabilitiesResponse, String... expectedFields) {
+        Map<String, Map<String, FieldCapabilities>> responseMap = fieldCapabilitiesResponse.get();
+        Set<String> builtInMetaDataFields = IndicesModule.getBuiltInMetaDataFields();
+        for (String field : builtInMetaDataFields) {
+            Map<String, FieldCapabilities> remove = responseMap.remove(field);
+            assertNotNull(" expected field [" + field + "] not found", remove);
+        }
+        for (String field : expectedFields) {
+            Map<String, FieldCapabilities> remove = responseMap.remove(field);
+            assertNotNull(" expected field [" + field + "] not found", remove);
+        }
+        assertEquals("Some unexpected fields were returned: " + responseMap.keySet(), 0, responseMap.size());
+    }
+
+    private static void assertExpectedFields(Map<String, Map<String, GetFieldMappingsResponse.FieldMappingMetaData>> mappings,
+                                            String... expectedFields) {
+        assertEquals(1, mappings.size());
+        Map<String, GetFieldMappingsResponse.FieldMappingMetaData> fields = new HashMap<>(mappings.get("type1"));
+        Set<String> builtInMetaDataFields = IndicesModule.getBuiltInMetaDataFields();
+        for (String field : builtInMetaDataFields) {
+            GetFieldMappingsResponse.FieldMappingMetaData fieldMappingMetaData = fields.remove(field);
+            assertNotNull(" expected field [" + field + "] not found", fieldMappingMetaData);
+        }
+        for (String field : expectedFields) {
+            GetFieldMappingsResponse.FieldMappingMetaData fieldMappingMetaData = fields.remove(field);
+            assertNotNull("expected field [" + field + "] not found", fieldMappingMetaData);
+        }
+        assertEquals("Some unexpected fields were returned: " + fields.keySet(), 0, fields.size());
+    }
 }

plugin/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java

@@ -5,16 +5,6 @@
  */
 package org.elasticsearch.xpack.security;
 
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.BiConsumer;
-
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterName;
@@ -24,16 +14,18 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.license.License;
 import org.elasticsearch.license.TestUtils;
-import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
@@ -46,7 +38,26 @@ import org.elasticsearch.xpack.security.authc.Realm;
 import org.elasticsearch.xpack.security.authc.Realms;
 import org.elasticsearch.xpack.security.authc.file.FileRealm;
 import org.elasticsearch.xpack.security.authc.ldap.support.SessionFactory;
+import org.elasticsearch.xpack.security.authz.AuthorizationService;
+import org.elasticsearch.xpack.security.authz.accesscontrol.IndicesAccessControl;
+import org.elasticsearch.xpack.security.authz.permission.FieldPermissions;
+import org.elasticsearch.xpack.security.authz.permission.FieldPermissionsDefinition;
 import org.elasticsearch.xpack.ssl.SSLService;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.Function;
+import java.util.function.Predicate;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_FORMAT_SETTING;
 import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME;
@@ -60,6 +71,8 @@ import static org.mockito.Mockito.when;
 public class SecurityTests extends ESTestCase {
 
     private Security security = null;
+    private ThreadContext threadContext = null;
+    private TestUtils.UpdatableLicenseState licenseState;
 
     public static class DummyExtension extends XPackExtension {
         private String realmType;
@@ -86,7 +99,8 @@ public class SecurityTests extends ESTestCase {
         }
         Settings settings = Settings.builder().put(testSettings).put("path.home", createTempDir()).build();
         Environment env = TestEnvironment.newEnvironment(settings);
-        security = new Security(settings, env, new XPackLicenseState(), new SSLService(settings, env));
+        licenseState = new TestUtils.UpdatableLicenseState();
+        security = new Security(settings, env, licenseState, new SSLService(settings, env));
         ThreadPool threadPool = mock(ThreadPool.class);
         ClusterService clusterService = mock(ClusterService.class);
         settings = Security.additionalSettings(settings, false);
@@ -95,6 +109,8 @@ public class SecurityTests extends ESTestCase {
         ClusterSettings clusterSettings = new ClusterSettings(settings, allowedSettings);
         when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
         when(threadPool.relativeTimeInMillis()).thenReturn(1L);
+        threadContext = new ThreadContext(Settings.EMPTY);
+        when(threadPool.getThreadContext()).thenReturn(threadContext);
         Client client = mock(Client.class);
         when(client.threadPool()).thenReturn(threadPool);
         when(client.settings()).thenReturn(settings);
@@ -102,7 +118,7 @@ public class SecurityTests extends ESTestCase {
                 Arrays.asList(extensions));
     }
 
-    private <T> T findComponent(Class<T> type, Collection<Object> components) {
+    private static <T> T findComponent(Class<T> type, Collection<Object> components) {
         for (Object obj : components) {
             if (type.isInstance(obj)) {
                 return type.cast(obj);
@@ -111,6 +127,15 @@ public class SecurityTests extends ESTestCase {
         return null;
     }
 
+    @Before
+    public void cleanup() throws IOException {
+        if (threadContext != null) {
+            threadContext.stashContext();
+            threadContext.close();
+            threadContext = null;
+        }
+    }
+
     public void testCustomRealmExtension() throws Exception {
         Collection<Object> components = createComponents(Settings.EMPTY, new DummyExtension("myrealm"));
         Realms realms = findComponent(Realms.class, components);
@@ -163,7 +188,7 @@ public class SecurityTests extends ESTestCase {
         assertEquals(LoggingAuditTrail.NAME, service.getAuditTrails().get(1).name());
     }
 
-    public void testUnknownOutput() throws Exception {
+    public void testUnknownOutput() {
         Settings settings = Settings.builder()
             .put(XPackSettings.AUDIT_ENABLED.getKey(), true)
             .put(Security.AUDIT_OUTPUTS_SETTING.getKey(), "foo").build();
@@ -312,4 +337,48 @@ public class SecurityTests extends ESTestCase {
             .nodes(discoveryNodes).build();
         joinValidator.accept(node, clusterState);
     }
+    
+    public void testGetFieldFilterSecurityEnabled() throws Exception {
+        createComponents(Settings.EMPTY);
+        Function<String, Predicate<String>> fieldFilter = security.getFieldFilter();
+        assertNotSame(MapperPlugin.NOOP_FIELD_FILTER, fieldFilter);
+        Map<String, IndicesAccessControl.IndexAccessControl> permissionsMap = new HashMap<>();
+
+        FieldPermissions permissions = new FieldPermissions(
+                new FieldPermissionsDefinition(new String[]{"field_granted"}, Strings.EMPTY_ARRAY));
+        IndicesAccessControl.IndexAccessControl indexGrantedAccessControl = new IndicesAccessControl.IndexAccessControl(true, permissions,
+                Collections.emptySet());
+        permissionsMap.put("index_granted", indexGrantedAccessControl);
+        IndicesAccessControl.IndexAccessControl indexAccessControl = new IndicesAccessControl.IndexAccessControl(false,
+                FieldPermissions.DEFAULT, Collections.emptySet());
+        permissionsMap.put("index_not_granted", indexAccessControl);
+        IndicesAccessControl.IndexAccessControl nullFieldPermissions =
+                new IndicesAccessControl.IndexAccessControl(true, null, Collections.emptySet());
+        permissionsMap.put("index_null", nullFieldPermissions);
+        IndicesAccessControl index = new IndicesAccessControl(true, permissionsMap);
+        threadContext.putTransient(AuthorizationService.INDICES_PERMISSIONS_KEY, index);
+
+        assertTrue(fieldFilter.apply("index_granted").test("field_granted"));
+        assertFalse(fieldFilter.apply("index_granted").test(randomAlphaOfLengthBetween(3, 10)));
+        assertTrue(fieldFilter.apply(randomAlphaOfLengthBetween(3, 6)).test("field_granted"));
+        assertTrue(fieldFilter.apply(randomAlphaOfLengthBetween(3, 6)).test(randomAlphaOfLengthBetween(3, 10)));
+        assertEquals(MapperPlugin.NOOP_FIELD_PREDICATE, fieldFilter.apply(randomAlphaOfLengthBetween(3, 10)));
+        expectThrows(IllegalStateException.class, () -> fieldFilter.apply("index_not_granted"));
+        assertTrue(fieldFilter.apply("index_null").test(randomAlphaOfLengthBetween(3, 6)));
+        assertEquals(MapperPlugin.NOOP_FIELD_PREDICATE, fieldFilter.apply("index_null"));
+    }
+
+    public void testGetFieldFilterSecurityDisabled() throws Exception {
+        createComponents(Settings.builder().put(XPackSettings.SECURITY_ENABLED.getKey(), false).build());
+        assertSame(MapperPlugin.NOOP_FIELD_FILTER, security.getFieldFilter());
+    }
+
+    public void testGetFieldFilterSecurityEnabledLicenseNoFLS() throws Exception {
+        createComponents(Settings.EMPTY);
+        Function<String, Predicate<String>> fieldFilter = security.getFieldFilter();
+        assertNotSame(MapperPlugin.NOOP_FIELD_FILTER, fieldFilter);
+        licenseState.update(randomFrom(License.OperationMode.BASIC, License.OperationMode.STANDARD, License.OperationMode.GOLD), true);
+        assertNotSame(MapperPlugin.NOOP_FIELD_FILTER, fieldFilter);
+        assertSame(MapperPlugin.NOOP_FIELD_PREDICATE, fieldFilter.apply(randomAlphaOfLengthBetween(3, 6)));
+    }
 }

plugin/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/FieldSubsetReaderTests.java

@@ -46,9 +46,20 @@ import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.apache.lucene.util.automaton.Operations;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.security.authz.permission.FieldPermissions;
+import org.elasticsearch.xpack.security.authz.permission.FieldPermissionsDefinition;
 import org.elasticsearch.xpack.security.support.Automatons;
 
 import java.io.IOException;
@@ -971,4 +982,196 @@ public class FieldSubsetReaderTests extends ESTestCase {
         directoryReader.close();
         dir.close();
     }
+
+    @SuppressWarnings("unchecked")
+    public void testMappingsFilteringDuelWithSourceFiltering() throws Exception {
+        MetaData metaData = MetaData.builder()
+                .put(IndexMetaData.builder("index")
+                        .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
+                        .putMapping("doc", MAPPING_TEST_ITEM)).build();
+
+        {
+            FieldPermissionsDefinition definition = new FieldPermissionsDefinition(new String[]{"*inner1"}, Strings.EMPTY_ARRAY);
+            FieldPermissions fieldPermissions = new FieldPermissions(definition);
+            ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = metaData.findMappings(new String[]{"index"},
+                    new String[]{"doc"}, index -> fieldPermissions::grantsAccessTo);
+            ImmutableOpenMap<String, MappingMetaData> index = mappings.get("index");
+            Map<String, Object> sourceAsMap = index.get("doc").getSourceAsMap();
+            assertEquals(1, sourceAsMap.size());
+            Map<String, Object> properties = (Map<String, Object>) sourceAsMap.get("properties");
+            assertEquals(2, properties.size());
+            Map<String, Object> objectMapping = (Map<String, Object>) properties.get("object");
+            assertEquals(1, objectMapping.size());
+            Map<String, Object> objectProperties = (Map<String, Object>) objectMapping.get("properties");
+            assertEquals(1, objectProperties.size());
+            assertTrue(objectProperties.containsKey("inner1"));
+            Map<String, Object> nestedMapping = (Map<String, Object>) properties.get("nested");
+            assertEquals(2, nestedMapping.size());
+            assertEquals("nested", nestedMapping.get("type"));
+            Map<String, Object> nestedProperties = (Map<String, Object>) nestedMapping.get("properties");
+            assertEquals(1, nestedProperties.size());
+            assertTrue(nestedProperties.containsKey("inner1"));
+
+            Automaton automaton = FieldPermissions.initializePermittedFieldsAutomaton(definition);
+            CharacterRunAutomaton include = new CharacterRunAutomaton(automaton);
+            Map<String, Object> stringObjectMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), DOC_TEST_ITEM, false);
+            Map<String, Object> filtered = FieldSubsetReader.filter(stringObjectMap, include, 0);
+            assertEquals(2, filtered.size());
+            Map<String, Object> object = (Map<String, Object>)filtered.get("object");
+            assertEquals(1, object.size());
+            assertTrue(object.containsKey("inner1"));
+            List<Map<String, Object>> nested = (List<Map<String, Object>>)filtered.get("nested");
+            assertEquals(2, nested.size());
+            for (Map<String, Object> objectMap : nested) {
+                assertEquals(1, objectMap.size());
+                assertTrue(objectMap.containsKey("inner1"));
+            }
+        }
+        {
+            FieldPermissionsDefinition definition = new FieldPermissionsDefinition(new String[]{"object*"}, Strings.EMPTY_ARRAY);
+            FieldPermissions fieldPermissions = new FieldPermissions(definition);
+            ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = metaData.findMappings(new String[]{"index"},
+                    new String[]{"doc"}, index -> fieldPermissions::grantsAccessTo);
+            ImmutableOpenMap<String, MappingMetaData> index = mappings.get("index");
+            Map<String, Object> sourceAsMap = index.get("doc").getSourceAsMap();
+            assertEquals(1, sourceAsMap.size());
+            Map<String, Object> properties = (Map<String, Object>) sourceAsMap.get("properties");
+            assertEquals(1, properties.size());
+            Map<String, Object> objectMapping = (Map<String, Object>) properties.get("object");
+            assertEquals(1, objectMapping.size());
+            Map<String, Object> objectProperties = (Map<String, Object>) objectMapping.get("properties");
+            assertEquals(2, objectProperties.size());
+            Map<String, Object> inner1 = (Map<String, Object>) objectProperties.get("inner1");
+            assertEquals(2, inner1.size());
+            assertEquals("text", inner1.get("type"));
+            Map<String, Object> inner1Fields = (Map<String, Object>) inner1.get("fields");
+            assertEquals(1, inner1Fields.size());
+            Map<String, Object> inner1Keyword = (Map<String, Object>) inner1Fields.get("keyword");
+            assertEquals(1, inner1Keyword.size());
+            assertEquals("keyword", inner1Keyword.get("type"));
+            Map<String, Object> inner2 = (Map<String, Object>) objectProperties.get("inner2");
+            assertEquals(1, inner2.size());
+            assertEquals("keyword", inner2.get("type"));
+
+            Automaton automaton = FieldPermissions.initializePermittedFieldsAutomaton(definition);
+            CharacterRunAutomaton include = new CharacterRunAutomaton(automaton);
+            Map<String, Object> stringObjectMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), DOC_TEST_ITEM, false);
+            Map<String, Object> filtered = FieldSubsetReader.filter(stringObjectMap, include, 0);
+            assertEquals(1, filtered.size());
+            Map<String, Object> object = (Map<String, Object>)filtered.get("object");
+            assertEquals(2, object.size());
+            assertTrue(object.containsKey("inner1"));
+            assertTrue(object.containsKey("inner2"));
+        }
+        {
+            FieldPermissionsDefinition definition = new FieldPermissionsDefinition(new String[]{"object"}, Strings.EMPTY_ARRAY);
+            FieldPermissions fieldPermissions = new FieldPermissions(definition);
+            ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = metaData.findMappings(new String[]{"index"},
+                    new String[]{"doc"}, index -> fieldPermissions::grantsAccessTo);
+            ImmutableOpenMap<String, MappingMetaData> index = mappings.get("index");
+            Map<String, Object> sourceAsMap = index.get("doc").getSourceAsMap();
+            assertEquals(1, sourceAsMap.size());
+            Map<String, Object> properties = (Map<String, Object>) sourceAsMap.get("properties");
+            assertEquals(1, properties.size());
+            Map<String, Object> objectMapping = (Map<String, Object>) properties.get("object");
+            assertEquals(1, objectMapping.size());
+            Map<String, Object> objectProperties = (Map<String, Object>) objectMapping.get("properties");
+            assertEquals(0, objectProperties.size());
+
+            Automaton automaton = FieldPermissions.initializePermittedFieldsAutomaton(definition);
+            CharacterRunAutomaton include = new CharacterRunAutomaton(automaton);
+            Map<String, Object> stringObjectMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), DOC_TEST_ITEM, false);
+            Map<String, Object> filtered = FieldSubsetReader.filter(stringObjectMap, include, 0);
+            //TODO FLS filters out empty objects from source, although they are granted access.
+            //When filtering mappings though we keep them.
+            assertEquals(0, filtered.size());
+            /*assertEquals(1, filtered.size());
+            Map<String, Object> object = (Map<String, Object>)filtered.get("object");
+            assertEquals(0, object.size());*/
+        }
+        {
+            FieldPermissionsDefinition definition = new FieldPermissionsDefinition(new String[]{"nested.inner2"}, Strings.EMPTY_ARRAY);
+            FieldPermissions fieldPermissions = new FieldPermissions(definition);
+            ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = metaData.findMappings(new String[]{"index"},
+                    new String[]{"doc"}, index -> fieldPermissions::grantsAccessTo);
+            ImmutableOpenMap<String, MappingMetaData> index = mappings.get("index");
+            Map<String, Object> sourceAsMap = index.get("doc").getSourceAsMap();
+            assertEquals(1, sourceAsMap.size());
+            Map<String, Object> properties = (Map<String, Object>) sourceAsMap.get("properties");
+            assertEquals(1, properties.size());
+            Map<String, Object> nestedMapping = (Map<String, Object>) properties.get("nested");
+            assertEquals(2, nestedMapping.size());
+            assertEquals("nested", nestedMapping.get("type"));
+            Map<String, Object> nestedProperties = (Map<String, Object>) nestedMapping.get("properties");
+            assertEquals(1, nestedProperties.size());
+            assertTrue(nestedProperties.containsKey("inner2"));
+
+            Automaton automaton = FieldPermissions.initializePermittedFieldsAutomaton(definition);
+            CharacterRunAutomaton include = new CharacterRunAutomaton(automaton);
+            Map<String, Object> stringObjectMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), DOC_TEST_ITEM, false);
+            Map<String, Object> filtered = FieldSubsetReader.filter(stringObjectMap, include, 0);
+            assertEquals(1, filtered.size());
+            List<Map<String, Object>> nested = (List<Map<String, Object>>)filtered.get("nested");
+            assertEquals(2, nested.size());
+            for (Map<String, Object> objectMap : nested) {
+                assertEquals(1, objectMap.size());
+                assertTrue(objectMap.containsKey("inner2"));
+            }
+        }
+    }
+
+    private static final String DOC_TEST_ITEM = "{\n" +
+            "  \"field_text\" : \"text\",\n" +
+            "  \"object\" : {\n" +
+            "    \"inner1\" : \"text\",\n" +
+            "    \"inner2\" : \"keyword\"\n" +
+            "  },\n" +
+            "  \"nested\" : [\n" +
+            "    {\n" +
+            "      \"inner1\" : 1,\n" +
+            "      \"inner2\" : \"2017/12/12\"\n" +
+            "    },\n" +
+            "    {\n" +
+            "      \"inner1\" : 2,\n" +
+            "      \"inner2\" : \"2017/11/11\"\n" +
+            "    }\n" +
+            "  ]\n" +
+            "}";
+
+    private static final String MAPPING_TEST_ITEM = "{\n" +
+            "  \"doc\": {\n" +
+            "    \"properties\" : {\n" +
+            "      \"field_text\" : {\n" +
+            "        \"type\":\"text\"\n" +
+            "      },\n" +
+            "      \"object\" : {\n" +
+            "        \"properties\" : {\n" +
+            "          \"inner1\" : {\n" +
+            "            \"type\": \"text\",\n" +
+            "            \"fields\" : {\n" +
+            "              \"keyword\" : {\n" +
+            "                \"type\" : \"keyword\"\n" +
+            "              }\n" +
+            "            }\n" +
+            "          },\n" +
+            "          \"inner2\" : {\n" +
+            "            \"type\": \"keyword\"\n" +
+            "          }\n" +
+            "        }\n" +
+            "      },\n" +
+            "      \"nested\" : {\n" +
+            "        \"type\":\"nested\",\n" +
+            "        \"properties\" : {\n" +
+            "          \"inner1\" : {\n" +
+            "            \"type\": \"integer\"\n" +
+            "          },\n" +
+            "          \"inner2\" : {\n" +
+            "            \"type\": \"date\"\n" +
+            "          }\n" +
+            "        }\n" +
+            "      }\n" +
+            "    }\n" +
+            "  }\n" +
+            "}";
 }

plugin/src/test/java/org/elasticsearch/xpack/security/authz/permission/FieldPermissionsTests.java

@@ -12,8 +12,6 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.security.authz.RoleDescriptor;
 
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicReferenceArray;