Permission for restricted indices (#37577)

This grants the capability to grant privileges over certain restricted
indices (.security and .security-6 at the moment).
It also removes the special status of the superuser role.

IndicesPermission.Group is extended by adding the `allow_restricted_indices`
boolean flag. By default the flag is false. When it is toggled, you acknowledge
that the indices under the scope of the permission group can cover the
restricted indices as well. Otherwise, by default, restricted indices are ignored
when granting privileges, thus rendering them hidden for authorization purposes.
This effectively adds a confirmation "check-box" for roles that might grant
privileges to restricted indices.

The "special status" of the superuser role has been removed and coded as
any other role:
```
new RoleDescriptor("superuser",
    new String[] { "all" },
    new RoleDescriptor.IndicesPrivileges[] {
        RoleDescriptor.IndicesPrivileges.builder()
            .indices("*")
            .privileges("all")
            .allowRestrictedIndices(true)
// this ----^
            .build() },
            new RoleDescriptor.ApplicationResourcePrivileges[] {
                RoleDescriptor.ApplicationResourcePrivileges.builder()
                    .application("*")
                    .privileges("*")
                    .resources("*")
                    .build()
            },
            null, new String[] { "*" },
    MetadataUtils.DEFAULT_RESERVED_METADATA,
    Collections.emptyMap());
```
In the context of the Backup .security work, this allows the creation of a
"curator role" that would permit listing (get settings) for all indices
(including the restricted ones). That way the curator role would be able to 
ist and snapshot all indices, but not read or restore any of them.

Supersedes #36765
Relates #34454

client/rest-high-level/src/main/java/org/elasticsearch/client/security/user/privileges/AbstractIndicesPrivileges.java

@@ -39,14 +39,16 @@ import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optiona
 
 public abstract class AbstractIndicesPrivileges {
     static final ParseField NAMES = new ParseField("names");
+    static final ParseField ALLOW_RESTRICTED_INDICES = new ParseField("allow_restricted_indices");
     static final ParseField PRIVILEGES = new ParseField("privileges");
     static final ParseField FIELD_PERMISSIONS = new ParseField("field_security");
     static final ParseField QUERY = new ParseField("query");
 
     protected final Set<String> indices;
     protected final Set<String> privileges;
+    protected final boolean allowRestrictedIndices;
 
-    AbstractIndicesPrivileges(Collection<String> indices, Collection<String> privileges) {
+    AbstractIndicesPrivileges(Collection<String> indices, Collection<String> privileges, boolean allowRestrictedIndices) {
         if (null == indices || indices.isEmpty()) {
             throw new IllegalArgumentException("indices privileges must refer to at least one index name or index name pattern");
         }
@@ -55,6 +57,7 @@ public abstract class AbstractIndicesPrivileges {
         }
         this.indices = Collections.unmodifiableSet(new HashSet<>(indices));
         this.privileges = Collections.unmodifiableSet(new HashSet<>(privileges));
+        this.allowRestrictedIndices = allowRestrictedIndices;
     }
 
     /**
@@ -73,6 +76,15 @@ public abstract class AbstractIndicesPrivileges {
         return this.privileges;
     }
 
+    /**
+     * True if the privileges cover restricted internal indices too. Certain indices are reserved for internal services and should be
+     * transparent to ordinary users. For that matter, when granting privileges, you also have to toggle this flag to confirm that all
+     * indices, including restricted ones, are in the scope of this permission. By default this is false.
+     */
+    public boolean allowRestrictedIndices() {
+        return this.allowRestrictedIndices;
+    }
+
     /**
      * If {@code true} some documents might not be visible. Only the documents
      * matching {@code query} will be readable.

client/rest-high-level/src/main/java/org/elasticsearch/client/security/user/privileges/IndicesPrivileges.java

@@ -50,14 +50,16 @@ public final class IndicesPrivileges extends AbstractIndicesPrivileges implement
             int i = 0;
             final Collection<String> indices = (Collection<String>) constructorObjects[i++];
             final Collection<String> privileges = (Collection<String>) constructorObjects[i++];
+            final boolean allowRestrictedIndices = (Boolean) constructorObjects[i++];
             final FieldSecurity fields = (FieldSecurity) constructorObjects[i++];
             final String query = (String) constructorObjects[i];
-            return new IndicesPrivileges(indices, privileges, fields, query);
+            return new IndicesPrivileges(indices, privileges, allowRestrictedIndices, fields, query);
         });
 
     static {
         PARSER.declareStringArray(constructorArg(), NAMES);
         PARSER.declareStringArray(constructorArg(), PRIVILEGES);
+        PARSER.declareBoolean(constructorArg(), ALLOW_RESTRICTED_INDICES);
         PARSER.declareObject(optionalConstructorArg(), FieldSecurity::parse, FIELD_PERMISSIONS);
         PARSER.declareStringOrNull(optionalConstructorArg(), QUERY);
     }
@@ -66,9 +68,9 @@ public final class IndicesPrivileges extends AbstractIndicesPrivileges implement
     // missing query means all documents, i.e. no restrictions
     private final @Nullable String query;
 
-    private IndicesPrivileges(Collection<String> indices, Collection<String> privileges, @Nullable FieldSecurity fieldSecurity,
+    private IndicesPrivileges(Collection<String> indices, Collection<String> privileges, boolean allowRestrictedIndices,
-                              @Nullable String query) {
+                              @Nullable FieldSecurity fieldSecurity, @Nullable String query) {
-        super(indices, privileges);
+        super(indices, privileges, allowRestrictedIndices);
         this.fieldSecurity = fieldSecurity;
         this.query = query;
     }
@@ -118,13 +120,14 @@ public final class IndicesPrivileges extends AbstractIndicesPrivileges implement
         IndicesPrivileges that = (IndicesPrivileges) o;
         return indices.equals(that.indices)
             && privileges.equals(that.privileges)
+            && allowRestrictedIndices == that.allowRestrictedIndices
             && Objects.equals(this.fieldSecurity, that.fieldSecurity)
             && Objects.equals(query, that.query);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(indices, privileges, fieldSecurity, query);
+        return Objects.hash(indices, privileges, allowRestrictedIndices, fieldSecurity, query);
     }
 
     @Override
@@ -141,6 +144,7 @@ public final class IndicesPrivileges extends AbstractIndicesPrivileges implement
         builder.startObject();
         builder.field(NAMES.getPreferredName(), indices);
         builder.field(PRIVILEGES.getPreferredName(), privileges);
+        builder.field(ALLOW_RESTRICTED_INDICES.getPreferredName(), allowRestrictedIndices);
         if (fieldSecurity != null) {
             builder.field(FIELD_PERMISSIONS.getPreferredName(), fieldSecurity, params);
         }
@@ -170,6 +174,7 @@ public final class IndicesPrivileges extends AbstractIndicesPrivileges implement
         Collection<String> deniedFields = null;
         private @Nullable
         String query = null;
+        boolean allowRestrictedIndices = false;
 
         public Builder() {
         }
@@ -223,6 +228,11 @@ public final class IndicesPrivileges extends AbstractIndicesPrivileges implement
             return this;
         }
 
+        public Builder allowRestrictedIndices(boolean allow) {
+            this.allowRestrictedIndices = allow;
+            return this;
+        }
+
         public IndicesPrivileges build() {
             final FieldSecurity fieldSecurity;
             if (grantedFields == null && deniedFields == null) {
@@ -230,7 +240,7 @@ public final class IndicesPrivileges extends AbstractIndicesPrivileges implement
             } else {
                 fieldSecurity = new FieldSecurity(grantedFields, deniedFields);
             }
-            return new IndicesPrivileges(indices, privileges, fieldSecurity, query);
+            return new IndicesPrivileges(indices, privileges, allowRestrictedIndices, fieldSecurity, query);
         }
     }
 

client/rest-high-level/src/main/java/org/elasticsearch/client/security/user/privileges/UserIndicesPrivileges.java

@@ -52,6 +52,7 @@ public class UserIndicesPrivileges extends AbstractIndicesPrivileges {
     static {
         PARSER.declareStringArray(constructorArg(), IndicesPrivileges.NAMES);
         PARSER.declareStringArray(constructorArg(), IndicesPrivileges.PRIVILEGES);
+        PARSER.declareBoolean(constructorArg(), IndicesPrivileges.ALLOW_RESTRICTED_INDICES);
         PARSER.declareObjectArray(optionalConstructorArg(), IndicesPrivileges.FieldSecurity::parse, IndicesPrivileges.FIELD_PERMISSIONS);
         PARSER.declareStringArray(optionalConstructorArg(), IndicesPrivileges.QUERY);
     }
@@ -61,8 +62,9 @@ public class UserIndicesPrivileges extends AbstractIndicesPrivileges {
         return new UserIndicesPrivileges(
             (List<String>) args[0],
             (List<String>) args[1],
-            (List<IndicesPrivileges.FieldSecurity>) args[2],
+            (Boolean) args[2],
-            (List<String>) args[3]
+            (List<IndicesPrivileges.FieldSecurity>) args[3],
+            (List<String>) args[4]
         );
     }
 
@@ -70,21 +72,13 @@ public class UserIndicesPrivileges extends AbstractIndicesPrivileges {
         return PARSER.parse(parser, null);
     }
 
-    public UserIndicesPrivileges(Collection<String> indices, Collection<String> privileges,
+    public UserIndicesPrivileges(Collection<String> indices, Collection<String> privileges, boolean allowRestrictedIndices,
                                  Collection<IndicesPrivileges.FieldSecurity> fieldSecurity, Collection<String> query) {
-        super(indices, privileges);
+        super(indices, privileges, allowRestrictedIndices);
         this.fieldSecurity = fieldSecurity == null ? Collections.emptySet() : Collections.unmodifiableSet(new HashSet<>(fieldSecurity));
         this.query = query == null ? Collections.emptySet() : Collections.unmodifiableSet(new HashSet<>(query));
     }
 
-    public Set<String> getIndices() {
-        return indices;
-    }
-
-    public Set<String> getPrivileges() {
-        return privileges;
-    }
-
     public Set<IndicesPrivileges.FieldSecurity> getFieldSecurity() {
         return fieldSecurity;
     }
@@ -114,13 +108,14 @@ public class UserIndicesPrivileges extends AbstractIndicesPrivileges {
         final UserIndicesPrivileges that = (UserIndicesPrivileges) o;
         return Objects.equals(indices, that.indices) &&
             Objects.equals(privileges, that.privileges) &&
+            allowRestrictedIndices == that.allowRestrictedIndices &&
             Objects.equals(fieldSecurity, that.fieldSecurity) &&
             Objects.equals(query, that.query);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(indices, privileges, fieldSecurity, query);
+        return Objects.hash(indices, privileges, allowRestrictedIndices, fieldSecurity, query);
     }
 
     @Override
@@ -128,6 +123,7 @@ public class UserIndicesPrivileges extends AbstractIndicesPrivileges {
         return "UserIndexPrivilege{" +
             "indices=" + indices +
             ", privileges=" + privileges +
+            ", allow_restricted_indices=" + allowRestrictedIndices +
             ", fieldSecurity=" + fieldSecurity +
             ", query=" + query +
             '}';

client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityRequestConvertersTests.java

@@ -393,7 +393,8 @@ public class SecurityRequestConvertersTests extends ESTestCase {
         final List<String> indicesPrivilegeDeniedName = Arrays.asList(randomArray(3, String[]::new, () -> randomAlphaOfLength(5)));
         final String indicesPrivilegeQuery = randomAlphaOfLengthBetween(0, 7);
         final IndicesPrivileges indicesPrivilege = IndicesPrivileges.builder().indices(indicesName).privileges(indicesPrivilegeName)
-                .grantedFields(indicesPrivilegeGrantedName).deniedFields(indicesPrivilegeDeniedName).query(indicesPrivilegeQuery).build();
+                .allowRestrictedIndices(randomBoolean()).grantedFields(indicesPrivilegeGrantedName).deniedFields(indicesPrivilegeDeniedName)
+                .query(indicesPrivilegeQuery).build();
         final RefreshPolicy refreshPolicy = randomFrom(RefreshPolicy.values());
         final Map<String, String> expectedParams;
         if (refreshPolicy != RefreshPolicy.NONE) {

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java

@@ -742,8 +742,10 @@ public class SecurityDocumentationIT extends ESRestHighLevelClientTestCase {
             HasPrivilegesRequest request = new HasPrivilegesRequest(
                 Sets.newHashSet("monitor", "manage"),
                 Sets.newHashSet(
-                    IndicesPrivileges.builder().indices("logstash-2018-10-05").privileges("read", "write").build(),
+                    IndicesPrivileges.builder().indices("logstash-2018-10-05").privileges("read", "write")
-                    IndicesPrivileges.builder().indices("logstash-2018-*").privileges("read").build()
+                        .allowRestrictedIndices(false).build(),
+                    IndicesPrivileges.builder().indices("logstash-2018-*").privileges("read")
+                        .allowRestrictedIndices(true).build()
                 ),
                 null
             );

client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetRolesResponseTests.java

@@ -48,6 +48,7 @@ public class GetRolesResponseTests extends ESTestCase {
                 "      {\n" +
                 "        \"names\" : [ \"index1\", \"index2\" ],\n" +
                 "        \"privileges\" : [ \"all\" ],\n" +
+                "        \"allow_restricted_indices\" : true,\n" +
                 "        \"field_security\" : {\n" +
                 "          \"grant\" : [ \"title\", \"body\" ]}\n" +
                 "      }\n" +
@@ -81,6 +82,7 @@ public class GetRolesResponseTests extends ESTestCase {
             .indices("index1", "index2")
             .privileges("all")
             .grantedFields("title", "body")
+            .allowRestrictedIndices(true)
             .build();
         assertThat(role.getIndicesPrivileges().contains(expectedIndicesPrivileges), equalTo(true));
         final Map<String, Object> expectedMetadata = new HashMap<>();
@@ -106,6 +108,7 @@ public class GetRolesResponseTests extends ESTestCase {
             .privileges("write", "monitor", "delete")
             .grantedFields("field1", "field2")
             .deniedFields("field3", "field4")
+            .allowRestrictedIndices(true)
             .build();
         Map<String, Object> metadata = new HashMap<>();
         metadata.put("key", "value");
@@ -125,9 +128,10 @@ public class GetRolesResponseTests extends ESTestCase {
             .privileges("write", "monitor", "delete")
             .grantedFields("other_field1", "other_field2")
             .deniedFields("other_field3", "other_field4")
+            .allowRestrictedIndices(false)
             .build();
         Map<String, Object> metadata2 = new HashMap<>();
-        metadata.put("other_key", "other_value");
+        metadata2.put("other_key", "other_value");
         final Role role2 = Role.builder()
             .name("role2_name")
             .clusterPrivileges("monitor", "manage", "manage_saml")
@@ -158,6 +162,7 @@ public class GetRolesResponseTests extends ESTestCase {
                 .privileges("write", "monitor", "delete")
                 .grantedFields("field1", "field2")
                 .deniedFields("field3", "field4")
+                .allowRestrictedIndices(true)
                 .build();
             Map<String, Object> metadata = new HashMap<String, Object>();
             metadata.put("key", "value");
@@ -179,6 +184,7 @@ public class GetRolesResponseTests extends ESTestCase {
                 .privileges("write", "monitor", "delete")
                 .grantedFields("field1", "field2")
                 .deniedFields("field3", "field4")
+                .allowRestrictedIndices(false)
                 .build();
             Map<String, Object> metadata = new HashMap<String, Object>();
             metadata.put("key", "value");

client/rest-high-level/src/test/java/org/elasticsearch/client/security/GetUserPrivilegesResponseTests.java

@@ -48,17 +48,18 @@ public class GetUserPrivilegesResponseTests extends ESTestCase {
             " {\"application\":{\"manage\":{\"applications\":[\"apps-*\"]}}}" +
             "]," +
             "\"indices\":[" +
-            " {\"names\":[\"test-1-*\"],\"privileges\":[\"read\"]}," +
+            " {\"names\":[\"test-1-*\"],\"privileges\":[\"read\"],\"allow_restricted_indices\": false}," +
-            " {\"names\":[\"test-4-*\"],\"privileges\":[\"read\"],\"field_security\":[{\"grant\":[\"*\"],\"except\":[\"private-*\"]}]}," +
+            " {\"names\":[\"test-4-*\"],\"privileges\":[\"read\"],\"allow_restricted_indices\": true," +
-            " {\"names\":[\"test-6-*\",\"test-7-*\"],\"privileges\":[\"read\"]," +
+            "  \"field_security\":[{\"grant\":[\"*\"],\"except\":[\"private-*\"]}]}," +
+            " {\"names\":[\"test-6-*\",\"test-7-*\"],\"privileges\":[\"read\"],\"allow_restricted_indices\": true," +
             "  \"query\":[\"{\\\"term\\\":{\\\"test\\\":true}}\"]}," +
-            " {\"names\":[\"test-2-*\"],\"privileges\":[\"read\"]," +
+            " {\"names\":[\"test-2-*\"],\"privileges\":[\"read\"],\"allow_restricted_indices\": false," +
             "  \"field_security\":[{\"grant\":[\"*\"],\"except\":[\"secret-*\",\"private-*\"]},{\"grant\":[\"apps-*\"]}]," +
             "  \"query\":[\"{\\\"term\\\":{\\\"test\\\":true}}\",\"{\\\"term\\\":{\\\"apps\\\":true}}\"]}," +
-            " {\"names\":[\"test-3-*\",\"test-6-*\"],\"privileges\":[\"read\",\"write\"]}," +
+            " {\"names\":[\"test-3-*\",\"test-6-*\"],\"privileges\":[\"read\",\"write\"],\"allow_restricted_indices\": true}," +
-            " {\"names\":[\"test-3-*\",\"test-4-*\",\"test-5-*\"],\"privileges\":[\"read\"]," +
+            " {\"names\":[\"test-3-*\",\"test-4-*\",\"test-5-*\"],\"privileges\":[\"read\"],\"allow_restricted_indices\": false," +
             "  \"field_security\":[{\"grant\":[\"test-*\"]}]}," +
-            " {\"names\":[\"test-1-*\",\"test-9-*\"],\"privileges\":[\"all\"]}" +
+            " {\"names\":[\"test-1-*\",\"test-9-*\"],\"privileges\":[\"all\"],\"allow_restricted_indices\": true}" +
             "]," +
             "\"applications\":[" +
             " {\"application\":\"app-dne\",\"privileges\":[\"all\"],\"resources\":[\"*\"]}," +
@@ -80,12 +81,14 @@ public class GetUserPrivilegesResponseTests extends ESTestCase {
         assertThat(response.getIndicesPrivileges().size(), equalTo(7));
         assertThat(Iterables.get(response.getIndicesPrivileges(), 0).getIndices(), contains("test-1-*"));
         assertThat(Iterables.get(response.getIndicesPrivileges(), 0).getPrivileges(), contains("read"));
+        assertThat(Iterables.get(response.getIndicesPrivileges(), 0).allowRestrictedIndices(), equalTo(false));
         assertThat(Iterables.get(response.getIndicesPrivileges(), 0).getFieldSecurity(), emptyIterable());
         assertThat(Iterables.get(response.getIndicesPrivileges(), 0).getQueries(), emptyIterable());
 
         final UserIndicesPrivileges test4Privilege = Iterables.get(response.getIndicesPrivileges(), 1);
         assertThat(test4Privilege.getIndices(), contains("test-4-*"));
         assertThat(test4Privilege.getPrivileges(), contains("read"));
+        assertThat(test4Privilege.allowRestrictedIndices(), equalTo(true));
         assertThat(test4Privilege.getFieldSecurity(), iterableWithSize(1));
         final IndicesPrivileges.FieldSecurity test4FLS = test4Privilege.getFieldSecurity().iterator().next();
         assertThat(test4FLS.getGrantedFields(), contains("*"));
@@ -95,6 +98,7 @@ public class GetUserPrivilegesResponseTests extends ESTestCase {
         final UserIndicesPrivileges test2Privilege = Iterables.get(response.getIndicesPrivileges(), 3);
         assertThat(test2Privilege.getIndices(), contains("test-2-*"));
         assertThat(test2Privilege.getPrivileges(), contains("read"));
+        assertThat(test2Privilege.allowRestrictedIndices(), equalTo(false));
         assertThat(test2Privilege.getFieldSecurity(), iterableWithSize(2));
         final Iterator<IndicesPrivileges.FieldSecurity> test2FLSIter = test2Privilege.getFieldSecurity().iterator();
         final IndicesPrivileges.FieldSecurity test2FLS1 = test2FLSIter.next();
@@ -110,6 +114,7 @@ public class GetUserPrivilegesResponseTests extends ESTestCase {
 
         assertThat(Iterables.get(response.getIndicesPrivileges(), 6).getIndices(), contains("test-1-*", "test-9-*"));
         assertThat(Iterables.get(response.getIndicesPrivileges(), 6).getPrivileges(), contains("all"));
+        assertThat(Iterables.get(response.getIndicesPrivileges(), 6).allowRestrictedIndices(), equalTo(true));
         assertThat(Iterables.get(response.getIndicesPrivileges(), 6).getFieldSecurity(), emptyIterable());
         assertThat(Iterables.get(response.getIndicesPrivileges(), 6).getQueries(), emptyIterable());
 

client/rest-high-level/src/test/java/org/elasticsearch/client/security/HasPrivilegesRequestTests.java

@@ -41,8 +41,10 @@ public class HasPrivilegesRequestTests extends ESTestCase {
         final HasPrivilegesRequest request = new HasPrivilegesRequest(
             new LinkedHashSet<>(Arrays.asList("monitor", "manage_watcher", "manage_ml")),
             new LinkedHashSet<>(Arrays.asList(
-                IndicesPrivileges.builder().indices("index-001", "index-002").privileges("all").build(),
+                IndicesPrivileges.builder().indices("index-001", "index-002").privileges("all")
-                IndicesPrivileges.builder().indices("index-003").privileges("read").build()
+                    .allowRestrictedIndices(true).build(),
+                IndicesPrivileges.builder().indices("index-003").privileges("read")
+                    .build()
             )),
             new LinkedHashSet<>(Arrays.asList(
                 new ApplicationResourcePrivileges("myapp", Arrays.asList("read", "write"), Arrays.asList("*")),
@@ -56,10 +58,12 @@ public class HasPrivilegesRequestTests extends ESTestCase {
             " \"cluster\":[\"monitor\",\"manage_watcher\",\"manage_ml\"]," +
             " \"index\":[{" +
             "   \"names\":[\"index-001\",\"index-002\"]," +
-            "   \"privileges\":[\"all\"]" +
+            "   \"privileges\":[\"all\"]," +
+            "   \"allow_restricted_indices\":true" +
             "  },{" +
             "   \"names\":[\"index-003\"]," +
-            "   \"privileges\":[\"read\"]" +
+            "   \"privileges\":[\"read\"]," +
+            "   \"allow_restricted_indices\":false" +
             " }]," +
             " \"application\":[{" +
             "   \"application\":\"myapp\"," +
@@ -81,6 +85,7 @@ public class HasPrivilegesRequestTests extends ESTestCase {
             () -> IndicesPrivileges.builder()
                 .indices(generateRandomStringArray(5, 12, false, false))
                 .privileges(generateRandomStringArray(3, 8, false, false))
+                .allowRestrictedIndices(randomBoolean())
                 .build()));
         final Set<ApplicationResourcePrivileges> application = Sets.newHashSet(randomArray(1, 5, ApplicationResourcePrivileges[]::new,
             () -> new ApplicationResourcePrivileges(

client/rest-high-level/src/test/java/org/elasticsearch/client/security/user/privileges/IndicesPrivilegesTests.java

@@ -37,6 +37,7 @@ public class IndicesPrivilegesTests extends AbstractXContentTestCase<IndicesPriv
         final IndicesPrivileges.Builder indicesPrivilegesBuilder = IndicesPrivileges.builder()
             .indices(generateRandomStringArray(4, 4, false, false))
             .privileges(randomSubsetOf(randomIntBetween(1, 4), Role.IndexPrivilegeName.ALL_ARRAY))
+            .allowRestrictedIndices(randomBoolean())
             .query(query);
         if (randomBoolean()) {
             final List<String> fields = Arrays.asList(generateRandomStringArray(4, 4, false));
@@ -49,7 +50,8 @@ public class IndicesPrivilegesTests extends AbstractXContentTestCase<IndicesPriv
     }
 
     public void testToXContentWithNullFieldSecurity() {
-        final IndicesPrivileges privileges = IndicesPrivileges.builder().indices("abc").privileges("all").build();
+        final IndicesPrivileges privileges = IndicesPrivileges.builder().indices("abc").privileges("all")
+                .allowRestrictedIndices(randomBoolean()).build();
         final String json = Strings.toString(privileges);
         assertThat(json, not(containsString("field_security")));
     }
@@ -60,6 +62,7 @@ public class IndicesPrivilegesTests extends AbstractXContentTestCase<IndicesPriv
             .privileges("all")
             .grantedFields(Collections.emptyList())
             .deniedFields(Collections.emptyList())
+            .allowRestrictedIndices(randomBoolean())
             .build();
         final String json = Strings.toString(privileges);
         assertThat(json, containsString("field_security"));
@@ -71,6 +74,7 @@ public class IndicesPrivilegesTests extends AbstractXContentTestCase<IndicesPriv
             .indices("abc")
             .privileges("all")
             .deniedFields("secret.*")
+            .allowRestrictedIndices(randomBoolean())
             .build();
         final String json = Strings.toString(privileges);
         assertThat(json, containsString("field_security"));

x-pack/docs/en/rest-api/security/get-roles.asciidoc

@@ -57,9 +57,10 @@ role. If the role is not defined in the native realm, the request returns 404.
       {
         "names" : [ "index1", "index2" ],
         "privileges" : [ "all" ],
-        "field_security" : { 
+        "allow_restricted_indices" : false,
+        "field_security" : {
           "grant" : [ "title", "body" ]}
-      }   
+      }
     ],
     "applications" : [ ],
     "run_as" : [ "other_user" ],
@@ -69,7 +70,7 @@ role. If the role is not defined in the native realm, the request returns 404.
     "transient_metadata": {
       "enabled": true
     }
-  }  
+  }
 }
 --------------------------------------------------
 // TESTRESPONSE

x-pack/docs/en/rest-api/security/has-privileges.asciidoc

@@ -29,6 +29,11 @@ privilege is assigned to the user.
 
 `index`::
 `names`::: (list) A list of indices.
+`allow_restricted_indices`::: (boolean) If `names` contains internal restricted
+that also have to be covered by the has-privilege check, then this has to be
+set to `true`. By default this is `false` because restricted indices should
+generaly not be "visible" to APIs. For most use cases it is safe to ignore
+this parameter.
 `privileges`::: (list) A list of the privileges that you want to check for the
 specified indices.
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequest.java

@@ -92,13 +92,14 @@ public class PutRoleRequest extends ActionRequest implements WriteRequest<PutRol
     }
 
     public void addIndex(String[] indices, String[] privileges, String[] grantedFields, String[] deniedFields,
-                         @Nullable BytesReference query) {
+                         @Nullable BytesReference query, boolean allowRestrictedIndices) {
         this.indicesPrivileges.add(RoleDescriptor.IndicesPrivileges.builder()
                 .indices(indices)
                 .privileges(privileges)
                 .grantedFields(grantedFields)
                 .deniedFields(deniedFields)
                 .query(query)
+                .allowRestrictedIndices(allowRestrictedIndices)
                 .build());
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestBuilder.java

@@ -64,8 +64,8 @@ public class PutRoleRequestBuilder extends ActionRequestBuilder<PutRoleRequest,
     }
 
     public PutRoleRequestBuilder addIndices(String[] indices, String[] privileges, String[] grantedFields, String[] deniedFields,
-                                            @Nullable BytesReference query) {
+                                            @Nullable BytesReference query, boolean allowRestrictedIndices) {
-        request.addIndex(indices, privileges, grantedFields, deniedFields, query);
+        request.addIndex(indices, privileges, grantedFields, deniedFields, query, allowRestrictedIndices);
         return this;
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponse.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.core.security.action.user;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -121,14 +122,17 @@ public final class GetUserPrivilegesResponse extends ActionResponse {
         private final Set<String> privileges;
         private final Set<FieldPermissionsDefinition.FieldGrantExcludeGroup> fieldSecurity;
         private final Set<BytesReference> queries;
+        private final boolean allowRestrictedIndices;
 
         public Indices(Collection<String> indices, Collection<String> privileges,
-                       Set<FieldPermissionsDefinition.FieldGrantExcludeGroup> fieldSecurity, Set<BytesReference> queries) {
+                Set<FieldPermissionsDefinition.FieldGrantExcludeGroup> fieldSecurity, Set<BytesReference> queries,
+                boolean allowRestrictedIndices) {
             // The use of TreeSet is to provide a consistent order that can be relied upon in tests
             this.indices = Collections.unmodifiableSet(new TreeSet<>(Objects.requireNonNull(indices)));
             this.privileges = Collections.unmodifiableSet(new TreeSet<>(Objects.requireNonNull(privileges)));
             this.fieldSecurity = Collections.unmodifiableSet(Objects.requireNonNull(fieldSecurity));
             this.queries = Collections.unmodifiableSet(Objects.requireNonNull(queries));
+            this.allowRestrictedIndices = allowRestrictedIndices;
         }
 
         public Indices(StreamInput in) throws IOException {
@@ -141,6 +145,11 @@ public final class GetUserPrivilegesResponse extends ActionResponse {
                 return new FieldPermissionsDefinition.FieldGrantExcludeGroup(grant, exclude);
             }));
             queries = Collections.unmodifiableSet(in.readSet(StreamInput::readBytesReference));
+            if (in.getVersion().onOrAfter(Version.V_7_0_0)) {
+                this.allowRestrictedIndices = in.readBoolean();
+            } else {
+                this.allowRestrictedIndices = false;
+            }
         }
 
         public Set<String> getIndices() {
@@ -159,11 +168,16 @@ public final class GetUserPrivilegesResponse extends ActionResponse {
             return queries;
         }
 
+        public boolean allowRestrictedIndices() {
+            return allowRestrictedIndices;
+        }
+
         @Override
         public String toString() {
             StringBuilder sb = new StringBuilder(getClass().getSimpleName())
                 .append("[")
                 .append("indices=[").append(Strings.collectionToCommaDelimitedString(indices))
+                .append("], allow_restricted_indices=[").append(allowRestrictedIndices)
                 .append("], privileges=[").append(Strings.collectionToCommaDelimitedString(privileges))
                 .append("]");
             if (fieldSecurity.isEmpty() == false) {
@@ -188,12 +202,13 @@ public final class GetUserPrivilegesResponse extends ActionResponse {
             return this.indices.equals(that.indices)
                 && this.privileges.equals(that.privileges)
                 && this.fieldSecurity.equals(that.fieldSecurity)
-                && this.queries.equals(that.queries);
+                && this.queries.equals(that.queries)
+                && this.allowRestrictedIndices == that.allowRestrictedIndices;
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(indices, privileges, fieldSecurity, queries);
+            return Objects.hash(indices, privileges, fieldSecurity, queries, allowRestrictedIndices);
         }
 
         @Override
@@ -222,6 +237,7 @@ public final class GetUserPrivilegesResponse extends ActionResponse {
                 }
                 builder.endArray();
             }
+            builder.field(RoleDescriptor.Fields.ALLOW_RESTRICTED_INDICES.getPreferredName(), allowRestrictedIndices);
             return builder.endObject();
         }
 
@@ -238,6 +254,9 @@ public final class GetUserPrivilegesResponse extends ActionResponse {
                 output.writeOptionalStringArray(fields.getExcludedFields());
             });
             out.writeCollection(queries, StreamOutput::writeBytesReference);
+            if (out.getVersion().onOrAfter(Version.V_7_0_0)) {
+                out.writeBoolean(allowRestrictedIndices);
+            }
         }
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java

@@ -428,6 +428,7 @@ public class RoleDescriptor implements ToXContentObject {
         String[] privileges = null;
         String[] grantedFields = null;
         String[] deniedFields = null;
+        boolean allowRestrictedIndices = false;
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
@@ -444,6 +445,13 @@ public class RoleDescriptor implements ToXContentObject {
                     throw new ElasticsearchParseException("failed to parse indices privileges for role [{}]. expected field [{}] " +
                             "value to be a string or an array of strings, but found [{}] instead", roleName, currentFieldName, token);
                 }
+            } else if (Fields.ALLOW_RESTRICTED_INDICES.match(currentFieldName, parser.getDeprecationHandler())) {
+                if (token == XContentParser.Token.VALUE_BOOLEAN) {
+                    allowRestrictedIndices = parser.booleanValue();
+                } else {
+                    throw new ElasticsearchParseException("failed to parse indices privileges for role [{}]. expected field [{}] " +
+                            "value to be a boolean, but found [{}] instead", roleName, currentFieldName, token);
+                }
             } else if (Fields.QUERY.match(currentFieldName, parser.getDeprecationHandler())) {
                 if (token == XContentParser.Token.START_OBJECT) {
                     XContentBuilder builder = JsonXContent.contentBuilder();
@@ -543,6 +551,7 @@ public class RoleDescriptor implements ToXContentObject {
                 .grantedFields(grantedFields)
                 .deniedFields(deniedFields)
                 .query(query)
+                .allowRestrictedIndices(allowRestrictedIndices)
                 .build();
     }
 
@@ -590,6 +599,10 @@ public class RoleDescriptor implements ToXContentObject {
         private String[] grantedFields = null;
         private String[] deniedFields = null;
         private BytesReference query;
+        // by default certain restricted indices are exempted when granting privileges, as they should generally be hidden for ordinary
+        // users. Setting this flag eliminates this special status, and any index name pattern in the permission will cover restricted
+        // indices as well.
+        private boolean allowRestrictedIndices = false;
 
         private IndicesPrivileges() {
         }
@@ -600,6 +613,11 @@ public class RoleDescriptor implements ToXContentObject {
             this.deniedFields = in.readOptionalStringArray();
             this.privileges = in.readStringArray();
             this.query = in.readOptionalBytesReference();
+            if (in.getVersion().onOrAfter(Version.V_7_0_0)) {
+                allowRestrictedIndices = in.readBoolean();
+            } else {
+                allowRestrictedIndices = false;
+            }
         }
 
         @Override
@@ -609,6 +627,9 @@ public class RoleDescriptor implements ToXContentObject {
             out.writeOptionalStringArray(deniedFields);
             out.writeStringArray(privileges);
             out.writeOptionalBytesReference(query);
+            if (out.getVersion().onOrAfter(Version.V_7_0_0)) {
+                out.writeBoolean(allowRestrictedIndices);
+            }
         }
 
         public static Builder builder() {
@@ -646,6 +667,10 @@ public class RoleDescriptor implements ToXContentObject {
             return hasDeniedFields() || hasGrantedFields();
         }
 
+        public boolean allowRestrictedIndices() {
+            return allowRestrictedIndices;
+        }
+
         private boolean hasDeniedFields() {
             return deniedFields != null && deniedFields.length > 0;
         }
@@ -666,6 +691,7 @@ public class RoleDescriptor implements ToXContentObject {
         public String toString() {
             StringBuilder sb = new StringBuilder("IndicesPrivileges[");
             sb.append("indices=[").append(Strings.arrayToCommaDelimitedString(indices));
+            sb.append("], allowRestrictedIndices=[").append(allowRestrictedIndices);
             sb.append("], privileges=[").append(Strings.arrayToCommaDelimitedString(privileges));
             sb.append("], ");
             if (grantedFields != null || deniedFields != null) {
@@ -702,6 +728,7 @@ public class RoleDescriptor implements ToXContentObject {
             IndicesPrivileges that = (IndicesPrivileges) o;
 
             if (!Arrays.equals(indices, that.indices)) return false;
+            if (allowRestrictedIndices != that.allowRestrictedIndices) return false;
             if (!Arrays.equals(privileges, that.privileges)) return false;
             if (!Arrays.equals(grantedFields, that.grantedFields)) return false;
             if (!Arrays.equals(deniedFields, that.deniedFields)) return false;
@@ -711,6 +738,7 @@ public class RoleDescriptor implements ToXContentObject {
         @Override
         public int hashCode() {
             int result = Arrays.hashCode(indices);
+            result = 31 * result + (allowRestrictedIndices ? 1 : 0);
             result = 31 * result + Arrays.hashCode(privileges);
             result = 31 * result + Arrays.hashCode(grantedFields);
             result = 31 * result + Arrays.hashCode(deniedFields);
@@ -736,6 +764,7 @@ public class RoleDescriptor implements ToXContentObject {
             if (query != null) {
                 builder.field("query", query.utf8ToString());
             }
+            builder.field(RoleDescriptor.Fields.ALLOW_RESTRICTED_INDICES.getPreferredName(), allowRestrictedIndices);
             return builder.endObject();
         }
 
@@ -774,6 +803,11 @@ public class RoleDescriptor implements ToXContentObject {
                 return query(query == null ? null : new BytesArray(query));
             }
 
+            public Builder allowRestrictedIndices(boolean allow) {
+                indicesPrivileges.allowRestrictedIndices = allow;
+                return this;
+            }
+
             public Builder query(@Nullable BytesReference query) {
                 if (query == null) {
                     indicesPrivileges.query = null;
@@ -954,6 +988,7 @@ public class RoleDescriptor implements ToXContentObject {
         ParseField APPLICATIONS = new ParseField("applications");
         ParseField RUN_AS = new ParseField("run_as");
         ParseField NAMES = new ParseField("names");
+        ParseField ALLOW_RESTRICTED_INDICES = new ParseField("allow_restricted_indices");
         ParseField RESOURCES = new ParseField("resources");
         ParseField QUERY = new ParseField("query");
         ParseField PRIVILEGES = new ParseField("privileges");

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java

@@ -17,20 +17,21 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
+import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames;
 import org.elasticsearch.xpack.core.security.support.Automatons;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.function.Function;
+import java.util.concurrent.ConcurrentMap;
 import java.util.function.Predicate;
 
 import static java.util.Collections.unmodifiableMap;
@@ -40,30 +41,19 @@ import static java.util.Collections.unmodifiableSet;
  * A permission that is based on privileges for index related actions executed
  * on specific indices
  */
-public final class IndicesPermission implements Iterable<IndicesPermission.Group> {
+public final class IndicesPermission {
 
     public static final IndicesPermission NONE = new IndicesPermission();
 
-    private final Function<String, Predicate<String>> loadingFunction;
+    private final ConcurrentMap<String, Predicate<String>> allowedIndicesMatchersForAction = new ConcurrentHashMap<>();
-
-    private final ConcurrentHashMap<String, Predicate<String>> allowedIndicesMatchersForAction = new ConcurrentHashMap<>();
 
     private final Group[] groups;
 
     public IndicesPermission(Group... groups) {
         this.groups = groups;
-        loadingFunction = (action) -> {
-            List<String> indices = new ArrayList<>();
-            for (Group group : groups) {
-                if (group.actionMatcher.test(action)) {
-                    indices.addAll(Arrays.asList(group.indices));
-                }
-            }
-            return indexMatcher(indices);
-        };
     }
 
-    static Predicate<String> indexMatcher(List<String> indices) {
+    static Predicate<String> indexMatcher(Collection<String> indices) {
         Set<String> exactMatch = new HashSet<>();
         List<String> nonExactMatch = new ArrayList<>();
         for (String indexPattern : indices) {
@@ -106,11 +96,6 @@ public final class IndicesPermission implements Iterable<IndicesPermission.Group
         }
     }
 
-    @Override
-    public Iterator<Group> iterator() {
-        return Arrays.asList(groups).iterator();
-    }
-
     public Group[] groups() {
         return groups;
     }
@@ -120,7 +105,7 @@ public final class IndicesPermission implements Iterable<IndicesPermission.Group
      * has the privilege for executing the given action on.
      */
     public Predicate<String> allowedIndicesMatcher(String action) {
-        return allowedIndicesMatchersForAction.computeIfAbsent(action, loadingFunction);
+        return allowedIndicesMatchersForAction.computeIfAbsent(action, a -> Group.buildIndexMatcherPredicateForAction(a, groups));
     }
 
     /**
@@ -232,15 +217,15 @@ public final class IndicesPermission implements Iterable<IndicesPermission.Group
         private final Predicate<String> actionMatcher;
         private final String[] indices;
         private final Predicate<String> indexNameMatcher;
-
-        public FieldPermissions getFieldPermissions() {
-            return fieldPermissions;
-        }
-
         private final FieldPermissions fieldPermissions;
         private final Set<BytesReference> query;
+        // by default certain restricted indices are exempted when granting privileges, as they should generally be hidden for ordinary
+        // users. Setting this flag true eliminates the special status for the purpose of this permission - restricted indices still have
+        // to be covered by the the "indices"
+        private final boolean allowRestrictedIndices;
 
-        public Group(IndexPrivilege privilege, FieldPermissions fieldPermissions, @Nullable Set<BytesReference> query, String... indices) {
+        public Group(IndexPrivilege privilege, FieldPermissions fieldPermissions, @Nullable Set<BytesReference> query,
+                boolean allowRestrictedIndices, String... indices) {
             assert indices.length != 0;
             this.privilege = privilege;
             this.actionMatcher = privilege.predicate();
@@ -248,6 +233,7 @@ public final class IndicesPermission implements Iterable<IndicesPermission.Group
             this.indexNameMatcher = indexMatcher(Arrays.asList(indices));
             this.fieldPermissions = Objects.requireNonNull(fieldPermissions);
             this.query = query;
+            this.allowRestrictedIndices = allowRestrictedIndices;
         }
 
         public IndexPrivilege privilege() {
@@ -263,18 +249,66 @@ public final class IndicesPermission implements Iterable<IndicesPermission.Group
             return query;
         }
 
+        public FieldPermissions getFieldPermissions() {
+            return fieldPermissions;
+        }
+
         private boolean check(String action) {
             return actionMatcher.test(action);
         }
 
         private boolean check(String action, String index) {
             assert index != null;
-            return check(action) && indexNameMatcher.test(index);
+            return check(action) && (indexNameMatcher.test(index)
+                    && (allowRestrictedIndices
+                            // all good if it is not restricted
+                            || (false == RestrictedIndicesNames.NAMES_SET.contains(index))
+                            // allow monitor as a special case, even for restricted
+                            || IndexPrivilege.MONITOR.predicate().test(action)));
         }
 
         boolean hasQuery() {
             return query != null;
         }
+
+        public boolean allowRestrictedIndices() {
+            return allowRestrictedIndices;
+        }
+
+        public static Automaton buildIndexMatcherAutomaton(boolean allowRestrictedIndices, String... indices) {
+            final Automaton indicesAutomaton = Automatons.patterns(indices);
+            if (allowRestrictedIndices) {
+                return indicesAutomaton;
+            } else {
+                return Automatons.minusAndMinimize(indicesAutomaton, RestrictedIndicesNames.NAMES_AUTOMATON);
+            }
+        }
+
+        private static Predicate<String> buildIndexMatcherPredicateForAction(String action, Group... groups) {
+            final Set<String> ordinaryIndices = new HashSet<>();
+            final Set<String> restrictedIndices = new HashSet<>();
+            for (final Group group : groups) {
+                if (group.actionMatcher.test(action)) {
+                    if (group.allowRestrictedIndices) {
+                        restrictedIndices.addAll(Arrays.asList(group.indices()));
+                    } else {
+                        ordinaryIndices.addAll(Arrays.asList(group.indices()));
+                    }
+                }
+            }
+            final Predicate<String> predicate;
+            if (restrictedIndices.isEmpty()) {
+                predicate = indexMatcher(ordinaryIndices)
+                    .and(index -> false == RestrictedIndicesNames.NAMES_SET.contains(index));
+            } else if (ordinaryIndices.isEmpty()) {
+                predicate = indexMatcher(restrictedIndices);
+            } else {
+                predicate = indexMatcher(restrictedIndices)
+                    .or(indexMatcher(ordinaryIndices)
+                         .and(index -> false == RestrictedIndicesNames.NAMES_SET.contains(index)));
+            }
+            return predicate;
+        }
     }
 
     private static class DocumentLevelPermissions {

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java

@@ -155,12 +155,13 @@ public final class Role {
         }
 
         public Builder add(IndexPrivilege privilege, String... indices) {
-            groups.add(new IndicesPermission.Group(privilege, FieldPermissions.DEFAULT, null, indices));
+            groups.add(new IndicesPermission.Group(privilege, FieldPermissions.DEFAULT, null, false, indices));
             return this;
         }
 
-        public Builder add(FieldPermissions fieldPermissions, Set<BytesReference> query, IndexPrivilege privilege, String... indices) {
+        public Builder add(FieldPermissions fieldPermissions, Set<BytesReference> query, IndexPrivilege privilege,
-            groups.add(new IndicesPermission.Group(privilege, fieldPermissions, query, indices));
+                boolean allowRestrictedIndices, String... indices) {
+            groups.add(new IndicesPermission.Group(privilege, fieldPermissions, query, allowRestrictedIndices, indices));
             return this;
         }
 
@@ -189,11 +190,8 @@ public final class Role {
                         new FieldPermissionsDefinition(privilege.getGrantedFields(), privilege.getDeniedFields()));
                 }
                 final Set<BytesReference> query = privilege.getQuery() == null ? null : Collections.singleton(privilege.getQuery());
-                list.add(new IndicesPermission.Group(IndexPrivilege.get(Sets.newHashSet(privilege.getPrivileges())),
+                list.add(new IndicesPermission.Group(IndexPrivilege.get(Sets.newHashSet(privilege.getPrivileges())), fieldPermissions,
-                    fieldPermissions,
+                        query, privilege.allowRestrictedIndices(), privilege.getIndices()));
-                    query,
-                    privilege.getIndices()));
-
             }
             return list;
         }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java

@@ -32,7 +32,7 @@ public class ReservedRolesStore implements BiConsumer<Set<String>, ActionListene
     public static final RoleDescriptor SUPERUSER_ROLE_DESCRIPTOR = new RoleDescriptor("superuser",
             new String[] { "all" },
             new RoleDescriptor.IndicesPrivileges[] {
-                    RoleDescriptor.IndicesPrivileges.builder().indices("*").privileges("all").build()},
+                    RoleDescriptor.IndicesPrivileges.builder().indices("*").privileges("all").allowRestrictedIndices(true).build()},
             new RoleDescriptor.ApplicationResourcePrivileges[] {
                 RoleDescriptor.ApplicationResourcePrivileges.builder().application("*").privileges("*").resources("*").build()
             },
@@ -43,11 +43,7 @@ public class ReservedRolesStore implements BiConsumer<Set<String>, ActionListene
 
     private static Map<String, RoleDescriptor> initializeReservedRoles() {
         return MapBuilder.<String, RoleDescriptor>newMapBuilder()
-                .put("superuser", new RoleDescriptor("superuser", new String[] { "all" },
+                .put("superuser", SUPERUSER_ROLE_DESCRIPTOR)
-                        new RoleDescriptor.IndicesPrivileges[] {
-                                RoleDescriptor.IndicesPrivileges.builder().indices("*").privileges("all").build()},
-                        new String[] { "*" },
-                        MetadataUtils.DEFAULT_RESERVED_METADATA))
                 .put("transport_client", new RoleDescriptor("transport_client", new String[] { "transport_client" }, null, null,
                         MetadataUtils.DEFAULT_RESERVED_METADATA))
                 .put("kibana_user", new RoleDescriptor("kibana_user", null, new RoleDescriptor.IndicesPrivileges[] {
@@ -82,8 +78,10 @@ public class ReservedRolesStore implements BiConsumer<Set<String>, ActionListene
                             "monitor"
                         },
                         new RoleDescriptor.IndicesPrivileges[] {
-                            RoleDescriptor.IndicesPrivileges.builder().indices("*").privileges("monitor").build(),
+                            RoleDescriptor.IndicesPrivileges.builder()
-                            RoleDescriptor.IndicesPrivileges.builder().indices(".kibana*").privileges("read").build()
+                                .indices("*").privileges("monitor").allowRestrictedIndices(true).build(),
+                            RoleDescriptor.IndicesPrivileges.builder()
+                                .indices(".kibana*").privileges("read").build()
                         },
                         null,
                         null,

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/index/RestrictedIndicesNames.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.security.index;
+
+import org.apache.lucene.util.automaton.Automaton;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.xpack.core.security.support.Automatons;
+import org.elasticsearch.xpack.core.upgrade.IndexUpgradeCheckVersion;
+
+import java.util.Collections;
+import java.util.Set;
+
+public final class RestrictedIndicesNames {
+    public static final String AUDIT_INDEX_NAME_PREFIX = ".security_audit_log";
+    public static final String INTERNAL_SECURITY_INDEX = ".security-" + IndexUpgradeCheckVersion.UPRADE_VERSION;
+    public static final String SECURITY_INDEX_NAME = ".security";
+
+    public static final Set<String> NAMES_SET = Collections.unmodifiableSet(Sets.newHashSet(SECURITY_INDEX_NAME, INTERNAL_SECURITY_INDEX));
+    public static final Automaton NAMES_AUTOMATON = Automatons.patterns(NAMES_SET);
+
+    private RestrictedIndicesNames() {
+    }
+}

x-pack/plugin/core/src/main/resources/security-index-template.json

@@ -88,6 +88,9 @@
             },
             "query" : {
               "type" : "keyword"
+            },
+            "allow_restricted_indices" : {
+              "type" : "boolean"
             }
           }
         },

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestTests.java

@@ -20,6 +20,7 @@ import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.xpack.core.XPackClientPlugin;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.ApplicationResourcePrivileges;
 import org.elasticsearch.xpack.core.security.authz.privilege.ConditionalClusterPrivileges;
 
@@ -89,7 +90,7 @@ public class PutRoleRequestTests extends ESTestCase {
 
         assertThat(copy.name(), equalTo(original.name()));
         assertThat(copy.cluster(), equalTo(original.cluster()));
-        assertThat(copy.indices(), equalTo(original.indices()));
+        assertIndicesSerializedRestricted(copy.indices(), original.indices());
         assertThat(copy.runAs(), equalTo(original.runAs()));
         assertThat(copy.metadata(), equalTo(original.metadata()));
         assertThat(copy.getRefreshPolicy(), equalTo(original.getRefreshPolicy()));
@@ -98,6 +99,18 @@ public class PutRoleRequestTests extends ESTestCase {
         assertThat(copy.conditionalClusterPrivileges(), arrayWithSize(0));
     }
 
+    private void assertIndicesSerializedRestricted(RoleDescriptor.IndicesPrivileges[] copy, RoleDescriptor.IndicesPrivileges[] original) {
+        assertThat(copy.length, equalTo(original.length));
+        for (int i = 0; i < copy.length; i++) {
+            assertThat(copy[i].allowRestrictedIndices(), equalTo(false));
+            assertThat(copy[i].getIndices(), equalTo(original[i].getIndices()));
+            assertThat(copy[i].getPrivileges(), equalTo(original[i].getPrivileges()));
+            assertThat(copy[i].getDeniedFields(), equalTo(original[i].getDeniedFields()));
+            assertThat(copy[i].getGrantedFields(), equalTo(original[i].getGrantedFields()));
+            assertThat(copy[i].getQuery(), equalTo(original[i].getQuery()));
+        }
+    }
+
     private void assertSuccessfulValidation(PutRoleRequest request) {
         final ActionRequestValidationException exception = request.validate();
         assertThat(exception, nullValue());
@@ -135,7 +148,8 @@ public class PutRoleRequestTests extends ESTestCase {
                 randomSubsetOf(randomIntBetween(1, 2), "read", "write", "index", "all").toArray(Strings.EMPTY_ARRAY),
                 generateRandomStringArray(randomIntBetween(1, 3), randomIntBetween(3, 8), true),
                 generateRandomStringArray(randomIntBetween(1, 3), randomIntBetween(3, 8), true),
-                null
+                null,
+                randomBoolean()
             );
         }
 

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/user/GetUserPrivilegesResponseTests.java

@@ -77,8 +77,9 @@ public class GetUserPrivilegesResponseTests extends ESTestCase {
                     final Set<String> cluster = maybeMutate(random, 0, original.getClusterPrivileges(), () -> randomAlphaOfLength(5));
                     final Set<ConditionalClusterPrivilege> conditionalCluster = maybeMutate(random, 1,
                         original.getConditionalClusterPrivileges(), () -> new ManageApplicationPrivileges(randomStringSet(3)));
-                    final Set<GetUserPrivilegesResponse.Indices> index = maybeMutate(random, 2, original.getIndexPrivileges(),
+                        final Set<GetUserPrivilegesResponse.Indices> index = maybeMutate(random, 2, original.getIndexPrivileges(),
-                        () -> new GetUserPrivilegesResponse.Indices(randomStringSet(1), randomStringSet(1), emptySet(), emptySet()));
+                                () -> new GetUserPrivilegesResponse.Indices(randomStringSet(1), randomStringSet(1), emptySet(), emptySet(),
+                                        randomBoolean()));
                     final Set<ApplicationResourcePrivileges> application = maybeMutate(random, 3, original.getApplicationPrivileges(),
                         () -> ApplicationResourcePrivileges.builder().resources(generateRandomStringArray(3, 3, false, false))
                             .application(randomAlphaOfLength(5)).privileges(generateRandomStringArray(3, 5, false, false)).build());
@@ -110,7 +111,7 @@ public class GetUserPrivilegesResponseTests extends ESTestCase {
             () -> new GetUserPrivilegesResponse.Indices(randomStringSet(6), randomStringSet(8),
                 Sets.newHashSet(randomArray(3, FieldGrantExcludeGroup[]::new, () -> new FieldGrantExcludeGroup(
                     generateRandomStringArray(3, 5, false, false), generateRandomStringArray(3, 5, false, false)))),
-                randomStringSet(3).stream().map(BytesArray::new).collect(Collectors.toSet())
+                randomStringSet(3).stream().map(BytesArray::new).collect(Collectors.toSet()), randomBoolean()
             ))
         );
         final Set<ApplicationResourcePrivileges> application = Sets.newHashSet(randomArray(5, ApplicationResourcePrivileges[]::new,

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java

@@ -15,10 +15,15 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
 import org.elasticsearch.action.admin.indices.get.GetIndexAction;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryAction;
+import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsAction;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
+import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresAction;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
 import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateAction;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesAction;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction;
+import org.elasticsearch.action.admin.indices.upgrade.get.UpgradeStatusAction;
 import org.elasticsearch.action.bulk.BulkAction;
 import org.elasticsearch.action.delete.DeleteAction;
 import org.elasticsearch.action.get.GetAction;
@@ -94,6 +99,7 @@ import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCa
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
+import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames;
 import org.elasticsearch.xpack.core.security.user.APMSystemUser;
 import org.elasticsearch.xpack.core.security.user.BeatsSystemUser;
 import org.elasticsearch.xpack.core.security.user.LogstashSystemUser;
@@ -115,6 +121,7 @@ import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
 
@@ -177,6 +184,8 @@ public class ReservedRolesStoreTests extends ESTestCase {
                 is(false));
         assertThat(ingestAdminRole.indices().allowedIndicesMatcher(GetAction.NAME).test(randomAlphaOfLengthBetween(8, 24)),
                 is(false));
+
+        assertNoAccessAllowed(ingestAdminRole, RestrictedIndicesNames.NAMES_SET);
     }
 
     public void testKibanaSystemRole() {
@@ -276,6 +285,8 @@ public class ReservedRolesStoreTests extends ESTestCase {
         assertThat(kibanaRole.indices().allowedIndicesMatcher(MultiSearchAction.NAME).test(index), is(true));
         assertThat(kibanaRole.indices().allowedIndicesMatcher(GetAction.NAME).test(index), is(true));
         assertThat(kibanaRole.indices().allowedIndicesMatcher(READ_CROSS_CLUSTER_NAME).test(index), is(false));
+
+        assertNoAccessAllowed(kibanaRole, RestrictedIndicesNames.NAMES_SET);
     }
 
     public void testKibanaUserRole() {
@@ -325,6 +336,8 @@ public class ReservedRolesStoreTests extends ESTestCase {
         final String applicationWithRandomIndex = "kibana-.kibana_" + randomAlphaOfLengthBetween(8, 24);
         assertThat(kibanaUserRole.application().grants(new ApplicationPrivilege(applicationWithRandomIndex, "app-random-index", "all"),
             "*"), is(false));
+
+        assertNoAccessAllowed(kibanaUserRole, RestrictedIndicesNames.NAMES_SET);
     }
 
     public void testMonitoringUserRole() {
@@ -366,6 +379,8 @@ public class ReservedRolesStoreTests extends ESTestCase {
         assertThat(monitoringUserRole.indices().allowedIndicesMatcher(SearchAction.NAME).test(index), is(true));
         assertThat(monitoringUserRole.indices().allowedIndicesMatcher(GetAction.NAME).test(index), is(true));
         assertThat(monitoringUserRole.indices().allowedIndicesMatcher(READ_CROSS_CLUSTER_NAME).test(index), is(true));
+
+        assertNoAccessAllowed(monitoringUserRole, RestrictedIndicesNames.NAMES_SET);
     }
 
     public void testRemoteMonitoringAgentRole() {
@@ -424,6 +439,7 @@ public class ReservedRolesStoreTests extends ESTestCase {
         assertThat(remoteMonitoringAgentRole.indices().allowedIndicesMatcher(SearchAction.NAME).test(metricbeatIndex), is(false));
         assertThat(remoteMonitoringAgentRole.indices().allowedIndicesMatcher(GetAction.NAME).test(metricbeatIndex), is(false));
 
+        assertNoAccessAllowed(remoteMonitoringAgentRole, RestrictedIndicesNames.NAMES_SET);
     }
 
     public void testRemoteMonitoringCollectorRole() {
@@ -470,6 +486,29 @@ public class ReservedRolesStoreTests extends ESTestCase {
             assertThat(remoteMonitoringAgentRole.indices().allowedIndicesMatcher(GetAction.NAME).test(index), is(false));
             assertThat(remoteMonitoringAgentRole.indices().allowedIndicesMatcher(GetIndexAction.NAME).test(index), is(false));
         });
+
+        assertThat(remoteMonitoringAgentRole.indices().allowedIndicesMatcher(GetSettingsAction.NAME)
+                .test(randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME)), is(true));
+        assertThat(remoteMonitoringAgentRole.indices().allowedIndicesMatcher(IndicesShardStoresAction.NAME)
+                .test(randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME)), is(true));
+        assertThat(remoteMonitoringAgentRole.indices().allowedIndicesMatcher(UpgradeStatusAction.NAME)
+                .test(randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME)), is(true));
+        assertThat(remoteMonitoringAgentRole.indices().allowedIndicesMatcher(RecoveryAction.NAME)
+                .test(randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME)), is(true));
+        assertThat(remoteMonitoringAgentRole.indices().allowedIndicesMatcher(IndicesStatsAction.NAME)
+                .test(randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME)), is(true));
+        assertThat(remoteMonitoringAgentRole.indices().allowedIndicesMatcher(IndicesSegmentsAction.NAME)
+                .test(randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME)), is(true));
+        assertThat(remoteMonitoringAgentRole.indices().allowedIndicesMatcher(SearchAction.NAME)
+                .test(randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME)), is(false));
+        assertThat(remoteMonitoringAgentRole.indices().allowedIndicesMatcher(GetAction.NAME)
+                .test(randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME)), is(false));
+        assertThat(remoteMonitoringAgentRole.indices().allowedIndicesMatcher(DeleteAction.NAME)
+                .test(randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME)), is(false));
+        assertThat(remoteMonitoringAgentRole.indices().allowedIndicesMatcher(IndexAction.NAME)
+                .test(randomFrom(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME)), is(false));
+
+        assertNoAccessAllowed(remoteMonitoringAgentRole, RestrictedIndicesNames.NAMES_SET);
     }
 
     public void testReportingUserRole() {
@@ -508,6 +547,8 @@ public class ReservedRolesStoreTests extends ESTestCase {
         assertThat(reportingUserRole.indices().allowedIndicesMatcher(UpdateAction.NAME).test(index), is(false));
         assertThat(reportingUserRole.indices().allowedIndicesMatcher(DeleteAction.NAME).test(index), is(false));
         assertThat(reportingUserRole.indices().allowedIndicesMatcher(BulkAction.NAME).test(index), is(false));
+
+        assertNoAccessAllowed(reportingUserRole, RestrictedIndicesNames.NAMES_SET);
     }
 
     public void testKibanaDashboardOnlyUserRole() {
@@ -553,6 +594,8 @@ public class ReservedRolesStoreTests extends ESTestCase {
         final String applicationWithRandomIndex = "kibana-.kibana_" + randomAlphaOfLengthBetween(8, 24);
         assertThat(dashboardsOnlyUserRole.application().grants(
             new ApplicationPrivilege(applicationWithRandomIndex, "app-random-index", "all"), "*"), is(false));
+
+        assertNoAccessAllowed(dashboardsOnlyUserRole, RestrictedIndicesNames.NAMES_SET);
     }
 
     public void testSuperuserRole() {
@@ -583,6 +626,12 @@ public class ReservedRolesStoreTests extends ESTestCase {
                         .putAlias(new AliasMetaData.Builder("ab").build())
                         .putAlias(new AliasMetaData.Builder("ba").build())
                         .build(), true)
+                .put(new IndexMetaData.Builder(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX)
+                        .settings(indexSettings)
+                        .numberOfShards(1)
+                        .numberOfReplicas(0)
+                        .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_INDEX_NAME).build())
+                        .build(), true)
                 .build();
 
         FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY);
@@ -600,10 +649,19 @@ public class ReservedRolesStoreTests extends ESTestCase {
                 .authorize(UpdateSettingsAction.NAME, Sets.newHashSet("aaaaaa", "ba"), metaData, fieldPermissionsCache);
         assertThat(authzMap.get("aaaaaa").isGranted(), is(true));
         assertThat(authzMap.get("b").isGranted(), is(true));
+        authzMap = superuserRole.indices().authorize(randomFrom(IndexAction.NAME, DeleteIndexAction.NAME, SearchAction.NAME),
+                Sets.newHashSet(RestrictedIndicesNames.SECURITY_INDEX_NAME), metaData, fieldPermissionsCache);
+        assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_INDEX_NAME).isGranted(), is(true));
+        assertThat(authzMap.get(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX).isGranted(), is(true));
         assertTrue(superuserRole.indices().check(SearchAction.NAME));
         assertFalse(superuserRole.indices().check("unknown"));
 
         assertThat(superuserRole.runAs().check(randomAlphaOfLengthBetween(1, 30)), is(true));
+
+        assertThat(superuserRole.indices().allowedIndicesMatcher(randomFrom(IndexAction.NAME, DeleteIndexAction.NAME, SearchAction.NAME))
+                .test(RestrictedIndicesNames.SECURITY_INDEX_NAME), is(true));
+        assertThat(superuserRole.indices().allowedIndicesMatcher(randomFrom(IndexAction.NAME, DeleteIndexAction.NAME, SearchAction.NAME))
+                .test(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX), is(true));
     }
 
     public void testLogstashSystemRole() {
@@ -628,6 +686,8 @@ public class ReservedRolesStoreTests extends ESTestCase {
         assertThat(logstashSystemRole.indices().allowedIndicesMatcher(IndexAction.NAME).test(".reporting"), is(false));
         assertThat(logstashSystemRole.indices().allowedIndicesMatcher("indices:foo").test(randomAlphaOfLengthBetween(8, 24)),
                 is(false));
+
+        assertNoAccessAllowed(logstashSystemRole, RestrictedIndicesNames.NAMES_SET);
     }
 
     public void testBeatsAdminRole() {
@@ -664,6 +724,8 @@ public class ReservedRolesStoreTests extends ESTestCase {
         assertThat(beatsAdminRole.indices().allowedIndicesMatcher(SearchAction.NAME).test(index), is(true));
         assertThat(beatsAdminRole.indices().allowedIndicesMatcher(MultiSearchAction.NAME).test(index), is(true));
         assertThat(beatsAdminRole.indices().allowedIndicesMatcher(GetAction.NAME).test(index), is(true));
+
+        assertNoAccessAllowed(beatsAdminRole, RestrictedIndicesNames.NAMES_SET);
     }
 
     public void testBeatsSystemRole() {
@@ -688,6 +750,8 @@ public class ReservedRolesStoreTests extends ESTestCase {
         assertThat(logstashSystemRole.indices().allowedIndicesMatcher(IndexAction.NAME).test(".reporting"), is(false));
         assertThat(logstashSystemRole.indices().allowedIndicesMatcher("indices:foo").test(randomAlphaOfLengthBetween(8, 24)),
                 is(false));
+
+        assertNoAccessAllowed(logstashSystemRole, RestrictedIndicesNames.NAMES_SET);
     }
 
     public void testAPMSystemRole() {
@@ -712,6 +776,8 @@ public class ReservedRolesStoreTests extends ESTestCase {
         assertThat(APMSystemRole.indices().allowedIndicesMatcher(IndexAction.NAME).test(".reporting"), is(false));
         assertThat(APMSystemRole.indices().allowedIndicesMatcher("indices:foo").test(randomAlphaOfLengthBetween(8, 24)),
                 is(false));
+
+        assertNoAccessAllowed(APMSystemRole, RestrictedIndicesNames.NAMES_SET);
     }
 
     public void testMachineLearningAdminRole() {
@@ -764,6 +830,8 @@ public class ReservedRolesStoreTests extends ESTestCase {
         assertOnlyReadAllowed(role, AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX);
         assertOnlyReadAllowed(role, AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT);
         assertOnlyReadAllowed(role, AuditorField.NOTIFICATIONS_INDEX);
+
+        assertNoAccessAllowed(role, RestrictedIndicesNames.NAMES_SET);
     }
 
     public void testMachineLearningUserRole() {
@@ -816,6 +884,8 @@ public class ReservedRolesStoreTests extends ESTestCase {
         assertNoAccessAllowed(role, AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX);
         assertOnlyReadAllowed(role, AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT);
         assertOnlyReadAllowed(role, AuditorField.NOTIFICATIONS_INDEX);
+
+        assertNoAccessAllowed(role, RestrictedIndicesNames.NAMES_SET);
     }
 
     public void testWatcherAdminRole() {
@@ -843,6 +913,8 @@ public class ReservedRolesStoreTests extends ESTestCase {
         for (String index : new String[]{ Watch.INDEX, historyIndex, TriggeredWatchStoreField.INDEX_NAME }) {
             assertOnlyReadAllowed(role, index);
         }
+
+        assertNoAccessAllowed(role, RestrictedIndicesNames.NAMES_SET);
     }
 
     public void testWatcherUserRole() {
@@ -871,6 +943,8 @@ public class ReservedRolesStoreTests extends ESTestCase {
         for (String index : new String[]{ Watch.INDEX, historyIndex }) {
             assertOnlyReadAllowed(role, index);
         }
+
+        assertNoAccessAllowed(role, RestrictedIndicesNames.NAMES_SET);
     }
 
     private void assertOnlyReadAllowed(Role role, String index) {
@@ -883,6 +957,14 @@ public class ReservedRolesStoreTests extends ESTestCase {
         assertThat(role.indices().allowedIndicesMatcher(UpdateAction.NAME).test(index), is(false));
         assertThat(role.indices().allowedIndicesMatcher(DeleteAction.NAME).test(index), is(false));
         assertThat(role.indices().allowedIndicesMatcher(BulkAction.NAME).test(index), is(false));
+
+        assertNoAccessAllowed(role, RestrictedIndicesNames.NAMES_SET);
+    }
+
+    private void assertNoAccessAllowed(Role role, Collection<String> indices) {
+        for (String index : indices) {
+            assertNoAccessAllowed(role, index);
+        }
     }
 
     private void assertNoAccessAllowed(Role role, String index) {

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUserPrivilegesAction.java

@@ -90,7 +90,7 @@ public class TransportGetUserPrivilegesAction extends HandledTransportAction<Get
         }
 
         final Set<GetUserPrivilegesResponse.Indices> indices = new LinkedHashSet<>();
-        for (IndicesPermission.Group group : userRole.indices()) {
+        for (IndicesPermission.Group group : userRole.indices().groups()) {
             final Set<BytesReference> queries = group.getQuery() == null ? Collections.emptySet() : group.getQuery();
             final Set<FieldPermissionsDefinition.FieldGrantExcludeGroup> fieldSecurity = group.getFieldPermissions().hasFieldLevelSecurity()
                 ? group.getFieldPermissions().getFieldPermissionsDefinition().getFieldGrantExcludeGroups() : Collections.emptySet();
@@ -98,7 +98,8 @@ public class TransportGetUserPrivilegesAction extends HandledTransportAction<Get
                 Arrays.asList(group.indices()),
                 group.privilege().name(),
                 fieldSecurity,
-                queries
+                queries,
+                group.allowRestrictedIndices()
             ));
         }
 

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java

@@ -122,7 +122,7 @@ public class TransportHasPrivilegesAction extends HandledTransportAction<HasPriv
                     privileges.putAll(existing.getPrivileges());
                 }
                 for (String privilege : check.getPrivileges()) {
-                    if (testIndexMatch(index, privilege, userRole, predicateCache)) {
+                    if (testIndexMatch(index, check.allowRestrictedIndices(), privilege, userRole, predicateCache)) {
                         logger.debug(() -> new ParameterizedMessage("Role [{}] has [{}] on index [{}]",
                             Strings.arrayToCommaDelimitedString(userRole.names()), privilege, index));
                         privileges.put(privilege, true);
@@ -171,16 +171,17 @@ public class TransportHasPrivilegesAction extends HandledTransportAction<HasPriv
         listener.onResponse(new HasPrivilegesResponse(request.username(), allMatch, cluster, indices.values(), privilegesByApplication));
     }
 
-    private boolean testIndexMatch(String checkIndex, String checkPrivilegeName, Role userRole,
+    private boolean testIndexMatch(String checkIndexPattern, boolean allowRestrictedIndices, String checkPrivilegeName, Role userRole,
-                                   Map<IndicesPermission.Group, Automaton> predicateCache) {
+            Map<IndicesPermission.Group, Automaton> predicateCache) {
         final IndexPrivilege checkPrivilege = IndexPrivilege.get(Collections.singleton(checkPrivilegeName));
 
-        final Automaton checkIndexAutomaton = Automatons.patterns(checkIndex);
+        final Automaton checkIndexAutomaton = IndicesPermission.Group.buildIndexMatcherAutomaton(allowRestrictedIndices, checkIndexPattern);
 
         List<Automaton> privilegeAutomatons = new ArrayList<>();
         for (IndicesPermission.Group group : userRole.indices().groups()) {
-            final Automaton groupIndexAutomaton = predicateCache.computeIfAbsent(group, g -> Automatons.patterns(g.indices()));
+            final Automaton groupIndexAutomaton = predicateCache.computeIfAbsent(group,
-            if (testIndex(checkIndexAutomaton, groupIndexAutomaton)) {
+                    g -> IndicesPermission.Group.buildIndexMatcherAutomaton(g.allowRestrictedIndices(), g.indices()));
+            if (Operations.subsetOf(checkIndexAutomaton, groupIndexAutomaton)) {
                 final IndexPrivilege rolePrivilege = group.privilege();
                 if (rolePrivilege.name().contains(checkPrivilegeName)) {
                     return true;
@@ -191,10 +192,6 @@ public class TransportHasPrivilegesAction extends HandledTransportAction<HasPriv
         return testPrivilege(checkPrivilege, Automatons.unionAndMinimize(privilegeAutomatons));
     }
 
-    private static boolean testIndex(Automaton checkIndex, Automaton roleIndex) {
-        return Operations.subsetOf(checkIndex, roleIndex);
-    }
-
     private static boolean testPrivilege(Privilege checkPrivilege, Automaton roleAutomaton) {
         return Operations.subsetOf(checkPrivilege.getAutomaton(), roleAutomaton);
     }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java

@@ -67,9 +67,7 @@ import org.elasticsearch.xpack.security.audit.AuditUtil;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
 import org.elasticsearch.xpack.security.authz.IndicesAndAliasesResolver.ResolvedIndices;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
-import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -87,7 +85,6 @@ public class AuthorizationService {
     public static final String ORIGINATING_ACTION_KEY = "_originating_action_name";
     public static final String ROLE_NAMES_KEY = "_effective_role_names";
 
-    private static final Predicate<String> MONITOR_INDEX_PREDICATE = IndexPrivilege.MONITOR.predicate();
     private static final Predicate<String> SAME_USER_PRIVILEGE = Automatons.predicate(
         ChangePasswordAction.NAME, AuthenticateAction.NAME, HasPrivilegesAction.NAME, GetUserPrivilegesAction.NAME);
 
@@ -290,7 +287,7 @@ public class AuthorizationService {
         }
 
         final MetaData metaData = clusterService.state().metaData();
-        final AuthorizedIndices authorizedIndices = new AuthorizedIndices(authentication.getUser(), permission, action, metaData);
+        final AuthorizedIndices authorizedIndices = new AuthorizedIndices(permission, action, metaData);
         final ResolvedIndices resolvedIndices = resolveIndexNames(auditId, authentication, action, request, metaData,
             authorizedIndices, permission);
         assert !resolvedIndices.isEmpty()
@@ -312,18 +309,10 @@ public class AuthorizationService {
 
         final Set<String> localIndices = new HashSet<>(resolvedIndices.getLocal());
         IndicesAccessControl indicesAccessControl = permission.authorize(action, localIndices, metaData, fieldPermissionsCache);
-        if (!indicesAccessControl.isGranted()) {
+        if (indicesAccessControl.isGranted()) {
-            throw denial(auditId, authentication, action, request, permission.names());
-        } else if (hasSecurityIndexAccess(indicesAccessControl)
-            && MONITOR_INDEX_PREDICATE.test(action) == false
-            && isSuperuser(authentication.getUser()) == false) {
-            // only the XPackUser is allowed to work with this index, but we should allow indices monitoring actions through for debugging
-            // purposes. These monitor requests also sometimes resolve indices concretely and then requests them
-            logger.debug("user [{}] attempted to directly perform [{}] against the security index [{}]",
-                authentication.getUser().principal(), action, SecurityIndexManager.SECURITY_INDEX_NAME);
-            throw denial(auditId, authentication, action, request, permission.names());
-        } else {
             putTransientIfNonExisting(AuthorizationServiceField.INDICES_PERMISSIONS_KEY, indicesAccessControl);
+        } else {
+            throw denial(auditId, authentication, action, request, permission.names());
         }
 
         //if we are creating an index we need to authorize potential aliases created at the same time
@@ -359,16 +348,6 @@ public class AuthorizationService {
         return SystemUser.is(user) || XPackUser.is(user) || XPackSecurityUser.is(user);
     }
 
-    private boolean hasSecurityIndexAccess(IndicesAccessControl indicesAccessControl) {
-        for (String index : SecurityIndexManager.indexNames()) {
-            final IndicesAccessControl.IndexAccessControl indexPermissions = indicesAccessControl.getIndexPermissions(index);
-            if (indexPermissions != null && indexPermissions.isGranted()) {
-                return true;
-            }
-        }
-        return false;
-    }
-
     /**
      * Performs authorization checks on the items within a {@link BulkShardRequest}.
      * This inspects the {@link BulkItemRequest items} within the request, computes
@@ -602,11 +581,6 @@ public class AuthorizationService {
         return authorizationError("action [{}] is unauthorized for user [{}]", action, authUser.principal());
     }
 
-    static boolean isSuperuser(User user) {
-        return Arrays.stream(user.roles())
-            .anyMatch(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR.getName()::equals);
-    }
-
     public static void addSettings(List<Setting<?>> settings) {
         settings.add(ANONYMOUS_AUTHORIZATION_EXCEPTION_SETTING);
     }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizedIndices.java

@@ -8,8 +8,6 @@ package org.elasticsearch.xpack.security.authz;
 import org.elasticsearch.cluster.metadata.AliasOrIndex;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
-import org.elasticsearch.xpack.core.security.user.User;
-import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -17,21 +15,17 @@ import java.util.List;
 import java.util.Map;
 import java.util.function.Predicate;
 
-import static org.elasticsearch.xpack.security.authz.AuthorizationService.isSuperuser;
-
 /**
  * Abstraction used to make sure that we lazily load authorized indices only when requested and only maximum once per request. Also
  * makes sure that authorized indices don't get updated throughout the same request for the same user.
  */
 class AuthorizedIndices {
-    private final User user;
     private final String action;
     private final MetaData metaData;
     private final Role userRoles;
     private List<String> authorizedIndices;
 
-    AuthorizedIndices(User user, Role userRoles, String action, MetaData metaData) {
+    AuthorizedIndices(Role userRoles, String action, MetaData metaData) {
-        this.user = user;
         this.userRoles = userRoles;
         this.action = action;
         this.metaData = metaData;
@@ -56,10 +50,6 @@ class AuthorizedIndices {
             }
         }
 
-        if (isSuperuser(user) == false) {
-            // we should filter out all of the security indices from wildcards
-            indicesAndAliases.removeAll(SecurityIndexManager.indexNames());
-        }
         return Collections.unmodifiableList(indicesAndAliases);
     }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java

@@ -240,7 +240,8 @@ public class CompositeRolesStore {
         Set<String> clusterPrivileges = new HashSet<>();
         final List<ConditionalClusterPrivilege> conditionalClusterPrivileges = new ArrayList<>();
         Set<String> runAs = new HashSet<>();
-        Map<Set<String>, MergeableIndicesPrivilege> indicesPrivilegesMap = new HashMap<>();
+        final Map<Set<String>, MergeableIndicesPrivilege> restrictedIndicesPrivilegesMap = new HashMap<>();
+        final Map<Set<String>, MergeableIndicesPrivilege> indicesPrivilegesMap = new HashMap<>();
 
         // Keyed by application + resource
         Map<Tuple<String, Set<String>>, Set<String>> applicationPrivilegesMap = new HashMap<>();
@@ -257,26 +258,8 @@ public class CompositeRolesStore {
             if (descriptor.getRunAs() != null) {
                 runAs.addAll(Arrays.asList(descriptor.getRunAs()));
             }
-            IndicesPrivileges[] indicesPrivileges = descriptor.getIndicesPrivileges();
+            MergeableIndicesPrivilege.collatePrivilegesByIndices(descriptor.getIndicesPrivileges(), true, restrictedIndicesPrivilegesMap);
-            for (IndicesPrivileges indicesPrivilege : indicesPrivileges) {
+            MergeableIndicesPrivilege.collatePrivilegesByIndices(descriptor.getIndicesPrivileges(), false, indicesPrivilegesMap);
-                Set<String> key = newHashSet(indicesPrivilege.getIndices());
-                // if a index privilege is an explicit denial, then we treat it as non-existent since we skipped these in the past when
-                // merging
-                final boolean isExplicitDenial =
-                        indicesPrivileges.length == 1 && "none".equalsIgnoreCase(indicesPrivilege.getPrivileges()[0]);
-                if (isExplicitDenial == false) {
-                    indicesPrivilegesMap.compute(key, (k, value) -> {
-                        if (value == null) {
-                            return new MergeableIndicesPrivilege(indicesPrivilege.getIndices(), indicesPrivilege.getPrivileges(),
-                                    indicesPrivilege.getGrantedFields(), indicesPrivilege.getDeniedFields(), indicesPrivilege.getQuery());
-                        } else {
-                            value.merge(new MergeableIndicesPrivilege(indicesPrivilege.getIndices(), indicesPrivilege.getPrivileges(),
-                                    indicesPrivilege.getGrantedFields(), indicesPrivilege.getDeniedFields(), indicesPrivilege.getQuery()));
-                            return value;
-                        }
-                    });
-                }
-            }
             for (RoleDescriptor.ApplicationResourcePrivileges appPrivilege : descriptor.getApplicationPrivileges()) {
                 Tuple<String, Set<String>> key = new Tuple<>(appPrivilege.getApplication(), newHashSet(appPrivilege.getResources()));
                 applicationPrivilegesMap.compute(key, (k, v) -> {
@@ -297,7 +280,12 @@ public class CompositeRolesStore {
         indicesPrivilegesMap.entrySet().forEach((entry) -> {
             MergeableIndicesPrivilege privilege = entry.getValue();
             builder.add(fieldPermissionsCache.getFieldPermissions(privilege.fieldPermissionsDefinition), privilege.query,
-                    IndexPrivilege.get(privilege.privileges), privilege.indices.toArray(Strings.EMPTY_ARRAY));
+                    IndexPrivilege.get(privilege.privileges), false, privilege.indices.toArray(Strings.EMPTY_ARRAY));
+        });
+        restrictedIndicesPrivilegesMap.entrySet().forEach((entry) -> {
+            MergeableIndicesPrivilege privilege = entry.getValue();
+            builder.add(fieldPermissionsCache.getFieldPermissions(privilege.fieldPermissionsDefinition), privilege.query,
+                    IndexPrivilege.get(privilege.privileges), true, privilege.indices.toArray(Strings.EMPTY_ARRAY));
         });
 
         if (applicationPrivilegesMap.isEmpty()) {
@@ -412,6 +400,30 @@ public class CompositeRolesStore {
                 this.query.addAll(other.query);
             }
         }
+
+        private static void collatePrivilegesByIndices(IndicesPrivileges[] indicesPrivileges, boolean allowsRestrictedIndices,
+                Map<Set<String>, MergeableIndicesPrivilege> indicesPrivilegesMap) {
+            for (final IndicesPrivileges indicesPrivilege : indicesPrivileges) {
+                // if a index privilege is an explicit denial, then we treat it as non-existent since we skipped these in the past when
+                // merging
+                final boolean isExplicitDenial = indicesPrivileges.length == 1
+                        && "none".equalsIgnoreCase(indicesPrivilege.getPrivileges()[0]);
+                if (isExplicitDenial || (indicesPrivilege.allowRestrictedIndices() != allowsRestrictedIndices)) {
+                    continue;
+                }
+                final Set<String> key = newHashSet(indicesPrivilege.getIndices());
+                indicesPrivilegesMap.compute(key, (k, value) -> {
+                    if (value == null) {
+                        return new MergeableIndicesPrivilege(indicesPrivilege.getIndices(), indicesPrivilege.getPrivileges(),
+                                indicesPrivilege.getGrantedFields(), indicesPrivilege.getDeniedFields(), indicesPrivilege.getQuery());
+                    } else {
+                        value.merge(new MergeableIndicesPrivilege(indicesPrivilege.getIndices(), indicesPrivilege.getPrivileges(),
+                                indicesPrivilege.getGrantedFields(), indicesPrivilege.getDeniedFields(), indicesPrivilege.getQuery()));
+                        return value;
+                    }
+                });
+            }
+        }
     }
 
     private static final class RolesRetrievalResult {

x-pack/plugin/security/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java

@@ -51,7 +51,7 @@ public class ClearRolesCacheTests extends NativeRealmIntegTestCase {
         for (String role : roles) {
             c.preparePutRole(role)
                     .cluster("none")
-                    .addIndices(new String[] { "*" }, new String[] { "ALL" }, null, null, null)
+                    .addIndices(new String[] { "*" }, new String[] { "ALL" }, null, null, null, randomBoolean())
                     .get();
             logger.debug("--> created role [{}]", role);
         }
@@ -83,7 +83,7 @@ public class ClearRolesCacheTests extends NativeRealmIntegTestCase {
         for (String role : toModify) {
             PutRoleResponse response = securityClient.preparePutRole(role)
                     .cluster("none")
-                    .addIndices(new String[] { "*" }, new String[] { "ALL" }, null, null, null)
+                    .addIndices(new String[] { "*" }, new String[] { "ALL" }, null, null, null, randomBoolean())
                     .runAs(role)
                     .setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE)
                     .get();

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/IndicesAliasesRequestInterceptorTests.java

@@ -57,7 +57,7 @@ public class IndicesAliasesRequestInterceptorTests extends ESTestCase {
         } else {
             queries = null;
         }
-        Role role = Role.builder().add(fieldPermissions, queries, IndexPrivilege.ALL, "foo").build();
+        Role role = Role.builder().add(fieldPermissions, queries, IndexPrivilege.ALL, randomBoolean(), "foo").build();
         final String action = IndicesAliasesAction.NAME;
         IndicesAccessControl accessControl = new IndicesAccessControl(true, Collections.singletonMap("foo",
                 new IndicesAccessControl.IndexAccessControl(true, fieldPermissions, queries)));

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/interceptor/ResizeRequestInterceptorTests.java

@@ -60,7 +60,7 @@ public class ResizeRequestInterceptorTests extends ESTestCase {
         } else {
             queries = null;
         }
-        Role role = Role.builder().add(fieldPermissions, queries, IndexPrivilege.ALL, "foo").build();
+        Role role = Role.builder().add(fieldPermissions, queries, IndexPrivilege.ALL, randomBoolean(), "foo").build();
         final String action = randomFrom(ShrinkAction.NAME, ResizeAction.NAME);
         IndicesAccessControl accessControl = new IndicesAccessControl(true, Collections.singletonMap("foo",
                         new IndicesAccessControl.IndexAccessControl(true, fieldPermissions, queries)));

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUserPrivilegesActionTests.java

@@ -43,7 +43,7 @@ public class TransportGetUserPrivilegesActionTests extends ESTestCase {
             .add(
                 new FieldPermissions(new FieldPermissionsDefinition(new String[]{ "public.*" }, new String[0])),
                 Collections.singleton(query),
-                IndexPrivilege.READ, "index-4", "index-5")
+                IndexPrivilege.READ, randomBoolean(), "index-4", "index-5")
             .addApplicationPrivilege(new ApplicationPrivilege("app01", "read", "data:read"), Collections.singleton("*"))
             .runAs(new Privilege(Sets.newHashSet("user01", "user02"), "user01", "user02"))
             .build();

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java

@@ -130,8 +130,8 @@ public class ESNativeMigrateToolTests extends NativeRealmIntegTestCase {
             c.preparePutRole(rname)
                     .cluster("all", "none")
                     .runAs("root", "nobody")
-                    .addIndices(new String[]{"index"}, new String[]{"read"},
+                    .addIndices(new String[] { "index" }, new String[] { "read" }, new String[] { "body", "title" }, null,
-                            new String[]{"body", "title"}, null, new BytesArray("{\"query\": {\"match_all\": {}}}"))
+                            new BytesArray("{\"query\": {\"match_all\": {}}}"), randomBoolean())
                     .get();
             addedRoles.add(rname);
         }

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeRealmMigrateToolTests.java

@@ -77,7 +77,8 @@ public class ESNativeRealmMigrateToolTests extends CommandTestCase {
         assertThat(ESNativeRealmMigrateTool.MigrateUserOrRoles.createRoleJson(rd),
                 equalTo("{\"cluster\":[]," +
                         "\"indices\":[{\"names\":[\"i1\",\"i2\",\"i3\"]," +
-                        "\"privileges\":[\"all\"],\"field_security\":{\"grant\":[\"body\"]}}]," +
+                        "\"privileges\":[\"all\"],\"field_security\":{\"grant\":[\"body\"]}," +
+                        "\"allow_restricted_indices\":false}]," +
                         "\"applications\":[]," +
                         "\"run_as\":[],\"metadata\":{},\"type\":\"role\"}"));
     }

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java

@@ -104,7 +104,7 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
                 PutRoleResponse response = securityClient()
                         .preparePutRole("native_anonymous")
                         .cluster("ALL")
-                        .addIndices(new String[]{"*"}, new String[]{"ALL"}, null, null, null)
+                        .addIndices(new String[]{"*"}, new String[]{"ALL"}, null, null, null, randomBoolean())
                         .get();
                 assertTrue(response.isCreated());
             } else {
@@ -189,7 +189,7 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
                 .cluster("all", "none")
                 .runAs("root", "nobody")
                 .addIndices(new String[]{"index"}, new String[]{"read"}, new String[]{"body", "title"}, null,
-                        new BytesArray("{\"query\": {\"match_all\": {}}}"))
+                        new BytesArray("{\"query\": {\"match_all\": {}}}"), randomBoolean())
                 .metadata(metadata)
                 .get();
         logger.error("--> waiting for .security index");
@@ -206,13 +206,13 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
                 .cluster("all", "none")
                 .runAs("root", "nobody")
                 .addIndices(new String[]{"index"}, new String[]{"read"}, new String[]{"body", "title"}, null,
-                        new BytesArray("{\"query\": {\"match_all\": {}}}"))
+                        new BytesArray("{\"query\": {\"match_all\": {}}}"), randomBoolean())
                 .get();
         c.preparePutRole("test_role3")
                 .cluster("all", "none")
                 .runAs("root", "nobody")
                 .addIndices(new String[]{"index"}, new String[]{"read"}, new String[]{"body", "title"}, null,
-                        new BytesArray("{\"query\": {\"match_all\": {}}}"))
+                        new BytesArray("{\"query\": {\"match_all\": {}}}"), randomBoolean())
                 .get();
 
         logger.info("--> retrieving all roles");
@@ -239,7 +239,7 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
         c.preparePutRole("test_role")
                 .cluster("all")
                 .addIndices(new String[] { "*" }, new String[] { "read" }, new String[]{"body", "title"}, null,
-                        new BytesArray("{\"match_all\": {}}"))
+                        new BytesArray("{\"match_all\": {}}"), randomBoolean())
                 .get();
         logger.error("--> creating user");
         c.preparePutUser("joe", "s3krit".toCharArray(), hasher, "test_role").get();
@@ -334,7 +334,7 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
         c.preparePutRole("test_role")
                 .cluster("all")
                 .addIndices(new String[]{"*"}, new String[]{"read"}, new String[]{"body", "title"}, null,
-                        new BytesArray("{\"match_all\": {}}"))
+                        new BytesArray("{\"match_all\": {}}"), randomBoolean())
                 .get();
         logger.error("--> creating user");
         c.preparePutUser("joe", "s3krit".toCharArray(), hasher, "test_role").get();
@@ -349,7 +349,7 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
             c.preparePutRole("test_role")
                     .cluster("none")
                     .addIndices(new String[]{"*"}, new String[]{"read"}, new String[]{"body", "title"}, null,
-                            new BytesArray("{\"match_all\": {}}"))
+                            new BytesArray("{\"match_all\": {}}"), randomBoolean())
                     .get();
             if (anonymousEnabled && roleExists) {
                 assertNoTimeout(client()
@@ -369,7 +369,7 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
             c.preparePutRole("test_role")
                     .cluster("none")
                     .addIndices(new String[]{"*"}, new String[]{"read"}, new String[]{"body", "title"}, null,
-                            new BytesArray("{\"match_all\": {}}"))
+                            new BytesArray("{\"match_all\": {}}"), randomBoolean())
                     .get();
             getRolesResponse = c.prepareGetRoles().names("test_role").get();
             assertTrue("test_role does not exist!", getRolesResponse.hasRoles());
@@ -385,7 +385,7 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
         c.preparePutRole("test_role")
                 .cluster("all")
                 .addIndices(new String[]{"*"}, new String[]{"read"}, new String[]{"body", "title"}, null,
-                        new BytesArray("{\"match_all\": {}}"))
+                        new BytesArray("{\"match_all\": {}}"), randomBoolean())
                 .get();
         c.preparePutUser("joe", "s3krit".toCharArray(), hasher, "test_role").get();
         logger.error("--> waiting for .security index");
@@ -411,11 +411,11 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
         // create some roles
         client.preparePutRole("admin_role")
                 .cluster("all")
-                .addIndices(new String[]{"*"}, new String[]{"all"}, null, null, null)
+                .addIndices(new String[]{"*"}, new String[]{"all"}, null, null, null, randomBoolean())
                 .get();
         client.preparePutRole("read_role")
                 .cluster("none")
-                .addIndices(new String[]{"*"}, new String[]{"read"}, null, null, null)
+                .addIndices(new String[]{"*"}, new String[]{"read"}, null, null, null, randomBoolean())
                 .get();
 
         assertThat(client.prepareGetUsers("joes").get().hasUsers(), is(false));
@@ -516,7 +516,7 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
         } else {
             client.preparePutRole("read_role")
                     .cluster("none")
-                    .addIndices(new String[]{"*"}, new String[]{"read"}, null, null, null)
+                    .addIndices(new String[]{"*"}, new String[]{"read"}, null, null, null, randomBoolean())
                     .get();
         }
 
@@ -642,7 +642,7 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
         SecurityClient client = new SecurityClient(client());
         PutRoleResponse putRoleResponse = client.preparePutRole("admin_role")
                 .cluster("all")
-                .addIndices(new String[]{"*"}, new String[]{"all"}, null, null, null)
+                .addIndices(new String[]{"*"}, new String[]{"all"}, null, null, null, randomBoolean())
                 .get();
         assertThat(putRoleResponse.isCreated(), is(true));
         roles++;
@@ -660,7 +660,7 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
             }
             roleResponse = client.preparePutRole("admin_role_fls")
                     .cluster("all")
-                    .addIndices(new String[]{"*"}, new String[]{"all"}, grantedFields, deniedFields, null)
+                    .addIndices(new String[]{"*"}, new String[]{"all"}, grantedFields, deniedFields, null, randomBoolean())
                     .get();
             assertThat(roleResponse.isCreated(), is(true));
             roles++;
@@ -669,7 +669,7 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
         if (dls) {
             PutRoleResponse roleResponse = client.preparePutRole("admin_role_dls")
                     .cluster("all")
-                    .addIndices(new String[]{"*"}, new String[]{"all"}, null, null, new BytesArray("{ \"match_all\": {} }"))
+                    .addIndices(new String[]{"*"}, new String[]{"all"}, null, null, new BytesArray("{\"match_all\": {}}"), randomBoolean())
                     .get();
             assertThat(roleResponse.isCreated(), is(true));
             roles++;

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java

@@ -16,11 +16,13 @@ import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor.IndicesPrivileges;
+import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions;
 import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
-import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
+import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
@@ -28,19 +30,18 @@ import java.util.List;
 import java.util.Set;
 
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.contains;
 
 public class AuthorizedIndicesTests extends ESTestCase {
 
     public void testAuthorizedIndicesUserWithoutRoles() {
-        User user = new User("test user");
+        AuthorizedIndices authorizedIndices = new AuthorizedIndices(Role.EMPTY, "", MetaData.EMPTY_META_DATA);
-        AuthorizedIndices authorizedIndices = new AuthorizedIndices(user, Role.EMPTY, "",
-                MetaData.EMPTY_META_DATA);
         List<String> list = authorizedIndices.get();
         assertTrue(list.isEmpty());
     }
 
     public void testAuthorizedIndicesUserWithSomeRoles() {
-        User user = new User("test user", "a_star", "b");
         RoleDescriptor aStarRole = new RoleDescriptor("a_star", null,
                 new IndicesPrivileges[] { IndicesPrivileges.builder().indices("a*").privileges("all").build() }, null);
         RoleDescriptor bRole = new RoleDescriptor("b", null,
@@ -58,55 +59,82 @@ public class AuthorizedIndicesTests extends ESTestCase {
                         .putAlias(new AliasMetaData.Builder("ab").build())
                         .putAlias(new AliasMetaData.Builder("ba").build())
                         .build(), true)
+                .put(new IndexMetaData.Builder(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX)
+                        .settings(indexSettings)
+                        .numberOfShards(1)
+                        .numberOfReplicas(0)
+                        .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_INDEX_NAME).build())
+                        .build(), true)
                 .build();
         final PlainActionFuture<Role> future = new PlainActionFuture<>();
         final Set<RoleDescriptor> descriptors = Sets.newHashSet(aStarRole, bRole);
         CompositeRolesStore.buildRoleFromDescriptors(descriptors, new FieldPermissionsCache(Settings.EMPTY), null, future);
         Role roles = future.actionGet();
-        AuthorizedIndices authorizedIndices = new AuthorizedIndices(user, roles, SearchAction.NAME, metaData);
+        AuthorizedIndices authorizedIndices = new AuthorizedIndices(roles, SearchAction.NAME, metaData);
         List<String> list = authorizedIndices.get();
         assertThat(list, containsInAnyOrder("a1", "a2", "aaaaaa", "b", "ab"));
         assertFalse(list.contains("bbbbb"));
         assertFalse(list.contains("ba"));
+        assertThat(list, not(contains(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX)));
+        assertThat(list, not(contains(RestrictedIndicesNames.SECURITY_INDEX_NAME)));
     }
 
     public void testAuthorizedIndicesUserWithSomeRolesEmptyMetaData() {
-        User user = new User("test user", "role");
         Role role = Role.builder("role").add(IndexPrivilege.ALL, "*").build();
-        AuthorizedIndices authorizedIndices = new AuthorizedIndices(user, role, SearchAction.NAME, MetaData.EMPTY_META_DATA);
+        AuthorizedIndices authorizedIndices = new AuthorizedIndices(role, SearchAction.NAME, MetaData.EMPTY_META_DATA);
         List<String> list = authorizedIndices.get();
         assertTrue(list.isEmpty());
     }
 
-    public void testSecurityIndicesAreRemovedFromRegularUser() {
+    public void testSecurityIndicesAreRestrictedForDefaultRole() {
-        User user = new User("test user", "user_role");
+        Role role = Role.builder(randomFrom("user_role", ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR.getName()))
-        Role role = Role.builder("user_role").add(IndexPrivilege.ALL, "*").cluster(ClusterPrivilege.ALL).build();
+                .add(IndexPrivilege.ALL, "*")
+                .cluster(ClusterPrivilege.ALL)
+                .build();
         Settings indexSettings = Settings.builder().put("index.version.created", Version.CURRENT).build();
         MetaData metaData = MetaData.builder()
                 .put(new IndexMetaData.Builder("an-index").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true)
                 .put(new IndexMetaData.Builder("another-index").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true)
-                .put(new IndexMetaData.Builder(SecurityIndexManager.SECURITY_INDEX_NAME).settings(indexSettings)
+                .put(new IndexMetaData.Builder(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX)
-                        .numberOfShards(1).numberOfReplicas(0).build(), true)
+                        .settings(indexSettings)
+                        .numberOfShards(1)
+                        .numberOfReplicas(0)
+                        .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_INDEX_NAME).build())
+                        .build(), true)
                 .build();
 
-        AuthorizedIndices authorizedIndices = new AuthorizedIndices(user, role, SearchAction.NAME, metaData);
+        AuthorizedIndices authorizedIndices = new AuthorizedIndices(role, SearchAction.NAME, metaData);
         List<String> list = authorizedIndices.get();
         assertThat(list, containsInAnyOrder("an-index", "another-index"));
+        assertThat(list, not(contains(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX)));
+        assertThat(list, not(contains(RestrictedIndicesNames.SECURITY_INDEX_NAME)));
     }
 
-    public void testSecurityIndicesAreNotRemovedFromSuperUsers() {
+    public void testSecurityIndicesAreNotRemovedFromUnrestrictedRole() {
-        User user = new User("admin", "kibana_user", "superuser");
+        Role role = Role.builder(randomAlphaOfLength(8))
-        Role role = Role.builder("kibana_user+superuser").add(IndexPrivilege.ALL, "*").cluster(ClusterPrivilege.ALL).build();
+                .add(FieldPermissions.DEFAULT, null, IndexPrivilege.ALL, true, "*")
+                .cluster(ClusterPrivilege.ALL)
+                .build();
         Settings indexSettings = Settings.builder().put("index.version.created", Version.CURRENT).build();
         MetaData metaData = MetaData.builder()
                 .put(new IndexMetaData.Builder("an-index").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true)
                 .put(new IndexMetaData.Builder("another-index").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true)
-                .put(new IndexMetaData.Builder(SecurityIndexManager.SECURITY_INDEX_NAME).settings(indexSettings)
+                .put(new IndexMetaData.Builder(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX)
-                        .numberOfShards(1).numberOfReplicas(0).build(), true)
+                        .settings(indexSettings)
+                        .numberOfShards(1)
+                        .numberOfReplicas(0)
+                        .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_INDEX_NAME).build())
+                        .build(), true)
                 .build();
 
-        AuthorizedIndices authorizedIndices = new AuthorizedIndices(user, role, SearchAction.NAME, metaData);
+        AuthorizedIndices authorizedIndices = new AuthorizedIndices(role, SearchAction.NAME, metaData);
         List<String> list = authorizedIndices.get();
-        assertThat(list, containsInAnyOrder("an-index", "another-index", SecurityIndexManager.SECURITY_INDEX_NAME));
+        assertThat(list, containsInAnyOrder("an-index", "another-index", SecurityIndexManager.SECURITY_INDEX_NAME,
+                SecurityIndexManager.INTERNAL_SECURITY_INDEX));
+
+        AuthorizedIndices authorizedIndicesSuperUser = new AuthorizedIndices(ReservedRolesStore.SUPERUSER_ROLE, SearchAction.NAME,
+                metaData);
+        assertThat(authorizedIndicesSuperUser.get(), containsInAnyOrder("an-index", "another-index",
+                SecurityIndexManager.SECURITY_INDEX_NAME, SecurityIndexManager.INTERNAL_SECURITY_INDEX));
     }
 }

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java

@@ -1366,7 +1366,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
     private AuthorizedIndices buildAuthorizedIndices(User user, String action) {
         PlainActionFuture<Role> rolesListener = new PlainActionFuture<>();
         authzService.roles(user, rolesListener);
-        return new AuthorizedIndices(user, rolesListener.actionGet(), action, metaData);
+        return new AuthorizedIndices(rolesListener.actionGet(), action, metaData);
     }
 
     public static IndexMetaData.Builder indexBuilder(String index) {

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RoleDescriptorTests.java

@@ -48,10 +48,11 @@ public class RoleDescriptorTests extends ESTestCase {
         RoleDescriptor.IndicesPrivileges privs = RoleDescriptor.IndicesPrivileges.builder()
                 .indices("idx")
                 .privileges("priv")
+                .allowRestrictedIndices(true)
                 .build();
         XContentBuilder b = jsonBuilder();
         privs.toXContent(b, ToXContent.EMPTY_PARAMS);
-        assertEquals("{\"names\":[\"idx\"],\"privileges\":[\"priv\"]}", Strings.toString(b));
+        assertEquals("{\"names\":[\"idx\"],\"privileges\":[\"priv\"],\"allow_restricted_indices\":true}", Strings.toString(b));
     }
 
     public void testToString() throws Exception {
@@ -80,7 +81,7 @@ public class RoleDescriptorTests extends ESTestCase {
 
         assertThat(descriptor.toString(), is("Role[name=test, cluster=[all,none]" +
                 ", global=[{APPLICATION:manage:applications=app01,app02}]" +
-                ", indicesPrivileges=[IndicesPrivileges[indices=[i1,i2], privileges=[read]" +
+                ", indicesPrivileges=[IndicesPrivileges[indices=[i1,i2], allowRestrictedIndices=[false], privileges=[read]" +
                 ", field_security=[grant=[body,title], except=null], query={\"query\": {\"match_all\": {}}}],]" +
                 ", applicationPrivileges=[ApplicationResourcePrivileges[application=my_app, privileges=[read,write], resources=[*]],]" +
                 ", runAs=[sudo], metadata=[{}]]"));
@@ -92,6 +93,7 @@ public class RoleDescriptorTests extends ESTestCase {
                         .indices("i1", "i2")
                         .privileges("read")
                         .grantedFields("body", "title")
+                        .allowRestrictedIndices(randomBoolean())
                         .query("{\"query\": {\"match_all\": {}}}")
                         .build()
         };
@@ -131,9 +133,9 @@ public class RoleDescriptorTests extends ESTestCase {
         assertArrayEquals(new String[] { "m", "n" }, rd.getRunAs());
 
         q = "{\"cluster\":[\"a\", \"b\"], \"run_as\": [\"m\", \"n\"], \"index\": [{\"names\": \"idx1\", \"privileges\": [\"p1\", " +
-                "\"p2\"]}, {\"names\": \"idx2\", \"privileges\": [\"p3\"], \"field_security\": " +
+                "\"p2\"]}, {\"names\": \"idx2\", \"allow_restricted_indices\": true, \"privileges\": [\"p3\"], \"field_security\": " +
                 "{\"grant\": [\"f1\", \"f2\"]}}, {\"names\": " +
-                "\"idx2\", " +
+                "\"idx2\", \"allow_restricted_indices\": false," +
                 "\"privileges\": [\"p3\"], \"field_security\": {\"grant\": [\"f1\", \"f2\"]}, \"query\": \"{\\\"match_all\\\": {}}\"}]}";
         rd = RoleDescriptor.parse("test", new BytesArray(q), false, XContentType.JSON);
         assertEquals("test", rd.getName());
@@ -142,12 +144,13 @@ public class RoleDescriptorTests extends ESTestCase {
         assertArrayEquals(new String[] { "m", "n" }, rd.getRunAs());
 
         q = "{\"cluster\":[\"a\", \"b\"], \"run_as\": [\"m\", \"n\"], \"index\": [{\"names\": [\"idx1\",\"idx2\"], \"privileges\": " +
-                "[\"p1\", \"p2\"]}]}";
+                "[\"p1\", \"p2\"], \"allow_restricted_indices\": true}]}";
         rd = RoleDescriptor.parse("test", new BytesArray(q), false, XContentType.JSON);
         assertEquals("test", rd.getName());
         assertArrayEquals(new String[] { "a", "b" }, rd.getClusterPrivileges());
         assertEquals(1, rd.getIndicesPrivileges().length);
         assertArrayEquals(new String[] { "idx1", "idx2" }, rd.getIndicesPrivileges()[0].getIndices());
+        assertTrue(rd.getIndicesPrivileges()[0].allowRestrictedIndices());
         assertArrayEquals(new String[] { "m", "n" }, rd.getRunAs());
         assertNull(rd.getIndicesPrivileges()[0].getQuery());
 
@@ -162,7 +165,7 @@ public class RoleDescriptorTests extends ESTestCase {
         assertThat(rd.getMetadata().get("foo"), is("bar"));
 
         q = "{\"cluster\":[\"a\", \"b\"], \"run_as\": [\"m\", \"n\"]," +
-                " \"index\": [{\"names\": [\"idx1\",\"idx2\"], \"privileges\": [\"p1\", \"p2\"]}]," +
+                " \"index\": [{\"names\": [\"idx1\",\"idx2\"], \"allow_restricted_indices\": false, \"privileges\": [\"p1\", \"p2\"]}]," +
                 " \"applications\": [" +
                 "     {\"resources\": [\"object-123\",\"object-456\"], \"privileges\":[\"read\", \"delete\"], \"application\":\"app1\"}," +
                 "     {\"resources\": [\"*\"], \"privileges\":[\"admin\"], \"application\":\"app2\" }" +
@@ -174,8 +177,9 @@ public class RoleDescriptorTests extends ESTestCase {
         assertThat(rd.getClusterPrivileges(), arrayContaining("a", "b"));
         assertThat(rd.getIndicesPrivileges().length, equalTo(1));
         assertThat(rd.getIndicesPrivileges()[0].getIndices(), arrayContaining("idx1", "idx2"));
-        assertThat(rd.getRunAs(), arrayContaining("m", "n"));
+        assertThat(rd.getIndicesPrivileges()[0].allowRestrictedIndices(), is(false));
         assertThat(rd.getIndicesPrivileges()[0].getQuery(), nullValue());
+        assertThat(rd.getRunAs(), arrayContaining("m", "n"));
         assertThat(rd.getApplicationPrivileges().length, equalTo(2));
         assertThat(rd.getApplicationPrivileges()[0].getResources(), arrayContaining("object-123", "object-456"));
         assertThat(rd.getApplicationPrivileges()[0].getPrivileges(), arrayContaining("read", "delete"));

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/SecurityScrollTests.java

@@ -30,7 +30,7 @@ public class SecurityScrollTests extends SecurityIntegTestCase {
     public void testScrollIsPerUser() throws Exception {
         assertSecurityIndexActive();
         securityClient().preparePutRole("scrollable")
-                .addIndices(new String[] { randomAlphaOfLengthBetween(4, 12) }, new String[] { "read" }, null, null, null)
+                .addIndices(new String[] { randomAlphaOfLengthBetween(4, 12) }, new String[] { "read" }, null, null, null, randomBoolean())
                 .get();
         securityClient().preparePutUser("other", SecuritySettingsSourceField.TEST_PASSWORD.toCharArray(), getFastStoredHashAlgoForTests(),
             "scrollable")

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/IndicesPermissionTests.java

@@ -27,6 +27,7 @@ import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsDe
 import org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission;
 import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege;
+import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -39,6 +40,7 @@ import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.is;
 
 public class IndicesPermissionTests extends ESTestCase {
 
@@ -57,7 +59,8 @@ public class IndicesPermissionTests extends ESTestCase {
         Set<BytesReference> query = Collections.singleton(new BytesArray("{}"));
         String[] fields = new String[]{"_field"};
         Role role = Role.builder("_role")
-                .add(new FieldPermissions(fieldPermissionDef(fields, null)), query, IndexPrivilege.ALL, "_index").build();
+                .add(new FieldPermissions(fieldPermissionDef(fields, null)), query, IndexPrivilege.ALL, randomBoolean(), "_index")
+                .build();
         IndicesAccessControl permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_index"), md, fieldPermissionsCache);
         assertThat(permissions.getIndexPermissions("_index"), notNullValue());
         assertTrue(permissions.getIndexPermissions("_index").getFieldPermissions().grantsAccessTo("_field"));
@@ -67,7 +70,8 @@ public class IndicesPermissionTests extends ESTestCase {
 
         // no document level security:
         role = Role.builder("_role")
-                .add(new FieldPermissions(fieldPermissionDef(fields, null)), null, IndexPrivilege.ALL, "_index").build();
+                .add(new FieldPermissions(fieldPermissionDef(fields, null)), null, IndexPrivilege.ALL, randomBoolean(), "_index")
+                .build();
         permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_index"), md, fieldPermissionsCache);
         assertThat(permissions.getIndexPermissions("_index"), notNullValue());
         assertTrue(permissions.getIndexPermissions("_index").getFieldPermissions().grantsAccessTo("_field"));
@@ -75,7 +79,7 @@ public class IndicesPermissionTests extends ESTestCase {
         assertThat(permissions.getIndexPermissions("_index").getQueries(), nullValue());
 
         // no field level security:
-        role = Role.builder("_role").add(new FieldPermissions(), query, IndexPrivilege.ALL, "_index").build();
+        role = Role.builder("_role").add(new FieldPermissions(), query, IndexPrivilege.ALL, randomBoolean(), "_index").build();
         permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_index"), md, fieldPermissionsCache);
         assertThat(permissions.getIndexPermissions("_index"), notNullValue());
         assertFalse(permissions.getIndexPermissions("_index").getFieldPermissions().hasFieldLevelSecurity());
@@ -84,7 +88,7 @@ public class IndicesPermissionTests extends ESTestCase {
 
         // index group associated with an alias:
         role = Role.builder("_role")
-                .add(new FieldPermissions(fieldPermissionDef(fields, null)), query, IndexPrivilege.ALL, "_alias")
+                .add(new FieldPermissions(fieldPermissionDef(fields, null)), query, IndexPrivilege.ALL, randomBoolean(), "_alias")
                 .build();
         permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_alias"), md, fieldPermissionsCache);
         assertThat(permissions.getIndexPermissions("_index"), notNullValue());
@@ -103,7 +107,8 @@ public class IndicesPermissionTests extends ESTestCase {
         String[] allFields = randomFrom(new String[]{"*"}, new String[]{"foo", "*"},
         new String[]{randomAlphaOfLengthBetween(1, 10), "*"});
         role = Role.builder("_role")
-                .add(new FieldPermissions(fieldPermissionDef(allFields, null)), query, IndexPrivilege.ALL, "_alias").build();
+                .add(new FieldPermissions(fieldPermissionDef(allFields, null)), query, IndexPrivilege.ALL, randomBoolean(), "_alias")
+                .build();
         permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_alias"), md, fieldPermissionsCache);
         assertThat(permissions.getIndexPermissions("_index"), notNullValue());
         assertFalse(permissions.getIndexPermissions("_index").getFieldPermissions().hasFieldLevelSecurity());
@@ -130,8 +135,9 @@ public class IndicesPermissionTests extends ESTestCase {
         allFields = randomFrom(new String[]{"*"}, new String[]{"foo", "*"},
                 new String[]{randomAlphaOfLengthBetween(1, 10), "*"});
         role = Role.builder("_role")
-                .add(new FieldPermissions(fieldPermissionDef(allFields, null)), fooQuery, IndexPrivilege.ALL, "_alias")
+                .add(new FieldPermissions(fieldPermissionDef(allFields, null)), fooQuery, IndexPrivilege.ALL, randomBoolean(), "_alias")
-                .add(new FieldPermissions(fieldPermissionDef(allFields, null)), query, IndexPrivilege.ALL, "_alias").build();
+                .add(new FieldPermissions(fieldPermissionDef(allFields, null)), query, IndexPrivilege.ALL, randomBoolean(), "_alias")
+                .build();
         permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_alias"), md, fieldPermissionsCache);
         Set<BytesReference> bothQueries = Sets.union(fooQuery, query);
         assertThat(permissions.getIndexPermissions("_index"), notNullValue());
@@ -165,8 +171,8 @@ public class IndicesPermissionTests extends ESTestCase {
         Set<BytesReference> query = Collections.singleton(new BytesArray("{}"));
         String[] fields = new String[]{"_field"};
         Role role = Role.builder("_role")
-                .add(new FieldPermissions(fieldPermissionDef(fields, null)), query, IndexPrivilege.ALL, "_index")
+                .add(new FieldPermissions(fieldPermissionDef(fields, null)), query, IndexPrivilege.ALL, randomBoolean(), "_index")
-                .add(new FieldPermissions(fieldPermissionDef(null, null)), null, IndexPrivilege.ALL, "*")
+                .add(new FieldPermissions(fieldPermissionDef(null, null)), null, IndexPrivilege.ALL, randomBoolean(), "*")
                 .build();
         IndicesAccessControl permissions = role.authorize(SearchAction.NAME, Sets.newHashSet("_index"), md, fieldPermissionsCache);
         assertThat(permissions.getIndexPermissions("_index"), notNullValue());
@@ -219,9 +225,10 @@ public class IndicesPermissionTests extends ESTestCase {
                 .build();
 
         FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY);
-        IndicesPermission.Group group1 = new IndicesPermission.Group(IndexPrivilege.ALL, new FieldPermissions(), null, "a1");
+        IndicesPermission.Group group1 = new IndicesPermission.Group(IndexPrivilege.ALL, new FieldPermissions(), null, randomBoolean(),
+                "a1");
         IndicesPermission.Group group2 = new IndicesPermission.Group(IndexPrivilege.ALL,
-                new FieldPermissions(fieldPermissionDef(null, new String[]{"denied_field"})), null, "a1");
+                new FieldPermissions(fieldPermissionDef(null, new String[]{"denied_field"})), null, randomBoolean(), "a1");
         IndicesPermission core = new IndicesPermission(group1, group2);
         Map<String, IndicesAccessControl.IndexAccessControl> authzMap =
                 core.authorize(SearchAction.NAME, Sets.newHashSet("a1", "ba"), metaData, fieldPermissionsCache);
@@ -234,13 +241,15 @@ public class IndicesPermissionTests extends ESTestCase {
         assertFalse(core.check("unknown"));
 
         // test with two indices
-        group1 = new IndicesPermission.Group(IndexPrivilege.ALL, new FieldPermissions(), null, "a1");
+        group1 = new IndicesPermission.Group(IndexPrivilege.ALL, new FieldPermissions(), null, randomBoolean(), "a1");
         group2 = new IndicesPermission.Group(IndexPrivilege.ALL,
-                new FieldPermissions(fieldPermissionDef(null, new String[]{"denied_field"})), null, "a1");
+                new FieldPermissions(fieldPermissionDef(null, new String[]{"denied_field"})), null, randomBoolean(), "a1");
         IndicesPermission.Group group3 = new IndicesPermission.Group(IndexPrivilege.ALL,
-                new FieldPermissions(fieldPermissionDef(new String[]{"*_field"}, new String[]{"denied_field"})), null, "a2");
+                new FieldPermissions(fieldPermissionDef(new String[] { "*_field" }, new String[] { "denied_field" })), null,
+                randomBoolean(), "a2");
         IndicesPermission.Group group4 = new IndicesPermission.Group(IndexPrivilege.ALL,
-                new FieldPermissions(fieldPermissionDef(new String[]{"*_field2"}, new String[]{"denied_field2"})), null, "a2");
+                new FieldPermissions(fieldPermissionDef(new String[] { "*_field2" }, new String[] { "denied_field2" })), null,
+                randomBoolean(), "a2");
         core = new IndicesPermission(group1, group2, group3, group4);
         authzMap = core.authorize(SearchAction.NAME, Sets.newHashSet("a1", "a2"), metaData, fieldPermissionsCache);
         assertFalse(authzMap.get("a1").getFieldPermissions().hasFieldLevelSecurity());
@@ -262,11 +271,41 @@ public class IndicesPermissionTests extends ESTestCase {
             indices.add("*" + prefix + "*" + suffixBegin + "*");
         }
         final ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
-                () -> new IndicesPermission.Group(IndexPrivilege.ALL, new FieldPermissions(), null, indices.toArray(Strings.EMPTY_ARRAY)));
+                () -> new IndicesPermission.Group(IndexPrivilege.ALL, new FieldPermissions(), null, randomBoolean(),
+                        indices.toArray(Strings.EMPTY_ARRAY)));
         assertThat(e.getMessage(), containsString(indices.get(0)));
         assertThat(e.getMessage(), containsString("too complex to evaluate"));
     }
 
+    public void testSecurityIndicesPermissions() {
+        final Settings indexSettings = Settings.builder().put("index.version.created", Version.CURRENT).build();
+        final MetaData metaData = new MetaData.Builder()
+                .put(new IndexMetaData.Builder(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX)
+                        .settings(indexSettings)
+                        .numberOfShards(1)
+                        .numberOfReplicas(0)
+                        .putAlias(new AliasMetaData.Builder(RestrictedIndicesNames.SECURITY_INDEX_NAME).build())
+                        .build(), true)
+                .build();
+        FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY);
+
+        // allow_restricted_indices: false
+        IndicesPermission.Group group = new IndicesPermission.Group(IndexPrivilege.ALL, new FieldPermissions(), null, false, "*");
+        Map<String, IndicesAccessControl.IndexAccessControl> authzMap = new IndicesPermission(group).authorize(SearchAction.NAME,
+                Sets.newHashSet(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME), metaData,
+                fieldPermissionsCache);
+        assertThat(authzMap.get(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX).isGranted(), is(false));
+        assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_INDEX_NAME).isGranted(), is(false));
+
+        // allow_restricted_indices: true
+        group = new IndicesPermission.Group(IndexPrivilege.ALL, new FieldPermissions(), null, true, "*");
+        authzMap = new IndicesPermission(group).authorize(SearchAction.NAME,
+                Sets.newHashSet(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX, RestrictedIndicesNames.SECURITY_INDEX_NAME), metaData,
+                fieldPermissionsCache);
+        assertThat(authzMap.get(RestrictedIndicesNames.INTERNAL_SECURITY_INDEX).isGranted(), is(true));
+        assertThat(authzMap.get(RestrictedIndicesNames.SECURITY_INDEX_NAME).isGranted(), is(true));
+    }
+
     private static FieldPermissionsDefinition fieldPermissionDef(String[] granted, String[] denied) {
         return new FieldPermissionsDefinition(granted, denied);
     }

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/rest/action/user/RestGetUserPrivilegesActionTests.java

@@ -54,8 +54,6 @@ public class RestGetUserPrivilegesActionTests extends ESTestCase {
 
     public void testBuildResponse() throws Exception {
         final RestGetUserPrivilegesAction.RestListener listener = new RestGetUserPrivilegesAction.RestListener(null);
-
-
         final Set<String> cluster = new LinkedHashSet<>(Arrays.asList("monitor", "manage_ml", "manage_watcher"));
         final Set<ConditionalClusterPrivilege> conditionalCluster = Collections.singleton(
             new ConditionalClusterPrivileges.ManageApplicationPrivileges(new LinkedHashSet<>(Arrays.asList("app01", "app02"))));
@@ -68,10 +66,11 @@ public class RestGetUserPrivilegesActionTests extends ESTestCase {
                 new LinkedHashSet<>(Arrays.asList(
                     new BytesArray("{ \"term\": { \"access\": \"public\" } }"),
                     new BytesArray("{ \"term\": { \"access\": \"standard\" } }")
-                ))
+                )),
+                false
             ),
             new GetUserPrivilegesResponse.Indices(Arrays.asList("index-4"), Collections.singleton("all"),
-                Collections.emptySet(), Collections.emptySet()
+                Collections.emptySet(), Collections.emptySet(), true
             )
         ));
         final Set<ApplicationResourcePrivileges> application = Sets.newHashSet(
@@ -100,8 +99,10 @@ public class RestGetUserPrivilegesActionTests extends ESTestCase {
             "\"query\":[" +
             "\"{ \\\"term\\\": { \\\"access\\\": \\\"public\\\" } }\"," +
             "\"{ \\\"term\\\": { \\\"access\\\": \\\"standard\\\" } }\"" +
-            "]}," +
+            "]," +
-            "{\"names\":[\"index-4\"],\"privileges\":[\"all\"]}" +
+            "\"allow_restricted_indices\":false" +
+            "}," +
+            "{\"names\":[\"index-4\"],\"privileges\":[\"all\"],\"allow_restricted_indices\":true}" +
             "]," +
             "\"applications\":[" +
             "{\"application\":\"app01\",\"privileges\":[\"read\",\"write\"],\"resources\":[\"*\"]}," +