Add Shield HTTP APIs for users and roles

Relates to elastic/elasticsearch#33

Original commit: elastic/x-pack-elasticsearch@a0942c9334

dev-tools/ci

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# This script is used as a single command to run the x-plugins tests. It will
+# attempt to check out 'elasticsearch' into a sibling directory and then run the
+# tests.
+#
+# If `USE_EXISTING_ES` is set to anything, this script will allow the use of an
+# existing 'elasticsearch' sibling directory. It defaults to complaining if an
+# existing (dirty) ES directory exists.
+#
+# Any arguments will be passed to the `gradle check` command
+
+# Turn on semi-strict mode
+set -eo pipefail
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+XBASEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../" && pwd )"
+XSIBLINGDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../" && pwd )"
+
+# go to the x-plugins directory
+cd $XSIBLINGDIR
+
+if [ "x$USE_EXISTING_ES" = "x" ]; then
+    if [ -d "./elasticsearch" ]; then
+        echo "I expected a clean workspace but an 'elasticsearch' sibling \
+directory already exists in [$XSIBLINGDIR]!"
+        echo
+        echo "Either define 'USE_EXISTING_ES' or remove the \
+existing 'elasticsearch' sibling."
+        exit 1
+    fi
+    echo "Checking out Elasticsearch 'master' branch..."
+    git clone https://github.com/elastic/elasticsearch.git --depth=1
+else
+    if [ -d "./elasticsearch" ]; then
+        echo "Using existing 'elasticsearch' checkout"
+    else
+        echo "You have defined 'USE_EXISTING_ES' but no existing \
+Elasticsearch directory exists!"
+        exit 2
+    fi
+fi
+
+echo "Running X-Plugins tests..."
+cd "$SCRIPT_DIR/.." && echo "Running 'gradle check' in $PWD"
+
+# Actually run the tests, passing any arguments to gradle
+gradle check $*
+
+# ~*~ shell-script-mode ~*~

elasticsearch/qa/shield-example-realm/src/main/java/org/elasticsearch/example/realm/CustomRealm.java

@@ -62,7 +62,7 @@ public class CustomRealm extends Realm<UsernamePasswordToken> {
     public User authenticate(UsernamePasswordToken token) {
         final String actualUser = token.principal();
         if (KNOWN_USER.equals(actualUser) && SecuredString.constantTimeEquals(token.credentials(), KNOWN_PW)) {
-            return new User.Simple(actualUser, ROLES);
+            return new User(actualUser, ROLES);
         }
         return null;
     }

elasticsearch/x-pack/build.gradle

@@ -1,4 +1,5 @@
 import org.elasticsearch.gradle.MavenFilteringHack
+import org.elasticsearch.gradle.test.NodeInfo
 
 apply plugin: 'elasticsearch.esplugin'
 esplugin {
@@ -125,8 +126,17 @@ integTest {
   // TODO: fix this rest test to not depend on a hardcoded port!
   systemProperty 'tests.rest.blacklist', 'getting_started/10_monitor_cluster_health/*'
   cluster {
-    // TODO set up tests so that shield can be enabled or disabled
-    systemProperty 'es.shield.enabled', 'false'
+    setupCommand 'setupDummyUser', 'bin/x-pack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
+    waitCondition = { NodeInfo node, AntBuilder ant ->
+        File tmpFile = new File(node.cwd, 'wait.success')
+        ant.get(src: "http://${node.httpUri()}",
+                dest: tmpFile.toString(),
+                username: "test_user",
+                password: "changeme",
+                ignoreerrors: true, // do not fail on error, so logging buffers can be flushed by the wait task
+                retries: 10)
+        return tmpFile.exists()
+    }
   }
 }
 
@@ -143,6 +153,10 @@ artifacts {
     testArtifacts testJar
 }
 
+run {
+  setupCommand 'setupDummyUser', 'bin/x-pack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
+}
+
 // classes are missing, e.g. com.ibm.icu.lang.UCharacter
 thirdPartyAudit.excludes = [
     // uses internal java api: sun.misc.Unsafe

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/shield/InternalMarvelUser.java

@@ -15,7 +15,7 @@ import org.elasticsearch.shield.authz.Privilege;
 /**
  *
  */
-public class InternalMarvelUser extends User.Simple {
+public class InternalMarvelUser extends User {
 
     static final String NAME = "__marvel_user";
     static final String[] ROLE_NAMES = new String[] { "__marvel_role" };

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/ShieldLifecycleService.java

@@ -13,7 +13,10 @@ import org.elasticsearch.common.component.LifecycleListener;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.shield.audit.AuditTrailModule;
 import org.elasticsearch.shield.audit.index.IndexAuditTrail;
+import org.elasticsearch.shield.authc.esnative.ESNativeUsersStore;
+import org.elasticsearch.shield.authz.esnative.ESNativeRolesStore;
 import org.elasticsearch.threadpool.ThreadPool;
 
 /**
@@ -29,15 +32,25 @@ import org.elasticsearch.threadpool.ThreadPool;
  */
 public class ShieldLifecycleService extends AbstractComponent implements ClusterStateListener {
 
+    private final Settings settings;
     private final ThreadPool threadPool;
     private final IndexAuditTrail indexAuditTrail;
+    private final ESNativeUsersStore esUserStore;
+    private final ESNativeRolesStore esRolesStore;
 
     @Inject
-    public ShieldLifecycleService(Settings settings, ClusterService clusterService, ThreadPool threadPool, IndexAuditTrail indexAuditTrail) {
+    public ShieldLifecycleService(Settings settings, ClusterService clusterService, ThreadPool threadPool,
+                                  IndexAuditTrail indexAuditTrail, ESNativeUsersStore esUserStore,
+                                  ESNativeRolesStore esRolesStore) {
         super(settings);
+        this.settings = settings;
         this.threadPool = threadPool;
         this.indexAuditTrail = indexAuditTrail;
+        this.esUserStore = esUserStore;
+        this.esRolesStore = esRolesStore;
         clusterService.add(this);
+        clusterService.add(esUserStore);
+        clusterService.add(esRolesStore);
         clusterService.addLifecycleListener(new LifecycleListener() {
 
             @Override
@@ -54,32 +67,94 @@ public class ShieldLifecycleService extends AbstractComponent implements Cluster
 
     @Override
     public void clusterChanged(ClusterChangedEvent event) {
-        // TODO if/when we have more services this should not be checking the audit trail
-        if (indexAuditTrail.state() == IndexAuditTrail.State.INITIALIZED) {
-            final boolean master = event.localNodeMaster();
-            if (indexAuditTrail.canStart(event, master)) {
+        final boolean master = event.localNodeMaster();
+        try {
+            if (esUserStore.canStart(event.state(), master)) {
                 threadPool.generic().execute(new AbstractRunnable() {
-
                     @Override
                     public void onFailure(Throwable throwable) {
-                        logger.error("failed to start shield lifecycle services", throwable);
+                        logger.error("failed to start native user store service", throwable);
                         assert false : "shield lifecycle services startup failed";
                     }
 
                     @Override
                     public void doRun() {
-                        indexAuditTrail.start(master);
+                        esUserStore.start();
                     }
                 });
             }
+        } catch (Exception e) {
+            logger.error("failed to start native user store", e);
         }
+
+        try {
+            if (esRolesStore.canStart(event.state(), master)) {
+                threadPool.generic().execute(new AbstractRunnable() {
+                    @Override
+                    public void onFailure(Throwable throwable) {
+                        logger.error("failed to start native roles store services", throwable);
+                        assert false : "shield lifecycle services startup failed";
+                    }
+
+                    @Override
+                    public void doRun() {
+                        esRolesStore.start();
+                    }
+                });
+            }
+        } catch (Exception e) {
+            logger.error("failed to start native roles store", e);
+        }
+
+        try {
+            if (AuditTrailModule.indexAuditLoggingEnabled(settings) &&
+                    indexAuditTrail.state() == IndexAuditTrail.State.INITIALIZED) {
+                if (indexAuditTrail.canStart(event, master)) {
+                    threadPool.generic().execute(new AbstractRunnable() {
+
+                        @Override
+                        public void onFailure(Throwable throwable) {
+                            logger.error("failed to start index audit trail services", throwable);
+                            assert false : "shield lifecycle services startup failed";
+                        }
+
+                        @Override
+                        public void doRun() {
+                            indexAuditTrail.start(master);
+                        }
+                    });
+                }
+            }
+        } catch (Exception e) {
+            logger.error("failed to start index audit trail", e);
+        }
+
     }
 
     public void stop() {
-        indexAuditTrail.stop();
+        try {
+            esUserStore.stop();
+        } catch (Exception e) {
+            logger.error("failed to stop native user module", e);
+        }
+        try {
+            esRolesStore.stop();
+        } catch (Exception e) {
+            logger.error("failed to stop native roles module", e);
+        }
+        try {
+            indexAuditTrail.stop();
+        } catch (Exception e) {
+            logger.error("failed to stop audit trail module", e);
+        }
     }
 
     public void close() {
-        indexAuditTrail.close();
+        // There is no .close() method for the roles module
+        try {
+            indexAuditTrail.close();
+        } catch (Exception e) {
+            logger.error("failed to close audit trail module", e);
+        }
     }
 }

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/ShieldModule.java

@@ -20,6 +20,7 @@ public class ShieldModule extends AbstractShieldModule {
     @Override
     protected void configure(boolean clientMode) {
         if (!clientMode) {
+            bind(ShieldLifecycleService.class).asEagerSingleton();
             bind(ShieldSettingsFilter.class).asEagerSingleton();
         }
     }

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java

@@ -18,8 +18,30 @@ import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.shield.action.ShieldActionFilter;
 import org.elasticsearch.shield.action.ShieldActionModule;
+import org.elasticsearch.shield.action.admin.role.AddRoleAction;
+import org.elasticsearch.shield.action.admin.role.DeleteRoleAction;
+import org.elasticsearch.shield.action.admin.role.GetRolesAction;
+import org.elasticsearch.shield.action.admin.role.RestAddRoleAction;
+import org.elasticsearch.shield.action.admin.role.RestDeleteRoleAction;
+import org.elasticsearch.shield.action.admin.role.RestGetRolesAction;
+import org.elasticsearch.shield.action.admin.role.TransportAddRoleAction;
+import org.elasticsearch.shield.action.admin.role.TransportDeleteRoleAction;
+import org.elasticsearch.shield.action.admin.role.TransportGetRolesAction;
+import org.elasticsearch.shield.action.admin.user.AddUserAction;
+import org.elasticsearch.shield.action.admin.user.DeleteUserAction;
+import org.elasticsearch.shield.action.admin.user.GetUsersAction;
+import org.elasticsearch.shield.action.admin.user.RestAddUserAction;
+import org.elasticsearch.shield.action.admin.user.RestDeleteUserAction;
+import org.elasticsearch.shield.action.admin.user.RestGetUsersAction;
+import org.elasticsearch.shield.action.admin.user.TransportAddUserAction;
+import org.elasticsearch.shield.action.admin.user.TransportDeleteUserAction;
+import org.elasticsearch.shield.action.admin.user.TransportGetUsersAction;
 import org.elasticsearch.shield.action.authc.cache.ClearRealmCacheAction;
 import org.elasticsearch.shield.action.authc.cache.TransportClearRealmCacheAction;
+import org.elasticsearch.shield.action.authz.cache.ClearRolesCacheAction;
+import org.elasticsearch.shield.action.authz.cache.TransportClearRolesCacheAction;
+import org.elasticsearch.shield.admin.ShieldAdminModule;
+import org.elasticsearch.shield.admin.ShieldInternalUserHolder;
 import org.elasticsearch.shield.audit.AuditTrailModule;
 import org.elasticsearch.shield.audit.index.IndexAuditUserHolder;
 import org.elasticsearch.shield.audit.logfile.LoggingAuditTrail;
@@ -40,6 +62,7 @@ import org.elasticsearch.shield.rest.ShieldRestModule;
 import org.elasticsearch.shield.rest.action.RestAuthenticateAction;
 import org.elasticsearch.shield.rest.action.RestShieldInfoAction;
 import org.elasticsearch.shield.rest.action.authc.cache.RestClearRealmCacheAction;
+import org.elasticsearch.shield.rest.action.authz.cache.RestClearRolesCacheAction;
 import org.elasticsearch.shield.ssl.SSLModule;
 import org.elasticsearch.shield.transport.ShieldClientTransportService;
 import org.elasticsearch.shield.transport.ShieldServerTransportService;
@@ -116,6 +139,7 @@ public class ShieldPlugin extends Plugin {
                     new ShieldRestModule(settings),
                     new ShieldActionModule(settings),
                     new ShieldTransportModule(settings),
+                    new ShieldAdminModule(settings),
                     new SSLModule(settings));
         }
     }
@@ -189,6 +213,13 @@ public class ShieldPlugin extends Plugin {
 
         // registering all shield actions
         module.registerAction(ClearRealmCacheAction.INSTANCE, TransportClearRealmCacheAction.class);
+        module.registerAction(ClearRolesCacheAction.INSTANCE, TransportClearRolesCacheAction.class);
+        module.registerAction(GetUsersAction.INSTANCE, TransportGetUsersAction.class);
+        module.registerAction(AddUserAction.INSTANCE, TransportAddUserAction.class);
+        module.registerAction(DeleteUserAction.INSTANCE, TransportDeleteUserAction.class);
+        module.registerAction(GetRolesAction.INSTANCE, TransportGetRolesAction.class);
+        module.registerAction(AddRoleAction.INSTANCE, TransportAddRoleAction.class);
+        module.registerAction(DeleteRoleAction.INSTANCE, TransportDeleteRoleAction.class);
     }
 
     public void onModule(NetworkModule module) {
@@ -209,15 +240,25 @@ public class ShieldPlugin extends Plugin {
         }
 
         if (clientMode == false) {
-            module.registerRestHandler(RestClearRealmCacheAction.class);
             module.registerRestHandler(RestAuthenticateAction.class);
+            module.registerRestHandler(RestClearRealmCacheAction.class);
+            module.registerRestHandler(RestClearRolesCacheAction.class);
+            module.registerRestHandler(RestGetUsersAction.class);
+            module.registerRestHandler(RestAddUserAction.class);
+            module.registerRestHandler(RestDeleteUserAction.class);
+            module.registerRestHandler(RestGetRolesAction.class);
+            module.registerRestHandler(RestAddRoleAction.class);
+            module.registerRestHandler(RestDeleteRoleAction.class);
             module.registerHttpTransport(ShieldPlugin.NAME, ShieldNettyHttpServerTransport.class);
         }
     }
 
     public void onModule(AuthorizationModule module) {
-        if (enabled && AuditTrailModule.auditingEnabled(settings)) {
-            module.registerReservedRole(IndexAuditUserHolder.ROLE);
+        if (enabled) {
+            module.registerReservedRole(ShieldInternalUserHolder.ROLE);
+            if (AuditTrailModule.auditingEnabled(settings)) {
+                module.registerReservedRole(IndexAuditUserHolder.ROLE);
+            }
         }
     }
 

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/User.java

@@ -19,130 +19,137 @@ import java.util.Arrays;
 /**
  * An authenticated user
  */
-public abstract class User implements ToXContent {
+public class User implements ToXContent {
 
     public static final User SYSTEM = new System();
 
-    /**
-     * @return  The principal of this user - effectively serving as the unique identity of of the user.
-     */
-    public abstract String principal();
+    private final String username;
+    private final String[] roles;
+    private final User runAs;
+
+    public User(String username, String... roles) {
+        this.username = username;
+        this.roles = roles == null ? Strings.EMPTY_ARRAY : roles;
+        this.runAs = null;
+    }
+
+    public User(String username, String[] roles, User runAs) {
+        this.username = username;
+        this.roles = roles == null ? Strings.EMPTY_ARRAY : roles;
+        assert (runAs == null || runAs.runAs() == null) : "the runAs user should not be a user that can run as";
+        if (runAs == SYSTEM) {
+            throw new ElasticsearchSecurityException("the runAs user cannot be the internal system user");
+        }
+        this.runAs = runAs;
+    }
 
     /**
-     * @return  The roles this user is associated with. The roles are identified by their unique names
-     *          and each represents as set of permissions
+     * @return  The principal of this user - effectively serving as the
+     *          unique identity of of the user.
      */
-    public abstract String[] roles();
+    public String principal() {
+        return this.username;
+    }
 
     /**
-     * @return The user that will be used for run as functionality. If run as functionality is not being
-     *         used, then <code>null</code> will be returned
+     * @return  The roles this user is associated with. The roles are
+     *          identified by their unique names and each represents as
+     *          set of permissions
      */
-    public abstract User runAs();
+    public String[] roles() {
+        return this.roles;
+    }
+
+    /**
+     * @return The user that will be used for run as functionality. If run as
+     *         functionality is not being used, then <code>null</code> will be
+     *         returned
+     */
+    public User runAs() {
+        return runAs;
+    }
 
     public final boolean isSystem() {
         return this == SYSTEM;
     }
 
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("User[username=").append(username);
+        sb.append(",roles=[");
+        if (roles != null) {
+            for (String role : roles) {
+                sb.append(role).append(",");
+            }
+        }
+        sb.append("]");
+        if (runAs != null) {
+            sb.append(",runAs=[").append(runAs.toString()).append("]");
+        }
+        sb.append("]");
+        return sb.toString();
+    }
+
     public static User readFrom(StreamInput input) throws IOException {
         if (input.readBoolean()) {
             String name = input.readString();
-            if (!System.NAME.equals(name)) {
+            if (System.NAME.equals(name)) {
+                return SYSTEM;
+            } else {
                 throw new IllegalStateException("invalid system user");
             }
-            return SYSTEM;
         }
         String username = input.readString();
         String[] roles = input.readStringArray();
         if (input.readBoolean()) {
             String runAsUsername = input.readString();
             String[] runAsRoles = input.readStringArray();
-            return new Simple(username, roles, new Simple(runAsUsername, runAsRoles));
+            return new User(username, roles, new User(runAsUsername, runAsRoles));
         }
-        return new Simple(username, roles);
+        return new User(username, roles);
     }
 
     public static void writeTo(User user, StreamOutput output) throws IOException {
         if (user.isSystem()) {
             output.writeBoolean(true);
             output.writeString(System.NAME);
-            return;
-        }
-        output.writeBoolean(false);
-        Simple simple = (Simple) user;
-        output.writeString(simple.username);
-        output.writeStringArray(simple.roles);
-        if (simple.runAs == null) {
-            output.writeBoolean(false);
         } else {
-            output.writeBoolean(true);
-            output.writeString(simple.runAs.principal());
-            output.writeStringArray(simple.runAs.roles());
+            output.writeBoolean(false);
+            output.writeString(user.principal());
+            output.writeStringArray(user.roles());
+            if (user.runAs == null) {
+                output.writeBoolean(false);
+            } else {
+                output.writeBoolean(true);
+                output.writeString(user.runAs.principal());
+                output.writeStringArray(user.runAs.roles());
+            }
         }
     }
 
-    public static class Simple extends User {
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null) return false;
+        if (!(o instanceof User)) return false;
 
-        private final String username;
-        private final String[] roles;
-        private final User runAs;
-
-        public Simple(String username, String[] roles) {
-            this(username, roles, null);
+        User user = (User) o;
+        if (!principal().equals(user.principal())) return false;
+        if (!Arrays.equals(roles(), user.roles())) return false;
+        if (runAs != null ? !runAs.equals(user.runAs) : user.runAs != null) {
+            return false;
         }
 
-        public Simple(String username, String[] roles, User runAs) {
-            this.username = username;
-            this.roles = roles == null ? Strings.EMPTY_ARRAY : roles;
-            assert (runAs == null || runAs.runAs() == null) : "the runAs user should not be a user that can run as";
-            if (runAs == SYSTEM) {
-                throw new ElasticsearchSecurityException("the runAs user cannot be the internal system user");
-            }
-            this.runAs = runAs;
-        }
+        return true;
+    }
 
-        @Override
-        public String principal() {
-            return username;
-        }
-
-        @Override
-        public String[] roles() {
-            return roles;
-        }
-
-        @Override
-        public User runAs() {
-            return runAs;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-
-            Simple simple = (Simple) o;
-
-            if (username != null ? !username.equals(simple.username) : simple.username != null) {
-                return false;
-            }
-            if (!Arrays.equals(roles, simple.roles)) {
-                return false;
-            }
-            if (runAs != null ? !runAs.equals(simple.runAs) : simple.runAs != null) {
-                return false;
-            }
-
-            return true;
-        }
-
-        @Override
-        public int hashCode() {
-            int result = username != null ? username.hashCode() : 0;
-            result = 31 * result + Arrays.hashCode(roles);
-            result = 31 * result + (runAs != null ? runAs.hashCode() : 0);
-            return result;
-        }
+    @Override
+    public int hashCode() {
+        int result = principal().hashCode();
+        result = 31 * result + Arrays.hashCode(roles());
+        result = 31 * result + (runAs != null ? runAs.hashCode() : 0);
+        return result;
     }
 
     @Override
@@ -155,26 +162,11 @@ public abstract class User implements ToXContent {
     }
 
     private static class System extends User {
-
         private static final String NAME = "__es_system_user";
         private static final String[] ROLES = new String[] { SystemRole.NAME };
 
         private System() {
-        }
-
-        @Override
-        public String principal() {
-            return NAME;
-        }
-
-        @Override
-        public String[] roles() {
-            return ROLES;
-        }
-
-        @Override
-        public User runAs() {
-            return null;
+            super(NAME, ROLES);
         }
     }
 }

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/role/AddRoleAction.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.role;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.client.ElasticsearchClient;
+
+/**
+ * Action for adding a role to the shield administrative index
+ */
+public class AddRoleAction extends Action<AddRoleRequest, AddRoleResponse, AddRoleRequestBuilder> {
+
+    public static final AddRoleAction INSTANCE = new AddRoleAction();
+    public static final String NAME = "cluster:admin/shield/role/add";
+
+
+    protected AddRoleAction() {
+        super(NAME);
+    }
+
+    @Override
+    public AddRoleRequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new AddRoleRequestBuilder(client, this);
+    }
+
+    @Override
+    public AddRoleResponse newResponse() {
+        return new AddRoleResponse();
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/role/AddRoleRequest.java

@@ -0,0 +1,155 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.role;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.shield.authz.RoleDescriptor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * Request object for adding a role to the shield index
+ */
+public class AddRoleRequest extends ActionRequest<AddRoleRequest> implements ToXContent {
+
+    private String name;
+    private List<String> clusterPriv;
+    // List of index names to privileges
+    private List<RoleDescriptor.IndicesPrivileges> indices = new ArrayList<>();
+    private List<String> runAs = new ArrayList<>();
+    private RoleDescriptor roleDescriptor;
+    
+    public AddRoleRequest() {
+    }
+
+    public AddRoleRequest(BytesReference source) throws Exception {
+        this.roleDescriptor = RoleDescriptor.source(source);
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (name == null) {
+            validationException = addValidationError("role name is missing", validationException);
+        }
+        return validationException;
+    }
+
+    public void name(String name) {
+        this.name = name;
+    }
+
+    public void cluster(String clusterPrivilege) {
+        this.clusterPriv = Collections.singletonList(clusterPrivilege);
+    }
+
+    public void cluster(List<String> clusterPrivileges) {
+        this.clusterPriv = clusterPrivileges;
+    }
+
+    public void addIndex(String[] indices, String[] privileges, @Nullable String[] fields, @Nullable BytesReference query) {
+        this.indices.add(RoleDescriptor.IndicesPrivileges.builder()
+                .indices(indices)
+                .privileges(privileges)
+                .fields(fields)
+                .query(query)
+                .build());
+    }
+
+    public void runAs(List<String> usernames) {
+        this.runAs = usernames;
+    }
+
+    public String name() {
+        return name;
+    }
+
+    public List<String> cluster() {
+        return clusterPriv;
+    }
+
+    public List<RoleDescriptor.IndicesPrivileges> indices() {
+        return indices;
+    }
+
+    public List<String> runAs() {
+        return runAs;
+    }
+
+    private RoleDescriptor roleDescriptor() {
+        if (this.roleDescriptor != null) {
+            return this.roleDescriptor;
+        }
+        this.roleDescriptor = new RoleDescriptor(name, this.clusterPriv.toArray(Strings.EMPTY_ARRAY),
+                this.indices, this.runAs.toArray(Strings.EMPTY_ARRAY));
+        return this.roleDescriptor;
+    }
+    
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        name = in.readString();
+        int clusterSize = in.readVInt();
+        List<String> tempCluster = new ArrayList<>(clusterSize);
+        for (int i = 0; i < clusterSize; i++) {
+            tempCluster.add(in.readString());
+        }
+        clusterPriv = tempCluster;
+        int indicesSize = in.readVInt();
+        indices = new ArrayList<>(indicesSize);
+        for (int i = 0; i < indicesSize; i++) {
+            indices.add(RoleDescriptor.IndicesPrivileges.readIndicesPrivileges(in));
+        }
+        if (in.readBoolean()) {
+            int runAsSize = in.readVInt();
+            runAs = new ArrayList<>(runAsSize);
+            for (int i = 0; i < runAsSize; i++) {
+                runAs.add(in.readString());
+            }
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(name);
+        out.writeVInt(clusterPriv.size());
+        for (String cluster : clusterPriv) {
+            out.writeString(cluster);
+        }
+        out.writeVInt(indices.size());
+        for (RoleDescriptor.IndicesPrivileges index : indices) {
+            index.writeTo(out);
+        }
+        if (runAs.isEmpty() == false) {
+            out.writeBoolean(true);
+            out.writeVInt(runAs.size());
+            for (String runAsUser : runAs) {
+                out.writeString(runAsUser);
+            }
+        } else {
+            out.writeBoolean(false);
+        }
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return this.roleDescriptor().toXContent(builder, params);
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/role/AddRoleRequestBuilder.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.role;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.shield.authz.RoleDescriptor;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Builder for requests to add a role to the administrative index
+ */
+public class AddRoleRequestBuilder extends ActionRequestBuilder<AddRoleRequest, AddRoleResponse, AddRoleRequestBuilder> {
+
+    public AddRoleRequestBuilder(ElasticsearchClient client) {
+        this(client, AddRoleAction.INSTANCE);
+    }
+
+    public AddRoleRequestBuilder(ElasticsearchClient client, AddRoleAction action) {
+        super(client, action, new AddRoleRequest());
+    }
+
+    public AddRoleRequestBuilder name(String name) {
+        request.name(name);
+        return this;
+    }
+
+    public AddRoleRequestBuilder cluster(String... cluster) {
+        request.cluster(Arrays.asList(cluster));
+        return this;
+    }
+
+    public AddRoleRequestBuilder runAs(String... runAsUsers) {
+        request.runAs(Arrays.asList(runAsUsers));
+        return this;
+    }
+
+    public AddRoleRequestBuilder addIndices(String[] indices, String[] privileges, @Nullable String[] fields, @Nullable BytesReference query) {
+        request.addIndex(indices, privileges, fields, query);
+        return this;
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/role/AddRoleResponse.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.role;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+/**
+ * Response when adding a role, includes a boolean for whether the role was
+ * created or updated.
+ */
+public class AddRoleResponse extends ActionResponse implements ToXContent {
+
+    private boolean created;
+
+    public AddRoleResponse() {
+    }
+    
+    public AddRoleResponse(boolean created) {
+        this.created = created;
+    }
+
+    public boolean isCreated() {
+        return created;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject().field("created", created).endObject();
+        return builder;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeBoolean(created);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        this.created = in.readBoolean();
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/role/DeleteRoleAction.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.role;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.client.ElasticsearchClient;
+
+/**
+ * Action for deleting a role from the shield administrative index
+ */
+public class DeleteRoleAction extends Action<DeleteRoleRequest, DeleteRoleResponse, DeleteRoleRequestBuilder> {
+
+    public static final DeleteRoleAction INSTANCE = new DeleteRoleAction();
+    public static final String NAME = "cluster:admin/shield/role/delete";
+
+
+    protected DeleteRoleAction() {
+        super(NAME);
+    }
+
+    @Override
+    public DeleteRoleRequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new DeleteRoleRequestBuilder(client, this);
+    }
+
+    @Override
+    public DeleteRoleResponse newResponse() {
+        return new DeleteRoleResponse();
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/role/DeleteRoleRequest.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.role;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * A request delete a role from the shield index
+ */
+public class DeleteRoleRequest extends ActionRequest<DeleteRoleRequest> {
+
+    private String role;
+
+    public DeleteRoleRequest() {
+    }
+
+    public DeleteRoleRequest(String roleName) {
+        this.role = roleName;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (role == null) {
+            validationException = addValidationError("role is missing", validationException);
+        }
+        return validationException;
+    }
+
+    public void role(String role) {
+        this.role = role;
+    }
+
+    public String role() {
+        return role;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        role = in.readString();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(role);
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/role/DeleteRoleRequestBuilder.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.role;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+
+/**
+ * A builder for requests to delete a role from the shield index
+ */
+public class DeleteRoleRequestBuilder extends ActionRequestBuilder<DeleteRoleRequest, DeleteRoleResponse, DeleteRoleRequestBuilder> {
+
+    public DeleteRoleRequestBuilder(ElasticsearchClient client) {
+        this(client, DeleteRoleAction.INSTANCE);
+    }
+
+    public DeleteRoleRequestBuilder(ElasticsearchClient client, DeleteRoleAction action) {
+        super(client, action, new DeleteRoleRequest());
+    }
+
+    public DeleteRoleRequestBuilder role(String roleName) {
+        request.role(roleName);
+        return this;
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/role/DeleteRoleResponse.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.role;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+/**
+ * Response for a role being deleted from the shield index
+ */
+public class DeleteRoleResponse extends ActionResponse implements ToXContent {
+
+    private boolean found = false;
+
+    public DeleteRoleResponse() {}
+
+    public DeleteRoleResponse(boolean found) {
+        this.found = found;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject().field("found", found).endObject();
+        return builder;
+    }
+
+    public boolean found() {
+        return this.found;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        found = in.readBoolean();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeBoolean(found);
+    }
+
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/role/GetRolesAction.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.role;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.client.ElasticsearchClient;
+
+/**
+ * Action to retrieve a role from the shield index
+ */
+public class GetRolesAction extends Action<GetRolesRequest, GetRolesResponse, GetRolesRequestBuilder> {
+
+    public static final GetRolesAction INSTANCE = new GetRolesAction();
+    public static final String NAME = "cluster:admin/shield/role/get";
+
+
+    protected GetRolesAction() {
+        super(NAME);
+    }
+
+    @Override
+    public GetRolesRequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new GetRolesRequestBuilder(client, this);
+    }
+
+    @Override
+    public GetRolesResponse newResponse() {
+        return new GetRolesResponse();
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/role/GetRolesRequest.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.role;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * Request to retrieve roles from the shield index
+ */
+public class GetRolesRequest extends ActionRequest<GetRolesRequest> {
+
+    private String[] roles;
+
+    public GetRolesRequest() {
+        roles = Strings.EMPTY_ARRAY;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (roles == null) {
+            validationException = addValidationError("role is missing", validationException);
+        }
+        return validationException;
+    }
+
+    public void roles(String... roles) {
+        this.roles = roles;
+    }
+
+    public String[] roles() {
+        return roles;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        roles = in.readStringArray();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeStringArray(roles);
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/role/GetRolesRequestBuilder.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.role;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+
+/**
+ * Builder for requests to retrieve a role from the shield index
+ */
+public class GetRolesRequestBuilder extends ActionRequestBuilder<GetRolesRequest, GetRolesResponse, GetRolesRequestBuilder> {
+
+    public GetRolesRequestBuilder(ElasticsearchClient client) {
+        this(client, GetRolesAction.INSTANCE);
+    }
+
+    public GetRolesRequestBuilder(ElasticsearchClient client, GetRolesAction action) {
+        super(client, action, new GetRolesRequest());
+    }
+
+    public GetRolesRequestBuilder roles(String... roles) {
+        request.roles(roles);
+        return this;
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/role/GetRolesResponse.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.role;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.shield.authz.RoleDescriptor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Response when retrieving a role from the shield index. Does not contain a
+ * real {@code Role} object, only a {@code RoleDescriptor}.
+ */
+public class GetRolesResponse extends ActionResponse {
+    private List<RoleDescriptor> roles;
+
+    public GetRolesResponse() {
+        roles = Collections.emptyList();
+    }
+
+    public GetRolesResponse(RoleDescriptor role) {
+        this.roles = Collections.singletonList(role);
+    }
+
+    public GetRolesResponse(List<RoleDescriptor> roles) {
+        this.roles = roles;
+    }
+
+    public List<RoleDescriptor> roles() {
+        return roles;
+    }
+
+    public boolean isExists() {
+        return roles != null && roles.size() > 0;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        int size = in.readVInt();
+        roles = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            roles.add(RoleDescriptor.readFrom(in));
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeVInt(roles.size());
+        for (RoleDescriptor role : roles) {
+            RoleDescriptor.writeTo(role, out);
+        }
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/role/RestAddRoleAction.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.role;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.support.RestBuilderListener;
+import org.elasticsearch.shield.authz.RoleDescriptor;
+import org.elasticsearch.shield.client.ShieldClient;
+
+/**
+ * Rest endpoint to add a Role to the shield index
+ */
+public class RestAddRoleAction extends BaseRestHandler {
+
+    @Inject
+    public RestAddRoleAction(Settings settings, RestController controller, Client client) {
+        super(settings, controller, client);
+        controller.registerHandler(RestRequest.Method.POST, "/_shield/role/{role}", this);
+        controller.registerHandler(RestRequest.Method.PUT, "/_shield/role/{role}", this);
+    }
+
+    @Override
+    protected void handleRequest(RestRequest request, final RestChannel channel, Client client) throws Exception {
+        AddRoleRequest addRoleReq = new AddRoleRequest(request.content());
+        addRoleReq.name(request.param("role"));
+
+        new ShieldClient(client).addRole(addRoleReq, new RestBuilderListener<AddRoleResponse>(channel) {
+            @Override
+            public RestResponse buildResponse(AddRoleResponse addRoleResponse, XContentBuilder builder) throws Exception {
+                return new BytesRestResponse(RestStatus.OK,
+                        builder.startObject()
+                        .field("role", addRoleResponse)
+                        .endObject());
+            }
+        });
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/role/RestDeleteRoleAction.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.role;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.support.RestBuilderListener;
+import org.elasticsearch.shield.client.ShieldClient;
+
+/**
+ * Rest endpoint to delete a Role from the shield index
+ */
+public class RestDeleteRoleAction extends BaseRestHandler {
+
+    @Inject
+    public RestDeleteRoleAction(Settings settings, RestController controller, Client client) {
+        super(settings, controller, client);
+        controller.registerHandler(RestRequest.Method.DELETE, "/_shield/role/{role}", this);
+    }
+
+    @Override
+    protected void handleRequest(RestRequest request, final RestChannel channel, Client client) throws Exception {
+        String role = request.param("role");
+        DeleteRoleRequest delRoleRequest = new DeleteRoleRequest(role);
+
+        new ShieldClient(client).deleteRole(delRoleRequest, new RestBuilderListener<DeleteRoleResponse>(channel) {
+            @Override
+            public RestResponse buildResponse(DeleteRoleResponse response, XContentBuilder builder) throws Exception {
+                return new BytesRestResponse(response.found() ? RestStatus.OK : RestStatus.NOT_FOUND,
+                        builder.startObject()
+                        .field("found", response.found())
+                        .endObject());
+            }
+        });
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/role/RestGetRolesAction.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.role;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.support.RestBuilderListener;
+import org.elasticsearch.shield.client.ShieldClient;
+
+/**
+ * Rest endpoint to retrieve a Role from the shield index
+ */
+public class RestGetRolesAction extends BaseRestHandler {
+
+    @Inject
+    public RestGetRolesAction(Settings settings, RestController controller, Client client) {
+        super(settings, controller, client);
+        controller.registerHandler(RestRequest.Method.GET, "/_shield/role/", this);
+        controller.registerHandler(RestRequest.Method.GET, "/_shield/role/{roles}", this);
+        controller.registerHandler(RestRequest.Method.GET, "/_shield/roles/", this);
+        controller.registerHandler(RestRequest.Method.GET, "/_shield/roles/{roles}", this);
+    }
+
+    @Override
+    protected void handleRequest(RestRequest request, final RestChannel channel, Client client) throws Exception {
+        String[] roles = Strings.splitStringByCommaToArray(request.param("roles"));
+
+        new ShieldClient(client).prepareGetRoles().roles(roles).execute(new RestBuilderListener<GetRolesResponse>(channel) {
+            @Override
+            public RestResponse buildResponse(GetRolesResponse getRolesResponse, XContentBuilder builder) throws Exception {
+                builder.startObject();
+                builder.field("found", getRolesResponse.isExists());
+                builder.startArray("roles");
+                for (ToXContent role : getRolesResponse.roles()) {
+                    role.toXContent(builder, ToXContent.EMPTY_PARAMS);
+                }
+                builder.endArray();
+                builder.endObject();
+                return new BytesRestResponse(RestStatus.OK, builder);
+            }
+        });
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/role/TransportAddRoleAction.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.role;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.shield.authz.esnative.ESNativeRolesStore;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+public class TransportAddRoleAction extends HandledTransportAction<AddRoleRequest, AddRoleResponse> {
+
+    private final ESNativeRolesStore rolesStore;
+
+    @Inject
+    public TransportAddRoleAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters,
+                                  IndexNameExpressionResolver indexNameExpressionResolver,
+                                  ESNativeRolesStore rolesStore, TransportService transportService) {
+        super(settings, AddRoleAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, AddRoleRequest::new);
+        this.rolesStore = rolesStore;
+    }
+
+    @Override
+    protected void doExecute(AddRoleRequest request, ActionListener<AddRoleResponse> listener) {
+        rolesStore.addRole(request, new ActionListener<Boolean>() {
+            @Override
+            public void onResponse(Boolean created) {
+                if (created) {
+                    logger.info("added role [{}]", request.name());
+                } else {
+                    logger.info("updated role [{}]", request.name());
+                }
+                listener.onResponse(new AddRoleResponse(created));
+            }
+
+            @Override
+            public void onFailure(Throwable t) {
+                listener.onFailure(t);
+            }
+        });
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/role/TransportDeleteRoleAction.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.role;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.shield.authz.esnative.ESNativeRolesStore;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+public class TransportDeleteRoleAction extends HandledTransportAction<DeleteRoleRequest, DeleteRoleResponse> {
+
+    private final ESNativeRolesStore rolesStore;
+
+    @Inject
+    public TransportDeleteRoleAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters,
+                                  IndexNameExpressionResolver indexNameExpressionResolver,
+                                  ESNativeRolesStore rolesStore, TransportService transportService) {
+        super(settings, DeleteRoleAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, DeleteRoleRequest::new);
+        this.rolesStore = rolesStore;
+    }
+
+    @Override
+    protected void doExecute(DeleteRoleRequest request, ActionListener<DeleteRoleResponse> listener) {
+        try {
+            rolesStore.removeRole(request, new ActionListener<Boolean>() {
+                @Override
+                public void onResponse(Boolean found) {
+                    listener.onResponse(new DeleteRoleResponse(found));
+                }
+
+                @Override
+                public void onFailure(Throwable t) {
+                    listener.onFailure(t);
+                }
+            });
+        } catch (Exception e) {
+            logger.error("failed to delete role [{}]", e);
+            listener.onFailure(e);
+        }
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/role/TransportGetRolesAction.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.role;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.shield.authz.RoleDescriptor;
+import org.elasticsearch.shield.authz.esnative.ESNativeRolesStore;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.Collections;
+import java.util.List;
+
+public class TransportGetRolesAction extends HandledTransportAction<GetRolesRequest, GetRolesResponse> {
+
+    private final ESNativeRolesStore rolesStore;
+
+    @Inject
+    public TransportGetRolesAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters,
+                                   IndexNameExpressionResolver indexNameExpressionResolver,
+                                   ESNativeRolesStore rolesStore, TransportService transportService) {
+        super(settings, GetRolesAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, GetRolesRequest::new);
+        this.rolesStore = rolesStore;
+    }
+
+    @Override
+    protected void doExecute(GetRolesRequest request, ActionListener<GetRolesResponse> listener) {
+        if (request.roles().length == 1) {
+            final String rolename = request.roles()[0];
+            // We can fetch a single role with a get, much easier
+            rolesStore.getRoleDescriptor(rolename, new ActionListener<RoleDescriptor>() {
+                @Override
+                public void onResponse(RoleDescriptor roleD) {
+                    if (roleD == null) {
+                        listener.onResponse(new GetRolesResponse());
+                    } else {
+                        listener.onResponse(new GetRolesResponse(roleD));
+                    }
+                }
+
+                @Override
+                public void onFailure(Throwable t) {
+                    logger.error("failed to retrieve role [{}]", t, rolename);
+                    listener.onFailure(t);
+                }
+            });
+        } else {
+            rolesStore.getRoleDescriptors(request.roles(), new ActionListener<List<RoleDescriptor>>() {
+                @Override
+                public void onResponse(List<RoleDescriptor> roles) {
+                    listener.onResponse(new GetRolesResponse(roles));
+                }
+
+                @Override
+                public void onFailure(Throwable t) {
+                    logger.error("failed to retrieve role [{}]", t,
+                            Strings.arrayToDelimitedString(request.roles(), ","));
+                }
+            });
+        }
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/user/AddUserAction.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.user;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.client.ElasticsearchClient;
+
+/**
+ * Action for adding a user to the shield administrative index
+ */
+public class AddUserAction extends Action<AddUserRequest, AddUserResponse, AddUserRequestBuilder> {
+
+    public static final AddUserAction INSTANCE = new AddUserAction();
+    public static final String NAME = "cluster:admin/shield/user/add";
+
+
+    protected AddUserAction() {
+        super(NAME);
+    }
+
+    @Override
+    public AddUserRequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new AddUserRequestBuilder(client, this);
+    }
+
+    @Override
+    public AddUserResponse newResponse() {
+        return new AddUserResponse();
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/user/AddUserRequest.java

@@ -0,0 +1,130 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.user;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.shield.authc.support.CharArrays;
+import org.elasticsearch.shield.authc.support.Hasher;
+import org.elasticsearch.shield.authc.support.SecuredString;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * Request object to add a {@code User} to the shield administrative index
+ */
+public class AddUserRequest extends ActionRequest<AddUserRequest> {
+
+    private final Hasher hasher = Hasher.BCRYPT;
+
+    private String username;
+    private String roles[];
+    private char[] passwordHash;
+
+    public AddUserRequest() {
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (username == null) {
+            validationException = addValidationError("user is missing", validationException);
+        }
+        if (roles == null) {
+            validationException = addValidationError("roles are missing", validationException);
+        }
+        if (passwordHash == null) {
+            validationException = addValidationError("passwordHash is missing", validationException);
+        }
+        return validationException;
+    }
+
+    public void username(String username) {
+        this.username = username;
+    }
+
+    public void roles(String... roles) {
+        this.roles = roles;
+    }
+
+    public void passwordHash(char[] passwordHash) {
+        this.passwordHash = passwordHash;
+    }
+
+    public String username() {
+        return username;
+    }
+
+    public String[] roles() {
+        return roles;
+    }
+
+    public char[] passwordHash() {
+        return passwordHash;
+    }
+
+    public AddUserRequest source(BytesReference source) throws Exception {
+        List<String> parsedRoles = new ArrayList<>();
+        try (XContentParser parser = XContentHelper.createParser(source)) {
+            XContentParser.Token token;
+            String currentFieldName = null;
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if (token.isValue()) {
+                    if ("username".equals(currentFieldName)) {
+                        username = parser.text();
+                    } else if ("password".equals(currentFieldName)) {
+                        // It's assumed the password is plaintext and needs to be hashed
+                        passwordHash = hasher.hash(new SecuredString(parser.text().toCharArray()));
+                    } else if ("roles".equals(currentFieldName)) {
+                        parsedRoles.add(parser.text());
+                    } else {
+                        throw new ElasticsearchParseException("unexpected field in add user request [{}]", currentFieldName);
+                    }
+                } else if (token == XContentParser.Token.START_OBJECT) {
+                    // expected
+                } else if (token == XContentParser.Token.START_ARRAY || token == XContentParser.Token.END_ARRAY) {
+                    if ("roles".equals(currentFieldName) == false) {
+                        throw new ElasticsearchParseException("unexpected array for field [{}]", currentFieldName);
+                    }
+                } else {
+                    throw new ElasticsearchParseException("failed to parse add user request, got value with wrong type [{}]", currentFieldName);
+                }
+            }
+        }
+        roles = parsedRoles.toArray(Strings.EMPTY_ARRAY);
+        return this;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        username = in.readString();
+        passwordHash = CharArrays.utf8BytesToChars(in.readByteArray());
+        roles = in.readStringArray();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(username);
+        out.writeByteArray(CharArrays.toUtf8Bytes(passwordHash));
+        out.writeStringArray(roles);
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/user/AddUserRequestBuilder.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.user;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.shield.authc.support.Hasher;
+import org.elasticsearch.shield.authc.support.SecuredString;
+
+public class AddUserRequestBuilder extends ActionRequestBuilder<AddUserRequest, AddUserResponse, AddUserRequestBuilder> {
+
+    private final Hasher hasher = Hasher.BCRYPT;
+
+    public AddUserRequestBuilder(ElasticsearchClient client) {
+        this(client, AddUserAction.INSTANCE);
+    }
+
+    public AddUserRequestBuilder(ElasticsearchClient client, AddUserAction action) {
+        super(client, action, new AddUserRequest());
+    }
+
+    public AddUserRequestBuilder username(String username) {
+        request.username(username);
+        return this;
+    }
+
+    public AddUserRequestBuilder roles(String... roles) {
+        request.roles(roles);
+        return this;
+    }
+
+    public AddUserRequestBuilder password(String password) {
+        request.passwordHash(hasher.hash(new SecuredString(password.toCharArray())));
+        return this;
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/user/AddUserResponse.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.user;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+/**
+ * Response when adding a user to the shield administrative index. Returns a
+ * single boolean field for whether the user was created or updated.
+ */
+public class AddUserResponse extends ActionResponse implements ToXContent {
+
+    private boolean created;
+
+    public AddUserResponse() {
+
+    }
+
+    public AddUserResponse(boolean created) {
+        this.created = created;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject().field("created", created).endObject();
+        return builder;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeBoolean(created);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        this.created = in.readBoolean();
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/user/DeleteUserAction.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.user;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.client.ElasticsearchClient;
+
+/**
+ * Action for deleting a user from the shield administrative index
+ */
+public class DeleteUserAction extends Action<DeleteUserRequest, DeleteUserResponse, DeleteUserRequestBuilder> {
+
+    public static final DeleteUserAction INSTANCE = new DeleteUserAction();
+    public static final String NAME = "cluster:admin/shield/user/delete";
+
+
+    protected DeleteUserAction() {
+        super(NAME);
+    }
+
+    @Override
+    public DeleteUserRequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new DeleteUserRequestBuilder(client, this);
+    }
+
+    @Override
+    public DeleteUserResponse newResponse() {
+        return new DeleteUserResponse();
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/user/DeleteUserRequest.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.user;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * A request to delete a user from the shield administrative index by username
+ */
+public class DeleteUserRequest extends ActionRequest<DeleteUserRequest> {
+
+    private String username;
+
+    public DeleteUserRequest() {
+    }
+
+    public DeleteUserRequest(String user) {
+        this.username = user;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (username == null) {
+            validationException = addValidationError("user is missing", validationException);
+        }
+        return validationException;
+    }
+
+    public String user() {
+        return this.username;
+    }
+
+    public void user(String username) {
+        this.username = username;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        username = in.readString();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(username);
+    }
+
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/user/DeleteUserRequestBuilder.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.user;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+
+public class DeleteUserRequestBuilder extends ActionRequestBuilder<DeleteUserRequest, DeleteUserResponse, DeleteUserRequestBuilder> {
+
+    public DeleteUserRequestBuilder(ElasticsearchClient client) {
+        this(client, DeleteUserAction.INSTANCE);
+    }
+
+    public DeleteUserRequestBuilder(ElasticsearchClient client, DeleteUserAction action) {
+        super(client, action, new DeleteUserRequest());
+    }
+
+    public DeleteUserRequestBuilder user(String username) {
+        request.user(username);
+        return this;
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/user/DeleteUserResponse.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.user;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
+/**
+ * Response when deleting a user from the shield administrative index. Returns a
+ * single boolean field for whether the user was found (and deleted) or not
+ * found.
+ */
+public class DeleteUserResponse extends ActionResponse implements ToXContent {
+
+    private boolean found;
+
+    public DeleteUserResponse() {
+
+    }
+
+    public DeleteUserResponse(boolean found) {
+        this.found = found;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject().field("found", found).endObject();
+        return builder;
+    }
+
+    public boolean found() {
+        return this.found;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        found = in.readBoolean();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeBoolean(found);
+    }
+
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/user/GetUsersAction.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.user;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.client.ElasticsearchClient;
+
+/**
+ * Action for retrieving a user from the shield administrative index
+ */
+public class GetUsersAction extends Action<GetUsersRequest, GetUsersResponse, GetUsersRequestBuilder> {
+
+    public static final GetUsersAction INSTANCE = new GetUsersAction();
+    public static final String NAME = "cluster:admin/shield/user/get";
+
+
+    protected GetUsersAction() {
+        super(NAME);
+    }
+
+    @Override
+    public GetUsersRequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new GetUsersRequestBuilder(client, this);
+    }
+
+    @Override
+    public GetUsersResponse newResponse() {
+        return new GetUsersResponse();
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/user/GetUsersRequest.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.user;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.Collections;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * Request to retrieve a user from the shield administrative index from a username
+ */
+public class GetUsersRequest extends ActionRequest<GetUsersRequest> {
+
+    private String[] users;
+
+    public GetUsersRequest() {
+        users = Strings.EMPTY_ARRAY;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (users == null) {
+            validationException = addValidationError("users cannot be null", validationException);
+        }
+        return validationException;
+    }
+
+    public void users(String... usernames) {
+        this.users = usernames;
+    }
+
+    public String[] users() {
+        return users;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        users = in.readStringArray();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeStringArray(users);
+    }
+
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/user/GetUsersRequestBuilder.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.user;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+
+public class GetUsersRequestBuilder extends ActionRequestBuilder<GetUsersRequest, GetUsersResponse, GetUsersRequestBuilder> {
+
+    public GetUsersRequestBuilder(ElasticsearchClient client) {
+        this(client, GetUsersAction.INSTANCE);
+    }
+
+    public GetUsersRequestBuilder(ElasticsearchClient client, GetUsersAction action) {
+        super(client, action, new GetUsersRequest());
+    }
+
+    public GetUsersRequestBuilder users(String... usernames) {
+        request.users(usernames);
+        return this;
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/user/GetUsersResponse.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.user;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.shield.User;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Response containing a User retrieved from the shield administrative index
+ */
+public class GetUsersResponse extends ActionResponse {
+    private List<User> users;
+
+    public GetUsersResponse() {
+        this.users = Collections.emptyList();
+    }
+
+    public GetUsersResponse(User user) {
+        this.users = Collections.singletonList(user);
+    }
+
+    public GetUsersResponse(List<User> users) {
+        this.users = users;
+    }
+
+    public List<User> users() {
+        return users;
+    }
+
+    public boolean isExists() {
+        return users != null && users.size() > 0;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        int size = in.readVInt();
+        users = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            users.add(User.readFrom(in));
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeVInt(users == null ? 0 : users.size());
+        for (User u : users) {
+            User.writeTo(u, out);
+        }
+    }
+
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/user/RestAddUserAction.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.user;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.support.RestBuilderListener;
+import org.elasticsearch.shield.client.ShieldClient;
+
+import java.io.IOException;
+
+/**
+ * Rest endpoint to add a User to the shield index
+ */
+public class RestAddUserAction extends BaseRestHandler {
+
+    @Inject
+    public RestAddUserAction(Settings settings, RestController controller, Client client) {
+        super(settings, controller, client);
+        controller.registerHandler(RestRequest.Method.POST, "/_shield/user/{username}", this);
+        controller.registerHandler(RestRequest.Method.PUT, "/_shield/user/{username}", this);
+    }
+
+    @Override
+    protected void handleRequest(RestRequest request, final RestChannel channel, Client client) throws Exception {
+        AddUserRequest addUserReq = new AddUserRequest();
+        addUserReq.username(request.param("username"));
+        addUserReq.source(request.content());
+
+        new ShieldClient(client).addUser(addUserReq, new RestBuilderListener<AddUserResponse>(channel) {
+            @Override
+            public RestResponse buildResponse(AddUserResponse addUserResponse, XContentBuilder builder) throws Exception {
+                return new BytesRestResponse(RestStatus.OK,
+                        builder.startObject()
+                                .field("user", (ToXContent) addUserResponse)
+                                .endObject());
+            }
+        });
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/user/RestDeleteUserAction.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.user;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.support.RestBuilderListener;
+import org.elasticsearch.shield.client.ShieldClient;
+
+import java.io.IOException;
+
+/**
+ * Rest action to delete a user from the shield index
+ */
+public class RestDeleteUserAction extends BaseRestHandler {
+
+    @Inject
+    public RestDeleteUserAction(Settings settings, RestController controller, Client client) {
+        super(settings, controller, client);
+        controller.registerHandler(RestRequest.Method.DELETE, "/_shield/user/{username}", this);
+    }
+
+    @Override
+    protected void handleRequest(RestRequest request, final RestChannel channel, Client client) throws Exception {
+        String user = request.param("username");
+        DeleteUserRequest delUserRequest = new DeleteUserRequest(user);
+
+        new ShieldClient(client).deleteUser(delUserRequest, new RestBuilderListener<DeleteUserResponse>(channel) {
+            @Override
+            public RestResponse buildResponse(DeleteUserResponse response, XContentBuilder builder) throws Exception {
+                return new BytesRestResponse(response.found() ? RestStatus.OK : RestStatus.NOT_FOUND,
+                        builder.startObject()
+                                .field("found", response.found())
+                                .endObject());
+            }
+        });
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/user/RestGetUsersAction.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.user;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.support.RestBuilderListener;
+import org.elasticsearch.shield.User;
+import org.elasticsearch.shield.client.ShieldClient;
+
+/**
+ * Rest action to retrieve a user from the shield index
+ */
+public class RestGetUsersAction extends BaseRestHandler {
+
+    @Inject
+    public RestGetUsersAction(Settings settings, RestController controller, Client client) {
+        super(settings, controller, client);
+        controller.registerHandler(RestRequest.Method.GET, "/_shield/user/", this);
+        controller.registerHandler(RestRequest.Method.GET, "/_shield/user/{user}", this);
+        controller.registerHandler(RestRequest.Method.GET, "/_shield/users/", this);
+        controller.registerHandler(RestRequest.Method.GET, "/_shield/users/{user}", this);
+    }
+
+    @Override
+    protected void handleRequest(RestRequest request, final RestChannel channel, Client client) throws Exception {
+        String[] users = Strings.splitStringByCommaToArray(request.param("user"));
+
+        new ShieldClient(client).prepareGetUsers().users(users).execute(new RestBuilderListener<GetUsersResponse>(channel) {
+            @Override
+            public RestResponse buildResponse(GetUsersResponse getUsersResponse, XContentBuilder builder) throws Exception {
+                builder.startObject();
+                builder.field("found", getUsersResponse.isExists());
+                builder.startArray("users");
+                for (User user : getUsersResponse.users()) {
+                    user.toXContent(builder, ToXContent.EMPTY_PARAMS);
+                }
+                builder.endArray();
+                builder.endObject();
+                return new BytesRestResponse(RestStatus.OK, builder);
+            }
+        });
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/user/TransportAddUserAction.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.user;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.shield.authc.esnative.ESNativeUsersStore;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+public class TransportAddUserAction extends HandledTransportAction<AddUserRequest, AddUserResponse> {
+
+    private final ESNativeUsersStore usersStore;
+
+    @Inject
+    public TransportAddUserAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters,
+                                  IndexNameExpressionResolver indexNameExpressionResolver,
+                                  ESNativeUsersStore usersStore, TransportService transportService) {
+        super(settings, AddUserAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, AddUserRequest::new);
+        this.usersStore = usersStore;
+    }
+
+    @Override
+    protected void doExecute(final AddUserRequest request, final ActionListener<AddUserResponse> listener) {
+        usersStore.addUser(request, new ActionListener<Boolean>() {
+            @Override
+            public void onResponse(Boolean created) {
+                if (created) {
+                    logger.info("added user [{}]", request.username());
+                } else {
+                    logger.info("updated user [{}]", request.username());
+                }
+                listener.onResponse(new AddUserResponse(created));
+            }
+
+            @Override
+            public void onFailure(Throwable e) {
+                logger.error("failed to add user: ", e);
+                listener.onFailure(e);
+            }
+        });
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/user/TransportDeleteUserAction.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.user;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.shield.authc.esnative.ESNativeUsersStore;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+public class TransportDeleteUserAction extends HandledTransportAction<DeleteUserRequest, DeleteUserResponse> {
+
+    private final ESNativeUsersStore usersStore;
+
+    @Inject
+    public TransportDeleteUserAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters,
+                                  IndexNameExpressionResolver indexNameExpressionResolver,
+                                  ESNativeUsersStore usersStore, TransportService transportService) {
+        super(settings, DeleteUserAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, DeleteUserRequest::new);
+        this.usersStore = usersStore;
+    }
+
+    @Override
+    protected void doExecute(DeleteUserRequest request, final ActionListener<DeleteUserResponse> listener) {
+        try {
+            usersStore.removeUser(request, new ActionListener<Boolean>() {
+                @Override
+                public void onResponse(Boolean found) {
+                    listener.onResponse(new DeleteUserResponse(found));
+                }
+
+                @Override
+                public void onFailure(Throwable e) {
+                    listener.onFailure(e);
+                }
+            });
+        } catch (Exception e) {
+            logger.error("failed to delete user [{}]", e);
+            listener.onFailure(e);
+        }
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/admin/user/TransportGetUsersAction.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.admin.user;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.shield.User;
+import org.elasticsearch.shield.authc.esnative.ESNativeRealm;
+import org.elasticsearch.shield.authc.esnative.ESNativeUsersStore;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.Collections;
+import java.util.List;
+
+public class TransportGetUsersAction extends HandledTransportAction<GetUsersRequest, GetUsersResponse> {
+
+    private final ESNativeUsersStore usersStore;
+
+    @Inject
+    public TransportGetUsersAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters,
+                                   IndexNameExpressionResolver indexNameExpressionResolver,
+                                   ESNativeUsersStore usersStore, TransportService transportService) {
+        super(settings, GetUsersAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, GetUsersRequest::new);
+        this.usersStore = usersStore;
+    }
+
+    @Override
+    protected void doExecute(final GetUsersRequest request, final ActionListener<GetUsersResponse> listener) {
+        if (request.users().length == 1) {
+            final String username = request.users()[0];
+            // We can fetch a single user with a get, much cheaper:
+            usersStore.getUser(username, new ActionListener<User>() {
+                @Override
+                public void onResponse(User user) {
+                    if (user == null) {
+                        listener.onResponse(new GetUsersResponse());
+                    } else {
+                        listener.onResponse(new GetUsersResponse(user));
+                    }
+                }
+
+                @Override
+                public void onFailure(Throwable e) {
+                    logger.error("failed to retrieve user [{}]", e, username);
+                    listener.onFailure(e);
+                }
+            });
+        } else {
+            usersStore.getUsers(request.users(), new ActionListener<List<User>>() {
+                @Override
+                public void onResponse(List<User> users) {
+                    listener.onResponse(new GetUsersResponse(users));
+                }
+
+                @Override
+                public void onFailure(Throwable e) {
+                    logger.error("failed to retrieve user [{}]", e,
+                            Strings.arrayToDelimitedString(request.users(), ","));
+                    listener.onFailure(e);
+                }
+            });
+        }
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/authc/cache/ClearRealmCacheRequestBuilder.java

@@ -7,23 +7,18 @@ package org.elasticsearch.shield.action.authc.cache;
 
 import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.shield.client.ShieldAuthcClient;
-import org.elasticsearch.shield.client.ShieldClient;
 
 /**
  *
  */
 public class ClearRealmCacheRequestBuilder extends NodesOperationRequestBuilder<ClearRealmCacheRequest, ClearRealmCacheResponse, ClearRealmCacheRequestBuilder> {
 
-    private final ShieldAuthcClient authcClient;
-
     public ClearRealmCacheRequestBuilder(ElasticsearchClient client) {
         this(client, ClearRealmCacheAction.INSTANCE);
     }
 
     public ClearRealmCacheRequestBuilder(ElasticsearchClient client, ClearRealmCacheAction action) {
         super(client, action, new ClearRealmCacheRequest());
-        authcClient = new ShieldClient(client).authc();
     }
 
     /**

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/authc/cache/TransportClearRealmCacheAction.java

@@ -47,6 +47,8 @@ public class TransportClearRealmCacheAction extends TransportNodesAction<ClearRe
             Object resp = responses.get(i);
             if (resp instanceof ClearRealmCacheResponse.Node) {
                 nodes.add((ClearRealmCacheResponse.Node) resp);
+            } else {
+                throw new IllegalArgumentException("node response [" + resp.getClass() + "] is not the correct type");
             }
         }
         return new ClearRealmCacheResponse(clusterName, nodes.toArray(new ClearRealmCacheResponse.Node[nodes.size()]));

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/authz/cache/ClearRolesCacheAction.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.authz.cache;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.shield.action.authc.cache.ClearRealmCacheRequestBuilder;
+
+/**
+ * The action for clearing the cache used by native roles that are stored in an index.
+ */
+public class ClearRolesCacheAction extends Action<ClearRolesCacheRequest, ClearRolesCacheResponse, ClearRolesCacheRequestBuilder> {
+
+    public static final ClearRolesCacheAction INSTANCE = new ClearRolesCacheAction();
+    public static final String NAME = "cluster:admin/shield/roles/cache/clear";
+
+    protected ClearRolesCacheAction() {
+        super(NAME);
+    }
+
+    @Override
+    public ClearRolesCacheRequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new ClearRolesCacheRequestBuilder(client, this, new ClearRolesCacheRequest());
+    }
+
+    @Override
+    public ClearRolesCacheResponse newResponse() {
+        return new ClearRolesCacheResponse();
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/authz/cache/ClearRolesCacheRequest.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.authz.cache;
+
+import org.elasticsearch.action.support.nodes.BaseNodeRequest;
+import org.elasticsearch.action.support.nodes.BaseNodesRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+/**
+ * The request used to clear the cache for native roles stored in an index.
+ */
+public class ClearRolesCacheRequest extends BaseNodesRequest<ClearRolesCacheRequest> {
+
+    String[] roles;
+
+    /**
+     * Sets the roles for which caches will be evicted. When not set all the roles will be evicted from the cache.
+     *
+     * @param roles    The role names
+     */
+    public ClearRolesCacheRequest roles(String... roles) {
+        this.roles = roles;
+        return this;
+    }
+
+    /**
+     * @return an array of role names that will have the cache evicted or <code>null</code> if all
+     */
+    public String[] roles() {
+        return roles;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        roles = in.readOptionalStringArray();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeOptionalStringArray(roles);
+    }
+
+    public static class Node extends BaseNodeRequest {
+        String[] roles;
+
+        public Node() {
+        }
+
+        public Node(ClearRolesCacheRequest request, String nodeId) {
+            super(request, nodeId);
+            this.roles = request.roles();
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            roles = in.readOptionalStringArray();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeOptionalStringArray(roles);
+        }
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/authz/cache/ClearRolesCacheRequestBuilder.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.authz.cache;
+
+import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+
+/**
+ * Request builder for the {@link ClearRolesCacheRequest}
+ */
+public class ClearRolesCacheRequestBuilder extends NodesOperationRequestBuilder<ClearRolesCacheRequest, ClearRolesCacheResponse, ClearRolesCacheRequestBuilder> {
+
+    public ClearRolesCacheRequestBuilder(ElasticsearchClient client) {
+        this(client, ClearRolesCacheAction.INSTANCE, new ClearRolesCacheRequest());
+    }
+
+    public ClearRolesCacheRequestBuilder(ElasticsearchClient client, ClearRolesCacheAction action, ClearRolesCacheRequest request) {
+        super(client, action, request);
+    }
+
+    /**
+     * Set the roles to be cleared
+     * @param roles the names of the roles that should be cleared
+     * @return the builder instance
+     */
+    public ClearRolesCacheRequestBuilder roles(String... roles) {
+        request.roles(roles);
+        return this;
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/authz/cache/ClearRolesCacheResponse.java

@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.authz.cache;
+
+import org.elasticsearch.action.support.nodes.BaseNodeResponse;
+import org.elasticsearch.action.support.nodes.BaseNodesResponse;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+
+import java.io.IOException;
+
+/**
+ * The response object that will be returned when clearing the cache of native roles
+ */
+public class ClearRolesCacheResponse extends BaseNodesResponse<ClearRolesCacheResponse.Node> implements ToXContent {
+
+    public ClearRolesCacheResponse() {
+    }
+
+    public ClearRolesCacheResponse(ClusterName clusterName, Node[] nodes) {
+        super(clusterName, nodes);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        nodes = new Node[in.readVInt()];
+        for (int i = 0; i < nodes.length; i++) {
+            nodes[i] = Node.readNodeResponse(in);
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeVInt(nodes.length);
+        for (Node node : nodes) {
+            node.writeTo(out);
+        }
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field("cluster_name", getClusterName().value());
+        builder.startObject("nodes");
+        for (ClearRolesCacheResponse.Node node: getNodes()) {
+            builder.startObject(node.getNode().id());
+            builder.field("name", node.getNode().name());
+            builder.endObject();
+        }
+        builder.endObject();
+        return builder.endObject();
+    }
+
+    @Override
+    public String toString() {
+        try {
+            XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
+            builder.startObject();
+            toXContent(builder, EMPTY_PARAMS);
+            builder.endObject();
+            return builder.string();
+        } catch (IOException e) {
+            return "{ \"error\" : \"" + e.getMessage() + "\"}";
+        }
+    }
+
+    public static class Node extends BaseNodeResponse {
+
+        Node() {
+        }
+
+        Node(DiscoveryNode node) {
+            super(node);
+        }
+
+        public static Node readNodeResponse(StreamInput in) throws IOException {
+            Node node = new Node();
+            node.readFrom(in);
+            return node;
+        }
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/action/authz/cache/TransportClearRolesCacheAction.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.action.authz.cache;
+
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.nodes.TransportNodesAction;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.shield.authz.esnative.ESNativeRolesStore;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReferenceArray;
+
+/**
+ *
+ */
+public class TransportClearRolesCacheAction extends TransportNodesAction<ClearRolesCacheRequest, ClearRolesCacheResponse, ClearRolesCacheRequest.Node, ClearRolesCacheResponse.Node> {
+
+    private final ESNativeRolesStore rolesStore;
+
+    @Inject
+    public TransportClearRolesCacheAction(Settings settings, ClusterName clusterName, ThreadPool threadPool,
+                                          ClusterService clusterService, TransportService transportService,
+                                          ActionFilters actionFilters, ESNativeRolesStore rolesStore,
+                                          IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(settings, ClearRolesCacheAction.NAME, clusterName, threadPool, clusterService, transportService,
+                actionFilters, indexNameExpressionResolver, ClearRolesCacheRequest::new, ClearRolesCacheRequest.Node::new,
+                ThreadPool.Names.MANAGEMENT);
+        this.rolesStore = rolesStore;
+    }
+
+    @Override
+    protected ClearRolesCacheResponse newResponse(ClearRolesCacheRequest request, AtomicReferenceArray nodesResponses) {
+        List<ClearRolesCacheResponse.Node> responses = new ArrayList<>(nodesResponses.length());
+        for (int i = 0; i < nodesResponses.length(); i++) {
+            Object resp = nodesResponses.get(i);
+            if (resp instanceof ClearRolesCacheResponse.Node) {
+                responses.add((ClearRolesCacheResponse.Node) resp);
+            } else {
+                throw new IllegalArgumentException("node response [" + resp.getClass() + "] is not the correct type");
+            }
+        }
+        return new ClearRolesCacheResponse(clusterName, responses.toArray(new ClearRolesCacheResponse.Node[responses.size()]));
+    }
+
+    @Override
+    protected ClearRolesCacheRequest.Node newNodeRequest(String nodeId, ClearRolesCacheRequest request) {
+        return new ClearRolesCacheRequest.Node(request, nodeId);
+    }
+
+    @Override
+    protected ClearRolesCacheResponse.Node newNodeResponse() {
+        return new ClearRolesCacheResponse.Node();
+    }
+
+    @Override
+    protected ClearRolesCacheResponse.Node nodeOperation(ClearRolesCacheRequest.Node request) {
+        if (request.roles == null || request.roles.length == 0) {
+            rolesStore.invalidateAll();
+        } else {
+            for (String role : request.roles) {
+                rolesStore.invalidate(role);
+            }
+        }
+        return new ClearRolesCacheResponse.Node(clusterService.localNode());
+    }
+
+    @Override
+    protected boolean accumulateExceptions() {
+        return false;
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/admin/ShieldAdminModule.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.admin;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.shield.support.AbstractShieldModule;
+
+/**
+ * TODO: document
+ */
+public class ShieldAdminModule extends AbstractShieldModule.Node {
+
+    private ShieldInternalUserHolder userHolder;
+
+    public ShieldAdminModule(Settings settings) {
+        super(settings);
+    }
+
+    @Override
+    protected void configureNode() {
+        userHolder = new ShieldInternalUserHolder();
+        bind(ShieldInternalUserHolder.class).toInstance(userHolder);
+        bind(ShieldTemplateService.class).asEagerSingleton();
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/admin/ShieldInternalUserHolder.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.admin;
+
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction;
+import org.elasticsearch.shield.User;
+import org.elasticsearch.shield.authz.Permission;
+import org.elasticsearch.shield.authz.Privilege;
+
+/**
+ * User holder for the shield internal user that manages the {@code .shield}
+ * index. Has permission to monitor the cluster as well as all actions that deal
+ * with the shield admin index.
+ */
+public class ShieldInternalUserHolder {
+
+    private static final String NAME = "__es_internal_user";
+    private static final String[] ROLES = new String[] { "__es_internal_role" };
+    public static final Permission.Global.Role ROLE = Permission.Global.Role.builder(ROLES[0])
+            .cluster(Privilege.Cluster.get(new Privilege.Name(PutIndexTemplateAction.NAME, "cluster:admin/shield/realm/cache/clear*", "cluster:admin/shield/roles/cache/clear*")))
+            .add(Privilege.Index.ALL, ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME)
+            .build();
+    private static final User SHIELD_INTERNAL_USER = new User(NAME, ROLES);
+
+    public User user() {
+        return SHIELD_INTERNAL_USER;
+    }
+
+    public static boolean isShieldInternalUser(User user) {
+        return SHIELD_INTERNAL_USER.equals(user);
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/admin/ShieldTemplateService.java

@@ -0,0 +1,122 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.admin;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterService;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.inject.Provider;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.gateway.GatewayService;
+import org.elasticsearch.shield.authc.AuthenticationService;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * ShieldTemplateService is responsible for adding the template needed for the
+ * {@code .shield} administrative index.
+ */
+public class ShieldTemplateService extends AbstractComponent implements ClusterStateListener {
+
+    public static final String SHIELD_ADMIN_INDEX_NAME = ".shield";
+    public static final String SHIELD_TEMPLATE_NAME = "shield-index-template";
+
+    private final ThreadPool threadPool;
+    private final Provider<Client> clientProvider;
+    private final Provider<AuthenticationService> authProvider;
+    private final ShieldInternalUserHolder adminUser;
+    private final AtomicBoolean templateCreationPending = new AtomicBoolean(false);
+
+    @Inject
+    public ShieldTemplateService(Settings settings, ClusterService clusterService,
+            Provider<Client> clientProvider, ThreadPool threadPool,
+            Provider<AuthenticationService> authProvider,
+            ShieldInternalUserHolder userHolder) {
+        super(settings);
+        this.threadPool = threadPool;
+        this.clientProvider = clientProvider;
+        this.authProvider = authProvider;
+        this.adminUser = userHolder;
+        clusterService.add(this);
+    }
+
+    private void createShieldTemplate() {
+        Client client = this.clientProvider.get();
+        AuthenticationService authService = this.authProvider.get();
+        try (InputStream is = getClass().getResourceAsStream("/" + SHIELD_TEMPLATE_NAME + ".json")) {
+            ByteArrayOutputStream out = new ByteArrayOutputStream();
+            Streams.copy(is, out);
+            final byte[] template = out.toByteArray();
+            logger.info("--> putting the shield index template");
+            PutIndexTemplateRequest putTemplateRequest = client.admin().indices()
+                    .preparePutTemplate(SHIELD_TEMPLATE_NAME).setSource(template).request();
+            authService.attachUserHeaderIfMissing(putTemplateRequest, adminUser.user());
+            PutIndexTemplateResponse templateResponse = client.admin().indices().putTemplate(putTemplateRequest).get();
+            if (templateResponse.isAcknowledged() == false) {
+                throw new ElasticsearchException("adding template for shield admin index was not acknowledged");
+            }
+        } catch (Exception e) {
+            logger.error("failed to create shield admin index template [{}]",
+                    e, SHIELD_ADMIN_INDEX_NAME);
+            throw new IllegalStateException("failed to create shield admin index template [" +
+                    SHIELD_ADMIN_INDEX_NAME + "]", e);
+        }
+
+    }
+
+    @Override
+    public void clusterChanged(ClusterChangedEvent event) {
+        if (event.state().blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
+            // wait until the gateway has recovered from disk, otherwise we think may not have .shield-audit-
+            // but they may not have been restored from the cluster state on disk
+            logger.debug("template service waiting until state has been recovered");
+            return;
+        }
+        
+        IndexRoutingTable shieldIndexRouting = event.state().routingTable().index(SHIELD_ADMIN_INDEX_NAME);
+
+        if (shieldIndexRouting == null) {
+            if (event.localNodeMaster()) {
+                ClusterState state = event.state();
+                // TODO for the future need to add some checking in the event the template needs to be updated...
+                IndexTemplateMetaData templateMeta = state.metaData().templates().get(SHIELD_TEMPLATE_NAME);
+                final boolean createTemplate = (templateMeta == null);
+
+                if (createTemplate && templateCreationPending.compareAndSet(false, true)) {
+                    threadPool.generic().execute(new AbstractRunnable() {
+                        @Override
+                        public void onFailure(Throwable t) {
+                            logger.warn("failed to create shield admin template", t);
+                            templateCreationPending.set(false);
+                        }
+
+                        @Override
+                        protected void doRun() throws Exception {
+                            if (createTemplate) {
+                                createShieldTemplate();
+                            }
+                            templateCreationPending.set(false);
+                        }
+                    });
+                }
+            }
+        }
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/audit/AuditTrailModule.java

@@ -54,8 +54,6 @@ public class AuditTrailModule extends AbstractShieldModule.Node {
                     bind(LoggingAuditTrail.class).asEagerSingleton();
                     break;
                 case IndexAuditTrail.NAME:
-                    // TODO should bind the lifecycle service in ShieldModule if we use it other places...
-                    bind(ShieldLifecycleService.class).asEagerSingleton();
                     bind(IndexAuditUserHolder.class).toInstance(indexAuditUser);
                     binder.addBinding().to(IndexAuditTrail.class);
                     bind(IndexAuditTrail.class).asEagerSingleton();
@@ -70,6 +68,18 @@ public class AuditTrailModule extends AbstractShieldModule.Node {
         return settings.getAsBoolean("shield.audit.enabled", false);
     }
 
+    public static boolean indexAuditLoggingEnabled(Settings settings) {
+        if (auditingEnabled(settings)) {
+            String[] outputs = settings.getAsArray("shield.audit.outputs");
+            for (String output : outputs) {
+                if (output.equals(IndexAuditTrail.NAME)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
     public static boolean fileAuditLoggingEnabled(Settings settings) {
         if (auditingEnabled(settings)) {
             String[] outputs = settings.getAsArray("shield.audit.outputs", new String[] { LoggingAuditTrail.NAME });

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/audit/index/IndexAuditTrail.java

@@ -40,9 +40,10 @@ import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentBuilderString;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.env.Environment;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.shield.admin.ShieldInternalUserHolder;
+import org.elasticsearch.xpack.XPackPlugin;
 import org.elasticsearch.shield.User;
 import org.elasticsearch.shield.audit.AuditTrail;
 import org.elasticsearch.shield.authc.AuthenticationService;
@@ -127,7 +128,6 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail, Cl
     private final IndexAuditUserHolder auditUser;
     private final Provider<Client> clientProvider;
     private final AuthenticationService authenticationService;
-    private final Environment environment;
     private final LinkedBlockingQueue<Message> eventQueue;
     private final QueueConsumer queueConsumer;
     private final Transport transport;
@@ -150,13 +150,12 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail, Cl
 
     @Inject
     public IndexAuditTrail(Settings settings, IndexAuditUserHolder indexingAuditUser,
-                           Environment environment, AuthenticationService authenticationService,
-                           Transport transport, Provider<Client> clientProvider, ThreadPool threadPool, ClusterService clusterService) {
+                           AuthenticationService authenticationService, Transport transport,
+                           Provider<Client> clientProvider, ThreadPool threadPool, ClusterService clusterService) {
         super(settings);
         this.auditUser = indexingAuditUser;
         this.authenticationService = authenticationService;
         this.clientProvider = clientProvider;
-        this.environment = environment;
         this.transport = transport;
         this.threadPool = threadPool;
         this.clusterService = clusterService;
@@ -418,7 +417,7 @@ public class IndexAuditTrail extends AbstractComponent implements AuditTrail, Cl
     public void accessGranted(User user, String action, TransportMessage<?> message) {
         if (!principalIsAuditor(user.principal())) {
             // special treatment for internal system actions - only log if explicitly told to
-            if (user.isSystem() && Privilege.SYSTEM.predicate().test(action)) {
+            if ((user.isSystem() && Privilege.SYSTEM.predicate().test(action)) || ShieldInternalUserHolder.isShieldInternalUser(user)) {
                 if (events.contains(SYSTEM_ACCESS_GRANTED)) {
                     try {
                         enqueue(message("access_granted", action, user, indices(message), message), "access_granted");

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/audit/index/IndexAuditUserHolder.java

@@ -20,8 +20,6 @@ public class IndexAuditUserHolder {
 
     private static final String NAME = "__indexing_audit_user";
     private static final String[] ROLE_NAMES = new String[] { "__indexing_audit_role" };
-
-    private final User user;
     public static final Permission.Global.Role ROLE = Permission.Global.Role.builder(ROLE_NAMES[0])
         .cluster(Privilege.Cluster.action(PutIndexTemplateAction.NAME))
         .add(Privilege.Index.CREATE_INDEX, IndexAuditTrail.INDEX_NAME_PREFIX + "*")
@@ -31,8 +29,10 @@ public class IndexAuditUserHolder {
         .add(Privilege.Index.action(PutMappingAction.NAME), IndexAuditTrail.INDEX_NAME_PREFIX + "*")
         .build();
 
+    private final User user;
+
     public IndexAuditUserHolder() {
-        this.user = new User.Simple(NAME, ROLE_NAMES);
+        this.user = new User(NAME, ROLE_NAMES);
     }
 
     public User user() {

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/audit/logfile/LoggingAuditTrail.java

@@ -17,6 +17,7 @@ import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.shield.User;
+import org.elasticsearch.shield.admin.ShieldInternalUserHolder;
 import org.elasticsearch.shield.audit.AuditTrail;
 import org.elasticsearch.shield.authc.AuthenticationToken;
 import org.elasticsearch.shield.authz.Privilege;
@@ -194,7 +195,7 @@ public class LoggingAuditTrail extends AbstractLifecycleComponent<LoggingAuditTr
         String indices = indicesString(message);
 
         // special treatment for internal system actions - only log on trace
-        if (user.isSystem() && Privilege.SYSTEM.predicate().test(action)) {
+        if ((user.isSystem() && Privilege.SYSTEM.predicate().test(action)) || ShieldInternalUserHolder.isShieldInternalUser(user)) {
             if (logger.isTraceEnabled()) {
                 if (indices != null) {
                     logger.trace("{}[transport] [access_granted]\t{}, {}, action=[{}], indices=[{}], request=[{}]", prefix, originAttributes(message, transport), principal(user), action, indices, message.getClass().getSimpleName());

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/AnonymousService.java

@@ -51,6 +51,6 @@ public class AnonymousService {
             return null;
         }
         String username = settings.get("shield.authc.anonymous.username", ANONYMOUS_USERNAME);
-        return new User.Simple(username, roles);
+        return new User(username, roles);
     }
 }

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/AuthenticationModule.java

@@ -8,6 +8,7 @@ package org.elasticsearch.shield.authc;
 import org.elasticsearch.common.inject.multibindings.MapBinder;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.shield.authc.activedirectory.ActiveDirectoryRealm;
+import org.elasticsearch.shield.authc.esnative.ESNativeRealm;
 import org.elasticsearch.shield.authc.esusers.ESUsersRealm;
 import org.elasticsearch.shield.authc.ldap.LdapRealm;
 import org.elasticsearch.shield.authc.pki.PkiRealm;
@@ -38,6 +39,7 @@ public class AuthenticationModule extends AbstractShieldModule.Node {
     protected void configureNode() {
         MapBinder<String, Realm.Factory> mapBinder = MapBinder.newMapBinder(binder(), String.class, Realm.Factory.class);
         mapBinder.addBinding(ESUsersRealm.TYPE).to(ESUsersRealm.Factory.class).asEagerSingleton();
+        mapBinder.addBinding(ESNativeRealm.TYPE).to(ESNativeRealm.Factory.class).asEagerSingleton();
         mapBinder.addBinding(ActiveDirectoryRealm.TYPE).to(ActiveDirectoryRealm.Factory.class).asEagerSingleton();
         mapBinder.addBinding(LdapRealm.TYPE).to(LdapRealm.Factory.class).asEagerSingleton();
         mapBinder.addBinding(PkiRealm.TYPE).to(PkiRealm.Factory.class).asEagerSingleton();

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/InternalAuthenticationService.java

@@ -8,6 +8,7 @@ package org.elasticsearch.shield.authc;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.common.Base64;
 import org.elasticsearch.common.ContextAndHeaderHolder;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -126,10 +127,10 @@ public class InternalAuthenticationService extends AbstractComponent implements
                 // wrap in a try catch because the user constructor could throw an exception if we are trying to runAs the system user
                 try {
                     if (runAsUser != null) {
-                        user = new User.Simple(user.principal(), user.roles(), runAsUser);
+                        user = new User(user.principal(), user.roles(), runAsUser);
                     } else {
                         // the requested run as user does not exist, but we don't throw an error here otherwise this could let information leak about users in the system... instead we'll just let the authz service fail throw an authorization error
-                        user = new User.Simple(user.principal(), user.roles(), new User.Simple(runAsUsername, null));
+                        user = new User(user.principal(), user.roles(), new User(runAsUsername, Strings.EMPTY_ARRAY));
                     }
                 } catch (Exception e) {
                     if (logger.isDebugEnabled()) {
@@ -303,10 +304,10 @@ public class InternalAuthenticationService extends AbstractComponent implements
                 // wrap in a try catch because the user constructor could throw an exception if we are trying to runAs the system user
                 try {
                     if (runAsUser != null) {
-                        user = new User.Simple(user.principal(), user.roles(), runAsUser);
+                        user = new User(user.principal(), user.roles(), runAsUser);
                     } else {
                         // the requested run as user does not exist, but we don't throw an error here otherwise this could let information leak about users in the system... instead we'll just let the authz service fail throw an authorization error
-                        user = new User.Simple(user.principal(), user.roles(), new User.Simple(runAsUsername, null));
+                        user = new User(user.principal(), user.roles(), new User(runAsUsername, Strings.EMPTY_ARRAY));
                     }
                 } catch (Exception e) {
                     if (logger.isDebugEnabled()) {

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/Realms.java

@@ -11,6 +11,7 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.shield.ShieldSettingsFilter;
+import org.elasticsearch.shield.authc.esnative.ESNativeRealm;
 import org.elasticsearch.shield.authc.esusers.ESUsersRealm;
 import org.elasticsearch.shield.license.ShieldLicenseState;
 
@@ -134,7 +135,14 @@ public class Realms extends AbstractLifecycleComponent<Realms> implements Iterab
 
         // there is no "realms" configuration, go over all the factories and try to create defaults
         // for all the internal realms
-        realms.add(factories.get(ESUsersRealm.TYPE).createDefault("default_" + ESUsersRealm.TYPE));
+        Realm.Factory indexRealmFactory = factories.get(ESNativeRealm.TYPE);
+        if (indexRealmFactory != null) {
+            realms.add(indexRealmFactory.createDefault("default_" + ESNativeRealm.TYPE));
+        }
+        Realm.Factory esUsersRealm = factories.get(ESUsersRealm.TYPE);
+        if (esUsersRealm != null) {
+            realms.add(esUsersRealm.createDefault("default_" + ESUsersRealm.TYPE));
+        }
         return realms;
     }
 

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/esnative/ESNativeRealm.java

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authc.esnative;
+
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.shield.User;
+import org.elasticsearch.shield.authc.Realm;
+import org.elasticsearch.shield.authc.RealmConfig;
+import org.elasticsearch.shield.authc.support.CachingUsernamePasswordRealm;
+import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
+
+import java.util.List;
+
+/**
+ * User/password realm that is backed by an Elasticsearch index
+ */
+public class ESNativeRealm extends CachingUsernamePasswordRealm {
+
+    public static final String TYPE = "esnative";
+
+    final ESNativeUsersStore userStore;
+
+    public ESNativeRealm(RealmConfig config, ESNativeUsersStore usersStore) {
+        super(TYPE, config);
+        this.userStore = usersStore;
+        usersStore.addListener(new Listener());
+    }
+
+    @Override
+    public boolean userLookupSupported() {
+        return true;
+    }
+
+    @Override
+    protected User doLookupUser(String username) {
+        return userStore.getUser(username);
+    }
+    
+    @Override
+    protected User doAuthenticate(UsernamePasswordToken token) {
+        return userStore.verifyPassword(token.principal(), token.credentials());
+    }
+
+    class Listener implements ESNativeUsersStore.ChangeListener {
+
+        @Override
+        public void onUsersChanged(List<String> usernames) {
+            for (String username : usernames) {
+                expire(username);
+            }
+        }
+    }
+
+    public static class Factory extends Realm.Factory<ESNativeRealm> {
+
+        private final Settings settings;
+        private final Environment env;
+        private final ESNativeUsersStore userStore;
+
+        @Inject
+        public Factory(Settings settings, Environment env, ESNativeUsersStore userStore) {
+            super(TYPE, true);
+            this.settings = settings;
+            this.env = env;
+            this.userStore = userStore;
+        }
+
+        @Override
+        public ESNativeRealm create(RealmConfig config) {
+            return new ESNativeRealm(config, userStore);
+        }
+
+        @Override
+        public ESNativeRealm createDefault(String name) {
+            RealmConfig config = new RealmConfig(name, Settings.EMPTY, settings, env);
+            return create(config);
+        }
+    }
+
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/esnative/ESNativeUsersStore.java

@@ -0,0 +1,669 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authc.esnative;
+
+import com.carrotsearch.hppc.ObjectHashSet;
+import com.carrotsearch.hppc.ObjectLongHashMap;
+import com.carrotsearch.hppc.ObjectLongMap;
+import com.carrotsearch.hppc.cursors.ObjectCursor;
+import com.carrotsearch.hppc.cursors.ObjectLongCursor;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.LatchedActionListener;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.search.ClearScrollRequest;
+import org.elasticsearch.action.search.ClearScrollResponse;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchScrollRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.inject.Provider;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.FutureUtils;
+import org.elasticsearch.gateway.GatewayService;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.shield.User;
+import org.elasticsearch.shield.action.admin.user.AddUserRequest;
+import org.elasticsearch.shield.action.admin.user.DeleteUserRequest;
+import org.elasticsearch.shield.action.authc.cache.ClearRealmCacheRequest;
+import org.elasticsearch.shield.action.authc.cache.ClearRealmCacheResponse;
+import org.elasticsearch.shield.admin.ShieldTemplateService;
+import org.elasticsearch.shield.admin.ShieldInternalUserHolder;
+import org.elasticsearch.shield.authc.AuthenticationService;
+import org.elasticsearch.shield.authc.support.Hasher;
+import org.elasticsearch.shield.authc.support.SecuredString;
+import org.elasticsearch.shield.client.ShieldClient;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportMessage;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * ESNativeUsersStore is a {@code UserStore} that, instead of reading from a
+ * file, reads from an Elasticsearch index instead. This {@code UserStore} in
+ * particular implements both a User store and a UserRoles store, which means it
+ * is responsible for fetching not only {@code User} objects, but also
+ * retrieving the roles for a given username.
+ *
+ * No caching is done by this class, it is handled at a higher level
+ */
+public class ESNativeUsersStore extends AbstractComponent implements ClusterStateListener {
+
+    // TODO - perhaps separate indices for users/roles instead of types?
+    public static final String INDEX_USER_TYPE = "user";
+
+    // this map contains the mapping for username -> version, which is used when polling the index to easily detect of
+    // any changes that may have been missed since the last update.
+    private final ObjectLongHashMap<String> versionMap = new ObjectLongHashMap<>();
+    private final Hasher hasher = Hasher.BCRYPT;
+    private final List<ChangeListener> listeners = new CopyOnWriteArrayList<>();
+    private final AtomicReference<State> state = new AtomicReference<>(State.INITIALIZED);
+    private final Provider<Client> clientProvider;
+    private final Provider<AuthenticationService> authProvider;
+    private final ShieldInternalUserHolder adminUser;
+    private final ThreadPool threadPool;
+
+    private ScheduledFuture<?> versionChecker;
+    private Client client;
+    private AuthenticationService authService;
+    private int scrollSize;
+    private TimeValue scrollKeepAlive;
+
+    private volatile boolean shieldIndexExists = false;
+
+    @Inject
+    public ESNativeUsersStore(Settings settings, Provider<Client> clientProvider,
+                              ShieldInternalUserHolder userHolder,
+                              Provider<AuthenticationService> authProvider, ThreadPool threadPool) {
+        super(settings);
+        this.clientProvider = clientProvider;
+        this.authProvider = authProvider;
+        this.adminUser = userHolder;
+        this.threadPool = threadPool;
+    }
+
+    private final void attachUser(TransportMessage message) {
+        try {
+            authService.attachUserHeaderIfMissing(message, adminUser.user());
+        } catch (IOException e) {
+            logger.error("failed to attach authorization to internal message!", e);
+            throw new ElasticsearchSecurityException("unable to attach administrative user to transport message",
+                    RestStatus.SERVICE_UNAVAILABLE, e);
+        }
+    }
+
+    @Nullable
+    private UserAndPassword transformUser(Map<String, Object> sourceMap) {
+        if (sourceMap == null) {
+            return null;
+        }
+        try {
+            String username = (String)sourceMap.get(Fields.USERNAME);
+            String password = (String)sourceMap.get(Fields.PASSWORD);
+            String[] roles = ((List<String>)sourceMap.get(Fields.ROLES)).toArray(Strings.EMPTY_ARRAY);
+            return new UserAndPassword(new User(username, roles),  password.toCharArray());
+        } catch (Exception e) {
+            logger.error("error in the format of get response for user", e);
+            return null;
+        }
+    }
+
+    /**
+     * Blocking version of {@code getUser} that blocks until the User is returned
+     */
+    public User getUser(String username) {
+        if (state() != State.STARTED) {
+            logger.trace("attempted to get user [{}] before service was started", username);
+            return null;
+        }
+        UserAndPassword uap = getUserAndPassword(username);
+        return uap == null ? null : uap.user();
+    }
+
+    /**
+     * Retrieve a single user, calling the listener when retrieved
+     */
+    public void getUser(String username, final ActionListener<User> listener) {
+        if (state() != State.STARTED) {
+            logger.trace("attempted to get user [{}] before service was started", username);
+            listener.onFailure(new IllegalStateException("user cannot be retrieved as native user service has not been started"));
+            return;
+        }
+        getUserAndPassword(username, new ActionListener<UserAndPassword>() {
+            @Override
+            public void onResponse(UserAndPassword uap) {
+                listener.onResponse(uap == null ? null : uap.user());
+            }
+
+            @Override
+            public void onFailure(Throwable t) {
+                if (t instanceof IndexNotFoundException) {
+                    logger.trace("failed to retrieve user", t);
+                } else {
+                    logger.info("failed to retrieve user", t);
+                }
+
+                // We don't invoke the onFailure listener here, instead
+                // we call the response with a null user
+                listener.onResponse(null);
+            }
+        });
+    }
+
+    /**
+     * Retrieve a list of users, if usernames is null or empty, fetch all users
+     */
+    public void getUsers(String[] usernames, final ActionListener<List<User>> listener) {
+        if (state() != State.STARTED) {
+            logger.trace("attempted to get users before service was started");
+            listener.onFailure(new IllegalStateException("users cannot be retrieved as native user service has not been started"));
+            return;
+        }
+        try {
+            final List<User> users = new ArrayList<>();
+            QueryBuilder query;
+            if (usernames == null || usernames.length == 0) {
+                query = QueryBuilders.boolQuery().filter(QueryBuilders.termQuery("_type", INDEX_USER_TYPE));
+            } else {
+                query = QueryBuilders.boolQuery().filter(QueryBuilders.idsQuery(INDEX_USER_TYPE).addIds(usernames));
+            }
+            SearchRequest request = client.prepareSearch(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME)
+                    .setScroll(scrollKeepAlive)
+                    .setQuery(query)
+                    .setSize(scrollSize)
+                    .setFetchSource(true)
+                    .request();
+            attachUser(request);
+            request.indicesOptions().ignoreUnavailable();
+
+            // This function is MADNESS! But it works, don't think about it too hard...
+            client.search(request, new ActionListener<SearchResponse>() {
+                @Override
+                public void onResponse(SearchResponse resp) {
+                    boolean hasHits = resp.getHits().getHits().length > 0;
+                    if (hasHits) {
+                        for (SearchHit hit : resp.getHits().getHits()) {
+                            UserAndPassword u = transformUser(hit.getSource());
+                            if (u != null) {
+                                users.add(u.user());
+                            }
+                        }
+                        SearchScrollRequest scrollRequest = client.prepareSearchScroll(resp.getScrollId())
+                                .setScroll(scrollKeepAlive).request();
+                        attachUser(scrollRequest);
+                        client.searchScroll(scrollRequest, this);
+                    } else {
+                        ClearScrollRequest clearScrollRequest = client.prepareClearScroll().addScrollId(resp.getScrollId()).request();
+                        attachUser(clearScrollRequest);
+                        client.clearScroll(clearScrollRequest, new ActionListener<ClearScrollResponse>() {
+                            @Override
+                            public void onResponse(ClearScrollResponse response) {
+                                // cool, it cleared, we don't really care though...
+                            }
+
+                            @Override
+                            public void onFailure(Throwable t) {
+                                // Not really much to do here except for warn about it...
+                                logger.warn("failed to clear scroll after retrieving all users", t);
+                            }
+                        });
+                        // Finally, return the list of users
+                        listener.onResponse(Collections.unmodifiableList(users));
+                    }
+                }
+
+                @Override
+                public void onFailure(Throwable t) {
+                    if (t instanceof IndexNotFoundException) {
+                        logger.trace("could not retrieve users because shield index does not exist");
+                    } else {
+                        logger.info("failed to retrieve users", t);
+                    }
+                    // We don't invoke the onFailure listener here, instead
+                    // we call the response with an empty list
+                    listener.onResponse(Collections.emptyList());
+                }
+            });
+        } catch (Exception e) {
+            logger.error("unable to retrieve users", e);
+            listener.onFailure(e);
+        }
+    }
+
+    private UserAndPassword getUserAndPassword(String username) {
+        final AtomicReference<UserAndPassword> userRef = new AtomicReference<>(null);
+        final CountDownLatch latch = new CountDownLatch(1);
+        getUserAndPassword(username, new LatchedActionListener<>(new ActionListener<UserAndPassword>() {
+            @Override
+            public void onResponse(UserAndPassword user) {
+                userRef.set(user);
+            }
+
+            @Override
+            public void onFailure(Throwable t) {
+                logger.info("failed to retrieve user", t);
+            }
+        }, latch));
+        try {
+            latch.await(30, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+            logger.info("timed out retrieving user");
+            return null;
+        }
+        return userRef.get();
+    }
+
+    private void getUserAndPassword(String user, final ActionListener<UserAndPassword> listener) {
+        try {
+            GetRequest request = client.prepareGet(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME, INDEX_USER_TYPE, user).request();
+            request.indicesOptions().ignoreUnavailable();
+            attachUser(request);
+            client.get(request, new ActionListener<GetResponse>() {
+                @Override
+                public void onResponse(GetResponse getFields) {
+                    listener.onResponse(transformUser(getFields.getSource()));
+                }
+
+                @Override
+                public void onFailure(Throwable t) {
+                    if (t instanceof IndexNotFoundException) {
+                        logger.trace("could not retrieve user because shield index does not exist", t);
+                    } else {
+                        logger.info("failed to retrieve user", t);
+                    }
+                    // We don't invoke the onFailure listener here, instead
+                    // we call the response with a null user
+                    listener.onResponse(null);
+                }
+            });
+        } catch (IndexNotFoundException infe) {
+            logger.trace("could not retrieve user because shield index does not exist");
+            listener.onResponse(null);
+        } catch (Exception e) {
+            logger.error("unable to retrieve user", e);
+            listener.onFailure(e);
+        }
+    }
+
+    public void addUser(final AddUserRequest addUserRequest, final ActionListener<Boolean> listener) {
+        if (state() != State.STARTED) {
+            listener.onFailure(new IllegalStateException("user cannot be added as native user service has not been started"));
+            return;
+        }
+
+        try {
+            IndexRequest request = client.prepareIndex(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME,
+                    INDEX_USER_TYPE, addUserRequest.username())
+                    .setSource("username", addUserRequest.username(),
+                            "password", String.valueOf(addUserRequest.passwordHash()),
+                            "roles", addUserRequest.roles())
+                    .request();
+            attachUser(request);
+
+            client.index(request, new ActionListener<IndexResponse>() {
+                @Override
+                public void onResponse(IndexResponse indexResponse) {
+                    // if the document was just created, then we don't need to clear cache
+                    if (indexResponse.isCreated()) {
+                        listener.onResponse(indexResponse.isCreated());
+                        return;
+                    }
+
+                    clearRealmCache(addUserRequest.username(), listener, indexResponse.isCreated());
+                }
+
+                @Override
+                public void onFailure(Throwable e) {
+                    listener.onFailure(e);
+                }
+            });
+        } catch (Exception e) {
+            logger.error("unable to add user", e);
+            listener.onFailure(e);
+        }
+    }
+
+    public void removeUser(final DeleteUserRequest deleteUserRequest, final ActionListener<Boolean> listener) {
+        if (state() != State.STARTED) {
+            listener.onFailure(new IllegalStateException("user cannot be deleted as native user service has not been started"));
+            return;
+        }
+
+        try {
+            DeleteRequest request = client.prepareDelete(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME,
+                    INDEX_USER_TYPE, deleteUserRequest.user()).request();
+            request.indicesOptions().ignoreUnavailable();
+            attachUser(request);
+            client.delete(request, new ActionListener<DeleteResponse>() {
+                @Override
+                public void onResponse(DeleteResponse deleteResponse) {
+                    clearRealmCache(deleteUserRequest.user(), listener, deleteResponse.isFound());
+                }
+
+                @Override
+                public void onFailure(Throwable e) {
+                    listener.onFailure(e);
+                }
+            });
+        } catch (Exception e) {
+            logger.error("unable to remove user", e);
+            listener.onFailure(e);
+        }
+    }
+
+    public boolean canStart(ClusterState clusterState, boolean master) {
+        if (state() != State.INITIALIZED) {
+            return false;
+        }
+
+        if (clusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
+            // wait until the gateway has recovered from disk, otherwise we
+            // think may not have the .shield index but they it may not have
+            // been restored from the cluster state on disk yet
+            logger.debug("native users store waiting until gateway has recovered from disk");
+            return false;
+        }
+
+        if (clusterState.metaData().templates().get(ShieldTemplateService.SHIELD_TEMPLATE_NAME) == null) {
+            logger.debug("native users template [{}] does not exist, so service cannot start",
+                    ShieldTemplateService.SHIELD_TEMPLATE_NAME);
+            return false;
+        }
+
+        IndexMetaData metaData = clusterState.metaData().index(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME);
+        if (metaData == null) {
+            logger.debug("shield user index [{}] does not exist, so service can start", ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME);
+            return true;
+        }
+
+        if (clusterState.routingTable().index(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME).allPrimaryShardsActive()) {
+            logger.debug("shield user index [{}] all primary shards started, so service can start", ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME);
+            return true;
+        }
+        return false;
+    }
+
+    public void start() {
+        try {
+            if (state.compareAndSet(State.INITIALIZED, State.STARTING)) {
+                this.client = clientProvider.get();
+                this.authService = authProvider.get();
+                this.scrollSize = settings.getAsInt("shield.authc.native.scroll.size", 1000);
+                this.scrollKeepAlive = settings.getAsTime("shield.authc.native.scroll.keep_alive", TimeValue.timeValueSeconds(10L));
+
+                // FIXME only start if a realm is using this
+                UserStorePoller poller = new UserStorePoller();
+                try {
+                    poller.doRun();
+                } catch (Exception e) {
+                    logger.warn("failed to do initial poll of shield users", e);
+                }
+                versionChecker = threadPool.scheduleWithFixedDelay(poller, settings.getAsTime("shield.authc.native.reload.interval", TimeValue.timeValueSeconds(30L)));
+                state.set(State.STARTED);
+            }
+        } catch (Exception e) {
+            logger.error("failed to start native user store", e);
+            state.set(State.FAILED);
+        }
+    }
+
+    public void stop() {
+        if (state.compareAndSet(State.STARTED, State.STOPPING)) {
+            try {
+                FutureUtils.cancel(versionChecker);
+            } catch (Throwable t) {
+                state.set(State.FAILED);
+                throw t;
+            } finally {
+                state.set(State.STOPPED);
+            }
+        }
+    }
+
+    /**
+     * This method is used to verify the username and credentials against those stored in the system.
+     * @param username username to lookup the user by
+     * @param password the plaintext password to verify
+     * @return {@link} User object if successful or {@code null} if verification fails
+     */
+    public User verifyPassword(String username, final SecuredString password) {
+        if (state() != State.STARTED) {
+            logger.trace("attempted to verify user credentials for [{}] but service was not started", username);
+            return null;
+        }
+
+        UserAndPassword user = getUserAndPassword(username);
+        if (user == null || user.passwordHash() == null) {
+            return null;
+        }
+        if (hasher.verify(password, user.passwordHash())) {
+            return user.user();
+        }
+        return null;
+    }
+
+    public void addListener(ChangeListener listener) {
+        listeners.add(listener);
+    }
+
+    private <Response> void clearRealmCache(String username, ActionListener<Response> listener, Response response) {
+        ShieldClient shieldClient = new ShieldClient(client);
+        ClearRealmCacheRequest request = shieldClient.prepareClearRealmCache()
+                .usernames(username).request();
+        attachUser(request);
+        shieldClient.clearRealmCache(request, new ActionListener<ClearRealmCacheResponse>() {
+            @Override
+            public void onResponse(ClearRealmCacheResponse nodes) {
+                listener.onResponse(response);
+            }
+
+            @Override
+            public void onFailure(Throwable e) {
+                logger.error("unable to clear realm cache for user [{}]", e, username);
+                ElasticsearchException exception = new ElasticsearchException("clearing the cache for [" + username
+                        + "] failed. please clear the realm cache manually", e);
+                listener.onFailure(exception);
+            }
+        });
+    }
+
+    @Override
+    public void clusterChanged(ClusterChangedEvent event) {
+        final boolean exists = event.state().metaData().indices().get(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME) != null;
+        // make sure all the primaries are active
+        if (exists && event.state().routingTable().index(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME).allPrimaryShardsActive()) {
+            logger.debug("shield user index [{}] all primary shards started, so polling can start", ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME);
+            shieldIndexExists = true;
+        } else {
+            // always set the value - it may have changed...
+            shieldIndexExists = false;
+        }
+    }
+
+    public State state() {
+        return state.get();
+    }
+
+    // FIXME hack for testing
+    public void reset() {
+        final State state = state();
+        if (state != State.STOPPED && state != State.FAILED) {
+            throw new IllegalStateException("can only reset if stopped!!!");
+        }
+        this.versionMap.clear();
+        this.listeners.clear();
+        this.client = null;
+        this.authService = null;
+        this.shieldIndexExists = false;
+        this.state.set(State.INITIALIZED);
+    }
+
+    public enum State {
+        INITIALIZED,
+        STARTING,
+        STARTED,
+        STOPPING,
+        STOPPED,
+        FAILED
+    }
+
+    private class UserStorePoller extends AbstractRunnable {
+
+        @Override
+        public void doRun() {
+            if (shieldIndexExists == false) {
+                logger.trace("cannot poll for user changes since shield admin index [{}] does not exist", ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME);
+                return;
+            }
+
+            // hold a reference to the client since the poller may run after the class is stopped (we don't interrupt it running) and
+            // we reset when we test which sets the client to null...
+            final Client client = ESNativeUsersStore.this.client;
+
+            logger.trace("starting polling of user index to check for changes");
+            // create a copy of all known users
+            ObjectHashSet<String> knownUsers = new ObjectHashSet<>(versionMap.keys());
+            List<String> changedUsers = new ArrayList<>();
+
+            ObjectLongMap<String> currentUsersMap = collectUsersAndVersions(client);
+            Iterator<ObjectLongCursor<String>> iterator = currentUsersMap.iterator();
+            while (iterator.hasNext()) {
+                ObjectLongCursor<String> cursor = iterator.next();
+                String username = cursor.key;
+                long version = cursor.value;
+                if (knownUsers.contains(username)) {
+                    final long lastKnownVersion = versionMap.get(username);
+                    if (version != lastKnownVersion) {
+                        // version is only changed by this method
+                        assert version > lastKnownVersion;
+                        versionMap.put(username, version);
+                        // there is a chance that the user's cache has already been cleared and we'll clear it again but
+                        // this should be ok in most cases as user changes should not be that frequent
+                        changedUsers.add(username);
+                    }
+                    knownUsers.remove(username);
+                } else {
+                    versionMap.put(username, version);
+                }
+            }
+
+            // we now have a list of users that were in our version map and have been deleted
+            Iterator<ObjectCursor<String>> userIter = knownUsers.iterator();
+            while (userIter.hasNext()) {
+                String user = userIter.next().value;
+                versionMap.remove(user);
+                changedUsers.add(user);
+            }
+
+            if (changedUsers.isEmpty()) {
+                return;
+            }
+
+            // make the list unmodifiable to prevent modifications by any listeners
+            changedUsers = Collections.unmodifiableList(changedUsers);
+            if (logger.isDebugEnabled()) {
+                logger.debug("changes detected for users [{}]", changedUsers);
+            }
+
+            // call listeners
+            Throwable th = null;
+            for (ChangeListener listener : listeners) {
+                try {
+                    listener.onUsersChanged(changedUsers);
+                } catch (Throwable t) {
+                    th = ExceptionsHelper.useOrSuppress(th, t);
+                }
+            }
+
+            ExceptionsHelper.reThrowIfNotNull(th);
+        }
+
+        @Override
+        public void onFailure(Throwable t) {
+            logger.error("error occurred while checking the native users for changes", t);
+        }
+
+        private ObjectLongMap<String> collectUsersAndVersions(Client client) {
+            final ObjectLongMap<String> map = new ObjectLongHashMap<>();
+            SearchResponse response = null;
+            try {
+                SearchRequest request = client.prepareSearch(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME)
+                        .setScroll(scrollKeepAlive)
+                        .setQuery(QueryBuilders.typeQuery(INDEX_USER_TYPE))
+                        .setSize(scrollSize)
+                        .setVersion(true)
+                        .setFetchSource(true)
+                        .request();
+                attachUser(request);
+                response = client.search(request).actionGet();
+
+                boolean keepScrolling = response.getHits().getHits().length > 0;
+                while (keepScrolling) {
+                    for (SearchHit hit : response.getHits().getHits()) {
+                        String username = hit.id();
+                        long version = hit.version();
+                        map.put(username, version);
+                    }
+                    SearchScrollRequest scrollRequest = client.prepareSearchScroll(response.getScrollId()).setScroll(scrollKeepAlive).request();
+                    attachUser(scrollRequest);
+                    response = client.searchScroll(scrollRequest).actionGet();
+                    keepScrolling = response.getHits().getHits().length > 0;
+                }
+            } catch (IndexNotFoundException e) {
+                logger.trace("shield user index does not exist", e);
+            } finally {
+                if (response != null) {
+                    ClearScrollRequest clearScrollRequest = client.prepareClearScroll().addScrollId(response.getScrollId()).request();
+                    attachUser(clearScrollRequest);
+                    client.clearScroll(clearScrollRequest).actionGet();
+                }
+            }
+            return map;
+        }
+    }
+
+    public static class Fields {
+        static String USERNAME = "username";
+        static String PASSWORD = "password";
+        static String ROLES = "roles";
+    }
+
+    interface ChangeListener {
+
+        void onUsersChanged(List<String> username);
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/esnative/UserAndPassword.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authc.esnative;
+
+import org.elasticsearch.shield.User;
+
+/**
+ * Like User, but includes the hashed password
+ *
+ * NOT to be used for password verification
+ *
+ * NOTE that this purposefully does not serialize the {@code passwordHash}
+ * field, because this is not meant to be used for security other than
+ * retrieving the UserAndPassword from the index before local authentication.
+ */
+class UserAndPassword {
+
+    private final User user;
+    private final char[] passwordHash;
+
+    UserAndPassword(User user, char[] passwordHash) {
+        this.user = user;
+        this.passwordHash = passwordHash;
+    }
+
+    public User user() {
+        return this.user;
+    }
+
+    public char[] passwordHash() {
+        return this.passwordHash;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        return false; // Don't use this for user comparison
+    }
+
+    @Override
+    public int hashCode() {
+        int result = this.user.hashCode();
+        result = 31 * result + passwordHash().hashCode();
+        return result;
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/esusers/ESUsersRealm.java

@@ -41,14 +41,15 @@ public class ESUsersRealm extends CachingUsernamePasswordRealm {
             return null;
         }
         String[] roles = userRolesStore.roles(token.principal());
-        return new User.Simple(token.principal(), roles);
+
+        return new User(token.principal(), roles);
     }
 
     @Override
     public User doLookupUser(String username) {
         if (userPasswdStore.userExists(username)){
             String[] roles = userRolesStore.roles(username);
-            return new User.Simple(username, roles);
+            return new User(username, roles);
         }
         return null;
     }
@@ -81,9 +82,7 @@ public class ESUsersRealm extends CachingUsernamePasswordRealm {
 
         @Override
         public ESUsersRealm create(RealmConfig config) {
-            FileUserPasswdStore userPasswdStore = new FileUserPasswdStore(config, watcherService);
-            FileUserRolesStore userRolesStore = new FileUserRolesStore(config, watcherService);
-            return new ESUsersRealm(config, userPasswdStore, userRolesStore);
+            return new ESUsersRealm(config, new FileUserPasswdStore(config, watcherService), new FileUserRolesStore(config, watcherService));
         }
 
         @Override

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/esusers/FileUserPasswdStore.java

@@ -75,11 +75,11 @@ public class FileUserPasswdStore {
         }
     }
 
-    void addListener(RefreshListener listener) {
+    public void addListener(RefreshListener listener) {
         listeners.add(listener);
     }
 
-    int usersCount() {
+    public int usersCount() {
         return users.size();
     }
 
@@ -180,7 +180,7 @@ public class FileUserPasswdStore {
         }
     }
 
-    protected void notifyRefresh() {
+    public void notifyRefresh() {
         for (RefreshListener listener : listeners) {
             listener.onRefresh();
         }

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/esusers/FileUserRolesStore.java

@@ -72,7 +72,7 @@ public class FileUserRolesStore {
         }
     }
 
-    synchronized void addListener(RefreshListener listener) {
+    public synchronized void addListener(RefreshListener listener) {
         listeners.add(listener);
     }
 

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/AbstractLdapRealm.java

@@ -80,7 +80,7 @@ public abstract class AbstractLdapRealm extends CachingUsernamePasswordRealm {
     private User createUser(String principal, LdapSession session) {
         List<String> groupDNs = session.groups();
         Set<String> roles = roleMapper.resolveRoles(session.userDn(), groupDNs);
-        return new User.Simple(principal, roles.toArray(new String[roles.size()]));
+        return new User(principal, roles.toArray(new String[roles.size()]));
     }
 
     class Listener implements RefreshListener {

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/pki/PkiRealm.java

@@ -82,7 +82,7 @@ public class PkiRealm extends Realm<X509AuthenticationToken> {
         }
 
         Set<String> roles = roleMapper.resolveRoles(token.dn(), Collections.<String>emptyList());
-        return new User.Simple(token.principal(), roles.toArray(new String[roles.size()]));
+        return new User(token.principal(), roles.toArray(new String[roles.size()]));
     }
 
     @Override

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealm.java

@@ -44,12 +44,14 @@ public abstract class CachingUsernamePasswordRealm extends UsernamePasswordRealm
 
     public final void expire(String username) {
         if (cache != null) {
+            logger.trace("invalidating cache for user [{}] in realm [{}]", username, name());
             cache.invalidate(username);
         }
     }
 
     public final void expireAll() {
         if (cache != null) {
+            logger.trace("invalidating cache for all users in realm [{}]", name());
             cache.invalidateAll();
         }
     }

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/support/SecuredString.java

@@ -20,6 +20,8 @@ import java.util.Arrays;
  */
 public class SecuredString implements CharSequence {
 
+    public static final SecuredString EMPTY = new SecuredString(new char[0]);
+
     private final char[] chars;
     private boolean cleared = false;
 
@@ -43,6 +45,8 @@ public class SecuredString implements CharSequence {
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null) return false;
+        // Require password to calculate equals
+        if (this == EMPTY || o == EMPTY) return false;
 
         if (o instanceof SecuredString) {
             SecuredString that = (SecuredString) o;
@@ -181,6 +185,21 @@ public class SecuredString implements CharSequence {
         return constantTimeEquals(securedString.internalChars(), string.toCharArray());
     }
 
+    /**
+     * This does a char by char comparison of the two Strings to provide protection against timing attacks. In other
+     * words it does not exit at the first character that does not match and only exits at the end of the comparison.
+     *
+     * NOTE: length will cause this function to exit early, which is OK as it is not considered feasible to prevent
+     * length attacks
+     *
+     * @param securedString the securedstring to compare to string char by char
+     * @param otherString the other securedstring to compare
+     * @return true if both match char for char
+     */
+    public static boolean constantTimeEquals(SecuredString securedString, SecuredString otherString) {
+        return constantTimeEquals(securedString.internalChars(), otherString.internalChars());
+    }
+
     /**
      * This does a char by char comparison of the two arrays to provide protection against timing attacks. In other
      * words it does not exit at the first character that does not match and only exits at the end of the comparison.

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/AuthorizationModule.java

@@ -7,6 +7,9 @@ package org.elasticsearch.shield.authz;
 
 import org.elasticsearch.common.inject.multibindings.Multibinder;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.shield.authc.esnative.ESNativeUsersStore;
+import org.elasticsearch.shield.authz.esnative.ESNativeRolesStore;
+import org.elasticsearch.shield.authz.store.CompositeRolesStore;
 import org.elasticsearch.shield.authz.store.FileRolesStore;
 import org.elasticsearch.shield.authz.store.RolesStore;
 import org.elasticsearch.shield.support.AbstractShieldModule;
@@ -15,7 +18,7 @@ import java.util.HashSet;
 import java.util.Set;
 
 /**
- *
+ * Module used to bind various classes necessary for authorization
  */
 public class AuthorizationModule extends AbstractShieldModule.Node {
 
@@ -37,8 +40,12 @@ public class AuthorizationModule extends AbstractShieldModule.Node {
             reservedRolesBinder.addBinding().toInstance(reservedRole);
         }
 
+        // First the file and native roles stores must be bound...
         bind(FileRolesStore.class).asEagerSingleton();
-        bind(RolesStore.class).to(FileRolesStore.class).asEagerSingleton();
+        bind(ESNativeRolesStore.class).asEagerSingleton();
+        // Then the composite roles store (which combines both) can be bound
+        bind(RolesStore.class).to(CompositeRolesStore.class).asEagerSingleton();
+        bind(ESNativeUsersStore.class).asEagerSingleton();
         bind(AuthorizationService.class).to(InternalAuthorizationService.class).asEagerSingleton();
     }
 

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/InternalAuthorizationService.java

@@ -97,7 +97,6 @@ public class InternalAuthorizationService extends AbstractComponent implements A
 
     @Override
     public void authorize(User user, String action, TransportRequest request) throws ElasticsearchSecurityException {
-
         // first we need to check if the user is the system. If it is, we'll just authorize the system access
         if (user.isSystem()) {
             if (SystemRole.INSTANCE.check(action)) {

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/Permission.java

@@ -139,6 +139,10 @@ public interface Permission {
                 return new Builder(name);
             }
 
+            public static Builder builder(RoleDescriptor rd) {
+                return new Builder(rd);
+            }
+
             public static class Builder {
 
                 private final String name;
@@ -150,6 +154,21 @@ public interface Permission {
                     this.name = name;
                 }
 
+                private Builder(RoleDescriptor rd) {
+                    this.name = rd.getName();
+                    this.cluster(Privilege.Cluster.get((new Privilege.Name(rd.getClusterPattern()))));
+                    for (RoleDescriptor.IndicesPrivileges iGroup : rd.getIndicesPrivileges()) {
+                        this.add(iGroup.getFields() == null ? null : Arrays.asList(iGroup.getFields()),
+                                iGroup.getQuery(),
+                                Privilege.Index.get(new Privilege.Name(iGroup.getPrivileges())),
+                                iGroup.getIndices());
+                    }
+                    String[] rdRunAs = rd.getRunAs();
+                    if (rdRunAs != null && rdRunAs.length > 0) {
+                        this.runAs(new Privilege.General(new Privilege.Name(rdRunAs), rdRunAs));
+                    }
+                }
+
                 // FIXME we should throw an exception if we have already set cluster or runAs...
                 public Builder cluster(Privilege.Cluster privilege) {
                     cluster = new Cluster.Core(privilege);

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/RoleDescriptor.java

@@ -0,0 +1,384 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authz;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * A holder for a Role that contains user-readable information about the Role
+ * without containing the actual Role object.
+ */
+public class RoleDescriptor implements ToXContent {
+
+    private final String name;
+    private final String[] clusterPattern;
+    private final List<IndicesPrivileges> indicesPrivileges;
+    private final String[] runAs;
+
+    public RoleDescriptor(String name, String[] clusterPattern,
+                          List<IndicesPrivileges> indicesPrivileges, String[] runAs) {
+        this.name = name;
+        this.clusterPattern = clusterPattern;
+        this.indicesPrivileges = indicesPrivileges;
+        this.runAs = runAs;
+    }
+
+    public String getName() {
+        return this.name;
+    }
+
+    public String[] getClusterPattern() {
+        return this.clusterPattern;
+    }
+
+    public List<IndicesPrivileges> getIndicesPrivileges() {
+        return this.indicesPrivileges;
+    }
+
+    public String[] getRunAs() {
+        return this.runAs;
+    }
+
+    private static void validateIndexName(String idxName) throws ElasticsearchParseException {
+        if (idxName == null) {
+            return;
+        }
+
+        if (idxName.indexOf(",") != -1) {
+            throw new ElasticsearchParseException("index name [" + idxName + "] may not contain ','");
+        }
+    }
+
+    private static RoleDescriptor.IndicesPrivileges parseIndex(XContentParser parser) throws Exception {
+        XContentParser.Token token;
+        String currentFieldName = null;
+        String[] idxNames = null;
+        String query = null;
+        List<String> privs = new ArrayList<>();
+        List<String> fields = new ArrayList<>();
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token.isValue()) {
+                if ("names".equals(currentFieldName)) {
+                    String idxName = parser.text();
+                    validateIndexName(idxName);
+                    idxNames = new String[]{idxName};
+                } else if ("query".equals(currentFieldName)) {
+                    query = parser.text();
+                } else {
+                    throw new ElasticsearchParseException("unexpected field in add role request [{}]", currentFieldName);
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                // expected
+            } else if (token == XContentParser.Token.START_ARRAY && "names".equals(currentFieldName)) {
+                List<String> idxNameList = new ArrayList<>();
+                while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                    if (token.isValue()) {
+                        String idxName = parser.text();
+                        validateIndexName(idxName);
+                        idxNameList.add(idxName);
+                    } else {
+                        throw new ElasticsearchParseException("unexpected object while parsing index names [{}]", token);
+                    }
+                }
+                idxNames = idxNameList.toArray(Strings.EMPTY_ARRAY);
+            } else if (token == XContentParser.Token.START_ARRAY && "privileges".equals(currentFieldName)) {
+                while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                    if (token.isValue()) {
+                        privs.add(parser.text());
+                    } else {
+                        throw new ElasticsearchParseException("unexpected object while parsing index privileges [{}]", token);
+                    }
+                }
+            } else if (token == XContentParser.Token.START_ARRAY && "fields".equals(currentFieldName)) {
+                while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                    if (token.isValue()) {
+                        fields.add(parser.text());
+                    } else {
+                        throw new ElasticsearchParseException("unexpected object while parsing index fields [{}]", token);
+                    }
+                }
+            } else {
+                throw new ElasticsearchParseException("failed to parse add role request indices, got value with wrong type [{}]", currentFieldName);
+            }
+        }
+        if (idxNames == null || idxNames.length == 0) {
+            throw new ElasticsearchParseException("'name' is a required field for index permissions");
+        }
+        if (privs.isEmpty()) {
+            throw new ElasticsearchParseException("'privileges' is a required field for index permissions");
+        }
+        return RoleDescriptor.IndicesPrivileges.builder()
+                .indices(idxNames)
+                .privileges(privs.toArray(Strings.EMPTY_ARRAY))
+                .fields(fields.isEmpty() ? null : fields.toArray(Strings.EMPTY_ARRAY))
+                .query(query == null ? null : new BytesArray(query))
+                .build();
+    }
+
+    private static List<RoleDescriptor.IndicesPrivileges> parseIndices(XContentParser parser) throws Exception {
+        XContentParser.Token token;
+        List<RoleDescriptor.IndicesPrivileges> tempIndices = new ArrayList<>();
+        while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+            if (token == XContentParser.Token.START_OBJECT) {
+                tempIndices.add(parseIndex(parser));
+            } else {
+                throw new ElasticsearchParseException("unexpected type parsing index sub object [{}]", token);
+            }
+        }
+        return tempIndices;
+    }
+
+    public static RoleDescriptor source(BytesReference source) throws Exception {
+        try (XContentParser parser = XContentHelper.createParser(source)) {
+            XContentParser.Token token;
+            String currentFieldName = null;
+            String roleName = null;
+            List<IndicesPrivileges> indicesPrivileges = new ArrayList<>();
+            List<String> runAsUsers = new ArrayList<>();
+            List<String> tempClusterPriv = new ArrayList<>();
+            parser.nextToken(); // remove object wrapping
+            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                if (token == XContentParser.Token.FIELD_NAME) {
+                    currentFieldName = parser.currentName();
+                } else if (token.isValue()) {
+                    if ("name".equals(currentFieldName)) {
+                        roleName = parser.text();
+                    } else {
+                        throw new ElasticsearchParseException("unexpected field in add role request [{}]", currentFieldName);
+                    }
+                } else if (token == XContentParser.Token.START_ARRAY && "indices".equals(currentFieldName)) {
+                    indicesPrivileges = parseIndices(parser);
+                } else if (token == XContentParser.Token.START_ARRAY && "run_as".equals(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        if (token.isValue()) {
+                            runAsUsers.add(parser.text());
+                        } else {
+                            throw new ElasticsearchParseException("unexpected value parsing run_as users [{}]", token);
+                        }
+                    }
+                } else if (token == XContentParser.Token.START_ARRAY && "cluster".equals(currentFieldName)) {
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        if (token.isValue()) {
+                            tempClusterPriv.add(parser.text());
+                        } else {
+                            throw new ElasticsearchParseException("unexpected value parsing cluster privileges [{}]", token);
+                        }
+                    }
+                } else {
+                    throw new ElasticsearchParseException("failed to parse add role request, got value with wrong type [{}]", currentFieldName);
+                }
+            }
+            if (roleName == null) {
+                throw new ElasticsearchParseException("field [name] required for role description");
+            }
+            return new RoleDescriptor(roleName, tempClusterPriv.toArray(Strings.EMPTY_ARRAY),
+                    indicesPrivileges, runAsUsers.toArray(Strings.EMPTY_ARRAY));
+        }
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder("Role[");
+        sb.append("name=").append(name);
+        sb.append(", cluster=[").append(Strings.arrayToCommaDelimitedString(clusterPattern));
+        sb.append("], indicesPrivileges=[");
+        for (IndicesPrivileges group : indicesPrivileges) {
+            sb.append(group.toString()).append(",");
+        }
+        sb.append("], runAs=[").append(Strings.arrayToCommaDelimitedString(runAs));
+        sb.append("]]");
+        return sb.toString();
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field("name", name);
+        builder.field("cluster", clusterPattern);
+        builder.field("indices", indicesPrivileges);
+        if (runAs != null) {
+            builder.field("run_as", runAs);
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    public static RoleDescriptor readFrom(StreamInput in) throws IOException {
+        String name = in.readString();
+        String[] clusterPattern = in.readStringArray();
+        int size = in.readVInt();
+        List<IndicesPrivileges> indicesPrivileges = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            IndicesPrivileges group = new IndicesPrivileges();
+            group.readFrom(in);
+            indicesPrivileges.add(group);
+        }
+        String[] runAs = in.readStringArray();
+        return new RoleDescriptor(name, clusterPattern, indicesPrivileges, runAs);
+    }
+
+    public static void writeTo(RoleDescriptor descriptor, StreamOutput out) throws IOException {
+        out.writeString(descriptor.getName());
+        out.writeStringArray(descriptor.getClusterPattern());
+        out.writeVInt(descriptor.getIndicesPrivileges().size());
+        for (IndicesPrivileges group : descriptor.getIndicesPrivileges()) {
+            group.writeTo(out);
+        }
+        out.writeStringArray(descriptor.getRunAs());
+    }
+
+    public static class IndicesPrivilegesBuilder {
+        private String[] privileges;
+        private String[] indices;
+        private String[] fields;
+        private BytesReference query;
+
+        IndicesPrivilegesBuilder() {}
+
+        public IndicesPrivilegesBuilder indices(String[] indices) {
+            this.indices = indices;
+            return this;
+        }
+
+        public IndicesPrivilegesBuilder privileges(String[] privileges) {
+            this.privileges = privileges;
+            return this;
+        }
+
+        public IndicesPrivilegesBuilder fields(@Nullable String[] fields) {
+            this.fields = fields;
+            return this;
+        }
+
+        public IndicesPrivilegesBuilder query(@Nullable BytesReference query) {
+            this.query = query;
+            return this;
+        }
+
+        public IndicesPrivileges build() {
+            return new IndicesPrivileges(privileges, indices, fields, query);
+        }
+    }
+
+    /**
+     * A class representing permissions for a group of indices mapped to
+     * privileges, fields, and a query.
+     */
+    public static class IndicesPrivileges implements ToXContent, Streamable {
+
+        private String[] privileges;
+        private String[] indices;
+        private String[] fields;
+        private BytesReference query;
+
+        private IndicesPrivileges() {}
+
+        IndicesPrivileges(String[] privileges, String[] indices,
+                                 @Nullable String[] fields, @Nullable BytesReference query) {
+            this.privileges = privileges;
+            this.indices = indices;
+            this.fields = fields;
+            this.query = query;
+        }
+
+        public static IndicesPrivilegesBuilder builder() {
+            return new IndicesPrivilegesBuilder();
+        }
+
+        public String[] getPrivileges() {
+            return this.privileges;
+        }
+
+        public String[] getIndices() {
+            return this.indices;
+        }
+
+        @Nullable
+        public String[] getFields() {
+            return this.fields;
+        }
+
+        @Nullable
+        public BytesReference getQuery() {
+            return this.query;
+        }
+
+        @Override
+        public String toString() {
+            StringBuilder sb = new StringBuilder("IndicesPrivileges[");
+            sb.append("privileges=[").append(Strings.arrayToCommaDelimitedString(privileges));
+            sb.append("], indices=[").append(Strings.arrayToCommaDelimitedString(indices));
+            sb.append("], fields=[").append(Strings.arrayToCommaDelimitedString(fields));
+            if (query != null) {
+                sb.append("], query=").append(query.toUtf8());
+            }
+            sb.append("]");
+            return sb.toString();
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject(); // start
+            builder.array("names", indices);
+            builder.array("privileges", privileges);
+            if (fields != null) {
+                builder.array("fields", fields);
+            }
+            if (query != null) {
+                builder.field("query", query.toUtf8());
+            }
+            builder.endObject(); // end start
+            return builder;
+        }
+
+        public static IndicesPrivileges readIndicesPrivileges(StreamInput in) throws IOException {
+            IndicesPrivileges ip = new IndicesPrivileges();
+            ip.readFrom(in);
+            return ip;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            this.privileges = in.readStringArray();
+            this.indices = in.readStringArray();
+            this.fields = in.readOptionalStringArray();
+            if (in.readBoolean()) {
+                this.query = new BytesArray(in.readByteArray());
+            }
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeStringArray(privileges);
+            out.writeStringArray(indices);
+            out.writeOptionalStringArray(fields);
+            if (query != null) {
+                out.writeBoolean(true);
+                out.writeByteArray(query.array());
+            } else {
+                out.writeBoolean(false);
+            }
+        }
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/esnative/ESNativeRolesStore.java

@@ -0,0 +1,585 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authz.esnative;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.LatchedActionListener;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.search.ClearScrollRequest;
+import org.elasticsearch.action.search.ClearScrollResponse;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchScrollRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.inject.Provider;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.FutureUtils;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.gateway.GatewayService;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.shield.action.admin.role.AddRoleRequest;
+import org.elasticsearch.shield.action.admin.role.DeleteRoleRequest;
+import org.elasticsearch.shield.action.authz.cache.ClearRolesCacheRequest;
+import org.elasticsearch.shield.action.authz.cache.ClearRolesCacheResponse;
+import org.elasticsearch.shield.admin.ShieldInternalUserHolder;
+import org.elasticsearch.shield.admin.ShieldTemplateService;
+import org.elasticsearch.shield.authc.AuthenticationService;
+import org.elasticsearch.shield.authz.Permission.Global.Role;
+import org.elasticsearch.shield.authz.RoleDescriptor;
+import org.elasticsearch.shield.authz.store.RolesStore;
+import org.elasticsearch.shield.client.ShieldClient;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportMessage;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
+/**
+ * ESNativeRolesStore is a {@code RolesStore} that, instead of reading from a
+ * file, reads from an Elasticsearch index instead. Unlike the file-based roles
+ * store, ESNativeRolesStore can be used to add a role to the store by inserting
+ * the document into the administrative index.
+ *
+ * No caching is done by this class, it is handled at a higher level
+ */
+public class ESNativeRolesStore extends AbstractComponent implements RolesStore, ClusterStateListener {
+
+    public static final String INDEX_ROLE_TYPE = "role";
+
+    private final Provider<Client> clientProvider;
+    private final Provider<AuthenticationService> authProvider;
+    private final ShieldInternalUserHolder adminUser;
+    private final ThreadPool threadPool;
+    private final AtomicReference<State> state = new AtomicReference<>(State.INITIALIZED);
+    private final ConcurrentHashMap<String, RoleAndVersion> roleCache = new ConcurrentHashMap<>();
+
+    private Client client;
+    private ShieldClient shieldClient;
+    private AuthenticationService authService;
+    private int scrollSize;
+    private TimeValue scrollKeepAlive;
+    private ScheduledFuture<?> versionChecker;
+
+    private volatile boolean shieldIndexExists = false;
+
+    @Inject
+    public ESNativeRolesStore(Settings settings, Provider<Client> clientProvider,
+                              ShieldInternalUserHolder userHolder,
+                              Provider<AuthenticationService> authProvider,
+                              ThreadPool threadPool) {
+        super(settings);
+        this.clientProvider = clientProvider;
+        this.authProvider = authProvider;
+        this.adminUser = userHolder;
+        this.threadPool = threadPool;
+    }
+
+    private final void attachUser(TransportMessage message) {
+        try {
+            authService.attachUserHeaderIfMissing(message, adminUser.user());
+        } catch (IOException e) {
+            logger.error("failed to attach authorization to internal message!", e);
+            throw new ElasticsearchSecurityException("unable to attach administrative user to transport message",
+                    RestStatus.SERVICE_UNAVAILABLE, e);
+        }
+    }
+
+    @Nullable
+    private RoleDescriptor transformRole(GetResponse response) {
+        if (response.isExists() == false) {
+            return null;
+        }
+        return transformRole(response.getSourceAsBytesRef());
+    }
+
+    @Nullable
+    private RoleDescriptor transformRole(BytesReference sourceBytes) {
+        try {
+            return RoleDescriptor.source(sourceBytes);
+        } catch (Exception e) {
+            logger.warn("unable to deserialize role from response", e);
+            return null;
+        }
+    }
+
+    /**
+     * Retrieve a list of roles, if rolesToGet is null or empty, fetch all roles
+     */
+    public void getRoleDescriptors(String[] rolesToGet, final ActionListener<List<RoleDescriptor>> listener) {
+        if (state() != State.STARTED) {
+            logger.trace("attempted to get roles before service was started");
+            listener.onFailure(new IllegalStateException("roles cannot be retrieved as native role service has not been started"));
+            return;
+        }
+        try {
+            final List<RoleDescriptor> roles = new ArrayList<>();
+            QueryBuilder query;
+            if (rolesToGet == null || rolesToGet.length == 0) {
+                query = QueryBuilders.boolQuery().filter(QueryBuilders.termQuery("_type", INDEX_ROLE_TYPE));
+            } else {
+                query = QueryBuilders.boolQuery().filter(QueryBuilders.idsQuery(INDEX_ROLE_TYPE).addIds(rolesToGet));
+            }
+            SearchRequest request = client.prepareSearch(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME)
+                    .setScroll(scrollKeepAlive)
+                    .setQuery(query)
+                    .setSize(scrollSize)
+                    .setFetchSource(true)
+                    .request();
+            attachUser(request);
+            request.indicesOptions().ignoreUnavailable();
+
+            // This function is MADNESS! But it works, don't think about it too hard...
+            client.search(request, new ActionListener<SearchResponse>() {
+                @Override
+                public void onResponse(SearchResponse resp) {
+                    boolean hasHits = resp.getHits().getHits().length > 0;
+                    if (hasHits) {
+                        for (SearchHit hit : resp.getHits().getHits()) {
+                            RoleDescriptor rd = transformRole(hit.getSourceRef());
+                            if (rd != null) {
+                                roles.add(rd);
+                            }
+                        }
+                        SearchScrollRequest scrollRequest = client.prepareSearchScroll(resp.getScrollId())
+                                .setScroll(scrollKeepAlive).request();
+                        attachUser(scrollRequest);
+                        client.searchScroll(scrollRequest, this);
+                    } else {
+                        ClearScrollRequest clearScrollRequest = client.prepareClearScroll().addScrollId(resp.getScrollId()).request();
+                        attachUser(clearScrollRequest);
+                        client.clearScroll(clearScrollRequest, new ActionListener<ClearScrollResponse>() {
+                            @Override
+                            public void onResponse(ClearScrollResponse response) {
+                                // cool, it cleared, we don't really care though...
+                            }
+
+                            @Override
+                            public void onFailure(Throwable t) {
+                                // Not really much to do here except for warn about it...
+                                logger.warn("failed to clear scroll after retrieving all roles", t);
+                            }
+                        });
+                        // Finally, return the list of users
+                        listener.onResponse(Collections.unmodifiableList(roles));
+                    }
+                }
+
+                @Override
+                public void onFailure(Throwable t) {
+                    if (t instanceof IndexNotFoundException) {
+                        logger.trace("could not retrieve roles because shield index does not exist");
+                    } else {
+                        logger.info("failed to retrieve roles", t);
+                    }
+                    // We don't invoke the onFailure listener here, instead
+                    // we call the response with an empty list
+                    listener.onResponse(Collections.emptyList());
+                }
+            });
+        } catch (Exception e) {
+            logger.error("unable to retrieve roles", e);
+            listener.onFailure(e);
+        }
+    }
+
+    public void getRoleDescriptor(final String role, final ActionListener<RoleDescriptor> listener) {
+        if (state() != State.STARTED) {
+            logger.trace("attempted to get role [{}] before service was started", role);
+            listener.onResponse(null);
+        }
+        RoleAndVersion roleAndVersion = getRoleAndVersion(role);
+        listener.onResponse(roleAndVersion == null ? null : roleAndVersion.getRoleDescriptor());
+    }
+
+    private void executeGetRoleRequest(String role, ActionListener<GetResponse> listener) {
+        try {
+            GetRequest request = client.prepareGet(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME, INDEX_ROLE_TYPE, role).request();
+            request.indicesOptions().ignoreUnavailable();
+            attachUser(request);
+            client.get(request, listener);
+        } catch (Exception e) {
+            logger.error("unable to retrieve role", e);
+            listener.onFailure(e);
+        }
+    }
+
+    public void removeRole(final DeleteRoleRequest deleteRoleRequest, final ActionListener<Boolean> listener) {
+        if (state() != State.STARTED) {
+            logger.trace("attempted to delete role [{}] before service was started", deleteRoleRequest.role());
+            listener.onResponse(false);
+        }
+        try {
+            DeleteRequest request = client.prepareDelete(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME,
+                    INDEX_ROLE_TYPE, deleteRoleRequest.role()).request();
+            request.indicesOptions().ignoreUnavailable();
+            attachUser(request);
+            client.delete(request, new ActionListener<DeleteResponse>() {
+                @Override
+                public void onResponse(DeleteResponse deleteResponse) {
+                    clearRoleCache(deleteRoleRequest.role(), listener, deleteResponse.isFound());
+                }
+
+                @Override
+                public void onFailure(Throwable e) {
+                    logger.error("failed to delete role from the index", e);
+                    listener.onFailure(e);
+                }
+            });
+        } catch (Exception e) {
+            logger.error("unable to remove role", e);
+            listener.onFailure(e);
+        }
+    }
+
+    private RoleAndVersion getRoleAndVersion(String roleId) {
+        RoleAndVersion roleAndVersion = null;
+        final AtomicReference<GetResponse> getRef = new AtomicReference<>(null);
+        final CountDownLatch latch = new CountDownLatch(1);
+        try {
+            roleAndVersion = roleCache.computeIfAbsent(roleId, new Function<String, RoleAndVersion>() {
+                @Override
+                public RoleAndVersion apply(String key) {
+                    logger.debug("attempting to load role [{}] from index", key);
+                    executeGetRoleRequest(roleId, new LatchedActionListener<>(new ActionListener<GetResponse>() {
+                        @Override
+                        public void onResponse(GetResponse role) {
+                            getRef.set(role);
+                        }
+
+                        @Override
+                        public void onFailure(Throwable t) {
+                            logger.info("failed to retrieve role", t);
+                        }
+                    }, latch));
+
+                    try {
+                        latch.await(30, TimeUnit.SECONDS);
+                    } catch (InterruptedException e) {
+                        logger.info("timed out retrieving role");
+                    }
+
+                    GetResponse response = getRef.get();
+                    if (response == null) {
+                        return null;
+                    }
+
+                    RoleDescriptor descriptor = transformRole(response);
+                    if (descriptor == null) {
+                        return null;
+                    }
+                    logger.debug("loaded role [{}] from index with version [{}]", key, response.getVersion());
+                    return new RoleAndVersion(descriptor, response.getVersion());
+                }
+            });
+        } catch (RuntimeException e) {
+            logger.error("could not get or load value from cache for role [{}]", e, roleId);
+        }
+
+        return roleAndVersion;
+    }
+
+    public void addRole(final AddRoleRequest addRoleRequest, final ActionListener<Boolean> listener) {
+        if (state() != State.STARTED) {
+            logger.trace("attempted to add role before service was started");
+            listener.onResponse(false);
+        }
+        try {
+            IndexRequest request = client.prepareIndex(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME,
+                    INDEX_ROLE_TYPE, addRoleRequest.name())
+                    .setSource(addRoleRequest.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS))
+                    .request();
+            attachUser(request);
+            client.index(request, new ActionListener<IndexResponse>() {
+                @Override
+                public void onResponse(IndexResponse indexResponse) {
+                    if (indexResponse.isCreated()) {
+                        listener.onResponse(indexResponse.isCreated());
+                        return;
+                    }
+                    clearRoleCache(addRoleRequest.name(), listener, indexResponse.isCreated());
+                }
+
+                @Override
+                public void onFailure(Throwable e) {
+                    logger.error("failed to add role to the index", e);
+                    listener.onFailure(e);
+                }
+            });
+        } catch (Exception e) {
+            logger.error("unable to add role", e);
+            listener.onFailure(e);
+        }
+
+    }
+
+    @Override
+    public Role role(String roleName) {
+        RoleAndVersion roleAndVersion = getRoleAndVersion(roleName);
+        return roleAndVersion == null ? null : roleAndVersion.getRole();
+    }
+
+    public boolean canStart(ClusterState clusterState, boolean master) {
+        if (state() != ESNativeRolesStore.State.INITIALIZED) {
+            return false;
+        }
+
+        if (clusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
+            // wait until the gateway has recovered from disk, otherwise we
+            // think may not have the .shield index but they it may not have
+            // been restored from the cluster state on disk yet
+            logger.debug("native roles store waiting until gateway has recovered from disk");
+            return false;
+        }
+
+        if (clusterState.metaData().templates().get(ShieldTemplateService.SHIELD_TEMPLATE_NAME) == null) {
+            logger.debug("native roles template [{}] does not exist, so service cannot start",
+                    ShieldTemplateService.SHIELD_TEMPLATE_NAME);
+            return false;
+        }
+        // Okay to start...
+        return true;
+    }
+
+    public void start() {
+        try {
+            if (state.compareAndSet(State.INITIALIZED, State.STARTING)) {
+                this.client = clientProvider.get();
+                this.shieldClient = new ShieldClient(client);
+                this.authService = authProvider.get();
+                this.scrollSize = settings.getAsInt("shield.authc.native.scroll.size", 1000);
+                this.scrollKeepAlive = settings.getAsTime("shield.authc.native.scroll.keep_alive", TimeValue.timeValueSeconds(10L));
+                TimeValue pollInterval = settings.getAsTime("shield.authc.native.reload.interval", TimeValue.timeValueSeconds(30L));
+                RolesStorePoller poller = new RolesStorePoller();
+                try {
+                    poller.doRun();
+                } catch (Exception e) {
+                    logger.warn("failed to perform initial poll of roles index [{}]. scheduling again in [{}]", e, ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME, pollInterval);
+                }
+                versionChecker = threadPool.scheduleWithFixedDelay(poller, pollInterval);
+                state.set(State.STARTED);
+            }
+        } catch (Exception e) {
+            logger.error("failed to start ESNativeRolesStore", e);
+            state.set(State.FAILED);
+        }
+    }
+
+    public void stop() {
+        if (state.compareAndSet(State.STARTED, State.STOPPING)) {
+            try {
+                FutureUtils.cancel(versionChecker);
+            } finally {
+                state.set(State.STOPPED);
+            }
+        }
+    }
+
+    // FIXME hack for testing
+    public void reset() {
+        final State state = state();
+        if (state != State.STOPPED && state != State.FAILED) {
+            throw new IllegalStateException("can only reset if stopped!!!");
+        }
+        this.roleCache.clear();
+        this.client = null;
+        this.authService = null;
+        this.shieldIndexExists = false;
+        this.state.set(State.INITIALIZED);
+    }
+
+    public void invalidateAll() {
+        logger.debug("invalidating all roles in cache");
+        roleCache.clear();
+    }
+
+    public void invalidate(String role) {
+        logger.debug("invalidating role [{}] in cache", role);
+        roleCache.remove(role);
+    }
+
+    private <Response> void clearRoleCache(final String role, ActionListener<Response> listener, Response response) {
+        ClearRolesCacheRequest request = new ClearRolesCacheRequest().roles(role);
+        attachUser(request);
+        shieldClient.clearRolesCache(request, new ActionListener<ClearRolesCacheResponse>() {
+            @Override
+            public void onResponse(ClearRolesCacheResponse nodes) {
+                listener.onResponse(response);
+            }
+
+            @Override
+            public void onFailure(Throwable e) {
+                logger.error("unable to clear cache for role [{}]", e, role);
+                ElasticsearchException exception = new ElasticsearchException("clearing the cache for [" + role
+                        + "] failed. please clear the role cache manually", e);
+                listener.onFailure(exception);
+            }
+        });
+    }
+
+    // TODO abstract this code rather than duplicating...
+    @Override
+    public void clusterChanged(ClusterChangedEvent event) {
+        final boolean exists = event.state().metaData().indices().get(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME) != null;
+        // make sure all the primaries are active
+        if (exists && event.state().routingTable().index(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME).allPrimaryShardsActive()) {
+            logger.debug("shield roles index [{}] all primary shards started, so polling can start", ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME);
+            shieldIndexExists = true;
+        } else {
+            // always set the value - it may have changed...
+            shieldIndexExists = false;
+        }
+    }
+
+    public State state() {
+        return state.get();
+    }
+
+    public enum State {
+        INITIALIZED,
+        STARTING,
+        STARTED,
+        STOPPING,
+        STOPPED,
+        FAILED
+    }
+
+    private class RolesStorePoller extends AbstractRunnable {
+
+        @Override
+        protected void doRun() throws Exception {
+            if (shieldIndexExists == false) {
+                logger.trace("cannot poll for role changes since shield admin index [{}] does not exist", ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME);
+                return;
+            }
+
+            // hold a reference to the client since the poller may run after the class is stopped (we don't interrupt it running) and
+            // we reset when we test which sets the client to null...
+            final Client client = ESNativeRolesStore.this.client;
+
+            logger.trace("starting polling of roles index to check for changes");
+            SearchResponse response = null;
+            // create a copy of the keys in the cache since we will be modifying this list
+            final Set<String> existingRoles = new HashSet<>(roleCache.keySet());
+            try {
+                SearchRequest request = client.prepareSearch(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME)
+                        .setScroll(scrollKeepAlive)
+                        .setQuery(QueryBuilders.typeQuery(INDEX_ROLE_TYPE))
+                        .setSize(scrollSize)
+                        .setFetchSource(true)
+                        .setVersion(true)
+                        .request();
+                attachUser(request);
+                response = client.search(request).actionGet();
+
+                boolean keepScrolling = response.getHits().getHits().length > 0;
+                while (keepScrolling) {
+                    for (final SearchHit hit : response.getHits().getHits()) {
+                        final String roleName = hit.getId();
+                        final long version = hit.version();
+                        existingRoles.remove(roleName);
+                        // we use the locking mechanisms provided by the map/cache to help protect against concurrent operations
+                        // that will leave the cache in a bad state
+                        roleCache.computeIfPresent(roleName, new BiFunction<String, RoleAndVersion, RoleAndVersion>() {
+                            @Override
+                            public RoleAndVersion apply(String roleName, RoleAndVersion existing) {
+                                if (version > existing.getVersion()) {
+                                    RoleDescriptor rd = transformRole(hit.getSourceRef());
+                                    if (rd != null) {
+                                        return new RoleAndVersion(rd, version);
+                                    }
+                                }
+                                return existing;
+                            }
+                        });
+                    }
+                    SearchScrollRequest scrollRequest = client.prepareSearchScroll(response.getScrollId()).setScroll(scrollKeepAlive).request();
+                    attachUser(scrollRequest);
+                    response = client.searchScroll(scrollRequest).actionGet();
+                    keepScrolling = response.getHits().getHits().length > 0;
+                }
+
+                // check to see if we had roles that do not exist in the index
+                if (existingRoles.isEmpty() == false) {
+                    for (String roleName : existingRoles) {
+                        invalidate(roleName);
+                    }
+                }
+            } catch (IndexNotFoundException e) {
+                logger.trace("shield roles index does not exist", e);
+            } finally {
+                if (response != null) {
+                    ClearScrollRequest clearScrollRequest = client.prepareClearScroll().addScrollId(response.getScrollId()).request();
+                    attachUser(clearScrollRequest);
+                    client.clearScroll(clearScrollRequest).actionGet();
+                }
+            }
+            logger.trace("completed polling of roles index");
+        }
+
+        @Override
+        public void onFailure(Throwable t) {
+            logger.error("error occurred while checking the native roles for changes", t);
+        }
+    }
+
+    private static class RoleAndVersion {
+        private final RoleDescriptor roleDescriptor;
+        private final Role role;
+        private final long version;
+
+        RoleAndVersion(RoleDescriptor roleDescriptor, long version) {
+            this.roleDescriptor = roleDescriptor;
+            this.role = Role.builder(roleDescriptor).build();
+            this.version = version;
+        }
+
+        RoleDescriptor getRoleDescriptor() {
+            return roleDescriptor;
+        }
+
+        Role getRole() {
+            return role;
+        }
+
+        long getVersion() {
+            return version;
+        }
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/store/CompositeRolesStore.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authz.store;
+
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.shield.authz.Permission;
+import org.elasticsearch.shield.authz.RoleDescriptor;
+import org.elasticsearch.shield.authz.esnative.ESNativeRolesStore;
+
+/**
+ * A composite roles store that combines file-based and index-based roles
+ * lookups. Checks the file first, then the index.
+ */
+public class CompositeRolesStore implements RolesStore {
+
+    private final FileRolesStore fileRolesStore;
+    private final ESNativeRolesStore nativeRolesStore;
+    
+    @Inject
+    public CompositeRolesStore(FileRolesStore fileRolesStore, ESNativeRolesStore nativeRolesStore) {
+        this.fileRolesStore = fileRolesStore;
+        this.nativeRolesStore = nativeRolesStore;
+    }
+    
+    public Permission.Global.Role role(String role) {
+        // Try the file first, then the index if it isn't there
+        Permission.Global.Role fileRole = fileRolesStore.role(role);
+        if (fileRole != null) {
+            return fileRole;
+        }
+
+        return nativeRolesStore.role(role);
+    }
+}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/store/FileRolesStore.java

@@ -24,6 +24,7 @@ import org.elasticsearch.shield.ShieldPlugin;
 import org.elasticsearch.shield.authc.support.RefreshListener;
 import org.elasticsearch.shield.authz.Permission;
 import org.elasticsearch.shield.authz.Privilege;
+import org.elasticsearch.shield.authz.RoleDescriptor;
 import org.elasticsearch.shield.authz.SystemRole;
 import org.elasticsearch.shield.support.NoOpLogger;
 import org.elasticsearch.shield.support.Validation;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/store/RolesStore.java

@@ -6,9 +6,10 @@
 package org.elasticsearch.shield.authz.store;
 
 import org.elasticsearch.shield.authz.Permission;
+import org.elasticsearch.shield.authz.RoleDescriptor;
 
 /**
- *
+ * An interface for looking up a role given a string role name
  */
 public interface RolesStore {
 

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/client/ShieldAuthcClient.java

@@ -15,7 +15,10 @@ import org.elasticsearch.shield.action.authc.cache.ClearRealmCacheResponse;
 
 /**
  * A client to manage Shield's authentication
+ *
+ * @deprecated Use {@link ShieldClient} directly instead
  */
+@Deprecated
 public class ShieldAuthcClient {
 
     private final ElasticsearchClient client;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/client/ShieldClient.java

@@ -5,24 +5,171 @@
  */
 package org.elasticsearch.shield.client;
 
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.shield.action.admin.role.AddRoleAction;
+import org.elasticsearch.shield.action.admin.role.AddRoleRequest;
+import org.elasticsearch.shield.action.admin.role.AddRoleRequestBuilder;
+import org.elasticsearch.shield.action.admin.role.AddRoleResponse;
+import org.elasticsearch.shield.action.admin.role.DeleteRoleAction;
+import org.elasticsearch.shield.action.admin.role.DeleteRoleRequest;
+import org.elasticsearch.shield.action.admin.role.DeleteRoleRequestBuilder;
+import org.elasticsearch.shield.action.admin.role.DeleteRoleResponse;
+import org.elasticsearch.shield.action.admin.role.GetRolesAction;
+import org.elasticsearch.shield.action.admin.role.GetRolesRequest;
+import org.elasticsearch.shield.action.admin.role.GetRolesRequestBuilder;
+import org.elasticsearch.shield.action.admin.role.GetRolesResponse;
+import org.elasticsearch.shield.action.admin.user.AddUserAction;
+import org.elasticsearch.shield.action.admin.user.AddUserRequest;
+import org.elasticsearch.shield.action.admin.user.AddUserRequestBuilder;
+import org.elasticsearch.shield.action.admin.user.AddUserResponse;
+import org.elasticsearch.shield.action.admin.user.DeleteUserAction;
+import org.elasticsearch.shield.action.admin.user.DeleteUserRequest;
+import org.elasticsearch.shield.action.admin.user.DeleteUserRequestBuilder;
+import org.elasticsearch.shield.action.admin.user.DeleteUserResponse;
+import org.elasticsearch.shield.action.admin.user.GetUsersAction;
+import org.elasticsearch.shield.action.admin.user.GetUsersRequest;
+import org.elasticsearch.shield.action.admin.user.GetUsersRequestBuilder;
+import org.elasticsearch.shield.action.admin.user.GetUsersResponse;
+import org.elasticsearch.shield.action.authc.cache.ClearRealmCacheAction;
+import org.elasticsearch.shield.action.authc.cache.ClearRealmCacheRequest;
+import org.elasticsearch.shield.action.authc.cache.ClearRealmCacheRequestBuilder;
+import org.elasticsearch.shield.action.authc.cache.ClearRealmCacheResponse;
+import org.elasticsearch.shield.action.authz.cache.ClearRolesCacheAction;
+import org.elasticsearch.shield.action.authz.cache.ClearRolesCacheRequest;
+import org.elasticsearch.shield.action.authz.cache.ClearRolesCacheRequestBuilder;
+import org.elasticsearch.shield.action.authz.cache.ClearRolesCacheResponse;
 
 /**
  * A wrapper to elasticsearch clients that exposes all Shield related APIs
  */
 public class ShieldClient {
 
+    private final ElasticsearchClient client;
     private final ShieldAuthcClient authcClient;
 
     public ShieldClient(ElasticsearchClient client) {
+        this.client = client;
         this.authcClient = new ShieldAuthcClient(client);
     }
 
-    /**
-     * @return  The Shield authentication client.
-     */
+    @Deprecated
     public ShieldAuthcClient authc() {
         return authcClient;
     }
 
+    /****************
+     * authc things *
+     ****************/
+
+    /**
+     * Clears the realm caches. It's possible to clear all user entries from all realms in the cluster or alternatively
+     * select the realms (by their unique names) and/or users (by their usernames) that should be evicted.
+     */
+    @SuppressWarnings("unchecked")
+    public ClearRealmCacheRequestBuilder prepareClearRealmCache() {
+        return new ClearRealmCacheRequestBuilder(client);
+    }
+
+    /**
+     * Clears the realm caches. It's possible to clear all user entries from all realms in the cluster or alternatively
+     * select the realms (by their unique names) and/or users (by their usernames) that should be evicted.
+     */
+    @SuppressWarnings("unchecked")
+    public void clearRealmCache(ClearRealmCacheRequest request, ActionListener<ClearRealmCacheResponse> listener) {
+        client.execute(ClearRealmCacheAction.INSTANCE, request, listener);
+    }
+
+    /**
+     * Clears the realm caches. It's possible to clear all user entries from all realms in the cluster or alternatively
+     * select the realms (by their unique names) and/or users (by their usernames) that should be evicted.
+     */
+    @SuppressWarnings("unchecked")
+    public ActionFuture<ClearRealmCacheResponse> clearRealmCache(ClearRealmCacheRequest request) {
+        return client.execute(ClearRealmCacheAction.INSTANCE, request);
+    }
+
+    /****************
+     * authz things *
+     ****************/
+
+    /**
+     * Clears the roles cache. This API only works for the naitve roles that are stored in an elasticsearch index. It is
+     * possible to clear the cache of all roles or to specify the names of individual roles that should have their cache
+     * cleared.
+     */
+    public ClearRolesCacheRequestBuilder prepareClearRolesCache() {
+        return new ClearRolesCacheRequestBuilder(client);
+    }
+
+    /**
+     * Clears the roles cache. This API only works for the naitve roles that are stored in an elasticsearch index. It is
+     * possible to clear the cache of all roles or to specify the names of individual roles that should have their cache
+     * cleared.
+     */
+    public void clearRolesCache(ClearRolesCacheRequest request, ActionListener<ClearRolesCacheResponse> listener) {
+        client.execute(ClearRolesCacheAction.INSTANCE, request, listener);
+    }
+
+    /**
+     * Clears the roles cache. This API only works for the naitve roles that are stored in an elasticsearch index. It is
+     * possible to clear the cache of all roles or to specify the names of individual roles that should have their cache
+     * cleared.
+     */
+    public ActionFuture<ClearRolesCacheResponse> clearRolesCache(ClearRolesCacheRequest request) {
+        return client.execute(ClearRolesCacheAction.INSTANCE, request);
+    }
+
+    /****************
+     * admin things *
+     ****************/
+
+    public GetUsersRequestBuilder prepareGetUsers() {
+        return new GetUsersRequestBuilder(client);
+    }
+
+    public void getUsers(GetUsersRequest request, ActionListener<GetUsersResponse> listener) {
+        client.execute(GetUsersAction.INSTANCE, request, listener);
+    }
+
+    public DeleteUserRequestBuilder prepareDeleteUser() {
+        return new DeleteUserRequestBuilder(client);
+    }
+
+    public void deleteUser(DeleteUserRequest request, ActionListener<DeleteUserResponse> listener) {
+        client.execute(DeleteUserAction.INSTANCE, request, listener);
+    }
+
+    public AddUserRequestBuilder prepareAddUser() {
+        return new AddUserRequestBuilder(client);
+    }
+
+    public void addUser(AddUserRequest request, ActionListener<AddUserResponse> listener) {
+        client.execute(AddUserAction.INSTANCE, request, listener);
+    }
+
+    public GetRolesRequestBuilder prepareGetRoles() {
+        return new GetRolesRequestBuilder(client);
+    }
+
+    public void getRoles(GetRolesRequest request, ActionListener<GetRolesResponse> listener) {
+        client.execute(GetRolesAction.INSTANCE, request, listener);
+    }
+
+    public DeleteRoleRequestBuilder prepareDeleteRole() {
+        return new DeleteRoleRequestBuilder(client);
+    }
+
+    public void deleteRole(DeleteRoleRequest request, ActionListener<DeleteRoleResponse> listener) {
+        client.execute(DeleteRoleAction.INSTANCE, request, listener);
+    }
+
+    public AddRoleRequestBuilder prepareAddRole() {
+        return new AddRoleRequestBuilder(client);
+    }
+
+    public void addRole(AddRoleRequest request, ActionListener<AddRoleResponse> listener) {
+        client.execute(AddRoleAction.INSTANCE, request, listener);
+    }
 }

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/rest/ShieldRestFilter.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.shield.rest;
 
+import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
@@ -23,6 +24,7 @@ import org.elasticsearch.shield.transport.netty.ShieldNettyHttpServerTransport;
 import org.jboss.netty.handler.ssl.SslHandler;
 
 import javax.net.ssl.SSLPeerUnverifiedException;
+
 import java.security.cert.Certificate;
 import java.security.cert.X509Certificate;
 

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/rest/action/authc/cache/RestClearRealmCacheAction.java

@@ -40,7 +40,7 @@ public class RestClearRealmCacheAction extends BaseRestHandler {
 
         ClearRealmCacheRequest req = new ClearRealmCacheRequest().realms(realms).usernames(usernames);
 
-        new ShieldClient(client).authc().clearRealmCache(req, new RestBuilderListener<ClearRealmCacheResponse>(channel) {
+        new ShieldClient(client).clearRealmCache(req, new RestBuilderListener<ClearRealmCacheResponse>(channel) {
             @Override
             public RestResponse buildResponse(ClearRealmCacheResponse response, XContentBuilder builder) throws Exception {
                 response.toXContent(builder, ToXContent.EMPTY_PARAMS);

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/rest/action/authz/cache/RestClearRolesCacheAction.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.rest.action.authz.cache;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.action.support.RestBuilderListener;
+import org.elasticsearch.shield.action.authz.cache.ClearRolesCacheRequest;
+import org.elasticsearch.shield.action.authz.cache.ClearRolesCacheResponse;
+import org.elasticsearch.shield.client.ShieldClient;
+
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+
+/**
+ *
+ */
+public class RestClearRolesCacheAction extends BaseRestHandler {
+
+    @Inject
+    public RestClearRolesCacheAction(Settings settings, RestController controller, Client client) {
+        super(settings, controller, client);
+        controller.registerHandler(POST, "/_shield/roles/{roles}/_cache/clear", this);
+    }
+
+    @Override
+    protected void handleRequest(RestRequest request, final RestChannel channel, Client client) throws Exception {
+
+        String[] roles = request.paramAsStringArrayOrEmptyIfAll("roles");
+
+        ClearRolesCacheRequest req = new ClearRolesCacheRequest().roles(roles);
+
+        new ShieldClient(client).clearRolesCache(req, new RestBuilderListener<ClearRolesCacheResponse>(channel) {
+            @Override
+            public RestResponse buildResponse(ClearRolesCacheResponse response, XContentBuilder builder) throws Exception {
+                response.toXContent(builder, ToXContent.EMPTY_PARAMS);
+                return new BytesRestResponse(RestStatus.OK, builder);
+            }
+        });
+    }
+}

elasticsearch/x-pack/shield/src/main/resources/shield-index-template.json

@@ -0,0 +1,66 @@
+{
+  "template": ".shield",
+  "order": 1000,
+  "settings": {
+    "number_of_shards": 1,
+    "number_of_replicas": 0,
+    "auto_expand_replicas": "0-all"
+  },
+  "mappings": {
+    "user": {
+      "dynamic": "strict",
+      "properties": {
+        "username": {
+          "type": "string",
+          "index": "not_analyzed"
+        },
+        "roles": {
+          "type": "string",
+          "index": "not_analyzed"
+        },
+        "password": {
+          "type": "string",
+          "index": "no"
+        }
+      }
+    },
+    "role" : {
+      "dynamic": "strict",
+      "properties" : {
+        "cluster" : {
+          "type" : "string",
+          "index": "not_analyzed"
+        },
+        "indices" : {
+          "type": "object",
+          "properties" : {
+            "fields" : {
+              "type" : "string",
+              "index": "not_analyzed"
+            },
+            "names" : {
+              "type" : "string",
+              "index": "not_analyzed"
+            },
+            "privileges" : {
+              "type" : "string",
+              "index": "not_analyzed"
+            },
+            "query" : {
+              "type" : "string",
+              "index": "not_analyzed"
+            }
+          }
+        },
+        "name" : {
+          "type" : "string",
+          "index": "not_analyzed"
+        },
+        "run_as" : {
+          "type" : "string",
+          "index": "not_analyzed"
+        }
+      }
+    }
+  }
+}

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java

@@ -143,7 +143,7 @@ public class ClearRealmsCacheTests extends ShieldIntegTestCase {
 
             final CountDownLatch latch = new CountDownLatch(1);
             final AtomicReference<Throwable> error = new AtomicReference<>();
-            client.authc().clearRealmCache(request, new ActionListener<ClearRealmCacheResponse>() {
+            client.clearRealmCache(request, new ActionListener<ClearRealmCacheResponse>() {
                 @Override
                 public void onResponse(ClearRealmCacheResponse response) {
                     assertThat(response.getNodes().length, equalTo(internalCluster().getNodeNames().length));

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java

@@ -0,0 +1,229 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.integration;
+
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.shield.action.admin.role.AddRoleResponse;
+import org.elasticsearch.shield.action.admin.role.GetRolesResponse;
+import org.elasticsearch.shield.admin.ShieldTemplateService;
+import org.elasticsearch.shield.authc.esnative.ESNativeUsersStore;
+import org.elasticsearch.shield.authc.support.SecuredString;
+import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
+import org.elasticsearch.shield.authz.RoleDescriptor;
+import org.elasticsearch.shield.authz.esnative.ESNativeRolesStore;
+import org.elasticsearch.shield.client.ShieldClient;
+import org.elasticsearch.test.ShieldIntegTestCase;
+import org.elasticsearch.test.ShieldSettingsSource;
+import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.elasticsearch.test.rest.client.http.HttpResponse;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.isOneOf;
+import static org.hamcrest.Matchers.notNullValue;
+
+/**
+ * Test for the Shield clear roles API that changes the polling aspect of shield to only run once an hour in order to
+ * test the cache clearing APIs.
+ */
+public class ClearRolesCacheTests extends ShieldIntegTestCase {
+
+    private static String[] roles;
+
+    @BeforeClass
+    public static void init() throws Exception {
+        roles = new String[randomIntBetween(5, 10)];
+        for (int i = 0; i < roles.length; i++) {
+            roles[i] = randomAsciiOfLength(6) + "_" + i;
+        }
+    }
+
+    @Before
+    public void setupForTest() throws Exception {
+        // Clear the realm cache for all realms since we use a SUITE scoped cluster
+        ShieldClient client = new ShieldClient(internalCluster().transportClient());
+        client.prepareClearRealmCache().get();
+
+        for (ESNativeUsersStore store : internalCluster().getInstances(ESNativeUsersStore.class)) {
+            assertBusy(new Runnable() {
+                @Override
+                public void run() {
+                    assertThat(store.state(), is(ESNativeUsersStore.State.STARTED));
+                }
+            });
+        }
+
+        for (ESNativeRolesStore store : internalCluster().getInstances(ESNativeRolesStore.class)) {
+            assertBusy(new Runnable() {
+                @Override
+                public void run() {
+                    assertThat(store.state(), is(ESNativeRolesStore.State.STARTED));
+                }
+            });
+        }
+
+        ShieldClient c = new ShieldClient(client());
+        // create roles
+        for (String role : roles) {
+            c.prepareAddRole().name(role)
+                    .cluster("none")
+                    .addIndices(new String[] { "*" }, new String[] { "ALL" }, null, null)
+                    .get();
+        }
+
+        ensureYellow(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME);
+
+        // warm up the caches on every node
+        for (ESNativeRolesStore rolesStore : internalCluster().getInstances(ESNativeRolesStore.class)) {
+            for (String role : roles) {
+                assertThat(rolesStore.role(role), notNullValue());
+            }
+        }
+    }
+
+    @Override
+    public Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+                .put(super.nodeSettings(nodeOrdinal))
+                .put("shield.authc.native.reload.interval", TimeValue.timeValueSeconds(2L))
+                .put(Node.HTTP_ENABLED, true)
+                .build();
+    }
+
+    @TestLogging("_root:DEBUG")
+    public void testModifyingViaApiClearsCache() throws Exception {
+        Client client = internalCluster().transportClient();
+        ShieldClient shieldClient = new ShieldClient(client);
+
+        int modifiedRolesCount = randomIntBetween(1, roles.length);
+        List<String> toModify = randomSubsetOf(modifiedRolesCount, roles);
+        for (String role : toModify) {
+            AddRoleResponse response = shieldClient.prepareAddRole().name(role)
+                    .cluster("none")
+                    .addIndices(new String[] { "*" }, new String[] { "ALL" }, null, null)
+                    .runAs(role)
+                    .get();
+            assertThat(response.isCreated(), is(false));
+        }
+
+        assertRolesAreCorrect(shieldClient, toModify);
+    }
+
+    public void testModifyingDocumentsDirectlyAndClearingCache() throws Exception {
+        Client client = internalCluster().transportClient();
+
+        int modifiedRolesCount = randomIntBetween(1, roles.length);
+        List<String> toModify = randomSubsetOf(modifiedRolesCount, roles);
+        for (String role : toModify) {
+            UpdateResponse response = client.prepareUpdate().setId(role).setIndex(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME)
+                    .setType(ESNativeRolesStore.INDEX_ROLE_TYPE)
+                    .setDoc("run_as", new String[] { role })
+                    .get();
+            assertThat(response.isCreated(), is(false));
+        }
+
+        ShieldClient shieldClient = new ShieldClient(client);
+        for (String role : roles) {
+            GetRolesResponse roleResponse = shieldClient.prepareGetRoles().roles(role).get();
+            assertThat(roleResponse.isExists(), is(true));
+            final String[] runAs = roleResponse.roles().get(0).getRunAs();
+            assertThat("role should be cached and no rules have run as set", runAs == null || runAs.length == 0, is(true));
+        }
+
+        boolean useHttp = randomBoolean();
+        boolean clearAll = randomBoolean();
+        String[] rolesToClear = clearAll ? (randomBoolean() ? roles : null) : toModify.toArray(Strings.EMPTY_ARRAY);
+        if (useHttp) {
+            String path;
+            if (rolesToClear == null) {
+                path = "/_shield/roles/" + (randomBoolean() ? "*" : "_all") + "/_cache/clear";
+            } else {
+                path = "/_shield/roles/" + Strings.arrayToCommaDelimitedString(rolesToClear) + "/_cache/clear";
+            }
+            HttpResponse response = httpClient().path(path).method("POST")
+                    .addHeader("Authorization",
+                            UsernamePasswordToken.basicAuthHeaderValue(ShieldSettingsSource.DEFAULT_USER_NAME,
+                                    new SecuredString(ShieldSettingsSource.DEFAULT_PASSWORD.toCharArray())))
+                    .execute();
+            assertThat(response.getStatusCode(), is(RestStatus.OK.getStatus()));
+        } else {
+            shieldClient.prepareClearRolesCache().roles(rolesToClear).get();
+        }
+
+        assertRolesAreCorrect(shieldClient, toModify);
+    }
+
+    public void testDeletingRoleDocumentDirectly() throws Exception {
+        Client client = internalCluster().transportClient();
+        ShieldClient shieldClient = new ShieldClient(client);
+
+        final String role = randomFrom(roles);
+        List<RoleDescriptor> foundRoles = shieldClient.prepareGetRoles().roles(role).get().roles();
+        assertThat(foundRoles.size(), is(1));
+        DeleteResponse response = client.prepareDelete(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME, ESNativeRolesStore.INDEX_ROLE_TYPE, role).get();
+        assertThat(response.isFound(), is(true));
+
+        assertBusy(new Runnable() {
+            @Override
+            public void run() {
+                assertThat(shieldClient.prepareGetRoles().roles(role).get().roles().isEmpty(), is(true));
+            }
+        });
+    }
+
+    private void assertRolesAreCorrect(ShieldClient shieldClient, List<String> toModify) {
+        for (String role : roles) {
+            GetRolesResponse roleResponse = shieldClient.prepareGetRoles().roles(role).get();
+            assertThat(roleResponse.isExists(), is(true));
+            final String[] runAs = roleResponse.roles().get(0).getRunAs();
+            if (toModify.contains(role)) {
+                assertThat("role should be modified and have run as", runAs == null || runAs.length == 0, is(false));
+                assertThat(Arrays.asList(runAs).contains(role), is(true));
+            } else {
+                assertThat("role should be cached and no rules have run as set", runAs == null || runAs.length == 0, is(true));
+            }
+        }
+    }
+
+    @After
+    public void stopESNativeStores() throws Exception {
+        for (ESNativeUsersStore store : internalCluster().getInstances(ESNativeUsersStore.class)) {
+            store.stop();
+            // the store may already be stopping so wait until it is stopped
+            assertBusy(new Runnable() {
+                @Override
+                public void run() {
+                    assertThat(store.state(), isOneOf(ESNativeUsersStore.State.STOPPED, ESNativeUsersStore.State.FAILED));
+                }
+            });
+            store.reset();
+        }
+
+        for (ESNativeRolesStore store : internalCluster().getInstances(ESNativeRolesStore.class)) {
+            store.stop();
+            // the store may already be stopping so wait until it is stopped
+            assertBusy(new Runnable() {
+                @Override
+                public void run() {
+                    assertThat(store.state(), isOneOf(ESNativeRolesStore.State.STOPPED, ESNativeRolesStore.State.FAILED));
+                }
+            });
+            store.reset();
+        }
+    }
+}

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ClusterPrivilegeTests.java

@@ -134,9 +134,9 @@ public class ClusterPrivilegeTests extends AbstractPrivilegeTestCase {
         assertAccessIsDenied("user_b", "PUT", "/someindex/bar/1", "{ \"name\" : \"elasticsearch\" }", params);
         assertAccessIsAllowed("user_c", "PUT", "/someindex/bar/1", "{ \"name\" : \"elasticsearch\" }", params);
 
-        assertAccessIsDenied("user_b", "PUT", "/_snapshot/my-repo/my-snapshot");
-        assertAccessIsDenied("user_c", "PUT", "/_snapshot/my-repo/my-snapshot");
-        assertAccessIsAllowed("user_a", "PUT", "/_snapshot/my-repo/my-snapshot");
+        assertAccessIsDenied("user_b", "PUT", "/_snapshot/my-repo/my-snapshot", "{ \"indices\": \"someindex\" }");
+        assertAccessIsDenied("user_c", "PUT", "/_snapshot/my-repo/my-snapshot", "{ \"indices\": \"someindex\" }");
+        assertAccessIsAllowed("user_a", "PUT", "/_snapshot/my-repo/my-snapshot", "{ \"indices\": \"someindex\" }");
 
         assertAccessIsDenied("user_b", "GET", "/_snapshot/my-repo/my-snapshot/_status");
         assertAccessIsDenied("user_c", "GET", "/_snapshot/my-repo/my-snapshot/_status");

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/LicensingTests.java

@@ -46,6 +46,7 @@ import java.util.List;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
@@ -178,7 +179,7 @@ public class LicensingTests extends ShieldIntegTestCase {
         assertThat(clusterStatsNodeResponse, notNullValue());
         ClusterStatsIndices indices = clusterStatsNodeResponse.getIndicesStats();
         assertThat(indices, notNullValue());
-        assertThat(indices.getIndexCount(), is(2));
+        assertThat(indices.getIndexCount(), greaterThanOrEqualTo(2));
 
         ClusterHealthResponse clusterIndexHealth = client.admin().cluster().prepareHealth().get();
         assertThat(clusterIndexHealth, notNullValue());

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/ShieldF.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.node.MockNode;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.shield.authc.esnative.ESNativeRealm;
+import org.elasticsearch.shield.authc.esusers.ESUsersRealm;
+import org.elasticsearch.shield.test.ShieldTestUtils;
+import org.elasticsearch.test.ShieldSettingsSource;
+import org.elasticsearch.xpack.XPackPlugin;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.concurrent.CountDownLatch;
+
+import static org.elasticsearch.shield.test.ShieldTestUtils.writeFile;
+
+/**
+ * Main class to easily run Shield from a IDE.
+ *
+ * During startup an error will be printed that the config directory can't be found, to fix this:
+ * set `-Des.path.home=` to a location where there is a config directory on your machine.
+ */
+public class ShieldF {
+
+    public static void main(String[] args) throws Throwable {
+        Settings.Builder settings = Settings.builder();
+        settings.put("http.cors.enabled", "true");
+        settings.put("http.cors.allow-origin", "*");
+        settings.put("script.inline", "on");
+        settings.put("shield.enabled", "true");
+        settings.put("security.manager.enabled", "false");
+        // Disable Marvel to prevent cluster activity
+        settings.put("marvel.enabled", "false");
+        settings.put(IndexModule.QUERY_CACHE_TYPE, ShieldPlugin.OPT_OUT_QUERY_CACHE);
+        settings.put("cluster.name", ShieldF.class.getSimpleName());
+
+        String homeDir = System.getProperty("es.path.home");
+        if (homeDir == null || Files.exists(PathUtils.get(homeDir)) == false) {
+            throw new IllegalStateException("es.path.home must be set and exist");
+        }
+        Path folder = ShieldTestUtils.createFolder(ShieldTestUtils.createFolder(PathUtils.get(homeDir), "config"), "shield");
+
+        settings.put("shield.authc.realms.esusers.type", ESUsersRealm.TYPE);
+        settings.put("shield.authc.realms.esusers.order", "0");
+        settings.put("shield.authc.realms.esusers.files.users", writeFile(folder, "users", ShieldSettingsSource.CONFIG_STANDARD_USER));
+        settings.put("shield.authc.realms.esusers.files.users_roles", writeFile(folder, "users_roles", ShieldSettingsSource.CONFIG_STANDARD_USER_ROLES));
+        settings.put("shield.authc.realms.esnative.type", ESNativeRealm.TYPE);
+        settings.put("shield.authc.realms.esnative.order", "1");
+        settings.put("shield.authz.store.files.roles", writeFile(folder, "roles.yml", ShieldSettingsSource.CONFIG_ROLE_ALLOW_ALL));
+
+        final CountDownLatch latch = new CountDownLatch(1);
+        final Node node = new MockNode(settings.build(), Version.CURRENT, Arrays.asList(XPackPlugin.class));
+        Runtime.getRuntime().addShutdownHook(new Thread() {
+
+            @Override
+            public void run() {
+                node.close();
+                latch.countDown();
+            }
+        });
+        node.start();
+        latch.await();
+    }
+}

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/UserTests.java

@@ -21,7 +21,8 @@ import static org.hamcrest.Matchers.sameInstance;
 
 public class UserTests extends ESTestCase {
     public void testWriteToAndReadFrom() throws Exception {
-        User user = new User.Simple(randomAsciiOfLengthBetween(4, 30), generateRandomStringArray(20, 30, false));
+        User user = new User(randomAsciiOfLengthBetween(4, 30),
+                generateRandomStringArray(20, 30, false));
         BytesStreamOutput output = new BytesStreamOutput();
 
         User.writeTo(user, output);
@@ -34,8 +35,10 @@ public class UserTests extends ESTestCase {
     }
 
     public void testWriteToAndReadFromWithRunAs() throws Exception {
-        User runAs = new User.Simple(randomAsciiOfLengthBetween(4, 30), randomBoolean() ? generateRandomStringArray(20, 30, false) : null);
-        User user = new User.Simple(randomAsciiOfLengthBetween(4, 30), generateRandomStringArray(20, 30, false), runAs);
+        User runAs = new User(randomAsciiOfLengthBetween(4, 30),
+                randomBoolean() ? generateRandomStringArray(20, 30, false) : null);
+        User user = new User(randomAsciiOfLengthBetween(4, 30),
+                generateRandomStringArray(20, 30, false), runAs);
         BytesStreamOutput output = new BytesStreamOutput();
 
         User.writeTo(user, output);
@@ -77,10 +80,19 @@ public class UserTests extends ESTestCase {
 
     public void testCreateUserRunningAsSystemUser() throws Exception {
         try {
-            new User.Simple(randomAsciiOfLengthBetween(3, 10), generateRandomStringArray(16, 30, false), User.SYSTEM);
+            new User(randomAsciiOfLengthBetween(3, 10),
+                    generateRandomStringArray(16, 30, false), User.SYSTEM);
             fail("should not be able to create a runAs user with the system user");
         } catch (ElasticsearchSecurityException e) {
             assertThat(e.getMessage(), containsString("system"));
         }
     }
+
+    public void testUserToString() throws Exception {
+        User sudo = new User("root", new String[]{"r3"});
+        User u = new User("bob", new String[]{"r1", "r2"}, sudo);
+        assertEquals("User[username=root,roles=[r3,]]", sudo.toString());
+        assertEquals("User[username=bob,roles=[r1,r2,],runAs=[User[username=root,roles=[r3,]]]]",
+                u.toString());
+    }
 }

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/action/ShieldActionFilterTests.java

@@ -64,7 +64,7 @@ public class ShieldActionFilterTests extends ESTestCase {
         ActionListener listener = mock(ActionListener.class);
         ActionFilterChain chain = mock(ActionFilterChain.class);
         Task task = mock(Task.class);
-        User user = new User.Simple("username", new String[] { "r1", "r2" });
+        User user = new User("username", "r1", "r2");
         when(authcService.authenticate("_action", request, User.SYSTEM)).thenReturn(user);
         doReturn(request).when(spy(filter)).unsign(user, "_action", request);
         filter.apply(task, "_action", request, listener, chain);
@@ -78,7 +78,7 @@ public class ShieldActionFilterTests extends ESTestCase {
         ActionFilterChain chain = mock(ActionFilterChain.class);
         RuntimeException exception = new RuntimeException("process-error");
         Task task = mock(Task.class);
-        User user = new User.Simple("username", new String[] { "r1", "r2" });
+        User user = new User("username", "r1", "r2");
         when(authcService.authenticate("_action", request, User.SYSTEM)).thenReturn(user);
         doThrow(exception).when(authzService).authorize(user, "_action", request);
         filter.apply(task, "_action", request, listener, chain);

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/admin/ESNativeTests.java

@@ -0,0 +1,421 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.admin;
+
+import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.shield.User;
+import org.elasticsearch.shield.action.admin.role.DeleteRoleResponse;
+import org.elasticsearch.shield.action.admin.role.GetRolesResponse;
+import org.elasticsearch.shield.action.admin.user.DeleteUserResponse;
+import org.elasticsearch.shield.action.admin.user.GetUsersResponse;
+import org.elasticsearch.shield.authc.esnative.ESNativeUsersStore;
+import org.elasticsearch.shield.authc.support.SecuredString;
+import org.elasticsearch.shield.authz.Permission;
+import org.elasticsearch.shield.authz.RoleDescriptor;
+import org.elasticsearch.shield.authz.esnative.ESNativeRolesStore;
+import org.elasticsearch.shield.client.ShieldClient;
+import org.elasticsearch.test.ShieldIntegTestCase;
+import org.elasticsearch.test.ShieldSettingsSource;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+import static org.hamcrest.Matchers.arrayContaining;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.isOneOf;
+
+/**
+ * Tests for the ESNativeUsersStore and ESNativeRolesStore
+ */
+public class ESNativeTests extends ShieldIntegTestCase {
+
+    public void testDeletingNonexistingUserAndRole() throws Exception {
+        ShieldClient c = new ShieldClient(client());
+        DeleteUserResponse resp = c.prepareDeleteUser().user("joe").get();
+        assertFalse("user shouldn't be found", resp.found());
+        DeleteRoleResponse resp2 = c.prepareDeleteRole().role("role").get();
+        assertFalse("role shouldn't be found", resp2.found());
+    }
+
+    public void testGettingUserThatDoesntExist() throws Exception {
+        ShieldClient c = new ShieldClient(client());
+        GetUsersResponse resp = c.prepareGetUsers().users("joe").get();
+        assertFalse("user should not exist", resp.isExists());
+        GetRolesResponse resp2 = c.prepareGetRoles().roles("role").get();
+        assertFalse("role should not exist", resp2.isExists());
+    }
+
+    public void testAddAndGetUser() throws Exception {
+        ShieldClient c = new ShieldClient(client());
+        logger.error("--> creating user");
+        c.prepareAddUser()
+                .username("joe")
+                .password("s3kirt")
+                .roles("role1", "user")
+                .get();
+        logger.error("--> waiting for .shield index");
+        ensureGreen(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME);
+        logger.info("--> retrieving user");
+        GetUsersResponse resp = c.prepareGetUsers().users("joe").get();
+        assertTrue("user should exist", resp.isExists());
+        User joe = resp.users().get(0);
+        assertEquals(joe.principal(), "joe");
+        assertArrayEquals(joe.roles(), new String[]{"role1", "user"});
+
+        logger.info("--> adding two more users");
+        c.prepareAddUser()
+                .username("joe2")
+                .password("s3kirt2")
+                .roles("role2", "user")
+                .get();
+        c.prepareAddUser()
+                .username("joe3")
+                .password("s3kirt3")
+                .roles("role3", "user")
+                .get();
+        // Since getting multiple users relies on them being visible to search, perform a refresh
+        refresh();
+        GetUsersResponse allUsersResp = c.prepareGetUsers().get();
+        assertTrue("users should exist", allUsersResp.isExists());
+        assertEquals("should be 3 users total", 3, allUsersResp.users().size());
+        List<String> names = new ArrayList<>(3);
+        for (User u : allUsersResp.users()) {
+            names.add(u.principal());
+        }
+        CollectionUtil.timSort(names);
+        assertArrayEquals(new String[]{"joe", "joe2", "joe3"}, names.toArray(Strings.EMPTY_ARRAY));
+
+        GetUsersResponse someUsersResp = c.prepareGetUsers().users("joe", "joe3").get();
+        assertTrue("users should exist", someUsersResp.isExists());
+        assertEquals("should be 2 users returned", 2, someUsersResp.users().size());
+        names = new ArrayList<>(2);
+        for (User u : someUsersResp.users()) {
+            names.add(u.principal());
+        }
+        CollectionUtil.timSort(names);
+        assertArrayEquals(new String[]{"joe", "joe3"}, names.toArray(Strings.EMPTY_ARRAY));
+
+        logger.info("--> deleting user");
+        DeleteUserResponse delResp = c.prepareDeleteUser().user("joe").get();
+        assertTrue(delResp.found());
+        logger.info("--> retrieving user");
+        resp = c.prepareGetUsers().users("joe").get();
+        assertFalse("user should not exist after being deleted", resp.isExists());
+    }
+
+    public void testAddAndGetRole() throws Exception {
+        ShieldClient c = new ShieldClient(client());
+        logger.error("--> creating role");
+        c.prepareAddRole()
+                .name("test_role")
+                .cluster("all", "none")
+                .runAs("root", "nobody")
+                .addIndices(new String[]{"index"}, new String[]{"read"},
+                        new String[]{"body", "title"}, new BytesArray("{\"query\": {\"match_all\": {}}}"))
+                .get();
+        logger.error("--> waiting for .shield index");
+        ensureGreen(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME);
+        logger.info("--> retrieving role");
+        GetRolesResponse resp = c.prepareGetRoles().roles("test_role").get();
+        assertTrue("role should exist", resp.isExists());
+        RoleDescriptor testRole = resp.roles().get(0);
+        assertNotNull(testRole);
+
+        c.prepareAddRole()
+                .name("test_role2")
+                .cluster("all", "none")
+                .runAs("root", "nobody")
+                .addIndices(new String[]{"index"}, new String[]{"read"},
+                        new String[]{"body", "title"}, new BytesArray("{\"query\": {\"match_all\": {}}}"))
+                .get();
+        c.prepareAddRole()
+                .name("test_role3")
+                .cluster("all", "none")
+                .runAs("root", "nobody")
+                .addIndices(new String[]{"index"}, new String[]{"read"},
+                        new String[]{"body", "title"}, new BytesArray("{\"query\": {\"match_all\": {}}}"))
+                .get();
+
+        // Refresh to make new roles visible
+        refresh();
+
+        logger.info("--> retrieving all roles");
+        GetRolesResponse allRolesResp = c.prepareGetRoles().get();
+        assertTrue("roles should exist", allRolesResp.isExists());
+        assertEquals("should be 3 roles total", 3, allRolesResp.roles().size());
+
+        logger.info("--> retrieving all roles");
+        GetRolesResponse someRolesResp = c.prepareGetRoles().roles("test_role", "test_role3").get();
+        assertTrue("roles should exist", someRolesResp.isExists());
+        assertEquals("should be 2 roles total", 2, someRolesResp.roles().size());
+
+        logger.info("--> deleting role");
+        DeleteRoleResponse delResp = c.prepareDeleteRole().role("test_role").get();
+        assertTrue(delResp.found());
+        logger.info("--> retrieving role");
+        GetRolesResponse resp2 = c.prepareGetRoles().roles("test_role").get();
+        assertFalse("role should not exist after being deleted", resp2.isExists());
+    }
+
+    public void testAddUserAndRoleThenAuth() throws Exception {
+        ShieldClient c = new ShieldClient(client());
+        logger.error("--> creating role");
+        c.prepareAddRole()
+                .name("test_role")
+                .cluster("all")
+                .addIndices(new String[]{"*"}, new String[]{"read"},
+                        new String[]{"body", "title"}, new BytesArray("{\"match_all\": {}}"))
+                .get();
+        logger.error("--> creating user");
+        c.prepareAddUser()
+                .username("joe")
+                .password("s3krit")
+                .roles("test_role")
+                .get();
+        refresh();
+        logger.error("--> waiting for .shield index");
+        ensureGreen(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME);
+        logger.info("--> retrieving user");
+        GetUsersResponse resp = c.prepareGetUsers().users("joe").get();
+        assertTrue("user should exist", resp.isExists());
+
+        createIndex("idx");
+        ensureGreen("idx");
+        // Index a document with the default test user
+        client().prepareIndex("idx", "doc", "1").setSource("body", "foo").setRefresh(true).get();
+
+        String token = basicAuthHeaderValue("joe", new SecuredString("s3krit".toCharArray()));
+        SearchResponse searchResp = client().prepareSearch("idx").putHeader("Authorization", token).get();
+
+        assertEquals(searchResp.getHits().getTotalHits(), 1L);
+    }
+
+    public void testUpdatingUserAndAuthentication() throws Exception {
+        ShieldClient c = new ShieldClient(client());
+        logger.error("--> creating user");
+        c.prepareAddUser()
+                .username("joe")
+                .password("s3krit")
+                .roles(ShieldSettingsSource.DEFAULT_ROLE)
+                .get();
+        refresh();
+        logger.error("--> waiting for .shield index");
+        ensureGreen(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME);
+        logger.info("--> retrieving user");
+        GetUsersResponse resp = c.prepareGetUsers().users("joe").get();
+        assertTrue("user should exist", resp.isExists());
+        assertThat(resp.users().get(0).roles(), arrayContaining(ShieldSettingsSource.DEFAULT_ROLE));
+
+        createIndex("idx");
+        ensureGreen("idx");
+        // Index a document with the default test user
+        client().prepareIndex("idx", "doc", "1").setSource("body", "foo").setRefresh(true).get();
+        String token = basicAuthHeaderValue("joe", new SecuredString("s3krit".toCharArray()));
+        SearchResponse searchResp = client().prepareSearch("idx").putHeader("Authorization", token).get();
+
+        assertEquals(searchResp.getHits().getTotalHits(), 1L);
+
+        c.prepareAddUser()
+                .username("joe")
+                .password("s3krit2")
+                .roles(ShieldSettingsSource.DEFAULT_ROLE)
+                .get();
+
+        try {
+            client().prepareSearch("idx").putHeader("Authorization", token).get();
+            fail("authentication with old credentials after an update to the user should fail!");
+        } catch (ElasticsearchSecurityException e) {
+            // expected
+            assertThat(e.status(), is(RestStatus.UNAUTHORIZED));
+        }
+
+        token = basicAuthHeaderValue("joe", new SecuredString("s3krit2".toCharArray()));
+        searchResp = client().prepareSearch("idx").putHeader("Authorization", token).get();
+        assertEquals(searchResp.getHits().getTotalHits(), 1L);
+    }
+
+    public void testCreateDeleteAuthenticate() {
+        ShieldClient c = new ShieldClient(client());
+        logger.error("--> creating user");
+        c.prepareAddUser()
+                .username("joe")
+                .password("s3krit")
+                .roles(ShieldSettingsSource.DEFAULT_ROLE)
+                .get();
+        refresh();
+        logger.error("--> waiting for .shield index");
+        ensureGreen(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME);
+        logger.info("--> retrieving user");
+        GetUsersResponse resp = c.prepareGetUsers().users("joe").get();
+        assertTrue("user should exist", resp.isExists());
+        assertThat(resp.users().get(0).roles(), arrayContaining(ShieldSettingsSource.DEFAULT_ROLE));
+
+        createIndex("idx");
+        ensureGreen("idx");
+        // Index a document with the default test user
+        client().prepareIndex("idx", "doc", "1").setSource("body", "foo").setRefresh(true).get();
+        String token = basicAuthHeaderValue("joe", new SecuredString("s3krit".toCharArray()));
+        SearchResponse searchResp = client().prepareSearch("idx").putHeader("Authorization", token).get();
+
+        assertEquals(searchResp.getHits().getTotalHits(), 1L);
+
+        DeleteUserResponse response = c.prepareDeleteUser().user("joe").get();
+        assertThat(response.found(), is(true));
+        try {
+            client().prepareSearch("idx").putHeader("Authorization", token).get();
+            fail("authentication with a deleted user should fail!");
+        } catch (ElasticsearchSecurityException e) {
+            // expected
+            assertThat(e.status(), is(RestStatus.UNAUTHORIZED));
+        }
+    }
+
+    public void testCreateAndUpdateRole() {
+        final boolean authenticate = randomBoolean();
+        ShieldClient c = new ShieldClient(client());
+        logger.error("--> creating role");
+        c.prepareAddRole()
+                .name("test_role")
+                .cluster("all")
+                .addIndices(new String[]{"*"}, new String[]{"read"},
+                        new String[]{"body", "title"}, new BytesArray("{\"match_all\": {}}"))
+                .get();
+        logger.error("--> creating user");
+        c.prepareAddUser()
+                .username("joe")
+                .password("s3krit")
+                .roles("test_role")
+                .get();
+        refresh();
+        logger.error("--> waiting for .shield index");
+        ensureGreen(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME);
+
+        if (authenticate) {
+            final String token = basicAuthHeaderValue("joe", new SecuredString("s3krit".toCharArray()));
+            ClusterHealthResponse response = client().admin().cluster().prepareHealth().putHeader("Authorization", token).get();
+            assertFalse(response.isTimedOut());
+            c.prepareAddRole()
+                    .name("test_role")
+                    .cluster("none")
+                    .addIndices(new String[]{"*"}, new String[]{"read"},
+                            new String[]{"body", "title"}, new BytesArray("{\"match_all\": {}}"))
+                    .get();
+            try {
+                client().admin().cluster().prepareHealth().putHeader("Authorization", token).get();
+                fail("user should not be able to execute any cluster actions!");
+            } catch (ElasticsearchSecurityException e) {
+                assertThat(e.status(), is(RestStatus.FORBIDDEN));
+            }
+        } else {
+            GetRolesResponse getRolesResponse = c.prepareGetRoles().roles("test_role").get();
+            assertTrue("test_role does not exist!", getRolesResponse.isExists());
+            assertTrue("any cluster permission should be authorized",
+                    Permission.Global.Role.builder(getRolesResponse.roles().get(0)).build().cluster().check("cluster:admin/foo"));
+            c.prepareAddRole()
+                    .name("test_role")
+                    .cluster("none")
+                    .addIndices(new String[]{"*"}, new String[]{"read"},
+                            new String[]{"body", "title"}, new BytesArray("{\"match_all\": {}}"))
+                    .get();
+            getRolesResponse = c.prepareGetRoles().roles("test_role").get();
+            assertTrue("test_role does not exist!", getRolesResponse.isExists());
+
+            assertFalse("no cluster permission should be authorized",
+                    Permission.Global.Role.builder(getRolesResponse.roles().get(0)).build().cluster().check("cluster:admin/bar"));
+        }
+    }
+
+    public void testAuthenticateWithDeletedRole() {
+        ShieldClient c = new ShieldClient(client());
+        logger.error("--> creating role");
+        c.prepareAddRole()
+                .name("test_role")
+                .cluster("all")
+                .addIndices(new String[]{"*"}, new String[]{"read"},
+                        new String[]{"body", "title"}, new BytesArray("{\"match_all\": {}}"))
+                .get();
+        c.prepareAddUser()
+                .username("joe")
+                .password("s3krit")
+                .roles("test_role")
+                .get();
+        refresh();
+        logger.error("--> waiting for .shield index");
+        ensureGreen(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME);
+
+        final String token = basicAuthHeaderValue("joe", new SecuredString("s3krit".toCharArray()));
+        ClusterHealthResponse response = client().admin().cluster().prepareHealth().putHeader("Authorization", token).get();
+        assertFalse(response.isTimedOut());
+        c.prepareDeleteRole().role("test_role").get();
+        try {
+            client().admin().cluster().prepareHealth().putHeader("Authorization", token).get();
+            fail("user should not be able to execute any actions!");
+        } catch (ElasticsearchSecurityException e) {
+            assertThat(e.status(), is(RestStatus.FORBIDDEN));
+        }
+    }
+
+    @Before
+    public void ensureStoresStarted() throws Exception {
+        // Clear the realm cache for all realms since we use a SUITE scoped cluster
+        ShieldClient client = new ShieldClient(client());
+        client.prepareClearRealmCache().get();
+
+        for (ESNativeUsersStore store : internalCluster().getInstances(ESNativeUsersStore.class)) {
+            assertBusy(new Runnable() {
+                @Override
+                public void run() {
+                    assertThat(store.state(), is(ESNativeUsersStore.State.STARTED));
+                }
+            });
+        }
+
+        for (ESNativeRolesStore store : internalCluster().getInstances(ESNativeRolesStore.class)) {
+            assertBusy(new Runnable() {
+                @Override
+                public void run() {
+                    assertThat(store.state(), is(ESNativeRolesStore.State.STARTED));
+                }
+            });
+        }
+    }
+
+    @After
+    public void stopESNativeStores() throws Exception {
+        for (ESNativeUsersStore store : internalCluster().getInstances(ESNativeUsersStore.class)) {
+            store.stop();
+            // the store may already be stopping so wait until it is stopped
+            assertBusy(new Runnable() {
+                @Override
+                public void run() {
+                    assertThat(store.state(), isOneOf(ESNativeUsersStore.State.STOPPED, ESNativeUsersStore.State.FAILED));
+                }
+            });
+            store.reset();
+        }
+
+        for (ESNativeRolesStore store : internalCluster().getInstances(ESNativeRolesStore.class)) {
+            store.stop();
+            // the store may already be stopping so wait until it is stopped
+            assertBusy(new Runnable() {
+                @Override
+                public void run() {
+                    assertThat(store.state(), isOneOf(ESNativeRolesStore.State.STOPPED, ESNativeRolesStore.State.FAILED));
+                }
+            });
+            store.reset();
+        }
+    }
+}

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/audit/AuditTrailServiceTests.java

@@ -97,7 +97,7 @@ public class AuditTrailServiceTests extends ESTestCase {
     }
 
     public void testAccessGranted() throws Exception {
-        User user = new User.Simple("_username", new String[] { "r1" });
+        User user = new User("_username", "r1");
         service.accessGranted(user, "_action", message);
         for (AuditTrail auditTrail : auditTrails) {
             verify(auditTrail).accessGranted(user, "_action", message);
@@ -105,7 +105,7 @@ public class AuditTrailServiceTests extends ESTestCase {
     }
 
     public void testAccessDenied() throws Exception {
-        User user = new User.Simple("_username", new String[] { "r1" });
+        User user = new User("_username", "r1");
         service.accessDenied(user, "_action", message);
         for (AuditTrail auditTrail : auditTrails) {
             verify(auditTrail).accessDenied(user, "_action", message);

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java

@@ -21,7 +21,6 @@ import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.rest.RestRequest;
@@ -204,9 +203,8 @@ public class IndexAuditTrailTests extends ShieldIntegTestCase {
         Transport transport = mock(Transport.class);
         when(transport.boundAddress()).thenReturn(new BoundTransportAddress(new TransportAddress[] { DummyTransportAddress.INSTANCE }, DummyTransportAddress.INSTANCE));
 
-        Environment env = new Environment(settings);
         threadPool = new ThreadPool("index audit trail tests");
-        auditor = new IndexAuditTrail(settings, user, env, authService, transport, Providers.of(client()), threadPool, mock(ClusterService.class));
+        auditor = new IndexAuditTrail(settings, user, authService, transport, Providers.of(client()), threadPool, mock(ClusterService.class));
         auditor.start(true);
     }
 
@@ -496,9 +494,10 @@ public class IndexAuditTrailTests extends ShieldIntegTestCase {
         final boolean runAs = randomBoolean();
         User user;
         if (runAs) {
-            user = new User.Simple("_username", new String[]{"r1"}, new User.Simple("running as", new String[] {"r2"}));
+            user = new User("_username", new String[]{"r1"},
+                    new User("running as", new String[] {"r2"}));
         } else {
-            user = new User.Simple("_username", new String[]{"r1"});
+            user = new User("_username", new String[]{"r1"});
         }
         auditor.accessGranted(user, "_action", message);
         awaitAuditDocumentCreation(resolveIndexName());
@@ -524,7 +523,7 @@ public class IndexAuditTrailTests extends ShieldIntegTestCase {
     public void testAccessGrantedMuted() throws Exception {
         initialize("access_granted");
         TransportMessage message = randomFrom(new RemoteHostMockMessage(), new LocalHostMockMessage(), new MockIndicesTransportMessage());
-        auditor.accessGranted(new User.Simple("_username", new String[]{"r1"}), "_action", message);
+        auditor.accessGranted(new User("_username", "r1"), "_action", message);
         try {
             getClient().prepareSearch(resolveIndexName()).setSize(0).setTerminateAfter(1).execute().actionGet();
             fail("Expected IndexNotFoundException");
@@ -565,9 +564,10 @@ public class IndexAuditTrailTests extends ShieldIntegTestCase {
         final boolean runAs = randomBoolean();
         User user;
         if (runAs) {
-            user = new User.Simple("_username", new String[]{"r1"}, new User.Simple("running as", new String[] {"r2"}));
+            user = new User("_username", new String[]{"r1"},
+                    new User("running as", new String[] {"r2"}));
         } else {
-            user = new User.Simple("_username", new String[]{"r1"});
+            user = new User("_username", new String[]{"r1"});
         }
         auditor.accessDenied(user, "_action", message);
         awaitAuditDocumentCreation(resolveIndexName());
@@ -593,7 +593,7 @@ public class IndexAuditTrailTests extends ShieldIntegTestCase {
     public void testAccessDenied_Muted() throws Exception {
         initialize("access_denied");
         TransportMessage message = randomFrom(new RemoteHostMockMessage(), new LocalHostMockMessage(), new MockIndicesTransportMessage());
-        auditor.accessDenied(new User.Simple("_username", new String[]{"r1"}), "_action", message);
+        auditor.accessDenied(new User("_username", "r1"), "_action", message);
         try {
             getClient().prepareSearch(resolveIndexName()).setSize(0).setTerminateAfter(1).execute().actionGet();
             fail("Expected IndexNotFoundException");
@@ -623,9 +623,10 @@ public class IndexAuditTrailTests extends ShieldIntegTestCase {
         final boolean runAs = randomBoolean();
         User user;
         if (runAs) {
-            user = new User.Simple("_username", new String[]{"r1"}, new User.Simple("running as", new String[] {"r2"}));
+            user = new User("_username", new String[]{"r1"},
+                    new User("running as", new String[] {"r2"}));
         } else {
-            user = new User.Simple("_username", new String[]{"r1"});
+            user = new User("_username", new String[]{"r1"});
         }
         auditor.tamperedRequest(user, "_action", message);
         awaitAuditDocumentCreation(resolveIndexName());
@@ -649,7 +650,7 @@ public class IndexAuditTrailTests extends ShieldIntegTestCase {
         initialize("tampered_request");
         TransportRequest message = new RemoteHostMockTransportRequest();
         if (randomBoolean()) {
-            auditor.tamperedRequest(new User.Simple("_username", new String[]{"r1"}), "_action", message);
+            auditor.tamperedRequest(new User("_username", new String[]{"r1"}), "_action", message);
         } else {
             auditor.tamperedRequest("_action", message);
         }
@@ -720,7 +721,7 @@ public class IndexAuditTrailTests extends ShieldIntegTestCase {
     public void testRunAsGranted() throws Exception {
         initialize();
         TransportMessage message = randomFrom(new RemoteHostMockMessage(), new LocalHostMockMessage(), new MockIndicesTransportMessage());
-        User user = new User.Simple("_username", new String[]{"r1"}, new User.Simple("running as", new String[] {"r2"}));
+        User user = new User("_username", new String[]{"r1"}, new User("running as", new String[] {"r2"}));
         auditor.runAsGranted(user, "_action", message);
         awaitAuditDocumentCreation(resolveIndexName());
 
@@ -737,7 +738,7 @@ public class IndexAuditTrailTests extends ShieldIntegTestCase {
     public void testRunAsGrantedMuted() throws Exception {
         initialize("run_as_granted");
         TransportMessage message = randomFrom(new RemoteHostMockMessage(), new LocalHostMockMessage(), new MockIndicesTransportMessage());
-        auditor.runAsGranted(new User.Simple("_username", new String[]{"r1"}, new User.Simple("running as", new String[]{"r2"})), "_action", message);
+        auditor.runAsGranted(new User("_username", new String[]{"r1"}, new User("running as", new String[]{"r2"})), "_action", message);
         try {
             getClient().prepareSearch(resolveIndexName()).setSize(0).setTerminateAfter(1).execute().actionGet();
             fail("Expected IndexNotFoundException");
@@ -749,7 +750,7 @@ public class IndexAuditTrailTests extends ShieldIntegTestCase {
     public void testRunAsDenied() throws Exception {
         initialize();
         TransportMessage message = randomFrom(new RemoteHostMockMessage(), new LocalHostMockMessage(), new MockIndicesTransportMessage());
-        User user = new User.Simple("_username", new String[]{"r1"}, new User.Simple("running as", new String[] {"r2"}));
+        User user = new User("_username", new String[]{"r1"}, new User("running as", new String[] {"r2"}));
         auditor.runAsDenied(user, "_action", message);
         awaitAuditDocumentCreation(resolveIndexName());
 
@@ -766,7 +767,7 @@ public class IndexAuditTrailTests extends ShieldIntegTestCase {
     public void testRunAsDeniedMuted() throws Exception {
         initialize("run_as_denied");
         TransportMessage message = randomFrom(new RemoteHostMockMessage(), new LocalHostMockMessage(), new MockIndicesTransportMessage());
-        auditor.runAsDenied(new User.Simple("_username", new String[]{"r1"}, new User.Simple("running as", new String[]{"r2"})), "_action", message);
+        auditor.runAsDenied(new User("_username", new String[]{"r1"}, new User("running as", new String[]{"r2"})), "_action", message);
         try {
             getClient().prepareSearch(resolveIndexName()).setSize(0).setTerminateAfter(1).execute().actionGet();
             fail("Expected IndexNotFoundException");

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailUpdateMappingTests.java

@@ -12,7 +12,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.env.Environment;
 import org.elasticsearch.shield.authc.AuthenticationService;
 import org.elasticsearch.test.ShieldIntegTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
@@ -37,6 +36,7 @@ import static org.mockito.Mockito.when;
  */
 public class IndexAuditTrailUpdateMappingTests extends ShieldIntegTestCase {
     private ThreadPool threadPool;
+    private IndexAuditTrail auditor;
 
     @Before
     public void setup() {
@@ -50,8 +50,7 @@ public class IndexAuditTrailUpdateMappingTests extends ShieldIntegTestCase {
         Settings settings = Settings.builder().put("shield.audit.index.rollover", rollover.name().toLowerCase(Locale.ENGLISH)).put("path.home", createTempDir()).build();
         Transport transport = mock(Transport.class);
         when(transport.boundAddress()).thenReturn(new BoundTransportAddress(new TransportAddress[] { DummyTransportAddress.INSTANCE }, DummyTransportAddress.INSTANCE));
-        Environment env = new Environment(settings);
-        IndexAuditTrail auditor = new IndexAuditTrail(settings, new IndexAuditUserHolder(), env, authService, transport, Providers.of(client()), threadPool, mock(ClusterService.class));
+        auditor = new IndexAuditTrail(settings, new IndexAuditUserHolder(), authService, transport, Providers.of(client()), threadPool, mock(ClusterService.class));
 
         // before starting we add an event
         auditor.authenticationFailed(new FakeRestRequest());
@@ -85,6 +84,9 @@ public class IndexAuditTrailUpdateMappingTests extends ShieldIntegTestCase {
 
     @After
     public void shutdown() {
+        if (auditor != null) {
+            auditor.stop();
+        }
         if (threadPool != null) {
             threadPool.shutdownNow();
         }

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/audit/logfile/LoggingAuditTrailTests.java

@@ -330,9 +330,10 @@ public class LoggingAuditTrailTests extends ESTestCase {
             boolean runAs = randomBoolean();
             User user;
             if (runAs) {
-                user = new User.Simple("_username", new String[]{"r1"}, new User.Simple("running as", new String[] {"r2"}));
+                user = new User("_username", new String[]{"r1"},
+                        new User("running as", new String[] {"r2"}));
             } else {
-                user = new User.Simple("_username", new String[]{"r1"});
+                user = new User("_username", new String[]{"r1"});
             }
             String userInfo = runAs ? "principal=[running as], run_by_principal=[_username]" : "principal=[_username]";
             auditTrail.accessGranted(user, "_action", message);
@@ -392,9 +393,10 @@ public class LoggingAuditTrailTests extends ESTestCase {
             boolean runAs = randomBoolean();
             User user;
             if (runAs) {
-                user = new User.Simple("_username", new String[]{"r1"}, new User.Simple("running as", new String[] {"r2"}));
+                user = new User("_username", new String[]{"r1"},
+                        new User("running as", new String[] {"r2"}));
             } else {
-                user = new User.Simple("_username", new String[]{"r1"});
+                user = new User("_username", new String[]{"r1"});
             }
             String userInfo = runAs ? "principal=[running as], run_by_principal=[_username]" : "principal=[_username]";
             auditTrail.accessGranted(user, "internal:_action", message);
@@ -430,9 +432,10 @@ public class LoggingAuditTrailTests extends ESTestCase {
             boolean runAs = randomBoolean();
             User user;
             if (runAs) {
-                user = new User.Simple("_username", new String[]{"r1"}, new User.Simple("running as", new String[] {"r2"}));
+                user = new User("_username", new String[]{"r1"},
+                        new User("running as", new String[] {"r2"}));
             } else {
-                user = new User.Simple("_username", new String[]{"r1"});
+                user = new User("_username", new String[]{"r1"});
             }
             String userInfo = runAs ? "principal=[running as], run_by_principal=[_username]" : "principal=[_username]";
             auditTrail.accessDenied(user, "_action", message);
@@ -493,9 +496,9 @@ public class LoggingAuditTrailTests extends ESTestCase {
         final boolean runAs = randomBoolean();
         User user;
         if (runAs) {
-            user = new User.Simple("_username", new String[]{"r1"}, new User.Simple("running as", new String[] {"r2"}));
+            user = new User("_username", new String[]{"r1"}, new User("running as", new String[] {"r2"}));
         } else {
-            user = new User.Simple("_username", new String[]{"r1"});
+            user = new User("_username", new String[]{"r1"});
         }
         String userInfo = runAs ? "principal=[running as], run_by_principal=[_username]" : "principal=[_username]";
         for (Level level : Level.values()) {
@@ -571,7 +574,7 @@ public class LoggingAuditTrailTests extends ESTestCase {
             LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, transport, logger).start();
             TransportMessage message = new MockMessage();
             String origins = LoggingAuditTrail.originAttributes(message, transport);
-            User user = new User.Simple("_username", new String[]{"r1"}, new User.Simple("running as", new String[] {"r2"}));
+            User user = new User("_username", new String[]{"r1"}, new User("running as", new String[] {"r2"}));
             auditTrail.runAsGranted(user, "_action", message);
             switch (level) {
                 case ERROR:
@@ -594,7 +597,7 @@ public class LoggingAuditTrailTests extends ESTestCase {
             LoggingAuditTrail auditTrail = new LoggingAuditTrail(settings, transport, logger).start();
             TransportMessage message = new MockMessage();
             String origins = LoggingAuditTrail.originAttributes(message, transport);
-            User user = new User.Simple("_username", new String[]{"r1"}, new User.Simple("running as", new String[] {"r2"}));
+            User user = new User("_username", new String[]{"r1"}, new User("running as", new String[] {"r2"}));
             auditTrail.runAsDenied(user, "_action", message);
             switch (level) {
                 case ERROR:

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/AnonymousUserHolderTests.java

@@ -50,7 +50,7 @@ public class AnonymousUserHolderTests extends ESTestCase {
     public void testWhenAnonymousDisabled() {
         AnonymousService anonymousService = new AnonymousService(Settings.EMPTY);
         assertThat(anonymousService.enabled(), is(false));
-        assertThat(anonymousService.isAnonymous(new User.Simple(randomAsciiOfLength(10), new String[] { randomAsciiOfLength(5) })), is(false));
+        assertThat(anonymousService.isAnonymous(new User(randomAsciiOfLength(10), randomAsciiOfLength(5))), is(false));
         assertThat(anonymousService.anonymousUser(), nullValue());
         assertThat(anonymousService.authorizationExceptionsEnabled(), is(true));
     }

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/AuthenticationModuleTests.java

@@ -5,29 +5,14 @@
  */
 package org.elasticsearch.shield.authc;
 
-import org.elasticsearch.ElasticsearchSecurityException;
-import org.elasticsearch.common.inject.Guice;
-import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsFilter;
-import org.elasticsearch.common.settings.SettingsModule;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.EnvironmentModule;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.shield.audit.AuditTrailModule;
 import org.elasticsearch.shield.authc.activedirectory.ActiveDirectoryRealm;
 import org.elasticsearch.shield.authc.esusers.ESUsersRealm;
 import org.elasticsearch.shield.authc.ldap.LdapRealm;
 import org.elasticsearch.shield.authc.pki.PkiRealm;
-import org.elasticsearch.shield.crypto.CryptoModule;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.threadpool.ThreadPoolModule;
-import org.elasticsearch.transport.TransportMessage;
 
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.notNullValue;
 
 /**
  * Unit tests for the AuthenticationModule
@@ -67,114 +52,4 @@ public class AuthenticationModuleTests extends ESTestCase {
             assertThat(e.getMessage(), containsString("null"));
         }
     }
-
-    public void testRegisteringCustomRealm() {
-        Settings settings = Settings.builder()
-                .put("name", "foo")
-                .put("path.home", createTempDir())
-                .put("client.type", "node").build();
-        AuthenticationModule module = new AuthenticationModule(settings);
-        // adding the same factory with a different type is valid the way realms are implemented...
-        module.addCustomRealm("custom", ESUsersRealm.Factory.class);
-        Environment env = new Environment(settings);
-        ThreadPool pool = new ThreadPool(settings);
-        try {
-            Injector injector = Guice.createInjector(module, new SettingsModule(settings, new SettingsFilter(settings)), new AuditTrailModule(settings), new CryptoModule(settings), new EnvironmentModule(env), new ThreadPoolModule(pool));
-            Realms realms = injector.getInstance(Realms.class);
-            Realm.Factory factory = realms.realmFactory("custom");
-            assertThat(factory, notNullValue());
-            assertThat(factory, instanceOf(ESUsersRealm.Factory.class));
-        } finally {
-            pool.shutdown();
-        }
-    }
-
-    public void testDefaultFailureHandler() {
-        Settings settings = Settings.builder()
-                .put("name", "foo")
-                .put("path.home", createTempDir())
-                .put("client.type", "node").build();
-        AuthenticationModule module = new AuthenticationModule(settings);
-        // setting it to null should have no effect
-        if (randomBoolean()) {
-            module.setAuthenticationFailureHandler(null);
-        }
-
-        if (randomBoolean()) {
-            module.addCustomRealm("custom", ESUsersRealm.Factory.class);
-        }
-        Environment env = new Environment(settings);
-        ThreadPool pool = new ThreadPool(settings);
-
-        try {
-            Injector injector = Guice.createInjector(module, new SettingsModule(settings, new SettingsFilter(settings)), new AuditTrailModule(settings), new CryptoModule(settings), new EnvironmentModule(env), new ThreadPoolModule(pool));
-            AuthenticationFailureHandler failureHandler = injector.getInstance(AuthenticationFailureHandler.class);
-            assertThat(failureHandler, notNullValue());
-            assertThat(failureHandler, instanceOf(DefaultAuthenticationFailureHandler.class));
-        } finally {
-            pool.shutdown();
-        }
-    }
-
-    public void testSettingFailureHandler() {
-        Settings settings = Settings.builder()
-                .put("name", "foo")
-                .put("path.home", createTempDir())
-                .put("client.type", "node").build();
-        AuthenticationModule module = new AuthenticationModule(settings);
-        module.setAuthenticationFailureHandler(NoOpFailureHandler.class);
-
-        if (randomBoolean()) {
-            module.addCustomRealm("custom", ESUsersRealm.Factory.class);
-        }
-        Environment env = new Environment(settings);
-        ThreadPool pool = new ThreadPool(settings);
-
-        try {
-            Injector injector = Guice.createInjector(module, new SettingsModule(settings, new SettingsFilter(settings)), new AuditTrailModule(settings), new CryptoModule(settings), new EnvironmentModule(env), new ThreadPoolModule(pool));
-            AuthenticationFailureHandler failureHandler = injector.getInstance(AuthenticationFailureHandler.class);
-            assertThat(failureHandler, notNullValue());
-            assertThat(failureHandler, instanceOf(NoOpFailureHandler.class));
-        } finally {
-            pool.shutdown();
-        }
-    }
-
-    // this class must be public for injection...
-    public static class NoOpFailureHandler implements AuthenticationFailureHandler {
-        @Override
-        public ElasticsearchSecurityException unsuccessfulAuthentication(RestRequest request, AuthenticationToken token) {
-            return null;
-        }
-
-        @Override
-        public ElasticsearchSecurityException unsuccessfulAuthentication(TransportMessage message, AuthenticationToken token, String action) {
-            return null;
-        }
-
-        @Override
-        public ElasticsearchSecurityException exceptionProcessingRequest(RestRequest request, Exception e) {
-            return null;
-        }
-
-        @Override
-        public ElasticsearchSecurityException exceptionProcessingRequest(TransportMessage message, Exception e) {
-            return null;
-        }
-
-        @Override
-        public ElasticsearchSecurityException missingToken(RestRequest request) {
-            return null;
-        }
-
-        @Override
-        public ElasticsearchSecurityException missingToken(TransportMessage message, String action) {
-            return null;
-        }
-
-        @Override
-        public ElasticsearchSecurityException authenticationRequired(String action) {
-            return null;
-        }
-    }
 }