Support Client and RoleMapping in custom Realms (#50950)

Previously custom realms were limited in what services and components
they had easy access to. It was possible to work around this because a
security extension is packaged within a Plugin, so there were ways to
store this components in static/SetOnce variables and access them from
the realm, but those techniques were fragile, undocumented and
difficult to discover.

This change includes key services as an argument to most of the methods
on SecurityExtension so that custom realm / role provider authors can
have easy access to them.

Backport of: #50534

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityExtension.java

@@ -7,10 +7,15 @@ package org.elasticsearch.xpack.core.security;
 
 import org.apache.lucene.util.SPIClassIterator;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationFailureHandler;
 import org.elasticsearch.xpack.core.security.authc.Realm;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult;
@@ -28,6 +33,26 @@ import java.util.function.BiConsumer;
  */
 public interface SecurityExtension {
 
+    /**
+     * This interface provides access to components (clients and services) that may be used
+     * within custom realms and role providers.
+     */
+    interface SecurityComponents {
+        /** Global settings for the current node */
+        Settings settings();
+        /** Provides access to key filesystem paths */
+        Environment environment();
+        /** An internal client for retrieving information/data from this cluster */
+        Client client();
+        /** The Elasticsearch thread pools */
+        ThreadPool threadPool();
+        /** Provides the ability to monitor files for changes */
+        ResourceWatcherService resourceWatcherService();
+        /** Access to listen to changes in cluster state and settings  */
+        ClusterService clusterService();
+        /** Provides support for mapping users' roles from groups and metadata */
+        UserRoleMapper roleMapper();
+    }
     /**
      * Returns authentication realm implementations added by this extension.
      *
@@ -35,9 +60,9 @@ public interface SecurityExtension {
      * is a {@link Realm.Factory} which will construct
      * that realm for use in authentication when that realm type is configured.
      *
-     * @param resourceWatcherService Use to watch configuration files for changes
+     * @param components Access to components that may be used to build realms
      */
-    default Map<String, Realm.Factory> getRealms(ResourceWatcherService resourceWatcherService) {
+    default Map<String, Realm.Factory> getRealms(SecurityComponents components) {
         return Collections.emptyMap();
     }
 
@@ -46,8 +71,10 @@ public interface SecurityExtension {
      *
      * Only one installed extension may have an authentication failure handler. If more than
      * one extension returns a non-null handler, an error is raised.
+     *
+     * @param components Access to components that may be used to build the handler
      */
-    default AuthenticationFailureHandler getAuthenticationFailureHandler() {
+    default AuthenticationFailureHandler getAuthenticationFailureHandler(SecurityComponents components) {
         return null;
     }
 
@@ -72,11 +99,10 @@ public interface SecurityExtension {
      *
      * By default, an empty list is returned.
      *
-     * @param settings The configured settings for the node
+     * @param components Access to components that may be used to build roles
-     * @param resourceWatcherService Use to watch configuration files for changes
      */
     default List<BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>>>
-        getRolesProviders(Settings settings, ResourceWatcherService resourceWatcherService) {
+        getRolesProviders(SecurityComponents components) {
         return Collections.emptyList();
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/CachingRealm.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.security.authc.support;
+package org.elasticsearch.xpack.core.security.authc.support;
 
 import org.elasticsearch.xpack.core.security.authc.Realm;
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/UserRoleMapper.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.xpack.security.authc.support;
+package org.elasticsearch.xpack.core.security.authc.support;
 
 import com.unboundid.ldap.sdk.DN;
 import com.unboundid.ldap.sdk.LDAPException;

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -232,6 +232,7 @@ import org.elasticsearch.xpack.security.rest.action.user.RestGetUsersAction;
 import org.elasticsearch.xpack.security.rest.action.user.RestHasPrivilegesAction;
 import org.elasticsearch.xpack.security.rest.action.user.RestPutUserAction;
 import org.elasticsearch.xpack.security.rest.action.user.RestSetEnabledAction;
+import org.elasticsearch.xpack.security.support.ExtensionComponents;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.elasticsearch.xpack.security.support.SecurityStatusChangeListener;
 import org.elasticsearch.xpack.security.transport.SecurityHttpSettings;
@@ -436,10 +437,12 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
         final AnonymousUser anonymousUser = new AnonymousUser(settings);
         final ReservedRealm reservedRealm = new ReservedRealm(env, settings, nativeUsersStore,
                 anonymousUser, securityIndex.get(), threadPool);
+        final SecurityExtension.SecurityComponents extensionComponents = new ExtensionComponents(env, client, clusterService,
+            resourceWatcherService, nativeRoleMappingStore);
         Map<String, Realm.Factory> realmFactories = new HashMap<>(InternalRealms.getFactories(threadPool, resourceWatcherService,
                 getSslService(), nativeUsersStore, nativeRoleMappingStore, securityIndex.get()));
         for (SecurityExtension extension : securityExtensions) {
-            Map<String, Realm.Factory> newRealms = extension.getRealms(resourceWatcherService);
+            Map<String, Realm.Factory> newRealms = extension.getRealms(extensionComponents);
             for (Map.Entry<String, Realm.Factory> entry : newRealms.entrySet()) {
                 if (realmFactories.put(entry.getKey(), entry.getValue()) != null) {
                     throw new IllegalArgumentException("Realm type [" + entry.getKey() + "] is already registered");
@@ -465,7 +468,7 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
         final ReservedRolesStore reservedRolesStore = new ReservedRolesStore();
         List<BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>>> rolesProviders = new ArrayList<>();
         for (SecurityExtension extension : securityExtensions) {
-            rolesProviders.addAll(extension.getRolesProviders(settings, resourceWatcherService));
+            rolesProviders.addAll(extension.getRolesProviders(extensionComponents));
         }
 
         final ApiKeyService apiKeyService = new ApiKeyService(settings, Clock.systemUTC(), client, getLicenseState(), securityIndex.get(),
@@ -481,7 +484,7 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
         getLicenseState().addListener(allRolesStore::invalidateAll);
         getLicenseState().addListener(new SecurityStatusChangeListener(getLicenseState()));
 
-        final AuthenticationFailureHandler failureHandler = createAuthenticationFailureHandler(realms);
+        final AuthenticationFailureHandler failureHandler = createAuthenticationFailureHandler(realms, extensionComponents);
         authcService.set(new AuthenticationService(settings, realms, auditTrailService, failureHandler, threadPool,
                 anonymousUser, tokenService, apiKeyService));
         components.add(authcService.get());
@@ -539,11 +542,12 @@ public class Security extends Plugin implements ActionPlugin, IngestPlugin, Netw
         return authorizationEngine;
     }
 
-    private AuthenticationFailureHandler createAuthenticationFailureHandler(final Realms realms) {
+    private AuthenticationFailureHandler createAuthenticationFailureHandler(final Realms realms,
+                                                                            final SecurityExtension.SecurityComponents components) {
         AuthenticationFailureHandler failureHandler = null;
         String extensionName = null;
         for (SecurityExtension extension : securityExtensions) {
-            AuthenticationFailureHandler extensionFailureHandler = extension.getAuthenticationFailureHandler();
+            AuthenticationFailureHandler extensionFailureHandler = extension.getAuthenticationFailureHandler(components);
             if (extensionFailureHandler != null && failureHandler != null) {
                 throw new IllegalStateException("Extensions [" + extensionName + "] and [" + extension.toString() + "] "
                         + "both set an authentication failure handler");

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/realm/TransportClearRealmCacheAction.java

@@ -20,7 +20,7 @@ import org.elasticsearch.xpack.core.security.action.realm.ClearRealmCacheRespons
 import org.elasticsearch.xpack.core.security.authc.Realm;
 import org.elasticsearch.xpack.security.authc.AuthenticationService;
 import org.elasticsearch.xpack.security.authc.Realms;
-import org.elasticsearch.xpack.security.authc.support.CachingRealm;
+import org.elasticsearch.xpack.core.security.authc.support.CachingRealm;
 
 import java.io.IOException;
 import java.util.List;

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealm.java

@@ -21,9 +21,9 @@ import org.elasticsearch.xpack.core.security.authc.Realm;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
 import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
 import org.elasticsearch.xpack.core.security.user.User;
-import org.elasticsearch.xpack.security.authc.support.CachingRealm;
+import org.elasticsearch.xpack.core.security.authc.support.CachingRealm;
 import org.elasticsearch.xpack.security.authc.support.DelegatedAuthorizationSupport;
-import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
 import org.ietf.jgss.GSSException;
 

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/LdapRealm.java

@@ -35,8 +35,8 @@ import org.elasticsearch.xpack.security.authc.ldap.support.LdapSession;
 import org.elasticsearch.xpack.security.authc.ldap.support.SessionFactory;
 import org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm;
 import org.elasticsearch.xpack.security.authc.support.DelegatedAuthorizationSupport;
-import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
-import org.elasticsearch.xpack.security.authc.support.UserRoleMapper.UserData;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper.UserData;
 import org.elasticsearch.xpack.security.authc.support.mapper.CompositeRoleMapper;
 import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
 

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectRealm.java

@@ -44,7 +44,7 @@ import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.security.authc.TokenService;
 import org.elasticsearch.xpack.security.authc.support.DelegatedAuthorizationSupport;
-import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 
 import java.net.URI;
 import java.net.URISyntaxException;

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/pki/PkiRealm.java

@@ -32,9 +32,9 @@ import org.elasticsearch.xpack.core.ssl.CertParsingUtils;
 import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings;
 import org.elasticsearch.xpack.security.authc.BytesKey;
 import org.elasticsearch.xpack.security.authc.TokenService;
-import org.elasticsearch.xpack.security.authc.support.CachingRealm;
+import org.elasticsearch.xpack.core.security.authc.support.CachingRealm;
 import org.elasticsearch.xpack.security.authc.support.DelegatedAuthorizationSupport;
-import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.security.authc.support.mapper.CompositeRoleMapper;
 import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
 

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/saml/SamlRealm.java

@@ -50,7 +50,7 @@ import org.elasticsearch.xpack.core.ssl.X509KeyPairSettings;
 import org.elasticsearch.xpack.security.authc.Realms;
 import org.elasticsearch.xpack.security.authc.TokenService;
 import org.elasticsearch.xpack.security.authc.support.DelegatedAuthorizationSupport;
-import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.opensaml.core.criterion.EntityIdCriterion;
 import org.opensaml.saml.common.xml.SAMLConstants;
 import org.opensaml.saml.criterion.EntityRoleCriterion;

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/CachingUsernamePasswordRealm.java

@@ -17,6 +17,7 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
+import org.elasticsearch.xpack.core.security.authc.support.CachingRealm;
 import org.elasticsearch.xpack.core.security.authc.support.CachingUsernamePasswordRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/DnRoleMapper.java

@@ -35,7 +35,9 @@ import org.elasticsearch.watcher.FileWatcher;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.XPackPlugin;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
+import org.elasticsearch.xpack.core.security.authc.support.CachingRealm;
 import org.elasticsearch.xpack.core.security.authc.support.DnRoleMapperSettings;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.unmodifiableMap;

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/CompositeRoleMapper.java

@@ -15,9 +15,9 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.GroupedActionListener;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
-import org.elasticsearch.xpack.security.authc.support.CachingRealm;
+import org.elasticsearch.xpack.core.security.authc.support.CachingRealm;
 import org.elasticsearch.xpack.security.authc.support.DnRoleMapper;
-import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 
 /**
  * A {@link UserRoleMapper} that composes one or more <i>delegate</i> role-mappers.

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java

@@ -37,8 +37,8 @@ import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRole
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.ExpressionModel;
 import org.elasticsearch.xpack.core.security.client.SecurityClient;
 import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames;
-import org.elasticsearch.xpack.security.authc.support.CachingRealm;
+import org.elasticsearch.xpack.core.security.authc.support.CachingRealm;
-import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
 import java.io.IOException;

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ExtensionComponents.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.security.support;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xpack.core.security.SecurityExtension;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
+
+/**
+ * Immutable implementation of {@link SecurityExtension.SecurityComponents}.
+ */
+public final class ExtensionComponents implements SecurityExtension.SecurityComponents {
+    private final Environment environment;
+    private final Client client;
+    private final ClusterService clusterService;
+    private final ResourceWatcherService resourceWatcherService;
+    private final UserRoleMapper roleMapper;
+
+    public ExtensionComponents(Environment environment, Client client, ClusterService clusterService,
+                               ResourceWatcherService resourceWatcherService, UserRoleMapper roleMapper) {
+        this.environment = environment;
+        this.client = client;
+        this.clusterService = clusterService;
+        this.resourceWatcherService = resourceWatcherService;
+        this.roleMapper = roleMapper;
+    }
+
+    @Override
+    public Settings settings() {
+        return environment.settings();
+    }
+
+    @Override
+    public Environment environment() {
+        return environment;
+    }
+
+    @Override
+    public Client client() {
+        return client;
+    }
+
+    @Override
+    public ThreadPool threadPool() {
+        return client.threadPool();
+    }
+
+    @Override
+    public ResourceWatcherService resourceWatcherService() {
+        return resourceWatcherService;
+    }
+
+    @Override
+    public ClusterService clusterService() {
+        return clusterService;
+    }
+
+    @Override
+    public UserRoleMapper roleMapper() {
+        return roleMapper;
+    }
+}

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java

@@ -89,7 +89,7 @@ public class SecurityTests extends ESTestCase {
         }
 
         @Override
-        public Map<String, Realm.Factory> getRealms(ResourceWatcherService resourceWatcherService) {
+        public Map<String, Realm.Factory> getRealms(SecurityComponents components) {
             return Collections.singletonMap(realmType, config -> null);
         }
     }

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/oidc/TransportOpenIdConnectLogoutActionTests.java

@@ -51,7 +51,7 @@ import org.elasticsearch.xpack.security.authc.Realms;
 import org.elasticsearch.xpack.security.authc.TokenService;
 import org.elasticsearch.xpack.security.authc.oidc.OpenIdConnectRealm;
 import org.elasticsearch.xpack.security.authc.oidc.OpenIdConnectTestCase;
-import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.junit.After;
 import org.junit.Before;

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java

@@ -61,7 +61,7 @@ import org.elasticsearch.xpack.security.authc.saml.SamlNameId;
 import org.elasticsearch.xpack.security.authc.saml.SamlRealm;
 import org.elasticsearch.xpack.security.authc.saml.SamlRealmTests;
 import org.elasticsearch.xpack.security.authc.saml.SamlTestCase;
-import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.junit.After;
 import org.junit.Before;

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmCacheTests.java

@@ -12,7 +12,7 @@ import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
 import org.elasticsearch.xpack.core.security.user.User;
-import org.elasticsearch.xpack.security.authc.support.UserRoleMapper.UserData;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper.UserData;
 import org.ietf.jgss.GSSException;
 
 import javax.security.auth.login.LoginException;

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTestCase.java

@@ -27,7 +27,7 @@ import org.elasticsearch.xpack.core.security.authc.RealmSettings;
 import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSettings;
 import org.elasticsearch.xpack.core.security.support.Exceptions;
 import org.elasticsearch.xpack.core.security.user.User;
-import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.junit.After;

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosRealmTests.java

@@ -22,7 +22,7 @@ import org.elasticsearch.xpack.core.security.authc.kerberos.KerberosRealmSetting
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.authc.support.MockLookupRealm;
-import org.elasticsearch.xpack.security.authc.support.UserRoleMapper.UserData;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper.UserData;
 import org.ietf.jgss.GSSException;
 
 import javax.security.auth.login.LoginException;

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectRealmTests.java

@@ -25,7 +25,7 @@ import org.elasticsearch.xpack.core.security.authc.oidc.OpenIdConnectRealmSettin
 import org.elasticsearch.xpack.core.security.authc.support.DelegatedAuthorizationSettings;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.authc.support.MockLookupRealm;
-import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.hamcrest.Matchers;
 import org.junit.Before;
 

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiRealmTests.java

@@ -29,7 +29,7 @@ import org.elasticsearch.xpack.core.security.support.NoOpLogger;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.authc.BytesKey;
 import org.elasticsearch.xpack.security.authc.support.MockLookupRealm;
-import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.junit.Before;
 import org.mockito.Mockito;
 

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlRealmTestHelper.java

@@ -13,7 +13,7 @@ import java.util.Collections;
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
-import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.opensaml.saml.common.xml.SAMLConstants;
 import org.opensaml.saml.saml2.metadata.EntityDescriptor;
 import org.opensaml.saml.saml2.metadata.IDPSSODescriptor;

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/saml/SamlRealmTests.java

@@ -35,7 +35,7 @@ import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.core.ssl.TestsSSLService;
 import org.elasticsearch.xpack.security.authc.Realms;
 import org.elasticsearch.xpack.security.authc.support.MockLookupRealm;
-import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.hamcrest.Matchers;
 import org.junit.Before;
 import org.mockito.Mockito;

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/DistinguishedNamePredicateTests.java

@@ -7,6 +7,7 @@ package org.elasticsearch.xpack.security.authc.support;
 
 import com.unboundid.ldap.sdk.DN;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression.FieldValue;
 
 import java.util.Locale;
@@ -73,4 +74,4 @@ public class DistinguishedNamePredicateTests extends ESTestCase {
     private void assertPredicate(Predicate<FieldValue> predicate, Object value, boolean expected) {
         assertThat("Predicate [" + predicate + "] match [" + value + "]", predicate.test(new FieldValue(value)), equalTo(expected));
     }
-}
+}

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/ExpressionRoleMappingTests.java

@@ -33,7 +33,7 @@ import org.elasticsearch.xpack.core.security.authc.support.mapper.TemplateRoleNa
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.AllExpression;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.AnyExpression;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression;
-import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.hamcrest.Matchers;
 import org.junit.Before;
 import org.mockito.Mockito;

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStoreTests.java

@@ -34,7 +34,7 @@ import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.
 import org.elasticsearch.xpack.core.security.index.RestrictedIndicesNames;
 import org.elasticsearch.xpack.core.security.user.User;
 import org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm;
-import org.elasticsearch.xpack.security.authc.support.UserRoleMapper;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 import org.hamcrest.Matchers;
 

x-pack/qa/security-example-spi-extension/build.gradle

@@ -29,6 +29,7 @@ testClusters.integTest {
   setting 'xpack.security.authc.realms.custom.custom.filtered_setting', 'should be filtered'
   setting 'xpack.security.authc.realms.file.esusers.order', '1'
   setting 'xpack.security.authc.realms.native.native.order', '2'
+  setting 'xpack.security.authc.realms.custom_role_mapping.role_map.order', '3'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.ilm.enabled', 'false'
   setting 'xpack.ml.enabled', 'false'

x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/ExampleSecurityExtension.java

@@ -6,14 +6,13 @@
 package org.elasticsearch.example;
 
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.example.realm.CustomAuthenticationFailureHandler;
 import org.elasticsearch.example.realm.CustomRealm;
+import org.elasticsearch.example.realm.CustomRoleMappingRealm;
 import org.elasticsearch.example.role.CustomInMemoryRolesProvider;
-import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xpack.core.security.SecurityExtension;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationFailureHandler;
 import org.elasticsearch.xpack.core.security.authc.Realm;
-import org.elasticsearch.xpack.core.security.SecurityExtension;
 import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult;
 
 import java.security.AccessController;
@@ -43,19 +42,21 @@ public class ExampleSecurityExtension implements SecurityExtension {
     }
 
     @Override
-    public Map<String, Realm.Factory> getRealms(ResourceWatcherService resourceWatcherService) {
+    public Map<String, Realm.Factory> getRealms(SecurityComponents components) {
-        return Collections.singletonMap(CustomRealm.TYPE, CustomRealm::new);
+        final Map<String, Realm.Factory> map = new HashMap<>();
+        map.put(CustomRealm.TYPE, CustomRealm::new);
+        map.put(CustomRoleMappingRealm.TYPE, config -> new CustomRoleMappingRealm(config, components.roleMapper()));
+        return map;
     }
 
     @Override
-    public AuthenticationFailureHandler getAuthenticationFailureHandler() {
+    public AuthenticationFailureHandler getAuthenticationFailureHandler(SecurityComponents components) {
         return new CustomAuthenticationFailureHandler();
     }
 
-
     @Override
     public List<BiConsumer<Set<String>, ActionListener<RoleRetrievalResult>>>
-    getRolesProviders(Settings settings, ResourceWatcherService resourceWatcherService) {
+    getRolesProviders(SecurityComponents components) {
         CustomInMemoryRolesProvider rp1 = new CustomInMemoryRolesProvider(Collections.singletonMap(ROLE_A, "read"));
         Map<String, String> roles = new HashMap<>();
         roles.put(ROLE_A, "all");

x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/SpiExtensionPlugin.java

@@ -7,6 +7,7 @@ package org.elasticsearch.example;
 
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.example.realm.CustomRealm;
+import org.elasticsearch.example.realm.CustomRoleMappingRealm;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestHeaderDefinition;
@@ -33,6 +34,7 @@ public class SpiExtensionPlugin extends Plugin implements ActionPlugin {
     public List<Setting<?>> getSettings() {
         List<Setting<?>> list = new ArrayList<>(RealmSettings.getStandardSettings(CustomRealm.TYPE));
         list.add(RealmSettings.simpleString(CustomRealm.TYPE, "filtered_setting", Setting.Property.NodeScope, Setting.Property.Filtered));
+        list.addAll(RealmSettings.getStandardSettings(CustomRoleMappingRealm.TYPE));
         return list;
     }
 }

x-pack/qa/security-example-spi-extension/src/main/java/org/elasticsearch/example/realm/CustomRoleMappingRealm.java

@@ -0,0 +1,96 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.example.realm;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.common.cache.Cache;
+import org.elasticsearch.common.cache.CacheBuilder;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
+import org.elasticsearch.xpack.core.security.authc.Realm;
+import org.elasticsearch.xpack.core.security.authc.RealmConfig;
+import org.elasticsearch.xpack.core.security.authc.support.CachingRealm;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
+import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
+import org.elasticsearch.xpack.core.security.user.User;
+
+import java.util.Collections;
+
+/**
+ * An example realm with specific behaviours:
+ * (1) It only supports lookup (that is, "run-as" and "authorization_realms") but not authentication
+ * (2) It performs role mapping to determine the roles for the looked-up user
+ * (3) It caches the looked-up User objects
+ */
+public class CustomRoleMappingRealm extends Realm implements CachingRealm {
+
+    public static final String TYPE = "custom_role_mapping";
+
+    static final String USERNAME = "role_mapped_user";
+    static final String USER_GROUP = "user_group";
+
+    private final Cache<String, User> cache;
+    private final UserRoleMapper roleMapper;
+
+    public CustomRoleMappingRealm(RealmConfig config, UserRoleMapper roleMapper) {
+        super(config);
+        this.cache = CacheBuilder.<String, User>builder().build();
+        this.roleMapper = roleMapper;
+        this.roleMapper.refreshRealmOnChange(this);
+    }
+
+    @Override
+    public boolean supports(AuthenticationToken token) {
+        return false;
+    }
+
+    @Override
+    public UsernamePasswordToken token(ThreadContext threadContext) {
+        return null;
+    }
+
+    @Override
+    public void authenticate(AuthenticationToken authToken, ActionListener<AuthenticationResult> listener) {
+        listener.onResponse(AuthenticationResult.notHandled());
+    }
+
+    @Override
+    public void lookupUser(String username, ActionListener<User> listener) {
+        final User user = cache.get(username);
+        if (user != null) {
+            listener.onResponse(user);
+            return;
+        }
+        if (USERNAME.equals(username)) {
+            buildUser(username, ActionListener.wrap(
+                u -> listener.onResponse(cache.computeIfAbsent(username, k -> u)),
+                listener::onFailure
+            ));
+        } else {
+            listener.onResponse(null);
+        }
+    }
+
+    private void buildUser(String username, ActionListener<User> listener) {
+        final UserRoleMapper.UserData data = new UserRoleMapper.UserData(username, null, Collections.singletonList(USER_GROUP),
+        Collections.emptyMap(), super.config);
+        roleMapper.resolveRoles(data, ActionListener.wrap(
+            roles -> listener.onResponse(new User(username, roles.toArray(new String[0]))),
+            listener::onFailure
+        ));
+    }
+
+    @Override
+    public void expire(String username) {
+        this.cache.invalidate(username);
+    }
+
+    @Override
+    public void expireAll() {
+        this.cache.invalidateAll();
+    }
+}

x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRoleMappingRealmIT.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.example.realm;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.RequestOptions;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.junit.Before;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
+
+/**
+ * Integration test to test authentication with the custom role-mapping realm
+ */
+public class CustomRoleMappingRealmIT extends ESRestTestCase {
+
+    private String expectedRole;
+
+    @Override
+    protected Settings restClientSettings() {
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + "." + CustomRealm.USER_HEADER, CustomRealm.KNOWN_USER)
+            .put(ThreadContext.PREFIX + "." + CustomRealm.PW_HEADER, CustomRealm.KNOWN_PW.toString())
+            .build();
+    }
+
+    @Before
+    public void setupRoleMapping() throws Exception {
+        expectedRole = randomAlphaOfLengthBetween(4, 16);
+        Request request = new Request("PUT", "/_security/role_mapping/test");
+        request.setJsonEntity("{" +
+            "\"enabled\": true," +
+            "\"roles\":[\"" +
+            expectedRole +
+            "\"]," +
+            "\"rules\":{\"field\":{\"groups\":\"" + CustomRoleMappingRealm.USER_GROUP + "\"} }" +
+            "}");
+        adminClient().performRequest(request);
+    }
+
+    public void testUserWithRoleMapping() throws Exception {
+        Request request = new Request("GET", "/_security/_authenticate");
+        RequestOptions.Builder options = request.getOptions().toBuilder();
+        // Authenticate as the custom realm superuser
+        options.addHeader(CustomRealm.USER_HEADER, CustomRealm.KNOWN_USER);
+        options.addHeader(CustomRealm.PW_HEADER, CustomRealm.KNOWN_PW.toString());
+        // But "run-as" the role mapped user
+        options.addHeader("es-security-runas-user", CustomRoleMappingRealm.USERNAME);
+        request.setOptions(options);
+
+        final Response response = client().performRequest(request);
+        final Map<String, Object> authenticate = entityAsMap(response);
+        assertThat(authenticate.get("username"), is(CustomRoleMappingRealm.USERNAME));
+        assertThat(authenticate.get("roles"), instanceOf(List.class));
+        assertThat(authenticate.get("roles"), equalTo(Collections.singletonList(expectedRole)));
+    }
+
+}

x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRoleMappingRealmTests.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.example.realm;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.core.security.authc.RealmConfig;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
+import org.elasticsearch.xpack.core.security.user.User;
+
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Supplier;
+
+import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+
+public class CustomRoleMappingRealmTests extends ESTestCase {
+
+    public void testCachingOfUserLookup() throws Exception {
+        final Environment env = TestEnvironment.newEnvironment(super.buildEnvSettings(Settings.EMPTY));
+        final UserRoleMapper roleMapper = mock(UserRoleMapper.class);
+        final RealmConfig realmConfig = new RealmConfig(
+            new RealmConfig.RealmIdentifier(CustomRoleMappingRealm.TYPE, "test"),
+            env.settings(), env, new ThreadContext(env.settings())
+        );
+        CustomRoleMappingRealm realm = new CustomRoleMappingRealm(realmConfig, roleMapper);
+
+        final AtomicInteger roleMappingCounter = new AtomicInteger(0);
+        mockRoleMapping(roleMapper, () -> {
+            roleMappingCounter.incrementAndGet();
+            return Sets.newHashSet("role1", "role2");
+        });
+
+        PlainActionFuture<User> future = new PlainActionFuture<>();
+        realm.lookupUser(CustomRoleMappingRealm.USERNAME, future);
+        final User user1 = future.get();
+        assertThat(user1.principal(), is(CustomRoleMappingRealm.USERNAME));
+        assertThat(user1.roles(), arrayContainingInAnyOrder("role1", "role2"));
+        assertThat(roleMappingCounter.get(), is(1));
+
+        future = new PlainActionFuture<>();
+        realm.lookupUser(CustomRoleMappingRealm.USERNAME, future);
+        final User user2 = future.get();
+        assertThat(user2, sameInstance(user1));
+        assertThat(roleMappingCounter.get(), is(1));
+    }
+
+    @SuppressWarnings("unchecked")
+    private void mockRoleMapping(UserRoleMapper roleMapper, Supplier<Set<String>> supplier) {
+        doAnswer(inv -> {
+            ActionListener<Set<String>> listener = (ActionListener<Set<String>>) inv.getArguments()[1];
+            listener.onResponse(supplier.get());
+            return null;
+        }).when(roleMapper).resolveRoles(any(UserRoleMapper.UserData.class), any(ActionListener.class));
+    }
+
+}