Add a short-lived token based access mechanism (elastic/x-pack-elasticsearch#1029)

This commit adds a token based access mechanism that is a subset of the OAuth 2.0 protocol. The
token mechanism takes the same values as a OAuth 2 standard (defined in RFC 6749 and RFC 6750),
but differs in that we use XContent for the body instead of form encoded values. Additionally, this
PR provides a mechanism for expiration of a token; this can be used to implement logout
functionality that prevents the token from being used again.

The actual tokens are encrypted using AES-GCM, which also provides authentication. The key for
encryption is derived from a salt value and a passphrase that is stored on each node in the
secure settings store. By default, the tokens have an expiration time of 20 minutes and is
configurable up to a maximum of one hour.

Relates elastic/x-pack-elasticsearch#8

Original commit: elastic/x-pack-elasticsearch@3d201ac2bf

plugin/src/main/java/org/elasticsearch/xpack/XPackPlugin.java

@@ -103,6 +103,7 @@ import javax.security.auth.DestroyFailedException;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.security.AccessController;
+import java.security.GeneralSecurityException;
 import java.security.KeyStoreException;
 import java.security.NoSuchAlgorithmException;
 import java.security.PrivilegedAction;
@@ -190,8 +191,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
     protected Graph graph;
     protected MachineLearning machineLearning;
 
-    public XPackPlugin(Settings settings) throws IOException, CertificateException, UnrecoverableKeyException, NoSuchAlgorithmException,
+    public XPackPlugin(Settings settings) throws IOException, DestroyFailedException, OperatorCreationException, GeneralSecurityException {
-            KeyStoreException, DestroyFailedException, OperatorCreationException {
         this.settings = settings;
         this.transportClientMode = transportClientMode(settings);
         this.env = transportClientMode ? null : new Environment(settings);
@@ -390,6 +390,7 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
         List<ExecutorBuilder<?>> executorBuilders = new ArrayList<ExecutorBuilder<?>>();
         executorBuilders.addAll(watcher.getExecutorBuilders(settings));
         executorBuilders.addAll(machineLearning.getExecutorBuilders(settings));
+        executorBuilders.addAll(security.getExecutorBuilders(settings));
         return executorBuilders;
     }
 

plugin/src/main/java/org/elasticsearch/xpack/XPackSettings.java

@@ -72,6 +72,9 @@ public class XPackSettings {
     public static final Setting<Boolean> RESERVED_REALM_ENABLED_SETTING =
             enabledSetting(XPackPlugin.SECURITY + ".authc.reserved_realm", true);
 
+    /** Setting for enabling or disabling the token service. Defaults to true */
+    public static final Setting<Boolean> TOKEN_SERVICE_ENABLED_SETTING = enabledSetting("security.authc.token", true);
+
     /*
      * SSL settings. These are the settings that are specifically registered for SSL. Many are private as we do not explicitly use them
      * but instead parse based on a prefix (eg *.ssl.*)

plugin/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -48,6 +48,8 @@ import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.NetworkPlugin;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
+import org.elasticsearch.threadpool.ExecutorBuilder;
+import org.elasticsearch.threadpool.FixedExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportInterceptor;
@@ -71,6 +73,10 @@ import org.elasticsearch.xpack.security.action.role.TransportClearRolesCacheActi
 import org.elasticsearch.xpack.security.action.role.TransportDeleteRoleAction;
 import org.elasticsearch.xpack.security.action.role.TransportGetRolesAction;
 import org.elasticsearch.xpack.security.action.role.TransportPutRoleAction;
+import org.elasticsearch.xpack.security.action.token.CreateTokenAction;
+import org.elasticsearch.xpack.security.action.token.InvalidateTokenAction;
+import org.elasticsearch.xpack.security.action.token.TransportCreateTokenAction;
+import org.elasticsearch.xpack.security.action.token.TransportInvalidateTokenAction;
 import org.elasticsearch.xpack.security.action.user.AuthenticateAction;
 import org.elasticsearch.xpack.security.action.user.ChangePasswordAction;
 import org.elasticsearch.xpack.security.action.user.DeleteUserAction;
@@ -97,6 +103,7 @@ import org.elasticsearch.xpack.security.authc.InternalRealms;
 import org.elasticsearch.xpack.security.authc.Realm;
 import org.elasticsearch.xpack.security.authc.RealmSettings;
 import org.elasticsearch.xpack.security.authc.Realms;
+import org.elasticsearch.xpack.security.authc.TokenService;
 import org.elasticsearch.xpack.security.authc.esnative.NativeRealm;
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
@@ -138,6 +145,8 @@ import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.time.Clock;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -165,7 +174,7 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin {
     public static final Setting<Optional<String>> USER_SETTING =
             new Setting<>(setting("user"), (String) null, Optional::ofNullable, Property.NodeScope);
 
-    public static final Setting<List<String>> AUDIT_OUTPUTS_SETTING =
+    static final Setting<List<String>> AUDIT_OUTPUTS_SETTING =
         Setting.listSetting(setting("audit.outputs"),
             s -> s.getAsMap().containsKey(setting("audit.outputs")) ?
                 Collections.emptyList() : Collections.singletonList(LoggingAuditTrail.NAME),
@@ -187,7 +196,8 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin {
     private final SetOnce<SecurityContext> securityContext = new SetOnce<>();
     private final SetOnce<ThreadContext> threadContext = new SetOnce<>();
 
-    public Security(Settings settings, Environment env, XPackLicenseState licenseState, SSLService sslService) throws IOException {
+    public Security(Settings settings, Environment env, XPackLicenseState licenseState, SSLService sslService)
+                    throws IOException, GeneralSecurityException {
         this.settings = settings;
         this.env = env;
         this.transportClientMode = XPackPlugin.transportClientMode(settings);
@@ -290,9 +300,10 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin {
                     auditTrails.stream().collect(Collectors.toList()), licenseState);
         components.add(auditTrailService);
 
-        SecurityLifecycleService securityLifecycleService =
+        final SecurityLifecycleService securityLifecycleService =
-            new SecurityLifecycleService(settings, clusterService, threadPool, client, licenseState,
+                new SecurityLifecycleService(settings, clusterService, threadPool, client, licenseState, indexAuditTrail);
-                    indexAuditTrail);
+        final TokenService tokenService = new TokenService(settings, Clock.systemUTC(), client, securityLifecycleService);
+        components.add(tokenService);
 
         // realms construction
         final NativeUsersStore nativeUsersStore = new NativeUsersStore(settings, client,
@@ -334,7 +345,7 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin {
             logger.debug("Using authentication failure handler from extension [" + extensionName + "]");
         }
 
-        authcService.set(new AuthenticationService(settings, realms, auditTrailService, failureHandler, threadPool, anonymousUser));
+        authcService.set(new AuthenticationService(settings, realms, auditTrailService, failureHandler, threadPool, anonymousUser, tokenService));
         components.add(authcService.get());
 
         final FileRolesStore fileRolesStore = new FileRolesStore(settings, env, resourceWatcherService, licenseState);
@@ -438,6 +449,9 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin {
         AuthorizationService.addSettings(settingsList);
         settingsList.add(CompositeRolesStore.CACHE_SIZE_SETTING);
         settingsList.add(FieldPermissionsCache.CACHE_SIZE_SETTING);
+        settingsList.add(TokenService.TOKEN_EXPIRATION);
+        settingsList.add(TokenService.TOKEN_PASSPHRASE);
+        settingsList.add(TokenService.DELETE_INTERVAL);
 
         // encryption settings
         CryptoService.addSettings(settingsList);
@@ -472,7 +486,9 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin {
         if (enabled) {
             return Arrays.asList(
                 new DefaultPasswordBootstrapCheck(settings),
-                new SSLBootstrapCheck(sslService, settings, env)
+                new SSLBootstrapCheck(sslService, settings, env),
+                new TokenPassphraseBootstrapCheck(settings),
+                new TokenSSLBootstrapCheck(settings)
             );
         } else {
             return Collections.emptyList();
@@ -524,7 +540,9 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin {
                 new ActionHandler<>(ChangePasswordAction.INSTANCE, TransportChangePasswordAction.class),
                 new ActionHandler<>(AuthenticateAction.INSTANCE, TransportAuthenticateAction.class),
                 new ActionHandler<>(SetEnabledAction.INSTANCE, TransportSetEnabledAction.class),
-                new ActionHandler<>(HasPrivilegesAction.INSTANCE, TransportHasPrivilegesAction.class));
+                new ActionHandler<>(HasPrivilegesAction.INSTANCE, TransportHasPrivilegesAction.class),
+                new ActionHandler<>(CreateTokenAction.INSTANCE, TransportCreateTokenAction.class),
+                new ActionHandler<>(InvalidateTokenAction.INSTANCE, TransportInvalidateTokenAction.class));
     }
 
     @Override
@@ -793,4 +811,11 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin {
         return handler -> new SecurityRestFilter(settings, licenseState, sslService, threadContext, authcService.get(), handler);
     }
 
+    public List<ExecutorBuilder<?>> getExecutorBuilders(final Settings settings) {
+        if (enabled && transportClientMode == false) {
+            return Collections.singletonList(
+                    new FixedExecutorBuilder(settings, TokenService.THREAD_POOL_NAME, 1, 1000, "xpack.security.authc.token.thread_pool"));
+        }
+        return Collections.emptyList();
+    }
 }

plugin/src/main/java/org/elasticsearch/xpack/security/SecurityLifecycleService.java

@@ -20,6 +20,7 @@ import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail;
+import org.elasticsearch.xpack.security.authc.TokenService;
 import org.elasticsearch.xpack.security.authc.esnative.NativeRealmMigrator;
 import org.elasticsearch.xpack.security.support.IndexLifecycleManager;
 
@@ -71,13 +72,12 @@ public class SecurityLifecycleService extends AbstractComponent implements Clust
         this.settings = settings;
         this.threadPool = threadPool;
         this.indexAuditTrail = indexAuditTrail;
-        this.securityIndex = new IndexLifecycleManager(settings, client,
+        this.securityIndex = new IndexLifecycleManager(settings, client, SECURITY_INDEX_NAME, SECURITY_TEMPLATE_NAME, migrator);
-                SECURITY_INDEX_NAME, SECURITY_TEMPLATE_NAME, migrator);
         clusterService.addListener(this);
         clusterService.addLifecycleListener(new LifecycleListener() {
             @Override
             public void beforeStop() {
-                stop();
+                close();
             }
         });
     }
@@ -93,12 +93,10 @@ public class SecurityLifecycleService extends AbstractComponent implements Clust
         }
 
         securityIndex.clusterChanged(event);
-
-        final boolean master = event.localNodeMaster();
         try {
             if (Security.indexAuditLoggingEnabled(settings) &&
                     indexAuditTrail.state() == IndexAuditTrail.State.INITIALIZED) {
-                if (indexAuditTrail.canStart(event, master)) {
+                if (indexAuditTrail.canStart(event, event.localNodeMaster())) {
                     threadPool.generic().execute(new AbstractRunnable() {
 
                         @Override
@@ -109,7 +107,7 @@ public class SecurityLifecycleService extends AbstractComponent implements Clust
 
                         @Override
                         public void doRun() {
-                            indexAuditTrail.start(master);
+                            indexAuditTrail.start(event.localNodeMaster());
                         }
                     });
                 }
@@ -119,19 +117,19 @@ public class SecurityLifecycleService extends AbstractComponent implements Clust
         }
     }
 
-    protected IndexLifecycleManager securityIndex() {
+    IndexLifecycleManager securityIndex() {
         return securityIndex;
     }
 
-    public boolean securityIndexExists() {
+    public boolean isSecurityIndexExisting() {
         return securityIndex.indexExists();
     }
 
-    public boolean securityIndexAvailable() {
+    public boolean isSecurityIndexAvailable() {
         return securityIndex.isAvailable();
     }
 
-    public boolean canWriteToSecurityIndex() {
+    public boolean isSecurityIndexWriteable() {
         return securityIndex.isWritable();
     }
 
@@ -146,7 +144,8 @@ public class SecurityLifecycleService extends AbstractComponent implements Clust
         return securityIndex.checkMappingVersion(requiredVersion);
     }
 
-    public void stop() {
+    // this is called in a lifecycle listener beforeStop on the cluster service
+    private void close() {
         if (indexAuditTrail != null) {
             try {
                 indexAuditTrail.stop();
@@ -175,6 +174,6 @@ public class SecurityLifecycleService extends AbstractComponent implements Clust
     }
 
     public static List<String> indexNames() {
-        return Collections.unmodifiableList(Arrays.asList(SECURITY_INDEX_NAME));
+        return Collections.singletonList(SECURITY_INDEX_NAME);
     }
 }

plugin/src/main/java/org/elasticsearch/xpack/security/TokenPassphraseBootstrapCheck.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.bootstrap.BootstrapCheck;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.XPackSettings;
+import org.elasticsearch.xpack.security.authc.TokenService;
+
+/**
+ * Bootstrap check to ensure that the user has set the token passphrase setting and is not using
+ * the default value in production
+ */
+final class TokenPassphraseBootstrapCheck implements BootstrapCheck {
+
+    static final int MINIMUM_PASSPHRASE_LENGTH = 8;
+
+    private final Settings settings;
+
+    TokenPassphraseBootstrapCheck(Settings settings) {
+        this.settings = settings;
+    }
+
+    @Override
+    public boolean check() {
+        if (XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.get(settings)) {
+            try (SecureString secureString = TokenService.TOKEN_PASSPHRASE.get(settings)) {
+                return secureString.length() < MINIMUM_PASSPHRASE_LENGTH || secureString.equals(TokenService.DEFAULT_PASSPHRASE);
+            }
+        }
+        // service is not enabled so no need to check
+        return false;
+    }
+
+    @Override
+    public String errorMessage() {
+        return "Please set a passphrase using the elasticsearch-keystore tool for the setting [" + TokenService.TOKEN_PASSPHRASE.getKey() +
+                "] that is at least " + MINIMUM_PASSPHRASE_LENGTH + " characters in length and does not match the default passphrase or " +
+                "disable the token service using the [" + XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey() + "] setting";
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/security/TokenSSLBootstrapCheck.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.bootstrap.BootstrapCheck;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xpack.XPackSettings;
+
+import static org.elasticsearch.xpack.XPackSettings.HTTP_SSL_ENABLED;
+
+/**
+ * Bootstrap check to ensure that the user has enabled HTTPS when using the token service
+ */
+final class TokenSSLBootstrapCheck implements BootstrapCheck {
+
+    private final Settings settings;
+
+    TokenSSLBootstrapCheck(Settings settings) {
+        this.settings = settings;
+    }
+
+    @Override
+    public boolean check() {
+        return XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.get(settings) && HTTP_SSL_ENABLED.get(settings) == false;
+    }
+
+    @Override
+    public String errorMessage() {
+        return "HTTPS is required in order to use the token service. Please enable HTTPS using the [" + HTTP_SSL_ENABLED.getKey() +
+                "] setting or disable the token service using the [" + XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey() + "] setting.";
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/security/action/token/CreateTokenAction.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.action.token;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.client.ElasticsearchClient;
+
+/**
+ * Action for creating a new token
+ */
+public final class CreateTokenAction extends Action<CreateTokenRequest, CreateTokenResponse, CreateTokenRequestBuilder> {
+
+    public static final String NAME = "cluster:admin/xpack/security/token/create";
+    public static final CreateTokenAction INSTANCE = new CreateTokenAction();
+
+    private CreateTokenAction() {
+        super(NAME);
+    }
+
+    @Override
+    public CreateTokenRequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new CreateTokenRequestBuilder(client);
+    }
+
+    @Override
+    public CreateTokenResponse newResponse() {
+        return new CreateTokenResponse();
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/security/action/token/CreateTokenRequest.java

@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.action.token;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.xpack.security.authc.support.CharArrays;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * Represents a request to create a token based on the provided information. This class accepts the
+ * fields for an OAuth 2.0 access token request that uses the <code>password</code> grant type.
+ */
+public final class CreateTokenRequest extends ActionRequest {
+
+    private String grantType;
+    private String username;
+    private SecureString password;
+    private String scope;
+
+    CreateTokenRequest() {}
+
+    public CreateTokenRequest(String grantType, String username, SecureString password, @Nullable String scope) {
+        this.grantType = grantType;
+        this.username = username;
+        this.password = password;
+        this.scope = scope;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if ("password".equals(grantType) == false) {
+            validationException = addValidationError("only [password] grant_type is supported", validationException);
+        }
+        if (Strings.isNullOrEmpty("username")) {
+            validationException = addValidationError("username is missing", validationException);
+        }
+        if (Strings.isNullOrEmpty("password")) {
+            validationException = addValidationError("password is missing", validationException);
+        }
+
+        return validationException;
+    }
+
+    public void setGrantType(String grantType) {
+        this.grantType = grantType;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public void setPassword(SecureString password) {
+        this.password = password;
+    }
+
+    public void setScope(@Nullable String scope) {
+        this.scope = scope;
+    }
+
+    public String getGrantType() {
+        return grantType;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public SecureString getPassword() {
+        return password;
+    }
+
+    @Nullable
+    public String getScope() {
+        return scope;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(grantType);
+        out.writeString(username);
+        final byte[] passwordBytes = CharArrays.toUtf8Bytes(password.getChars());
+        try {
+            out.writeByteArray(passwordBytes);
+        } finally {
+            Arrays.fill(passwordBytes, (byte) 0);
+        }
+        out.writeOptionalString(scope);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        grantType = in.readString();
+        username = in.readString();
+        final byte[] passwordBytes = in.readByteArray();
+        try {
+            password = new SecureString(CharArrays.utf8BytesToChars(passwordBytes));
+        } finally {
+            Arrays.fill(passwordBytes, (byte) 0);
+        }
+        scope = in.readOptionalString();
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/security/action/token/CreateTokenRequestBuilder.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.action.token;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.settings.SecureString;
+
+/**
+ * Request builder used to populate a {@link CreateTokenRequest}
+ */
+public final class CreateTokenRequestBuilder
+        extends ActionRequestBuilder<CreateTokenRequest, CreateTokenResponse, CreateTokenRequestBuilder> {
+
+    public CreateTokenRequestBuilder(ElasticsearchClient client) {
+        super(client, CreateTokenAction.INSTANCE, new CreateTokenRequest());
+    }
+
+    /**
+     * Specifies the grant type for this request. Currently only <code>password</code> is supported
+     */
+    public CreateTokenRequestBuilder setGrantType(String grantType) {
+        request.setGrantType(grantType);
+        return this;
+    }
+
+    /**
+     * Set the username to be used for authentication with a password grant
+     */
+    public CreateTokenRequestBuilder setUsername(String username) {
+        request.setUsername(username);
+        return this;
+    }
+
+    /**
+     * Set the password credentials associated with the user. These credentials will be used for
+     * authentication and the resulting token will be for this user
+     */
+    public CreateTokenRequestBuilder setPassword(SecureString password) {
+        request.setPassword(password);
+        return this;
+    }
+
+    /**
+     * Set the scope of the access token. A <code>null</code> scope implies the default scope. If
+     * the requested scope differs from the scope of the token, the token's scope will be returned
+     * in the response
+     */
+    public CreateTokenRequestBuilder setScope(@Nullable String scope) {
+        request.setScope(scope);
+        return this;
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/security/action/token/CreateTokenResponse.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.action.token;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Response containing the token string that was generated from a token creation request. This
+ * object also contains the scope and expiration date. If the scope was not provided or if the
+ * provided scope matches the scope of the token, then the scope value is <code>null</code>
+ */
+public final class CreateTokenResponse extends ActionResponse implements ToXContentObject {
+
+    private String tokenString;
+    private TimeValue expiresIn;
+    private String scope;
+
+    CreateTokenResponse() {}
+
+    public CreateTokenResponse(String tokenString, TimeValue expiresIn, String scope) {
+        this.tokenString = Objects.requireNonNull(tokenString);
+        this.expiresIn = Objects.requireNonNull(expiresIn);
+        this.scope = scope;
+    }
+
+    public String getTokenString() {
+        return tokenString;
+    }
+
+    public String getScope() {
+        return scope;
+    }
+
+    public TimeValue getExpiresIn() {
+        return expiresIn;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(tokenString);
+        expiresIn.writeTo(out);
+        out.writeOptionalString(scope);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        tokenString = in.readString();
+        expiresIn = new TimeValue(in);
+        scope = in.readOptionalString();
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject()
+            .field("access_token", tokenString)
+            .field("type", "Bearer")
+            .field("expires_in", expiresIn.seconds());
+        // only show the scope if it is not null
+        if (scope != null) {
+            builder.field("scope", scope);
+        }
+        return builder.endObject();
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/security/action/token/InvalidateTokenAction.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.action.token;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.client.ElasticsearchClient;
+
+/**
+ * Action for invalidating a given token
+ */
+public final class InvalidateTokenAction extends Action<InvalidateTokenRequest, InvalidateTokenResponse, InvalidateTokenRequestBuilder> {
+
+    public static final String NAME = "cluster:admin/xpack/security/token/invalidate";
+    public static final InvalidateTokenAction INSTANCE = new InvalidateTokenAction();
+
+    private InvalidateTokenAction() {
+        super(NAME);
+    }
+
+    @Override
+    public InvalidateTokenRequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new InvalidateTokenRequestBuilder(client);
+    }
+
+    @Override
+    public InvalidateTokenResponse newResponse() {
+        return new InvalidateTokenResponse();
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/security/action/token/InvalidateTokenRequest.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.action.token;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * Request for invalidating a token so that it can no longer be used
+ */
+public final class InvalidateTokenRequest extends ActionRequest {
+
+    private String tokenString;
+
+    InvalidateTokenRequest() {}
+
+    /**
+     * @param tokenString the string representation of the token
+     */
+    public InvalidateTokenRequest(String tokenString) {
+        this.tokenString = tokenString;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (Strings.isNullOrEmpty(tokenString)) {
+            validationException = addValidationError("token string must be provided", null);
+        }
+        return validationException;
+    }
+
+    String getTokenString() {
+        return tokenString;
+    }
+
+    void setTokenString(String token) {
+        this.tokenString = token;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(tokenString);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        tokenString = in.readString();
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/security/action/token/InvalidateTokenRequestBuilder.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.action.token;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+
+/**
+ * Request builder that is used to populate a {@link InvalidateTokenRequest}
+ */
+public final class InvalidateTokenRequestBuilder
+        extends ActionRequestBuilder<InvalidateTokenRequest, InvalidateTokenResponse, InvalidateTokenRequestBuilder> {
+
+    public InvalidateTokenRequestBuilder(ElasticsearchClient client) {
+        super(client, InvalidateTokenAction.INSTANCE, new InvalidateTokenRequest());
+    }
+
+    /**
+     * The string representation of the token that is being invalidated. This is the value returned
+     * from a create token request.
+     */
+    public InvalidateTokenRequestBuilder setTokenString(String token) {
+        request.setTokenString(token);
+        return this;
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/security/action/token/InvalidateTokenResponse.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.action.token;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+/**
+ * Response for a invalidation of a token.
+ */
+public final class InvalidateTokenResponse extends ActionResponse {
+
+    private boolean created;
+
+    InvalidateTokenResponse() {}
+
+    InvalidateTokenResponse(boolean created) {
+        this.created = created;
+    }
+
+    /**
+     * If the token is already invalidated then created will be <code>false</code>
+     */
+    public boolean isCreated() {
+        return created;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeBoolean(created);
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        created = in.readBoolean();
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenAction.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.action.token;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.security.authc.AuthenticationService;
+import org.elasticsearch.xpack.security.authc.TokenService;
+import org.elasticsearch.xpack.security.authc.UserToken;
+
+/**
+ * Transport action responsible for creating a token based on a request. Requests provide user
+ * credentials that can be different than those of the user that is currently authenticated so we
+ * always re-authenticate within this action. This authenticated user will be the user that the
+ * token represents
+ */
+public final class TransportCreateTokenAction extends HandledTransportAction<CreateTokenRequest, CreateTokenResponse> {
+
+    private static final String DEFAULT_SCOPE = "full";
+    private final TokenService tokenService;
+    private final AuthenticationService authenticationService;
+
+    @Inject
+    public TransportCreateTokenAction(Settings settings, ThreadPool threadPool, TransportService transportService,
+                                      ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                      TokenService tokenService, AuthenticationService authenticationService) {
+        super(settings, CreateTokenAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver,
+                CreateTokenRequest::new);
+        this.tokenService = tokenService;
+        this.authenticationService = authenticationService;
+    }
+
+    @Override
+    protected void doExecute(CreateTokenRequest request, ActionListener<CreateTokenResponse> listener) {
+        try (ThreadContext.StoredContext ignore = threadPool.getThreadContext().stashContext()) {
+            authenticationService.authenticate(CreateTokenAction.NAME, request,
+                    request.getUsername(), request.getPassword(),
+                    ActionListener.wrap(authentication -> {
+                        try (SecureString ignore1 = request.getPassword()) {
+                            final UserToken token = tokenService.createUserToken(authentication);
+                            final String tokenStr = tokenService.getUserTokenString(token);
+                            final String scope;
+                            // the OAuth2.0 RFC requires the scope to be provided in the
+                            // response if it differs from the user provided scope. If the
+                            // scope was not provided then it does not need to be returned.
+                            // if the scope is not supported, the value of the scope that the
+                            // token is for must be returned
+                            if (request.getScope() != null) {
+                                scope = DEFAULT_SCOPE; // this is the only non-null value that is currently supported
+                            } else {
+                                scope = null;
+                            }
+
+                            listener.onResponse(new CreateTokenResponse(tokenStr, tokenService.getExpirationDelay(), scope));
+                        }
+                    }, e -> {
+                        // clear the request password
+                        request.getPassword().close();
+                        listener.onFailure(e);
+                    }));
+        }
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/security/action/token/TransportInvalidateTokenAction.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.action.token;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.security.authc.TokenService;
+
+/**
+ * Transport action responsible for handling invalidation of tokens
+ */
+public final class TransportInvalidateTokenAction extends HandledTransportAction<InvalidateTokenRequest, InvalidateTokenResponse> {
+
+    private final TokenService tokenService;
+
+    @Inject
+    public TransportInvalidateTokenAction(Settings settings, ThreadPool threadPool, TransportService transportService,
+                                          ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                          TokenService tokenService) {
+        super(settings, InvalidateTokenAction.NAME, threadPool, transportService, actionFilters,
+                indexNameExpressionResolver, InvalidateTokenRequest::new);
+        this.tokenService = tokenService;
+    }
+
+    @Override
+    protected void doExecute(InvalidateTokenRequest request,
+                             ActionListener<InvalidateTokenResponse> listener) {
+        tokenService.invalidateToken(request.getTokenString(), ActionListener.wrap(
+                created -> listener.onResponse(new InvalidateTokenResponse(created)),
+                listener::onFailure));
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/security/authc/Authentication.java

@@ -112,22 +112,6 @@ public class Authentication {
         return authentication;
     }
 
-    void writeToContextIfMissing(ThreadContext context)
-            throws IOException, IllegalArgumentException {
-        if (context.getTransient(AUTHENTICATION_KEY) != null) {
-            if (context.getHeader(AUTHENTICATION_KEY) == null) {
-                throw new IllegalStateException("authentication present as a transient but not a header");
-            }
-            return;
-        }
-
-        if (context.getHeader(AUTHENTICATION_KEY) != null) {
-            deserializeHeaderAndPutInContext(context.getHeader(AUTHENTICATION_KEY), context);
-        } else {
-            writeToContext(context);
-        }
-    }
-
     /**
      * Writes the authentication to the context. There must not be an existing authentication in the context and if there is an
      * {@link IllegalStateException} will be thrown
@@ -167,6 +151,28 @@ public class Authentication {
         }
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        Authentication that = (Authentication) o;
+
+        if (!user.equals(that.user)) return false;
+        if (!authenticatedBy.equals(that.authenticatedBy)) return false;
+        if (lookedUpBy != null ? !lookedUpBy.equals(that.lookedUpBy) : that.lookedUpBy != null) return false;
+        return version.equals(that.version);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = user.hashCode();
+        result = 31 * result + authenticatedBy.hashCode();
+        result = 31 * result + (lookedUpBy != null ? lookedUpBy.hashCode() : 0);
+        result = 31 * result + version.hashCode();
+        return result;
+    }
+
     public static class RealmRef {
 
         private final String nodeName;
@@ -202,6 +208,26 @@ public class Authentication {
         public String getType() {
             return type;
         }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+
+            RealmRef realmRef = (RealmRef) o;
+
+            if (!nodeName.equals(realmRef.nodeName)) return false;
+            if (!name.equals(realmRef.name)) return false;
+            return type.equals(realmRef.type);
+        }
+
+        @Override
+        public int hashCode() {
+            int result = nodeName.hashCode();
+            result = 31 * result + name.hashCode();
+            result = 31 * result + type.hashCode();
+            return result;
+        }
     }
 }
 

plugin/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java

@@ -13,6 +13,7 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -25,10 +26,10 @@ import org.elasticsearch.xpack.common.IteratingActionListener;
 import org.elasticsearch.xpack.security.audit.AuditTrail;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.elasticsearch.xpack.security.authc.Authentication.RealmRef;
+import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.security.user.AnonymousUser;
 import org.elasticsearch.xpack.security.user.User;
 
-import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 import java.util.function.BiConsumer;
@@ -53,11 +54,13 @@ public class AuthenticationService extends AbstractComponent {
     private final ThreadContext threadContext;
     private final String nodeName;
     private final AnonymousUser anonymousUser;
+    private final TokenService tokenService;
     private final boolean runAsEnabled;
     private final boolean isAnonymousUserEnabled;
 
     public AuthenticationService(Settings settings, Realms realms, AuditTrailService auditTrail,
-                                 AuthenticationFailureHandler failureHandler, ThreadPool threadPool, AnonymousUser anonymousUser) {
+                                 AuthenticationFailureHandler failureHandler, ThreadPool threadPool,
+                                 AnonymousUser anonymousUser, TokenService tokenService) {
         super(settings);
         this.nodeName = Node.NODE_NAME_SETTING.get(settings);
         this.realms = realms;
@@ -67,6 +70,7 @@ public class AuthenticationService extends AbstractComponent {
         this.anonymousUser = anonymousUser;
         this.runAsEnabled = RUN_AS_ENABLED.get(settings);
         this.isAnonymousUserEnabled = AnonymousUser.isAnonymousEnabled(settings);
+        this.tokenService = tokenService;
     }
 
     /**
@@ -84,7 +88,7 @@ public class AuthenticationService extends AbstractComponent {
      * Authenticates the user that is associated with the given message. If the user was authenticated successfully (i.e.
      * a user was indeed associated with the request and the credentials were verified to be valid), the method returns
      * the user and that user is then "attached" to the message's context. If no user was found to be attached to the given
-     * message, the the given fallback user will be returned instead.
+     * message, then the given fallback user will be returned instead.
      *
      * @param action        The action of the message
      * @param message       The message to be authenticated
@@ -98,14 +102,17 @@ public class AuthenticationService extends AbstractComponent {
     }
 
     /**
-     * Checks if there's already a user header attached to the given message. If missing, a new header is
+     * Authenticates the username and password that are provided as parameters. This will not look
-     * set on the message with the given user (encoded).
+     * at the values in the ThreadContext for Authentication.
      *
-     * @param user      The user to be attached if the header is missing
+     * @param action        The action of the message
+     * @param message       The message that resulted in this authenticate call
+     * @param username      The username to be used for authentication
+     * @param password      The password to be used for authentication
      */
-    void attachUserIfMissing(User user) throws IOException {
+    public void authenticate(String action, TransportMessage message, String username,
-        Authentication authentication = new Authentication(user, new RealmRef("__attach", "__attach", nodeName), null);
+                             SecureString password, ActionListener<Authentication> listener) {
-        authentication.writeToContextIfMissing(threadContext);
+        new Authenticator(action, message, null, listener, username, password).authenticateAsync();
     }
 
     // pkg private method for testing
@@ -140,6 +147,13 @@ public class AuthenticationService extends AbstractComponent {
             this(new AuditableTransportRequest(auditTrail, failureHandler, threadContext, action, message), fallbackUser, listener);
         }
 
+        Authenticator(String action, TransportMessage message, User fallbackUser,
+                      ActionListener<Authentication> listener, String username,
+                      SecureString password) {
+            this(new AuditableTransportRequest(auditTrail, failureHandler, threadContext, action, message), fallbackUser, listener);
+            this.authenticationToken = new UsernamePasswordToken(username, password);
+        }
+
         private Authenticator(AuditableRequest auditableRequest, User fallbackUser, ActionListener<Authentication> listener) {
             this.request = auditableRequest;
             this.fallbackUser = fallbackUser;
@@ -152,6 +166,7 @@ public class AuthenticationService extends AbstractComponent {
          *
          * <ol>
          *     <li>look for existing authentication {@link #lookForExistingAuthentication(Consumer)}</li>
+         *     <li>look for a user token</li>
          *     <li>token extraction {@link #extractToken(Consumer)}</li>
          *     <li>token authentication {@link #consumeToken(AuthenticationToken)}</li>
          *     <li>user lookup for run as if necessary {@link #consumeUser(User)} and
@@ -163,9 +178,23 @@ public class AuthenticationService extends AbstractComponent {
             lookForExistingAuthentication((authentication) -> {
                 if (authentication != null) {
                     listener.onResponse(authentication);
+                } else {
+                    tokenService.getAndValidateToken(threadContext, ActionListener.wrap(userToken -> {
+                        if (userToken != null) {
+                            writeAuthToContext(userToken.getAuthentication());
                         } else {
                             extractToken(this::consumeToken);
                         }
+                    }, e -> {
+                        if (e instanceof ElasticsearchSecurityException &&
+                                tokenService.isExpiredTokenException((ElasticsearchSecurityException) e) == false) {
+                            // intentionally ignore the returned exception; we call this primarily
+                            // for the auditing as we already have a purpose built exception
+                            request.tamperedRequest();
+                        }
+                        listener.onFailure(e);
+                    }));
+                }
             });
         }
 
@@ -190,9 +219,8 @@ public class AuthenticationService extends AbstractComponent {
                 action = () -> listener.onFailure(request.tamperedRequest());
             }
 
-            // we use the success boolean as we need to know if the executed code block threw an exception and we already called on
+            // While we could place this call in the try block, the issue is that we catch all exceptions and could catch exceptions that
-            // failure; if we did call the listener we do not need to continue. While we could place this call in the try block, the
+            // have nothing to do with a tampered request.
-            // issue is that we catch all exceptions and could catch exceptions that have nothing to do with a tampered request.
             action.run();
         }
 
@@ -205,6 +233,9 @@ public class AuthenticationService extends AbstractComponent {
         void extractToken(Consumer<AuthenticationToken> consumer) {
             Runnable action = () -> consumer.accept(null);
             try {
+                if (authenticationToken != null) {
+                    action = () -> consumer.accept(authenticationToken);
+                } else {
                     for (Realm realm : realms) {
                         final AuthenticationToken token = realm.token(threadContext);
                         if (token != null) {
@@ -212,6 +243,7 @@ public class AuthenticationService extends AbstractComponent {
                             break;
                         }
                     }
+                }
             } catch (Exception e) {
                 action = () -> listener.onFailure(request.exceptionProcessingRequest(e, null));
             }
@@ -379,11 +411,20 @@ public class AuthenticationService extends AbstractComponent {
                 logger.debug("user [{}] is disabled. failing authentication", finalUser);
                 listener.onFailure(request.authenticationFailed(authenticationToken));
             } else {
-                request.authenticationSuccess(authenticatedBy.getName(), finalUser);
                 final Authentication finalAuth = new Authentication(finalUser, authenticatedBy, lookedupBy);
-                Runnable action = () -> listener.onResponse(finalAuth);
+                writeAuthToContext(finalAuth);
+            }
+        }
+
+        /**
+         * Writes the authentication to the {@link ThreadContext} and then calls the listener if
+         * successful
+         */
+        void writeAuthToContext(Authentication authentication) {
+            request.authenticationSuccess(authentication.getAuthenticatedBy().getName(), authentication.getUser());
+            Runnable action = () -> listener.onResponse(authentication);
             try {
-                    finalAuth.writeToContext(threadContext);
+                authentication.writeToContext(threadContext);
             } catch (Exception e) {
                 action = () -> listener.onFailure(request.exceptionProcessingRequest(e, authenticationToken));
             }
@@ -393,7 +434,6 @@ public class AuthenticationService extends AbstractComponent {
             action.run();
         }
     }
-    }
 
     abstract static class AuditableRequest {
 

plugin/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.authc;
+
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse;
+import org.elasticsearch.action.bulk.byscroll.DeleteByQueryRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.threadpool.ThreadPool.Names;
+import org.elasticsearch.xpack.common.action.XPackDeleteByQueryAction;
+import org.elasticsearch.xpack.security.InternalClient;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.elasticsearch.action.support.TransportActions.isShardNotAvailableException;
+
+/**
+ * Responsible for cleaning the invalidated tokens from the invalidated tokens index.
+ */
+final class ExpiredTokenRemover extends AbstractRunnable {
+
+    private final InternalClient client;
+    private final AtomicBoolean inProgress = new AtomicBoolean(false);
+    private final Logger logger;
+
+    ExpiredTokenRemover(Settings settings, InternalClient internalClient) {
+        this.client = internalClient;
+        this.logger = Loggers.getLogger(getClass(), settings);
+    }
+
+    @Override
+    public void doRun() {
+        SearchRequest searchRequest = new SearchRequest(TokenService.INDEX_NAME);
+        DeleteByQueryRequest dbq = new DeleteByQueryRequest(searchRequest);
+        searchRequest.source()
+                .query(QueryBuilders.boolQuery()
+                        .filter(QueryBuilders.termQuery("doc_type", TokenService.DOC_TYPE))
+                        .filter(QueryBuilders.rangeQuery("expiration_time").lte(DateTime.now(DateTimeZone.UTC))));
+        client.execute(XPackDeleteByQueryAction.INSTANCE, dbq, ActionListener.wrap(r -> markComplete(),
+                e -> {
+                    if (isShardNotAvailableException(e) == false) {
+                        logger.error("failed to delete expired tokens", e);
+                    }
+                    markComplete();
+                }));
+    }
+
+    void submit(ThreadPool threadPool) {
+        if (inProgress.compareAndSet(false, true)) {
+            threadPool.executor(Names.GENERIC).submit(this);
+        }
+    }
+
+    @Override
+    public void onFailure(Exception e) {
+        logger.error("failed to delete expired tokens", e);
+        markComplete();
+    }
+
+    private void markComplete() {
+        if (inProgress.compareAndSet(true, false) == false) {
+            throw new IllegalStateException("in progress was set to false but should have been true!");
+        }
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java

@@ -0,0 +1,553 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.authc;
+
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.StringHelper;
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.DocWriteRequest.OpType;
+import org.elasticsearch.action.DocWriteResponse.Result;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.support.TransportActions;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.cache.Cache;
+import org.elasticsearch.common.cache.CacheBuilder;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.io.stream.InputStreamStreamInput;
+import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.SecureSetting;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.index.engine.VersionConflictEngineException;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xpack.XPackPlugin;
+import org.elasticsearch.xpack.XPackSettings;
+import org.elasticsearch.xpack.security.InternalClient;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
+
+import javax.crypto.Cipher;
+import javax.crypto.CipherInputStream;
+import javax.crypto.CipherOutputStream;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.SecretKey;
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.GCMParameterSpec;
+import javax.crypto.spec.PBEKeySpec;
+import javax.crypto.spec.SecretKeySpec;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
+import java.security.spec.InvalidKeySpecException;
+import java.time.Clock;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Service responsible for the creation, validation, and other management of {@link UserToken}
+ * objects for authentication
+ */
+public final class TokenService extends AbstractComponent {
+
+    /**
+     * The parameters below are used to generate the cryptographic key that is used to encrypt the
+     * values returned by this service. These parameters are based off of the
+     * <a href="https://www.owasp.org/index.php/Password_Storage_Cheat_Sheet">OWASP Password Storage
+     * Cheat Sheet</a> and the <a href="https://pages.nist.gov/800-63-3/sp800-63b.html#sec5">
+     * NIST Digital Identity Guidelines</a>
+     */
+    private static final int ITERATIONS = 100000;
+    private static final String KDF_ALGORITHM = "PBKDF2withHMACSHA512";
+    private static final int SALT_BYTES = 32;
+    private static final int IV_BYTES = 12;
+    private static final int VERSION_BYTES = 4;
+    private static final String ENCRYPTION_CIPHER = "AES/GCM/NoPadding";
+    private static final DateTimeFormatter DEFAULT_DATE_PRINTER = DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneOffset.UTC);
+    private static final String EXPIRED_TOKEN_WWW_AUTH_VALUE = "Bearer realm=\"" + XPackPlugin.SECURITY +
+            "\", error=\"invalid_token\", error_description=\"The access token expired\"";
+    private static final String MALFORMED_TOKEN_WWW_AUTH_VALUE = "Bearer realm=\"" + XPackPlugin.SECURITY +
+            "\", error=\"invalid_token\", error_description=\"The access token is malformed\"";
+    private static final String TYPE = "doc";
+
+    public static final String INDEX_NAME = SecurityLifecycleService.SECURITY_INDEX_NAME;
+    public static final String THREAD_POOL_NAME = XPackPlugin.SECURITY + "-token-key";
+    public static final Setting<SecureString> TOKEN_PASSPHRASE = SecureSetting.secureString("xpack.security.authc.token.passphrase", null);
+    public static final Setting<TimeValue> TOKEN_EXPIRATION = Setting.timeSetting("xpack.security.authc.token.timeout",
+            TimeValue.timeValueMinutes(20L), TimeValue.timeValueSeconds(1L), Property.NodeScope);
+    public static final Setting<TimeValue> DELETE_INTERVAL = Setting.timeSetting("xpack.security.authc.token.delete.interval",
+                    TimeValue.timeValueMinutes(30L), Property.NodeScope);
+    public static final String DEFAULT_PASSPHRASE = "changeme is a terrible password, so let's not use it anymore!";
+
+    static final String DOC_TYPE = "invalidated-token";
+    static final int MINIMUM_BYTES = VERSION_BYTES + SALT_BYTES + IV_BYTES + 1;
+    static final int MINIMUM_BASE64_BYTES = Double.valueOf(Math.ceil((4 * MINIMUM_BYTES) / 3)).intValue();
+
+    private final SecureRandom secureRandom = new SecureRandom();
+    private final Cache<BytesKey, SecretKey> keyCache;
+    private final SecureString tokenPassphrase;
+    private final Clock clock;
+    private final TimeValue expirationDelay;
+    private final TimeValue deleteInterval;
+    private final BytesKey salt;
+    private final InternalClient internalClient;
+    private final SecurityLifecycleService lifecycleService;
+    private final ExpiredTokenRemover expiredTokenRemover;
+    private final boolean enabled;
+    private final byte[] currentVersionBytes;
+
+    private volatile long lastExpirationRunMs;
+
+
+    /**
+     * Creates a new token service
+     * @param settings the node settings
+     * @param clock the clock that will be used for comparing timestamps
+     * @param internalClient the client to use when checking for revocations
+     */
+    public TokenService(Settings settings, Clock clock, InternalClient internalClient,
+                        SecurityLifecycleService lifecycleService) throws GeneralSecurityException {
+        super(settings);
+        byte[] saltArr = new byte[SALT_BYTES];
+        secureRandom.nextBytes(saltArr);
+        this.salt = new BytesKey(saltArr);
+        this.keyCache = CacheBuilder.<BytesKey, SecretKey>builder()
+                .setExpireAfterAccess(TimeValue.timeValueMinutes(60L))
+                .setMaximumWeight(500L)
+                .build();
+        final SecureString tokenPassphraseValue = TOKEN_PASSPHRASE.get(settings);
+        if (tokenPassphraseValue.length() == 0) {
+            // setting didn't exist - we should only be in a non-production mode for this
+            this.tokenPassphrase = new SecureString(DEFAULT_PASSPHRASE.toCharArray());
+        } else {
+            this.tokenPassphrase = tokenPassphraseValue;
+        }
+
+        this.clock = clock.withZone(ZoneOffset.UTC);
+        this.expirationDelay = TOKEN_EXPIRATION.get(settings);
+        this.internalClient = internalClient;
+        this.lifecycleService = lifecycleService;
+        this.lastExpirationRunMs = internalClient.threadPool().relativeTimeInMillis();
+        this.deleteInterval = DELETE_INTERVAL.get(settings);
+        this.enabled = XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.get(settings);
+        this.expiredTokenRemover = new ExpiredTokenRemover(settings, internalClient);
+        this.currentVersionBytes = ByteBuffer.allocate(4).putInt(Version.CURRENT.id).array();
+        ensureEncryptionCiphersSupported();
+        try (SecureString closeableChars = tokenPassphrase.clone()) {
+            keyCache.put(salt, computeSecretKey(closeableChars.getChars(), salt.bytes));
+        }
+    }
+
+    /**
+     * Create a token based on the provided authentication
+     */
+    public UserToken createUserToken(Authentication authentication)
+            throws IOException, GeneralSecurityException {
+        ensureEnabled();
+        final ZonedDateTime expiration = getExpirationTime();
+        return new UserToken(authentication, expiration);
+    }
+
+    /**
+     * Looks in the context to see if the request provided a header with a user token
+     */
+    void getAndValidateToken(ThreadContext ctx, ActionListener<UserToken> listener) {
+        if (enabled) {
+            final String token = getFromHeader(ctx);
+            if (token == null) {
+                listener.onResponse(null);
+            } else {
+                try {
+                    decodeToken(token, ActionListener.wrap(userToken -> {
+                        if (userToken != null) {
+                            ZonedDateTime currentTime = clock.instant().atZone(ZoneOffset.UTC);
+                            if (currentTime.isAfter(userToken.getExpirationTime())) {
+                                // token expired
+                                listener.onFailure(expiredTokenException());
+                            } else {
+                                checkIfTokenIsRevoked(userToken, listener);
+                            }
+                        } else {
+                            listener.onResponse(null);
+                        }
+                    }, listener::onFailure));
+                } catch (IOException e) {
+                    // could happen with a token that is not ours
+                    logger.debug("invalid token", e);
+                    listener.onResponse(null);
+                }
+            }
+        } else {
+            listener.onResponse(null);
+        }
+    }
+
+    private void decodeToken(String token, ActionListener<UserToken> listener) throws IOException {
+        // We intentionally do not use try-with resources since we need to keep the stream open if we need to compute a key!
+        StreamInput in = new InputStreamStreamInput(
+                Base64.getDecoder().wrap(new ByteArrayInputStream(token.getBytes(StandardCharsets.UTF_8))));
+        if (in.available() < MINIMUM_BASE64_BYTES) {
+            logger.debug("invalid token");
+            listener.onResponse(null);
+        } else {
+            // the token exists and the value is at least as long as we'd expect
+            final Version version = Version.readVersion(in);
+            if (version.before(Version.V_5_5_0_UNRELEASED)) {
+                listener.onResponse(null);
+            } else {
+                final BytesKey decodedSalt = new BytesKey(in.readByteArray());
+                final SecretKey decodeKey = keyCache.get(decodedSalt);
+                final byte[] iv = in.readByteArray();
+                if (decodeKey != null) {
+                    try {
+                        decryptToken(in, getDecryptionCipher(iv, decodeKey, version, decodedSalt), version, listener);
+                    } catch (GeneralSecurityException e) {
+                        // could happen with a token that is not ours
+                        logger.debug("invalid token", e);
+                        listener.onResponse(null);
+                    }
+                } else {
+                    /* As a measure of protected against DOS, we can pass requests requiring a key
+                     * computation off to a single thread executor. For normal usage, the initial
+                     * request(s) that require a key computation will be delayed and there will be
+                     * some additional latency.
+                     */
+                    internalClient.threadPool().executor(THREAD_POOL_NAME)
+                            .submit(new KeyComputingRunnable(in, iv, version, decodedSalt, listener));
+                }
+            }
+        }
+    }
+
+    private void decryptToken(StreamInput in, Cipher cipher, Version version, ActionListener<UserToken> listener) throws IOException {
+        try (CipherInputStream cis = new CipherInputStream(in, cipher); StreamInput decryptedInput = new InputStreamStreamInput(cis)) {
+            decryptedInput.setVersion(version);
+            listener.onResponse(new UserToken(decryptedInput));
+        }
+    }
+
+    /**
+     * This method records an entry to indicate that a token with a given id has been expired.
+     */
+    public void invalidateToken(String tokenString, ActionListener<Boolean> listener) {
+        ensureEnabled();
+        if (lifecycleService.isSecurityIndexWriteable() == false) {
+            listener.onFailure(new IllegalStateException("cannot write to the tokens index"));
+        } else if (Strings.isNullOrEmpty(tokenString)) {
+            listener.onFailure(new IllegalArgumentException("token must be provided"));
+        } else {
+            maybeStartTokenRemover();
+            try {
+                decodeToken(tokenString, ActionListener.wrap(userToken -> {
+                    if (userToken == null) {
+                        listener.onFailure(malformedTokenException());
+                    } else if (userToken.getExpirationTime().isBefore(clock.instant().atZone(ZoneOffset.UTC))) {
+                        // no need to invalidate - it's already expired
+                        listener.onResponse(false);
+                    } else {
+                        final String id = userToken.getId();
+                        internalClient.prepareIndex(INDEX_NAME, TYPE, id)
+                                .setOpType(OpType.CREATE)
+                                .setSource("doc_type", DOC_TYPE, "expiration_time", DEFAULT_DATE_PRINTER.format(getExpirationTime()))
+                                .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL)
+                                .execute(new ActionListener<IndexResponse>() {
+                                    @Override
+                                    public void onResponse(IndexResponse indexResponse) {
+                                        listener.onResponse(indexResponse.getResult() == Result.CREATED);
+                                    }
+
+                                    @Override
+                                    public void onFailure(Exception e) {
+                                        if (e instanceof VersionConflictEngineException) {
+                                            // doc already exists
+                                            listener.onResponse(false);
+                                        } else {
+                                            listener.onFailure(e);
+                                        }
+                                    }
+                                });
+                    }
+                }, listener::onFailure));
+            } catch (IOException e) {
+                logger.error("received a malformed token as part of a invalidation request", e);
+                listener.onFailure(malformedTokenException());
+            }
+        }
+    }
+
+    private void ensureEnabled() {
+        if (enabled == false) {
+            throw new IllegalStateException("tokens are not enabled");
+        }
+    }
+
+    /**
+     * Checks if the token has been stored as a revoked token to ensure we do not allow tokens that
+     * have been explicitly cleared.
+     */
+    private void checkIfTokenIsRevoked(UserToken userToken, ActionListener<UserToken> listener) {
+        if (lifecycleService.isSecurityIndexAvailable()) {
+            internalClient.prepareGet(INDEX_NAME, TYPE, userToken.getId())
+                    .execute(new ActionListener<GetResponse>() {
+
+                        @Override
+                        public void onResponse(GetResponse response) {
+                            if (response.isExists()) {
+                                // this token is explicitly expired!
+                                listener.onFailure(expiredTokenException());
+                            } else {
+                                listener.onResponse(userToken);
+                            }
+                        }
+
+                        @Override
+                        public void onFailure(Exception e) {
+                            // if the index or the shard is not there / available we assume that
+                            // the token is not valid
+                            if (TransportActions.isShardNotAvailableException(e)) {
+                                logger.warn("failed to get token [{}] since index is not available", userToken.getId());
+                                listener.onResponse(null);
+                            } else {
+                                logger.error(new ParameterizedMessage("failed to get token [{}]", userToken.getId()), e);
+                                listener.onFailure(e);
+                            }
+                        }
+                    });
+        } else if (lifecycleService.isSecurityIndexExisting()) {
+            // index exists but the index isn't available, do not trust the token
+            logger.warn("could not validate token as the security index is not available");
+            listener.onResponse(null);
+        } else {
+            // index doesn't exist so the token is considered valid.
+            listener.onResponse(userToken);
+        }
+    }
+
+    public TimeValue getExpirationDelay() {
+        return expirationDelay;
+    }
+
+    private ZonedDateTime getExpirationTime() {
+        return clock.instant().plusSeconds(expirationDelay.getSeconds()).atZone(ZoneOffset.UTC);
+    }
+
+    private void maybeStartTokenRemover() {
+        if (lifecycleService.isSecurityIndexAvailable()) {
+            if (internalClient.threadPool().relativeTimeInMillis() - lastExpirationRunMs > deleteInterval.getMillis()) {
+                expiredTokenRemover.submit(internalClient.threadPool());
+                lastExpirationRunMs = internalClient.threadPool().relativeTimeInMillis();
+            }
+        }
+    }
+
+    /**
+     * Gets the token from the <code>Authorization</code> header if the header begins with
+     * <code>Bearer </code>
+     */
+    private String getFromHeader(ThreadContext threadContext) {
+        String header = threadContext.getHeader("Authorization");
+        if (Strings.hasLength(header) && header.startsWith("Bearer ")
+                && header.length() > "Bearer ".length()) {
+            return header.substring("Bearer ".length());
+        }
+        return null;
+    }
+
+    /**
+     * Serializes a token to a String containing an encrypted representation of the token
+     */
+    public String getUserTokenString(UserToken userToken) throws IOException, GeneralSecurityException {
+        // we know that the minimum length is larger than the default of the ByteArrayOutputStream so set the size to this explicitly
+        try (ByteArrayOutputStream os = new ByteArrayOutputStream(MINIMUM_BASE64_BYTES);
+             OutputStream base64 = Base64.getEncoder().wrap(os);
+             StreamOutput out = new OutputStreamStreamOutput(base64)) {
+            Version.writeVersion(Version.CURRENT, out);
+            out.writeByteArray(salt.bytes);
+            final byte[] initializationVector = getNewInitializationVector();
+            out.writeByteArray(initializationVector);
+            try (CipherOutputStream encryptedOutput = new CipherOutputStream(out, getEncryptionCipher(initializationVector));
+                 StreamOutput encryptedStreamOutput = new OutputStreamStreamOutput(encryptedOutput)) {
+                userToken.writeTo(encryptedStreamOutput);
+                encryptedStreamOutput.close();
+                return new String(os.toByteArray(), StandardCharsets.UTF_8);
+            }
+        }
+    }
+
+    private void ensureEncryptionCiphersSupported() throws NoSuchPaddingException, NoSuchAlgorithmException {
+        Cipher.getInstance(ENCRYPTION_CIPHER);
+        SecretKeyFactory.getInstance(KDF_ALGORITHM);
+    }
+
+    private Cipher getEncryptionCipher(byte[] iv) throws GeneralSecurityException {
+        Cipher cipher = Cipher.getInstance(ENCRYPTION_CIPHER);
+        cipher.init(Cipher.ENCRYPT_MODE, keyCache.get(salt), new GCMParameterSpec(128, iv), secureRandom);
+        cipher.updateAAD(currentVersionBytes);
+        cipher.updateAAD(salt.bytes);
+        return cipher;
+    }
+
+    private Cipher getDecryptionCipher(byte[] iv, SecretKey key, Version version,
+                                       BytesKey salt) throws GeneralSecurityException {
+        Cipher cipher = Cipher.getInstance(ENCRYPTION_CIPHER);
+        cipher.init(Cipher.DECRYPT_MODE, key, new GCMParameterSpec(128, iv), secureRandom);
+        cipher.updateAAD(ByteBuffer.allocate(4).putInt(version.id).array());
+        cipher.updateAAD(salt.bytes);
+        return cipher;
+    }
+
+    private byte[] getNewInitializationVector() {
+        final byte[] initializationVector = new byte[IV_BYTES];
+        secureRandom.nextBytes(initializationVector);
+        return initializationVector;
+    }
+
+    /**
+     * Generates a secret key based off of the provided password and salt.
+     * This method is computationally expensive.
+     */
+    static SecretKey computeSecretKey(char[] rawPassword, byte[] salt)
+            throws NoSuchAlgorithmException, InvalidKeySpecException {
+        SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance(KDF_ALGORITHM);
+        PBEKeySpec keySpec = new PBEKeySpec(rawPassword, salt, ITERATIONS, 128);
+        SecretKey tmp = secretKeyFactory.generateSecret(keySpec);
+        return new SecretKeySpec(tmp.getEncoded(), "AES");
+    }
+
+    /**
+     * Creates an {@link ElasticsearchSecurityException} that indicates the token was expired. It
+     * is up to the client to re-authenticate and obtain a new token
+     */
+    private static ElasticsearchSecurityException expiredTokenException() {
+        ElasticsearchSecurityException e =
+                new ElasticsearchSecurityException("token expired", RestStatus.UNAUTHORIZED);
+        e.addHeader("WWW-Authenticate", EXPIRED_TOKEN_WWW_AUTH_VALUE);
+        return e;
+    }
+
+    /**
+     * Creates an {@link ElasticsearchSecurityException} that indicates the token was expired. It
+     * is up to the client to re-authenticate and obtain a new token
+     */
+    private static ElasticsearchSecurityException malformedTokenException() {
+        ElasticsearchSecurityException e =
+                new ElasticsearchSecurityException("token malformed", RestStatus.UNAUTHORIZED);
+        e.addHeader("WWW-Authenticate", MALFORMED_TOKEN_WWW_AUTH_VALUE);
+        return e;
+    }
+
+    boolean isExpiredTokenException(ElasticsearchSecurityException e) {
+        final List<String> headers = e.getHeader("WWW-Authenticate");
+        return headers != null && headers.stream().anyMatch(EXPIRED_TOKEN_WWW_AUTH_VALUE::equals);
+    }
+
+    private class KeyComputingRunnable extends AbstractRunnable {
+
+        private final StreamInput in;
+        private final Version version;
+        private final BytesKey decodedSalt;
+        private final ActionListener<UserToken> listener;
+        private final byte[] iv;
+
+        KeyComputingRunnable(StreamInput input, byte[] iv, Version version, BytesKey decodedSalt, ActionListener<UserToken> listener) {
+            this.in = input;
+            this.version = version;
+            this.decodedSalt = decodedSalt;
+            this.listener = listener;
+            this.iv = iv;
+        }
+
+        @Override
+        protected void doRun() {
+            try {
+                final SecretKey computedKey = keyCache.computeIfAbsent(decodedSalt, (salt) -> {
+                    try (SecureString closeableChars = tokenPassphrase.clone()) {
+                        return computeSecretKey(closeableChars.getChars(), decodedSalt.bytes);
+                    }
+                });
+                decryptToken(in, getDecryptionCipher(iv, computedKey, version, decodedSalt), version, listener);
+            } catch (ExecutionException e) {
+                if (e.getCause() != null &&
+                        (e.getCause() instanceof GeneralSecurityException || e.getCause() instanceof IOException
+                                || e.getCause() instanceof IllegalArgumentException)) {
+                    // this could happen if another realm supports the Bearer token so we should
+                    // see if another realm can use this token!
+                    logger.debug("unable to decode bearer token", e);
+                    listener.onResponse(null);
+                } else {
+                    listener.onFailure(e);
+                }
+            } catch (GeneralSecurityException | IOException e) {
+                logger.debug("unable to decode bearer token", e);
+                listener.onResponse(null);
+            }
+        }
+
+        @Override
+        public void onFailure(Exception e) {
+            listener.onFailure(e);
+        }
+
+        @Override
+        public void onAfter() {
+            IOUtils.closeWhileHandlingException(in);
+        }
+    }
+
+    /**
+     * Simple wrapper around bytes so that it can be used as a cache key. The hashCode is computed
+     * once upon creation and cached.
+     */
+    static class BytesKey {
+
+        final byte[] bytes;
+        private final int hashCode;
+
+        BytesKey(byte[] bytes) {
+            this.bytes = bytes;
+            this.hashCode = StringHelper.murmurhash3_x86_32(bytes, 0, bytes.length, StringHelper.GOOD_FAST_HASH_SEED);
+        }
+
+        @Override
+        public int hashCode() {
+            return hashCode;
+        }
+
+        @Override
+        public boolean equals(Object other) {
+            if (other == null) {
+                return false;
+            }
+            if (other instanceof BytesKey == false) {
+                return false;
+            }
+
+            BytesKey otherBytes = (BytesKey) other;
+            return Arrays.equals(otherBytes.bytes, bytes);
+        }
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/security/authc/UserToken.java

@@ -0,0 +1,93 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.authc;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.util.Objects;
+
+/**
+ * This token is a combination of a {@link Authentication} object with an expiry. This token can be
+ * serialized for use later. Note, if serializing this token to a entity outside of the cluster,
+ * care must be taken to encrypt and validate the serialized bytes or they cannot be trusted.
+ *
+ * Additionally, care must also be used when transporting these tokens as a stolen token can be
+ * used by an adversary to gain access. For this reason, TLS must be enabled for these tokens to
+ * be used.
+ */
+public final class UserToken implements Writeable {
+
+    private final Version version;
+    private final String id;
+    private final Authentication authentication;
+    private final ZonedDateTime expirationTime;
+
+    /**
+     * Create a new token with an autogenerated id
+     */
+    UserToken(Authentication authentication, ZonedDateTime expirationTime) {
+        this.version = Version.CURRENT;
+        this.id = UUIDs.base64UUID();
+        this.authentication = Objects.requireNonNull(authentication);
+        this.expirationTime = Objects.requireNonNull(expirationTime);
+    }
+
+    /**
+     * Creates a new token based on the values from the stream
+     */
+    UserToken(StreamInput input) throws IOException {
+        this.version = input.getVersion();
+        this.id = input.readString();
+        this.authentication = new Authentication(input);
+        this.expirationTime = Instant.ofEpochSecond(input.readLong(), input.readInt())
+                .atZone(ZoneId.of(input.readString()));
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(id);
+        authentication.writeTo(out);
+        out.writeLong(expirationTime.toEpochSecond());
+        out.writeInt(expirationTime.getNano());
+        out.writeString(expirationTime.getZone().getId());
+    }
+
+    /**
+     * Get the authentication
+     */
+    Authentication getAuthentication() {
+        return authentication;
+    }
+
+    /**
+     * Get the expiration time
+     */
+    ZonedDateTime getExpirationTime() {
+        return expirationTime;
+    }
+
+    /**
+     * The ID of this token
+     */
+    String getId() {
+        return id;
+    }
+
+    /**
+     * The version of the node this token was created on
+     */
+    Version getVersion() {
+        return version;
+    }
+}

plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmMigrator.java

@@ -70,7 +70,7 @@ public class NativeRealmMigrator implements IndexLifecycleManager.IndexDataMigra
      *                 {@link ActionListener#onResponse(Object) onResponse(true)} if an upgrade is performed, or
      *                 {@link ActionListener#onResponse(Object) onResponse(false)} if no upgrade was required.
      * @see SecurityLifecycleService#securityIndexMappingAndTemplateSufficientToRead(ClusterState, Logger)
-     * @see SecurityLifecycleService#canWriteToSecurityIndex
+     * @see SecurityLifecycleService#isSecurityIndexWriteable
      * @see IndexLifecycleManager#mappingVersion
      */
     @Override

plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java

@@ -180,7 +180,7 @@ public class NativeUsersStore extends AbstractComponent {
         if (isTribeNode) {
             listener.onFailure(new UnsupportedOperationException("users may not be created or modified using a tribe node"));
             return;
-        } else if (securityLifecycleService.canWriteToSecurityIndex() == false) {
+        } else if (securityLifecycleService.isSecurityIndexWriteable() == false) {
             listener.onFailure(new IllegalStateException("password cannot be changed as user service cannot write until template and " +
                     "mappings are up to date"));
             return;
@@ -253,7 +253,7 @@ public class NativeUsersStore extends AbstractComponent {
         if (isTribeNode) {
             listener.onFailure(new UnsupportedOperationException("users may not be created or modified using a tribe node"));
             return;
-        }  else if (securityLifecycleService.canWriteToSecurityIndex() == false) {
+        }  else if (securityLifecycleService.isSecurityIndexWriteable() == false) {
             listener.onFailure(new IllegalStateException("user cannot be created or changed as the user service cannot write until " +
                     "template and mappings are up to date"));
             return;
@@ -344,7 +344,7 @@ public class NativeUsersStore extends AbstractComponent {
         if (isTribeNode) {
             listener.onFailure(new UnsupportedOperationException("users may not be created or modified using a tribe node"));
             return;
-        }  else if (securityLifecycleService.canWriteToSecurityIndex() == false) {
+        }  else if (securityLifecycleService.isSecurityIndexWriteable() == false) {
             listener.onFailure(new IllegalStateException("enabled status cannot be changed as user service cannot write until template " +
                     "and mappings are up to date"));
             return;
@@ -422,7 +422,7 @@ public class NativeUsersStore extends AbstractComponent {
         if (isTribeNode) {
             listener.onFailure(new UnsupportedOperationException("users may not be deleted using a tribe node"));
             return;
-        }  else if (securityLifecycleService.canWriteToSecurityIndex() == false) {
+        }  else if (securityLifecycleService.isSecurityIndexWriteable() == false) {
             listener.onFailure(new IllegalStateException("user cannot be deleted as user service cannot write until template and " +
                     "mappings are up to date"));
             return;
@@ -470,7 +470,7 @@ public class NativeUsersStore extends AbstractComponent {
     }
 
     void getReservedUserInfo(String username, ActionListener<ReservedUserInfo> listener) {
-        if (!securityLifecycleService.securityIndexExists()) {
+        if (!securityLifecycleService.isSecurityIndexExisting()) {
             listener.onFailure(new IllegalStateException("Attempt to get reserved user info but the security index does not exist"));
             return;
         }

plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java

@@ -205,7 +205,7 @@ public class ReservedRealm extends CachingUsernamePasswordRealm {
         if (userIsDefinedForCurrentSecurityMapping(username) == false) {
             logger.debug("Marking user [{}] as disabled because the security mapping is not at the required version", username);
             listener.onResponse(DISABLED_USER_INFO);
-        } else if (securityLifecycleService.securityIndexExists() == false) {
+        } else if (securityLifecycleService.isSecurityIndexExisting() == false) {
             listener.onResponse(DEFAULT_USER_INFO);
         } else {
             nativeUsersStore.getReservedUserInfo(username, ActionListener.wrap((userInfo) -> {

plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java

@@ -130,7 +130,7 @@ public class NativeRolesStore extends AbstractComponent {
         if (isTribeNode) {
             listener.onFailure(new UnsupportedOperationException("roles may not be deleted using a tribe node"));
             return;
-        } else if (securityLifecycleService.canWriteToSecurityIndex() == false) {
+        } else if (securityLifecycleService.isSecurityIndexWriteable() == false) {
             listener.onFailure(new IllegalStateException("role cannot be deleted as service cannot write until template and " +
                     "mappings are up to date"));
             return;
@@ -164,7 +164,7 @@ public class NativeRolesStore extends AbstractComponent {
     public void putRole(final PutRoleRequest request, final RoleDescriptor role, final ActionListener<Boolean> listener) {
         if (isTribeNode) {
             listener.onFailure(new UnsupportedOperationException("roles may not be created or modified using a tribe node"));
-        } else if (securityLifecycleService.canWriteToSecurityIndex() == false) {
+        } else if (securityLifecycleService.isSecurityIndexWriteable() == false) {
             listener.onFailure(new IllegalStateException("role cannot be created or modified as service cannot write until template and " +
                     "mappings are up to date"));
         } else if (licenseState.isDocumentAndFieldLevelSecurityAllowed()) {
@@ -203,7 +203,7 @@ public class NativeRolesStore extends AbstractComponent {
 
     public void usageStats(ActionListener<Map<String, Object>> listener) {
         Map<String, Object> usageStats = new HashMap<>();
-        if (securityLifecycleService.securityIndexExists() == false) {
+        if (securityLifecycleService.isSecurityIndexExisting() == false) {
             usageStats.put("size", 0L);
             usageStats.put("fls", false);
             usageStats.put("dls", false);
@@ -261,7 +261,7 @@ public class NativeRolesStore extends AbstractComponent {
     }
 
     private void getRoleDescriptor(final String roleId, ActionListener<RoleDescriptor> roleActionListener) {
-        if (securityLifecycleService.securityIndexExists() == false) {
+        if (securityLifecycleService.isSecurityIndexExisting() == false) {
             roleActionListener.onResponse(null);
         } else {
             executeGetRoleRequest(roleId, new ActionListener<GetResponse>() {

plugin/src/main/java/org/elasticsearch/xpack/security/client/SecurityClient.java

@@ -30,6 +30,14 @@ import org.elasticsearch.xpack.security.action.role.PutRoleAction;
 import org.elasticsearch.xpack.security.action.role.PutRoleRequest;
 import org.elasticsearch.xpack.security.action.role.PutRoleRequestBuilder;
 import org.elasticsearch.xpack.security.action.role.PutRoleResponse;
+import org.elasticsearch.xpack.security.action.token.CreateTokenAction;
+import org.elasticsearch.xpack.security.action.token.CreateTokenRequest;
+import org.elasticsearch.xpack.security.action.token.CreateTokenRequestBuilder;
+import org.elasticsearch.xpack.security.action.token.CreateTokenResponse;
+import org.elasticsearch.xpack.security.action.token.InvalidateTokenAction;
+import org.elasticsearch.xpack.security.action.token.InvalidateTokenRequest;
+import org.elasticsearch.xpack.security.action.token.InvalidateTokenRequestBuilder;
+import org.elasticsearch.xpack.security.action.token.InvalidateTokenResponse;
 import org.elasticsearch.xpack.security.action.user.ChangePasswordAction;
 import org.elasticsearch.xpack.security.action.user.ChangePasswordRequest;
 import org.elasticsearch.xpack.security.action.user.ChangePasswordRequestBuilder;
@@ -230,4 +238,20 @@ public class SecurityClient {
     public void putRole(PutRoleRequest request, ActionListener<PutRoleResponse> listener) {
         client.execute(PutRoleAction.INSTANCE, request, listener);
     }
+
+    public CreateTokenRequestBuilder prepareCreateToken() {
+        return new CreateTokenRequestBuilder(client);
+    }
+
+    public void createToken(CreateTokenRequest request, ActionListener<CreateTokenResponse> listener) {
+        client.execute(CreateTokenAction.INSTANCE, request, listener);
+    }
+
+    public InvalidateTokenRequestBuilder prepareInvalidateToken(String token) {
+        return new InvalidateTokenRequestBuilder(client).setTokenString(token);
+    }
+
+    public void invalidateToken(InvalidateTokenRequest request, ActionListener<InvalidateTokenResponse> listener) {
+        client.execute(InvalidateTokenAction.INSTANCE, request, listener);
+    }
 }

plugin/src/main/resources/security-index-template.json

@@ -125,6 +125,21 @@
           "type": "boolean"
         }
       }
+    },
+    "doc": {
+      "_meta": {
+        "security-version": "${security.template.version}"
+      },
+      "dynamic": "strict",
+      "properties": {
+        "doc_type": {
+          "type" : "keyword"
+        },
+        "expiration_time": {
+          "type": "date",
+          "format": "date_time"
+        }
+      }
     }
   }
 }

plugin/src/test/java/org/elasticsearch/xpack/TimeWarpedXPackPlugin.java

@@ -12,6 +12,7 @@ import org.elasticsearch.xpack.watcher.test.TimeWarpedWatcher;
 
 import javax.security.auth.DestroyFailedException;
 import java.io.IOException;
+import java.security.GeneralSecurityException;
 import java.security.KeyStoreException;
 import java.security.NoSuchAlgorithmException;
 import java.security.UnrecoverableKeyException;
@@ -21,8 +22,8 @@ import java.time.Clock;
 public class TimeWarpedXPackPlugin extends XPackPlugin {
     private final ClockMock clock = new ClockMock();
 
-    public TimeWarpedXPackPlugin(Settings settings) throws IOException, CertificateException, UnrecoverableKeyException,
+    public TimeWarpedXPackPlugin(Settings settings) throws IOException,
-            NoSuchAlgorithmException, KeyStoreException, DestroyFailedException, OperatorCreationException {
+            DestroyFailedException, OperatorCreationException, GeneralSecurityException {
         super(settings);
         watcher = new TimeWarpedWatcher(settings);
     }

plugin/src/test/java/org/elasticsearch/xpack/security/SecurityLifecycleServiceTests.java

@@ -223,7 +223,7 @@ public class SecurityLifecycleServiceTests extends ESTestCase {
     }
 
     private void checkMappingUpdateWorkCorrectly(ClusterState.Builder clusterStateBuilder, Version expectedOldVersion) {
-        final int expectedNumberOfListeners = 3; // we have three types in the mapping
+        final int expectedNumberOfListeners = 4; // we have four types in the mapping
 
         AtomicReference<Version> migratorVersionRef = new AtomicReference<>(null);
         AtomicReference<ActionListener<Boolean>> migratorListenerRef = new AtomicReference<>(null);

plugin/src/test/java/org/elasticsearch/xpack/security/SecuritySettingsTests.java

@@ -149,7 +149,8 @@ public class SecuritySettingsTests extends ESTestCase {
             assertThat(e.getMessage(), not(containsString(IndexAuditTrail.INDEX_NAME_PREFIX)));
         }
 
-        Security.validateAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".security").build());
+        Security.validateAutoCreateIndex(Settings.builder()
+                .putArray("action.auto_create_index", ".security", ".security-invalidated-tokens").build());
         Security.validateAutoCreateIndex(Settings.builder().put("action.auto_create_index", "*s*").build());
         Security.validateAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".s*").build());
 
@@ -169,7 +170,7 @@ public class SecuritySettingsTests extends ESTestCase {
         }
 
         Security.validateAutoCreateIndex(Settings.builder()
-                        .put("action.auto_create_index", ".security")
+                        .putArray("action.auto_create_index", ".security", ".security-invalidated-tokens")
                         .put(XPackSettings.AUDIT_ENABLED.getKey(), true)
                         .build());
 
@@ -186,7 +187,7 @@ public class SecuritySettingsTests extends ESTestCase {
         }
 
         Security.validateAutoCreateIndex(Settings.builder()
-                .put("action.auto_create_index", ".security_audit_log*,.security")
+                .put("action.auto_create_index", ".security_audit_log*,.security,.security-invalidated-tokens")
                 .put(XPackSettings.AUDIT_ENABLED.getKey(), true)
                 .put(Security.AUDIT_OUTPUTS_SETTING.getKey(), randomFrom("index", "logfile,index"))
                 .build());

plugin/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java

@@ -12,6 +12,7 @@ import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
+import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -69,7 +70,9 @@ public class SecurityTests extends ESTestCase {
         allowedSettings.addAll(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         ClusterSettings clusterSettings = new ClusterSettings(settings, allowedSettings);
         when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
-        return security.createComponents(null, threadPool, clusterService, mock(ResourceWatcherService.class), Arrays.asList(extensions));
+        InternalClient client = new InternalClient(Settings.EMPTY, threadPool, mock(Client.class));
+        when(threadPool.relativeTimeInMillis()).thenReturn(1L);
+        return security.createComponents(client, threadPool, clusterService, mock(ResourceWatcherService.class), Arrays.asList(extensions));
     }
 
     private <T> T findComponent(Class<T> type, Collection<Object> components) {

plugin/src/test/java/org/elasticsearch/xpack/security/TokenPassphraseBootstrapCheckTests.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.XPackSettings;
+import org.elasticsearch.xpack.security.TokenPassphraseBootstrapCheck;
+import org.elasticsearch.xpack.security.authc.TokenService;
+
+import static org.elasticsearch.xpack.security.TokenPassphraseBootstrapCheck.MINIMUM_PASSPHRASE_LENGTH;
+
+public class TokenPassphraseBootstrapCheckTests extends ESTestCase {
+
+    public void testTokenPassphraseCheck() throws Exception {
+        assertTrue(new TokenPassphraseBootstrapCheck(Settings.EMPTY).check());
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
+        assertTrue(new TokenPassphraseBootstrapCheck(settings).check());
+
+        secureSettings.setString(TokenService.TOKEN_PASSPHRASE.getKey(), randomAlphaOfLengthBetween(MINIMUM_PASSPHRASE_LENGTH, 30));
+        assertFalse(new TokenPassphraseBootstrapCheck(settings).check());
+
+        secureSettings.setString(TokenService.TOKEN_PASSPHRASE.getKey(), TokenService.DEFAULT_PASSPHRASE);
+        assertTrue(new TokenPassphraseBootstrapCheck(settings).check());
+
+        secureSettings.setString(TokenService.TOKEN_PASSPHRASE.getKey(), randomAlphaOfLengthBetween(1, MINIMUM_PASSPHRASE_LENGTH - 1));
+        assertTrue(new TokenPassphraseBootstrapCheck(settings).check());
+    }
+
+    public void testTokenPassphraseCheckServiceDisabled() throws Exception {
+        Settings settings = Settings.builder().put(XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey(), false).build();
+        assertFalse(new TokenPassphraseBootstrapCheck(settings).check());
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        settings = Settings.builder().put(settings).setSecureSettings(secureSettings).build();
+        assertFalse(new TokenPassphraseBootstrapCheck(settings).check());
+
+        secureSettings.setString(TokenService.TOKEN_PASSPHRASE.getKey(), randomAlphaOfLengthBetween(1, 30));
+        assertFalse(new TokenPassphraseBootstrapCheck(settings).check());
+
+        secureSettings.setString(TokenService.TOKEN_PASSPHRASE.getKey(), TokenService.DEFAULT_PASSPHRASE);
+        assertFalse(new TokenPassphraseBootstrapCheck(settings).check());
+    }
+}

plugin/src/test/java/org/elasticsearch/xpack/security/TokenSSLBootsrapCheckTests.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.XPackSettings;
+import org.elasticsearch.xpack.security.TokenSSLBootstrapCheck;
+
+public class TokenSSLBootsrapCheckTests extends ESTestCase {
+
+    public void testTokenSSLBootstrapCheck() {
+        Settings settings = Settings.EMPTY;
+        assertTrue(new TokenSSLBootstrapCheck(settings).check());
+
+        settings = Settings.builder().put(XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey(), false).build();
+        assertFalse(new TokenSSLBootstrapCheck(settings).check());
+
+        settings = Settings.builder().put(XPackSettings.HTTP_SSL_ENABLED.getKey(), true).build();
+        assertFalse(new TokenSSLBootstrapCheck(settings).check());
+
+        settings = Settings.builder().put(XPackSettings.HTTP_SSL_ENABLED.getKey(), false).build();
+        assertTrue(new TokenSSLBootstrapCheck(settings).check());
+    }
+}

plugin/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java

@@ -74,7 +74,7 @@ public class TransportGetUsersActionTests extends ESTestCase {
     public void testAnonymousUser() {
         NativeUsersStore usersStore = mock(NativeUsersStore.class);
         SecurityLifecycleService securityLifecycleService = mock(SecurityLifecycleService.class);
-        when(securityLifecycleService.securityIndexAvailable()).thenReturn(true);
+        when(securityLifecycleService.isSecurityIndexAvailable()).thenReturn(true);
         AnonymousUser anonymousUser = new AnonymousUser(settings);
         ReservedRealm reservedRealm =
             new ReservedRealm(mock(Environment.class), settings, usersStore, anonymousUser, securityLifecycleService, new ThreadContext(Settings.EMPTY));
@@ -144,7 +144,7 @@ public class TransportGetUsersActionTests extends ESTestCase {
     public void testReservedUsersOnly() {
         NativeUsersStore usersStore = mock(NativeUsersStore.class);
         SecurityLifecycleService securityLifecycleService = mock(SecurityLifecycleService.class);
-        when(securityLifecycleService.securityIndexAvailable()).thenReturn(true);
+        when(securityLifecycleService.isSecurityIndexAvailable()).thenReturn(true);
         when(securityLifecycleService.checkSecurityMappingVersion(any())).thenReturn(true);
 
         ReservedRealmTests.mockGetAllReservedUserInfo(usersStore, Collections.emptyMap());
@@ -190,7 +190,7 @@ public class TransportGetUsersActionTests extends ESTestCase {
                 Arrays.asList(new User("jane"), new User("fred")), randomUsers());
         NativeUsersStore usersStore = mock(NativeUsersStore.class);
         SecurityLifecycleService securityLifecycleService = mock(SecurityLifecycleService.class);
-        when(securityLifecycleService.securityIndexAvailable()).thenReturn(true);
+        when(securityLifecycleService.isSecurityIndexAvailable()).thenReturn(true);
         ReservedRealmTests.mockGetAllReservedUserInfo(usersStore, Collections.emptyMap());
         ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(settings),
                 securityLifecycleService, new ThreadContext(Settings.EMPTY));

plugin/src/test/java/org/elasticsearch/xpack/security/action/user/TransportPutUserActionTests.java

@@ -116,7 +116,7 @@ public class TransportPutUserActionTests extends ESTestCase {
     public void testReservedUser() {
         NativeUsersStore usersStore = mock(NativeUsersStore.class);
         SecurityLifecycleService securityLifecycleService = mock(SecurityLifecycleService.class);
-        when(securityLifecycleService.securityIndexAvailable()).thenReturn(true);
+        when(securityLifecycleService.isSecurityIndexAvailable()).thenReturn(true);
         ReservedRealmTests.mockGetAllReservedUserInfo(usersStore, Collections.emptyMap());
         Settings settings = Settings.builder().put("path.home", createTempDir()).build();
         ReservedRealm reservedRealm = new ReservedRealm(new Environment(settings), settings, usersStore,

plugin/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java

@@ -6,17 +6,24 @@
 package org.elasticsearch.xpack.security.authc;
 
 import java.io.IOException;
+import java.time.Clock;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.get.GetAction;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.SecureString;
@@ -25,10 +32,15 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
+import org.elasticsearch.threadpool.FixedExecutorBuilder;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportMessage;
+import org.elasticsearch.xpack.security.InternalClient;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.elasticsearch.xpack.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.security.authc.AuthenticationService.Authenticator;
@@ -38,6 +50,7 @@ import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.security.user.AnonymousUser;
 import org.elasticsearch.xpack.security.user.SystemUser;
 import org.elasticsearch.xpack.security.user.User;
+import org.junit.After;
 import org.junit.Before;
 
 import static org.elasticsearch.test.SecurityTestsUtils.assertAuthenticationException;
@@ -77,6 +90,9 @@ public class AuthenticationServiceTests extends ESTestCase {
     private AuthenticationToken token;
     private ThreadPool threadPool;
     private ThreadContext threadContext;
+    private TokenService tokenService;
+    private SecurityLifecycleService lifecycleService;
+    private Client client;
 
     @Before
     public void init() throws Exception {
@@ -102,10 +118,22 @@ public class AuthenticationServiceTests extends ESTestCase {
                 threadContext, mock(ReservedRealm.class), Arrays.asList(firstRealm, secondRealm), Collections.singletonList(firstRealm));
 
         auditTrail = mock(AuditTrailService.class);
-        threadPool = mock(ThreadPool.class);
+        client = mock(Client.class);
-        when(threadPool.getThreadContext()).thenReturn(threadContext);
+        threadPool = new ThreadPool(settings,
+                new FixedExecutorBuilder(settings, TokenService.THREAD_POOL_NAME, 1, 1000, "xpack.security.authc.token.thread_pool"));
+        threadContext = threadPool.getThreadContext();
+        InternalClient internalClient = new InternalClient(Settings.EMPTY, threadPool, client);
+        lifecycleService = mock(SecurityLifecycleService.class);
+        tokenService = new TokenService(settings, Clock.systemUTC(), internalClient, lifecycleService);
         service = new AuthenticationService(settings, realms, auditTrail,
-                new DefaultAuthenticationFailureHandler(), threadPool, new AnonymousUser(settings));
+                new DefaultAuthenticationFailureHandler(), threadPool, new AnonymousUser(settings), tokenService);
+    }
+
+    @After
+    public void shutdownThreadpool() throws InterruptedException {
+        if (threadPool != null) {
+            terminate(threadPool);
+        }
     }
 
     @SuppressWarnings("unchecked")
@@ -345,7 +373,9 @@ public class AuthenticationServiceTests extends ESTestCase {
         final AtomicBoolean completed = new AtomicBoolean(false);
         final SetOnce<Authentication> authRef = new SetOnce<>();
         final SetOnce<String> authHeaderRef = new SetOnce<>();
+        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
             service.authenticate("_action", message, SystemUser.INSTANCE, ActionListener.wrap(authentication -> {
+
                 assertThat(authentication, notNullValue());
                 assertThat(authentication.getUser(), sameInstance(user1));
                 assertThreadContextContainsAuthentication(authentication);
@@ -353,15 +383,18 @@ public class AuthenticationServiceTests extends ESTestCase {
                 authHeaderRef.set(threadContext.getHeader(Authentication.AUTHENTICATION_KEY));
                 setCompletedToTrue(completed);
             }, this::logAndFail));
+        }
         assertTrue(completed.compareAndSet(true, false));
         reset(firstRealm);
 
         // checking authentication from the context
         InternalMessage message1 = new InternalMessage();
-        final ThreadContext threadContext1 = new ThreadContext(Settings.EMPTY);
+        ThreadPool threadPool1 = new TestThreadPool("testAutheticateTransportContextAndHeader1");
-        when(threadPool.getThreadContext()).thenReturn(threadContext1);
+        try {
+            ThreadContext threadContext1 = threadPool1.getThreadContext();
             service = new AuthenticationService(Settings.EMPTY, realms, auditTrail,
-                new DefaultAuthenticationFailureHandler(), threadPool, new AnonymousUser(Settings.EMPTY));
+                    new DefaultAuthenticationFailureHandler(), threadPool1, new AnonymousUser(Settings.EMPTY), tokenService);
+
 
             threadContext1.putTransient(Authentication.AUTHENTICATION_KEY, authRef.get());
             threadContext1.putHeader(Authentication.AUTHENTICATION_KEY, authHeaderRef.get());
@@ -373,12 +406,18 @@ public class AuthenticationServiceTests extends ESTestCase {
             assertTrue(completed.compareAndSet(true, false));
             verifyZeroInteractions(firstRealm);
             reset(firstRealm);
+        } finally {
+            terminate(threadPool1);
+        }
 
         // checking authentication from the user header
-        ThreadContext threadContext2 = new ThreadContext(Settings.EMPTY);
+        ThreadPool threadPool2 = new TestThreadPool("testAutheticateTransportContextAndHeader2");
-        when(threadPool.getThreadContext()).thenReturn(threadContext2);
+        try {
+            ThreadContext threadContext2 = threadPool2.getThreadContext();
+            final String header;
+            try (ThreadContext.StoredContext ignore = threadContext2.stashContext()) {
                 service = new AuthenticationService(Settings.EMPTY, realms, auditTrail,
-                new DefaultAuthenticationFailureHandler(), threadPool, new AnonymousUser(Settings.EMPTY));
+                        new DefaultAuthenticationFailureHandler(), threadPool2, new AnonymousUser(Settings.EMPTY), tokenService);
                 threadContext2.putHeader(Authentication.AUTHENTICATION_KEY, authHeaderRef.get());
 
                 BytesStreamOutput output = new BytesStreamOutput();
@@ -386,10 +425,12 @@ public class AuthenticationServiceTests extends ESTestCase {
                 StreamInput input = output.bytes().streamInput();
                 threadContext2 = new ThreadContext(Settings.EMPTY);
                 threadContext2.readHeaders(input);
+                header = threadContext2.getHeader(Authentication.AUTHENTICATION_KEY);
+            }
 
-        when(threadPool.getThreadContext()).thenReturn(threadContext2);
+            threadPool2.getThreadContext().putHeader(Authentication.AUTHENTICATION_KEY, header);
             service = new AuthenticationService(Settings.EMPTY, realms, auditTrail,
-                new DefaultAuthenticationFailureHandler(), threadPool, new AnonymousUser(Settings.EMPTY));
+                    new DefaultAuthenticationFailureHandler(), threadPool2, new AnonymousUser(Settings.EMPTY), tokenService);
             service.authenticate("_action", new InternalMessage(), SystemUser.INSTANCE, ActionListener.wrap(result -> {
                 assertThat(result, notNullValue());
                 assertThat(result.getUser(), equalTo(user1));
@@ -397,6 +438,9 @@ public class AuthenticationServiceTests extends ESTestCase {
             }, this::logAndFail));
             assertTrue(completed.get());
             verifyZeroInteractions(firstRealm);
+        } finally {
+            terminate(threadPool2);
+        }
     }
 
     public void testAuthenticateTamperedUser() throws Exception {
@@ -412,35 +456,6 @@ public class AuthenticationServiceTests extends ESTestCase {
         }
     }
 
-    public void testAttachIfMissing() throws Exception {
-        User user;
-        if (randomBoolean()) {
-            user = SystemUser.INSTANCE;
-        } else {
-            user = new User("username", "r1", "r2");
-        }
-        assertThat(threadContext.getTransient(Authentication.AUTHENTICATION_KEY), nullValue());
-        assertThat(threadContext.getHeader(Authentication.AUTHENTICATION_KEY), nullValue());
-        service.attachUserIfMissing(user);
-
-        Authentication authentication = threadContext.getTransient(Authentication.AUTHENTICATION_KEY);
-        assertThat(authentication, notNullValue());
-        assertThat(authentication.getUser(), sameInstance((Object) user));
-        assertThat(authentication.getLookedUpBy(), nullValue());
-        assertThat(authentication.getAuthenticatedBy().getName(), is("__attach"));
-        assertThat(authentication.getAuthenticatedBy().getType(), is("__attach"));
-        assertThat(authentication.getAuthenticatedBy().getNodeName(), is("authc_test"));
-        assertThat(threadContext.getHeader(Authentication.AUTHENTICATION_KEY), equalTo((Object) authentication.encode()));
-    }
-
-    public void testAttachIfMissingExists() throws Exception {
-        Authentication authentication = new Authentication(new User("username", "r1", "r2"), new RealmRef("test", "test", "foo"), null);
-        threadContext.putTransient(Authentication.AUTHENTICATION_KEY, authentication);
-        threadContext.putHeader(Authentication.AUTHENTICATION_KEY, authentication.encode());
-        service.attachUserIfMissing(new User("username2", "r3", "r4"));
-        assertThreadContextContainsAuthentication(authentication);
-    }
-
     public void testAnonymousUserRest() throws Exception {
         String username = randomBoolean() ? AnonymousUser.DEFAULT_ANONYMOUS_USERNAME : "user1";
         Settings.Builder builder = Settings.builder()
@@ -451,7 +466,7 @@ public class AuthenticationServiceTests extends ESTestCase {
         Settings settings = builder.build();
         final AnonymousUser anonymousUser = new AnonymousUser(settings);
         service = new AuthenticationService(settings, realms, auditTrail, new DefaultAuthenticationFailureHandler(),
-                threadPool, anonymousUser);
+                threadPool, anonymousUser, tokenService);
         RestRequest request = new FakeRestRequest();
 
         Authentication result = authenticateBlocking(request);
@@ -469,7 +484,7 @@ public class AuthenticationServiceTests extends ESTestCase {
                 .build();
         final AnonymousUser anonymousUser = new AnonymousUser(settings);
         service = new AuthenticationService(settings, realms, auditTrail,
-                new DefaultAuthenticationFailureHandler(), threadPool, anonymousUser);
+                new DefaultAuthenticationFailureHandler(), threadPool, anonymousUser, tokenService);
         InternalMessage message = new InternalMessage();
 
         Authentication result = authenticateBlocking("_action", message, null);
@@ -484,7 +499,7 @@ public class AuthenticationServiceTests extends ESTestCase {
                 .build();
         final AnonymousUser anonymousUser = new AnonymousUser(settings);
         service = new AuthenticationService(settings, realms, auditTrail,
-                new DefaultAuthenticationFailureHandler(), threadPool, anonymousUser);
+                new DefaultAuthenticationFailureHandler(), threadPool, anonymousUser, tokenService);
 
         InternalMessage message = new InternalMessage();
 
@@ -764,6 +779,78 @@ public class AuthenticationServiceTests extends ESTestCase {
         assertAuthenticationException(e);
     }
 
+    public void testAuthenticateWithToken() throws Exception {
+        User user = new User("_username", "r1");
+        final AtomicBoolean completed = new AtomicBoolean(false);
+        final Authentication expected = new Authentication(user, new RealmRef("realm", "custom", "node"), null);
+        String token = tokenService.getUserTokenString(tokenService.createUserToken(expected));
+        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+            threadContext.putHeader("Authorization", "Bearer " + token);
+            service.authenticate("_action", message, null, ActionListener.wrap(result -> {
+                assertThat(result, notNullValue());
+                assertThat(result.getUser(), is(user));
+                assertThat(result.getLookedUpBy(), is(nullValue()));
+                assertThat(result.getAuthenticatedBy(), is(notNullValue()));
+                assertEquals(expected, result);
+                setCompletedToTrue(completed);
+            }, this::logAndFail));
+        }
+        assertTrue(completed.get());
+        verify(auditTrail).authenticationSuccess("realm", user, "_action", message);
+        verifyNoMoreInteractions(auditTrail);
+    }
+
+    public void testInvalidToken() throws Exception {
+        final User user = new User("_username", "r1");
+        when(firstRealm.token(threadContext)).thenReturn(token);
+        when(firstRealm.supports(token)).thenReturn(true);
+        mockAuthenticate(firstRealm, token, user);
+        final int numBytes = randomIntBetween(TokenService.MINIMUM_BYTES, TokenService.MINIMUM_BYTES + 32);
+        final byte[] randomBytes = new byte[numBytes];
+        random().nextBytes(randomBytes);
+        final CountDownLatch latch = new CountDownLatch(1);
+        final Authentication expected = new Authentication(user, new RealmRef(firstRealm.name(), firstRealm.type(), "authc_test"), null);
+        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+            threadContext.putHeader("Authorization", "Bearer " + Base64.getEncoder().encodeToString(randomBytes));
+            service.authenticate("_action", message, null, ActionListener.wrap(result -> {
+                assertThat(result, notNullValue());
+                assertThat(result.getUser(), is(user));
+                assertThat(result.getLookedUpBy(), is(nullValue()));
+                assertThat(result.getAuthenticatedBy(), is(notNullValue()));
+                assertThreadContextContainsAuthentication(result);
+                assertEquals(expected, result);
+                latch.countDown();
+            }, this::logAndFail));
+        }
+
+        // we need to use a latch here because the key computation goes async on another thread!
+        latch.await();
+        verify(auditTrail).authenticationSuccess(firstRealm.name(), user, "_action", message);
+        verifyNoMoreInteractions(auditTrail);
+    }
+
+    public void testExpiredToken() throws Exception {
+        User user = new User("_username", "r1");
+        final Authentication expected = new Authentication(user, new RealmRef("realm", "custom", "node"), null);
+        String token = tokenService.getUserTokenString(tokenService.createUserToken(expected));
+        when(lifecycleService.isSecurityIndexAvailable()).thenReturn(true);
+        doAnswer(invocationOnMock -> {
+            ActionListener<GetResponse> listener = (ActionListener<GetResponse>) invocationOnMock.getArguments()[2];
+            GetResponse response = mock(GetResponse.class);
+            when(response.isExists()).thenReturn(true);
+            listener.onResponse(response);
+            return Void.TYPE;
+        }).when(client).execute(eq(GetAction.INSTANCE), any(GetRequest.class), any(ActionListener.class));
+
+        try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
+            threadContext.putHeader("Authorization", "Bearer " + token);
+            ElasticsearchSecurityException e =
+                    expectThrows(ElasticsearchSecurityException.class, () -> authenticateBlocking("_action", message, null));
+            assertEquals(RestStatus.UNAUTHORIZED, e.status());
+            assertEquals("token expired", e.getMessage());
+        }
+    }
+
     private static class InternalMessage extends TransportMessage {
     }
 

plugin/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java

@@ -0,0 +1,110 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.authc;
+
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.settings.SecureString;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.test.SecurityIntegTestCase;
+import org.elasticsearch.test.SecuritySettingsSource;
+import org.elasticsearch.xpack.security.action.token.CreateTokenResponse;
+import org.elasticsearch.xpack.security.action.token.InvalidateTokenResponse;
+import org.elasticsearch.xpack.security.client.SecurityClient;
+import org.junit.After;
+
+import java.time.Instant;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+
+public class TokenAuthIntegTests extends SecurityIntegTestCase {
+
+    @Override
+    public Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+                .put(super.nodeSettings(nodeOrdinal))
+                // turn down token expiration interval and crank up the deletion interval
+                .put(TokenService.TOKEN_EXPIRATION.getKey(), TimeValue.timeValueSeconds(1L))
+                .put(TokenService.DELETE_INTERVAL.getKey(), TimeValue.timeValueSeconds(1L))
+                .build();
+    }
+
+    public void testExpiredTokensDeletedAfterExpiration() throws Exception {
+        final Client client = internalClient();
+        SecurityClient securityClient = new SecurityClient(client);
+        CreateTokenResponse response = securityClient.prepareCreateToken()
+                .setGrantType("password")
+                .setUsername(SecuritySettingsSource.DEFAULT_USER_NAME)
+                .setPassword(new SecureString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))
+                .get();
+        Instant created = Instant.now();
+
+        InvalidateTokenResponse invalidateResponse = securityClient.prepareInvalidateToken(response.getTokenString()).get();
+        assertTrue(invalidateResponse.isCreated());
+        assertBusy(() -> {
+            SearchResponse searchResponse = client.prepareSearch(TokenService.INDEX_NAME)
+                    .setSource(SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("doc_type", TokenService.DOC_TYPE)))
+                    .setSize(0)
+                    .setTerminateAfter(1)
+                    .get();
+            assertThat(searchResponse.getHits().getTotalHits(), greaterThan(0L));
+        });
+
+        AtomicBoolean deleteTriggered = new AtomicBoolean(false);
+        assertBusy(() -> {
+            assertTrue(Instant.now().isAfter(created.plusSeconds(1L).plusMillis(500L)));
+            if (deleteTriggered.compareAndSet(false, true)) {
+                // invalidate a invalid token... doesn't matter that it is bad... we just want this action to trigger the deletion
+                try {
+                    securityClient.prepareInvalidateToken("fooobar").execute().actionGet();
+                } catch (ElasticsearchSecurityException e) {
+                    assertEquals("token malformed", e.getMessage());
+                }
+            }
+            client.admin().indices().prepareRefresh(TokenService.INDEX_NAME).get();
+            SearchResponse searchResponse = client.prepareSearch(TokenService.INDEX_NAME)
+                    .setSource(SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("doc_type", TokenService.DOC_TYPE)))
+                    .setSize(0)
+                    .setTerminateAfter(1)
+                    .get();
+            assertThat(searchResponse.getHits().getTotalHits(), equalTo(0L));
+        }, 30, TimeUnit.SECONDS);
+    }
+
+    public void testExpireMultipleTimes() {
+        CreateTokenResponse response = securityClient().prepareCreateToken()
+                .setGrantType("password")
+                .setUsername(SecuritySettingsSource.DEFAULT_USER_NAME)
+                .setPassword(new SecureString(SecuritySettingsSource.DEFAULT_PASSWORD.toCharArray()))
+                .get();
+        Instant created = Instant.now();
+
+        InvalidateTokenResponse invalidateResponse = securityClient().prepareInvalidateToken(response.getTokenString()).get();
+        // if the token is expired then the API will return false for created so we need to handle that
+        final boolean correctResponse = invalidateResponse.isCreated() || created.plusSeconds(1L).isBefore(Instant.now());
+        assertTrue(correctResponse);
+        assertFalse(securityClient().prepareInvalidateToken(response.getTokenString()).get().isCreated());
+    }
+
+    @After
+    public void wipeSecurityIndex() {
+        try {
+            // this is a hack to clean up the .security index since only superusers can delete it and the default test user is not a
+            // superuser since the role used there is a file based role since we cannot guarantee the superuser role is always available
+            internalClient().admin().indices().prepareDelete(TokenService.INDEX_NAME).get();
+        } catch (IndexNotFoundException e) {
+            logger.warn("securirty index does not exist", e);
+        }
+    }
+}

plugin/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java

@@ -0,0 +1,294 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.authc;
+
+import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.get.GetAction;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.settings.MockSecureSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.EqualsHashCodeTestUtils;
+import org.elasticsearch.threadpool.FixedExecutorBuilder;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.XPackSettings;
+import org.elasticsearch.xpack.security.InternalClient;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
+import org.elasticsearch.xpack.security.authc.Authentication.RealmRef;
+import org.elasticsearch.xpack.security.authc.TokenService.BytesKey;
+import org.elasticsearch.xpack.security.user.User;
+import org.elasticsearch.xpack.support.clock.ClockMock;
+import org.junit.After;
+import org.junit.Before;
+
+import javax.crypto.SecretKey;
+import java.time.Clock;
+import java.util.Base64;
+
+import static org.elasticsearch.repositories.ESBlobStoreTestCase.randomBytes;
+import static org.hamcrest.Matchers.containsString;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TokenServiceTests extends ESTestCase {
+
+    private InternalClient internalClient;
+    private ThreadPool threadPool;
+    private Client client;
+    private SecurityLifecycleService lifecycleService;
+
+    @Before
+    public void setupClient() {
+        client = mock(Client.class);
+        Settings settings = Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "TokenServiceTests").build();
+        threadPool = new ThreadPool(settings,
+                new FixedExecutorBuilder(settings, TokenService.THREAD_POOL_NAME, 1, 1000, "xpack.security.authc.token.thread_pool"));
+        internalClient = new InternalClient(settings, threadPool, client);
+        lifecycleService = mock(SecurityLifecycleService.class);
+        when(lifecycleService.isSecurityIndexWriteable()).thenReturn(true);
+        doAnswer(invocationOnMock -> {
+            ActionListener<GetResponse> listener = (ActionListener<GetResponse>) invocationOnMock.getArguments()[2];
+            GetResponse response = mock(GetResponse.class);
+            when(response.isExists()).thenReturn(false);
+            listener.onResponse(response);
+            return Void.TYPE;
+        }).when(client).execute(eq(GetAction.INSTANCE), any(GetRequest.class), any(ActionListener.class));
+        when(client.threadPool()).thenReturn(threadPool);
+    }
+
+    @After
+    public void shutdownThreadpool() throws InterruptedException {
+        terminate(threadPool);
+    }
+
+    public void testAttachAndGetToken() throws Exception {
+        TokenService tokenService = new TokenService(Settings.EMPTY, Clock.systemUTC(), internalClient, lifecycleService);
+        Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null);
+        final UserToken token = tokenService.createUserToken(authentication);
+        assertNotNull(token);
+
+        ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
+        requestContext.putHeader("Authorization", "Bearer " + tokenService.getUserTokenString(token));
+
+        try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
+            PlainActionFuture<UserToken> future = new PlainActionFuture<>();
+            tokenService.getAndValidateToken(requestContext, future);
+            UserToken serialized = future.get();
+            assertEquals(authentication, serialized.getAuthentication());
+        }
+
+        try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
+            // verify a second separate token service with its own salt can also verify
+            TokenService anotherService = new TokenService(Settings.EMPTY, Clock.systemUTC(), internalClient, lifecycleService);
+            PlainActionFuture<UserToken> future = new PlainActionFuture<>();
+            anotherService.getAndValidateToken(requestContext, future);
+            UserToken fromOtherService = future.get();
+            assertEquals(authentication, fromOtherService.getAuthentication());
+        }
+    }
+
+    public void testPassphraseWorks() throws Exception {
+        TokenService tokenService = new TokenService(Settings.EMPTY, Clock.systemUTC(), internalClient, lifecycleService);
+        Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null);
+        final UserToken token = tokenService.createUserToken(authentication);
+        assertNotNull(token);
+
+        ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
+        requestContext.putHeader("Authorization", "Bearer " + tokenService.getUserTokenString(token));
+
+        try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
+            PlainActionFuture<UserToken> future = new PlainActionFuture<>();
+            tokenService.getAndValidateToken(requestContext, future);
+            UserToken serialized = future.get();
+            assertEquals(authentication, serialized.getAuthentication());
+        }
+
+        try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
+            // verify a second separate token service with its own passphrase cannot verify
+            MockSecureSettings secureSettings = new MockSecureSettings();
+            Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
+            secureSettings.setString(TokenService.TOKEN_PASSPHRASE.getKey(), randomAlphaOfLengthBetween(8, 30));
+            TokenService anotherService = new TokenService(settings, Clock.systemUTC(), internalClient, lifecycleService);
+            PlainActionFuture<UserToken> future = new PlainActionFuture<>();
+            anotherService.getAndValidateToken(requestContext, future);
+            assertNull(future.get());
+        }
+    }
+
+    public void testInvalidatedToken() throws Exception {
+        when(lifecycleService.isSecurityIndexAvailable()).thenReturn(true);
+        TokenService tokenService = new TokenService(Settings.EMPTY, Clock.systemUTC(), internalClient, lifecycleService);
+        Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null);
+        final UserToken token = tokenService.createUserToken(authentication);
+        assertNotNull(token);
+        doAnswer(invocationOnMock -> {
+            GetRequest request = (GetRequest) invocationOnMock.getArguments()[1];
+            assertEquals(token.getId(), request.id());
+            ActionListener<GetResponse> listener = (ActionListener<GetResponse>) invocationOnMock.getArguments()[2];
+            GetResponse response = mock(GetResponse.class);
+            when(response.isExists()).thenReturn(true);
+            listener.onResponse(response);
+            return Void.TYPE;
+        }).when(client).execute(eq(GetAction.INSTANCE), any(GetRequest.class), any(ActionListener.class));
+
+        ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
+        requestContext.putHeader("Authorization", "Bearer " + tokenService.getUserTokenString(token));
+
+        try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
+            PlainActionFuture<UserToken> future = new PlainActionFuture<>();
+            tokenService.getAndValidateToken(requestContext, future);
+            ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, future::actionGet);
+            final String headerValue = e.getHeader("WWW-Authenticate").get(0);
+            assertThat(headerValue, containsString("Bearer realm="));
+            assertThat(headerValue, containsString("expired"));
+        }
+    }
+
+    public void testComputeSecretKeyIsConsistent() throws Exception {
+        byte[] saltArr = new byte[32];
+        random().nextBytes(saltArr);
+        SecretKey key = TokenService.computeSecretKey(TokenService.DEFAULT_PASSPHRASE.toCharArray(), saltArr);
+        SecretKey key2 = TokenService.computeSecretKey(TokenService.DEFAULT_PASSPHRASE.toCharArray(), saltArr);
+        assertArrayEquals(key.getEncoded(), key2.getEncoded());
+    }
+
+    public void testTokenExpiry() throws Exception {
+        ClockMock clock = ClockMock.frozen();
+        TokenService tokenService = new TokenService(Settings.EMPTY, clock, internalClient, lifecycleService);
+        Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null);
+        final UserToken token = tokenService.createUserToken(authentication);
+
+        ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
+        requestContext.putHeader("Authorization", "Bearer " + tokenService.getUserTokenString(token));
+
+        try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
+            // the clock is still frozen, so the cookie should be valid
+            PlainActionFuture<UserToken> future = new PlainActionFuture<>();
+            tokenService.getAndValidateToken(requestContext, future);
+            assertEquals(authentication, future.get().getAuthentication());
+        }
+
+        final TimeValue defaultExpiration = TokenService.TOKEN_EXPIRATION.get(Settings.EMPTY);
+        final int fastForwardAmount = randomIntBetween(1, Math.toIntExact(defaultExpiration.getSeconds()));
+        try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
+            // move the clock forward but don't go to expiry
+            clock.fastForwardSeconds(fastForwardAmount);
+            PlainActionFuture<UserToken> future = new PlainActionFuture<>();
+            tokenService.getAndValidateToken(requestContext, future);
+            assertEquals(authentication, future.get().getAuthentication());
+        }
+
+        try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
+            // move to expiry
+            clock.fastForwardSeconds(Math.toIntExact(defaultExpiration.getSeconds()) - fastForwardAmount);
+            PlainActionFuture<UserToken> future = new PlainActionFuture<>();
+            tokenService.getAndValidateToken(requestContext, future);
+            assertEquals(authentication, future.get().getAuthentication());
+        }
+
+        try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
+            // move one second past expiry
+            clock.fastForwardSeconds(1);
+            PlainActionFuture<UserToken> future = new PlainActionFuture<>();
+            tokenService.getAndValidateToken(requestContext, future);
+            ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class, future::actionGet);
+            final String headerValue = e.getHeader("WWW-Authenticate").get(0);
+            assertThat(headerValue, containsString("Bearer realm="));
+            assertThat(headerValue, containsString("expired"));
+        }
+    }
+
+    public void testTokenServiceDisabled() throws Exception {
+        TokenService tokenService = new TokenService(Settings.builder()
+                .put(XPackSettings.TOKEN_SERVICE_ENABLED_SETTING.getKey(), false)
+                .build(),
+                Clock.systemUTC(), internalClient, lifecycleService);
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> tokenService.createUserToken(null));
+        assertEquals("tokens are not enabled", e.getMessage());
+
+        PlainActionFuture<UserToken> future = new PlainActionFuture<>();
+        tokenService.getAndValidateToken(null, future);
+        assertNull(future.get());
+
+        e = expectThrows(IllegalStateException.class, () -> {
+            PlainActionFuture<Boolean> invalidateFuture = new PlainActionFuture<>();
+            tokenService.invalidateToken(null, invalidateFuture);
+            invalidateFuture.actionGet();
+        });
+        assertEquals("tokens are not enabled", e.getMessage());
+    }
+
+    public void testBytesKeyEqualsHashCode() {
+        final int dataLength = randomIntBetween(2, 32);
+        final byte[] data = randomBytes(dataLength);
+        BytesKey bytesKey = new BytesKey(data);
+        EqualsHashCodeTestUtils.checkEqualsAndHashCode(bytesKey, (b) -> new BytesKey(b.bytes.clone()), (b) -> {
+            final byte[] copy = b.bytes.clone();
+            final int randomlyChangedValue = randomIntBetween(0, copy.length - 1);
+            final byte original = copy[randomlyChangedValue];
+            boolean loop;
+            do {
+                byte value = randomByte();
+                if (value == original) {
+                    loop = true;
+                } else {
+                    loop = false;
+                    copy[randomlyChangedValue] = value;
+                }
+            } while (loop);
+            return new BytesKey(copy);
+        });
+    }
+
+    public void testMalformedToken() throws Exception {
+        final int numBytes = randomIntBetween(1, TokenService.MINIMUM_BYTES + 32);
+        final byte[] randomBytes = new byte[numBytes];
+        random().nextBytes(randomBytes);
+        TokenService tokenService = new TokenService(Settings.EMPTY, Clock.systemUTC(), internalClient, lifecycleService);
+
+        ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
+        requestContext.putHeader("Authorization", "Bearer " + Base64.getEncoder().encodeToString(randomBytes));
+
+        try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
+            PlainActionFuture<UserToken> future = new PlainActionFuture<>();
+            tokenService.getAndValidateToken(requestContext, future);
+            assertNull(future.get());
+        }
+    }
+
+    public void testIndexNotAvailable() throws Exception {
+        TokenService tokenService = new TokenService(Settings.EMPTY, Clock.systemUTC(), internalClient, lifecycleService);
+        Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null);
+        final UserToken token = tokenService.createUserToken(authentication);
+        assertNotNull(token);
+
+        ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
+        requestContext.putHeader("Authorization", "Bearer " + tokenService.getUserTokenString(token));
+
+        try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
+            PlainActionFuture<UserToken> future = new PlainActionFuture<>();
+            tokenService.getAndValidateToken(requestContext, future);
+            UserToken serialized = future.get();
+            assertEquals(authentication, serialized.getAuthentication());
+
+            when(lifecycleService.isSecurityIndexAvailable()).thenReturn(false);
+            when(lifecycleService.isSecurityIndexExisting()).thenReturn(true);
+            future = new PlainActionFuture<>();
+            tokenService.getAndValidateToken(requestContext, future);
+            assertNull(future.get());
+        }
+    }
+}

plugin/src/test/java/org/elasticsearch/xpack/security/authc/UserTokenTests.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.security.authc;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.security.authc.Authentication.RealmRef;
+import org.elasticsearch.xpack.security.user.User;
+
+import java.io.IOException;
+import java.time.Clock;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+
+public class UserTokenTests extends ESTestCase {
+
+    public void testSerialization() throws IOException {
+        final Authentication authentication = new Authentication(new User("joe", "a role"), new RealmRef("realm", "native", "node1"), null);
+        final int seconds = randomIntBetween(0, Math.toIntExact(TimeValue.timeValueMinutes(30L).getSeconds()));
+        final ZonedDateTime expirationTime = Clock.systemUTC().instant().atZone(ZoneOffset.UTC).plusSeconds(seconds);
+        final UserToken userToken = new UserToken(authentication, expirationTime);
+
+        BytesStreamOutput output = new BytesStreamOutput();
+        userToken.writeTo(output);
+
+        final UserToken serialized = new UserToken(output.bytes().streamInput());
+        assertEquals(authentication, serialized.getAuthentication());
+        assertEquals(expirationTime, serialized.getExpirationTime());
+    }
+}

plugin/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java

@@ -129,9 +129,9 @@ public class NativeUsersStoreTests extends ESTestCase {
 
     private NativeUsersStore startNativeUsersStore() {
         SecurityLifecycleService securityLifecycleService = mock(SecurityLifecycleService.class);
-        when(securityLifecycleService.securityIndexAvailable()).thenReturn(true);
+        when(securityLifecycleService.isSecurityIndexAvailable()).thenReturn(true);
-        when(securityLifecycleService.securityIndexExists()).thenReturn(true);
+        when(securityLifecycleService.isSecurityIndexExisting()).thenReturn(true);
-        when(securityLifecycleService.canWriteToSecurityIndex()).thenReturn(true);
+        when(securityLifecycleService.isSecurityIndexWriteable()).thenReturn(true);
         final NativeUsersStore nativeUsersStore = new NativeUsersStore(Settings.EMPTY, internalClient, securityLifecycleService);
         return nativeUsersStore;
     }

plugin/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java

@@ -67,7 +67,7 @@ public class ReservedRealmTests extends ESTestCase {
     public void setupMocks() throws Exception {
         usersStore = mock(NativeUsersStore.class);
         securityLifecycleService = mock(SecurityLifecycleService.class);
-        when(securityLifecycleService.securityIndexAvailable()).thenReturn(true);
+        when(securityLifecycleService.isSecurityIndexAvailable()).thenReturn(true);
         when(securityLifecycleService.checkSecurityMappingVersion(any())).thenReturn(true);
         mockGetAllReservedUserInfo(usersStore, Collections.emptyMap());
     }
@@ -89,7 +89,7 @@ public class ReservedRealmTests extends ESTestCase {
         final String principal = expected.principal();
         final boolean securityIndexExists = randomBoolean();
         if (securityIndexExists) {
-            when(securityLifecycleService.securityIndexExists()).thenReturn(true);
+            when(securityLifecycleService.isSecurityIndexExisting()).thenReturn(true);
             doAnswer((i) -> {
                 ActionListener listener = (ActionListener) i.getArguments()[1];
                 listener.onResponse(null);
@@ -104,7 +104,7 @@ public class ReservedRealmTests extends ESTestCase {
         reservedRealm.doAuthenticate(new UsernamePasswordToken(principal, DEFAULT_PASSWORD), listener);
         final User authenticated = listener.actionGet();
         assertEquals(expected, authenticated);
-        verify(securityLifecycleService).securityIndexExists();
+        verify(securityLifecycleService).isSecurityIndexExisting();
         if (securityIndexExists) {
             verify(usersStore).getReservedUserInfo(eq(principal), any(ActionListener.class));
         }
@@ -142,7 +142,7 @@ public class ReservedRealmTests extends ESTestCase {
         Settings settings = Settings.builder().put(XPackSettings.RESERVED_REALM_ENABLED_SETTING.getKey(), false).build();
         final boolean securityIndexExists = randomBoolean();
         if (securityIndexExists) {
-            when(securityLifecycleService.securityIndexExists()).thenReturn(true);
+            when(securityLifecycleService.isSecurityIndexExisting()).thenReturn(true);
         }
         final ReservedRealm reservedRealm =
             new ReservedRealm(mock(Environment.class), settings, usersStore,
@@ -172,7 +172,7 @@ public class ReservedRealmTests extends ESTestCase {
         final User expectedUser = randomFrom(new ElasticUser(enabled), new KibanaUser(enabled), new LogstashSystemUser(enabled));
         final String principal = expectedUser.principal();
         final SecureString newPassword = new SecureString("foobar".toCharArray());
-        when(securityLifecycleService.securityIndexExists()).thenReturn(true);
+        when(securityLifecycleService.isSecurityIndexExisting()).thenReturn(true);
         doAnswer((i) -> {
             ActionListener callback = (ActionListener) i.getArguments()[1];
             callback.onResponse(new ReservedUserInfo(Hasher.BCRYPT.hash(newPassword), enabled, false));
@@ -199,7 +199,7 @@ public class ReservedRealmTests extends ESTestCase {
         assertEquals(expectedUser, authenticated);
         assertThat(expectedUser.enabled(), is(enabled));
 
-        verify(securityLifecycleService, times(2)).securityIndexExists();
+        verify(securityLifecycleService, times(2)).isSecurityIndexExisting();
         verify(usersStore, times(2)).getReservedUserInfo(eq(principal), any(ActionListener.class));
         final ArgumentCaptor<Predicate> predicateCaptor = ArgumentCaptor.forClass(Predicate.class);
         verify(securityLifecycleService, times(2)).checkSecurityMappingVersion(predicateCaptor.capture());
@@ -218,7 +218,7 @@ public class ReservedRealmTests extends ESTestCase {
         reservedRealm.doLookupUser(principal, listener);
         final User user = listener.actionGet();
         assertEquals(expectedUser, user);
-        verify(securityLifecycleService).securityIndexExists();
+        verify(securityLifecycleService).isSecurityIndexExisting();
 
         final ArgumentCaptor<Predicate> predicateCaptor = ArgumentCaptor.forClass(Predicate.class);
         verify(securityLifecycleService).checkSecurityMappingVersion(predicateCaptor.capture());
@@ -252,7 +252,7 @@ public class ReservedRealmTests extends ESTestCase {
                               new AnonymousUser(Settings.EMPTY), securityLifecycleService, new ThreadContext(Settings.EMPTY));
         final User expectedUser = randomFrom(new ElasticUser(true), new KibanaUser(true), new LogstashSystemUser(true));
         final String principal = expectedUser.principal();
-        when(securityLifecycleService.securityIndexExists()).thenReturn(true);
+        when(securityLifecycleService.isSecurityIndexExisting()).thenReturn(true);
         final RuntimeException e = new RuntimeException("store threw");
         doAnswer((i) -> {
             ActionListener callback = (ActionListener) i.getArguments()[1];
@@ -265,7 +265,7 @@ public class ReservedRealmTests extends ESTestCase {
         ElasticsearchSecurityException securityException = expectThrows(ElasticsearchSecurityException.class, future::actionGet);
         assertThat(securityException.getMessage(), containsString("failed to lookup"));
 
-        verify(securityLifecycleService).securityIndexExists();
+        verify(securityLifecycleService).isSecurityIndexExisting();
         verify(usersStore).getReservedUserInfo(eq(principal), any(ActionListener.class));
 
         final ArgumentCaptor<Predicate> predicateCaptor = ArgumentCaptor.forClass(Predicate.class);

plugin/src/test/java/org/elasticsearch/xpack/watcher/test/bench/WatcherExecutorServiceBenchmark.java

@@ -27,6 +27,7 @@ import org.elasticsearch.xpack.watcher.trigger.schedule.ScheduleRegistry;
 
 import javax.security.auth.DestroyFailedException;
 import java.io.IOException;
+import java.security.GeneralSecurityException;
 import java.security.KeyStoreException;
 import java.security.NoSuchAlgorithmException;
 import java.security.UnrecoverableKeyException;
@@ -207,8 +208,8 @@ public class WatcherExecutorServiceBenchmark {
     public static final class XPackBenchmarkPlugin extends XPackPlugin {
 
 
-        public XPackBenchmarkPlugin(Settings settings) throws IOException, CertificateException, UnrecoverableKeyException,
+        public XPackBenchmarkPlugin(Settings settings) throws IOException, DestroyFailedException, OperatorCreationException,
-                NoSuchAlgorithmException, KeyStoreException, DestroyFailedException, OperatorCreationException {
+                GeneralSecurityException {
             super(settings);
             watcher = new BenchmarkWatcher(settings);
         }

plugin/src/test/resources/org/elasticsearch/transport/actions

@@ -93,6 +93,8 @@ cluster:admin/xpack/security/user/has_privileges
 cluster:admin/xpack/security/role/put
 cluster:admin/xpack/security/role/delete
 cluster:admin/xpack/security/role/get
+cluster:admin/xpack/security/token/create
+cluster:admin/xpack/security/token/invalidate
 cluster:admin/xpack/watcher/service
 cluster:admin/xpack/watcher/watch/delete
 cluster:admin/xpack/watcher/watch/execute

plugin/src/test/resources/org/elasticsearch/transport/handlers

@@ -22,6 +22,8 @@ cluster:admin/xpack/security/user/put
 cluster:admin/xpack/security/user/delete
 cluster:admin/xpack/security/user/get
 cluster:admin/xpack/security/user/set_enabled
+cluster:admin/xpack/security/token/create
+cluster:admin/xpack/security/token/invalidate
 indices:admin/analyze[s]
 indices:admin/cache/clear[n]
 indices:admin/forcemerge[n]

qa/smoke-test-graph-with-security/src/test/java/org/elasticsearch/smoketest/GraphWithSecurityIT.java

@@ -52,8 +52,5 @@ public class GraphWithSecurityIT extends ESClientYamlSuiteTestCase {
             .put(ThreadContext.PREFIX + ".Authorization", token)
             .build();
     }
-    
-    
-
 }