Support concurrent refresh of refresh tokens (#39647)

This is a backport of #39631

Co-authored-by: Jay Modi jaymode@users.noreply.github.com

This change adds support for the concurrent refresh of access
tokens as described in #36872
In short it allows subsequent client requests to refresh the same token that
come within a predefined window of 60 seconds to be handled as duplicates
of the original one and thus receive the same response with the same newly
issued access token and refresh token.
In order to support that, two new fields are added in the token document. One
contains the instant (in epoqueMillis) when a given refresh token is refreshed
and one that contains a pointer to the token document that stores the new
refresh token and access token that was created by the original refresh.
A side effect of this change, that was however also a intended enhancement
for the token service, is that we needed to stop encrypting the string
representation of the UserToken while serializing. ( It was necessary as we
correctly used a new IV for every time we encrypted a token in serialization, so
subsequent serializations of the same exact UserToken would produce
different access token strings)

This change also handles the serialization/deserialization BWC logic:

    In mixed clusters we keep creating tokens in the old format and
    consume only old format tokens
    In upgraded clusters, we start creating tokens in the new format but
    still remain able to consume old format tokens (that could have been
    created during the rolling upgrade and are still valid)
    When reading/writing TokensInvalidationResult objects, we take into
    consideration that pre 7.1.0 these contained an integer field that carried
    the attempt count

Resolves #36872

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/TokensInvalidationResult.java

@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.core.security.authc.support;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -32,10 +33,9 @@ public class TokensInvalidationResult implements ToXContentObject, Writeable {
     private final List<String> invalidatedTokens;
     private final List<String> previouslyInvalidatedTokens;
     private final List<ElasticsearchException> errors;
-    private final int attemptCount;
 
     public TokensInvalidationResult(List<String> invalidatedTokens, List<String> previouslyInvalidatedTokens,
-                                    @Nullable List<ElasticsearchException> errors, int attemptCount) {
+                                    @Nullable List<ElasticsearchException> errors) {
         Objects.requireNonNull(invalidatedTokens, "invalidated_tokens must be provided");
         this.invalidatedTokens = invalidatedTokens;
         Objects.requireNonNull(previouslyInvalidatedTokens, "previously_invalidated_tokens must be provided");
@@ -45,18 +45,19 @@ public class TokensInvalidationResult implements ToXContentObject, Writeable {
         } else {
             this.errors = Collections.emptyList();
         }
-        this.attemptCount = attemptCount;
     }
 
     public TokensInvalidationResult(StreamInput in) throws IOException {
         this.invalidatedTokens = in.readStringList();
         this.previouslyInvalidatedTokens = in.readStringList();
         this.errors = in.readList(StreamInput::readException);
-        this.attemptCount = in.readVInt();
+        if (in.getVersion().before(Version.V_7_1_0)) {
+            in.readVInt();
+        }
     }
 
     public static TokensInvalidationResult emptyResult() {
-        return new TokensInvalidationResult(Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), 0);
+        return new TokensInvalidationResult(Collections.emptyList(), Collections.emptyList(), Collections.emptyList());
     }
 
 
@@ -72,10 +73,6 @@ public class TokensInvalidationResult implements ToXContentObject, Writeable {
         return errors;
     }
 
-    public int getAttemptCount() {
-        return attemptCount;
-    }
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject()
@@ -100,6 +97,8 @@ public class TokensInvalidationResult implements ToXContentObject, Writeable {
         out.writeStringCollection(invalidatedTokens);
         out.writeStringCollection(previouslyInvalidatedTokens);
         out.writeCollection(errors, StreamOutput::writeException);
-        out.writeVInt(attemptCount);
+        if (out.getVersion().before(Version.V_7_1_0)) {
+            out.writeVInt(5);
+        }
     }
 }

x-pack/plugin/core/src/main/resources/security-index-template.json

@@ -199,6 +199,13 @@
             "refreshed" : {
               "type" : "boolean"
             },
+            "refresh_time": {
+              "type": "date",
+              "format": "epoch_millis"
+            },
+            "superseded_by": {
+              "type": "keyword"
+            },
             "invalidated" : {
               "type" : "boolean"
             },

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/token/InvalidateTokenResponseTests.java

@@ -29,8 +29,7 @@ public class InvalidateTokenResponseTests extends ESTestCase {
         TokensInvalidationResult result = new TokensInvalidationResult(Arrays.asList(generateRandomStringArray(20, 15, false)),
             Arrays.asList(generateRandomStringArray(20, 15, false)),
             Arrays.asList(new ElasticsearchException("foo", new IllegalArgumentException("this is an error message")),
-                new ElasticsearchException("bar", new IllegalArgumentException("this is an error message2"))),
+                new ElasticsearchException("bar", new IllegalArgumentException("this is an error message2"))));
-            randomIntBetween(0, 5));
         InvalidateTokenResponse response = new InvalidateTokenResponse(result);
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             response.writeTo(output);
@@ -47,8 +46,7 @@ public class InvalidateTokenResponseTests extends ESTestCase {
         }
 
         result = new TokensInvalidationResult(Arrays.asList(generateRandomStringArray(20, 15, false)),
-            Arrays.asList(generateRandomStringArray(20, 15, false)),
+            Arrays.asList(generateRandomStringArray(20, 15, false)), Collections.emptyList());
-            Collections.emptyList(), randomIntBetween(0, 5));
         response = new InvalidateTokenResponse(result);
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             response.writeTo(output);
@@ -68,8 +66,7 @@ public class InvalidateTokenResponseTests extends ESTestCase {
         List previouslyInvalidatedTokens = Arrays.asList(generateRandomStringArray(20, 15, false));
         TokensInvalidationResult result = new TokensInvalidationResult(invalidatedTokens, previouslyInvalidatedTokens,
             Arrays.asList(new ElasticsearchException("foo", new IllegalArgumentException("this is an error message")),
-                new ElasticsearchException("bar", new IllegalArgumentException("this is an error message2"))),
+                new ElasticsearchException("bar", new IllegalArgumentException("this is an error message2"))));
-            randomIntBetween(0, 5));
         InvalidateTokenResponse response = new InvalidateTokenResponse(result);
         XContentBuilder builder = XContentFactory.jsonBuilder();
         response.toXContent(builder, ToXContent.EMPTY_PARAMS);

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/saml/TransportSamlAuthenticateAction.java

@@ -63,7 +63,7 @@ public final class TransportSamlAuthenticateAction extends HandledTransportActio
                 final Map<String, Object> tokenMeta = (Map<String, Object>) result.getMetadata().get(SamlRealm.CONTEXT_TOKEN_DATA);
                 tokenService.createUserToken(authentication, originatingAuthentication,
                         ActionListener.wrap(tuple -> {
-                            final String tokenString = tokenService.getUserTokenString(tuple.v1());
+                            final String tokenString = tokenService.getAccessTokenAsString(tuple.v1());
                             final TimeValue expiresIn = tokenService.getExpirationDelay();
                             listener.onResponse(
                                     new SamlAuthenticateResponse(authentication.getUser().principal(), tokenString, tuple.v2(), expiresIn));

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportCreateTokenAction.java

@@ -89,7 +89,7 @@ public final class TransportCreateTokenAction extends HandledTransportAction<Cre
                              boolean includeRefreshToken, ActionListener<CreateTokenResponse> listener) {
         try {
             tokenService.createUserToken(authentication, originatingAuth, ActionListener.wrap(tuple -> {
-                final String tokenStr = tokenService.getUserTokenString(tuple.v1());
+                final String tokenStr = tokenService.getAccessTokenAsString(tuple.v1());
                 final String scope = getResponseScopeValue(request.getScope());
 
                 final CreateTokenResponse response =

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/token/TransportRefreshTokenAction.java

@@ -31,7 +31,7 @@ public class TransportRefreshTokenAction extends HandledTransportAction<CreateTo
     @Override
     protected void doExecute(Task task, CreateTokenRequest request, ActionListener<CreateTokenResponse> listener) {
         tokenService.refreshToken(request.getRefreshToken(), ActionListener.wrap(tuple -> {
-            final String tokenStr = tokenService.getUserTokenString(tuple.v1());
+            final String tokenStr = tokenService.getAccessTokenAsString(tuple.v1());
             final String scope = getResponseScopeValue(request.getScope());
 
             final CreateTokenResponse response =

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/saml/TransportSamlLogoutActionTests.java

@@ -242,7 +242,7 @@ public class TransportSamlLogoutActionTests extends SamlTestCase {
         tokenService.createUserToken(authentication, authentication, future, tokenMetaData, true);
         final UserToken userToken = future.actionGet().v1();
         mockGetTokenFromId(userToken, false, client);
-        final String tokenString = tokenService.getUserTokenString(userToken);
+        final String tokenString = tokenService.getAccessTokenAsString(userToken);
 
         final SamlLogoutRequest request = new SamlLogoutRequest();
         request.setToken(tokenString);

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java

@@ -1109,7 +1109,7 @@ public class AuthenticationServiceTests extends ESTestCase {
             Authentication originatingAuth = new Authentication(new User("creator"), new RealmRef("test", "test", "test"), null);
             tokenService.createUserToken(expected, originatingAuth, tokenFuture, Collections.emptyMap(), true);
         }
-        String token = tokenService.getUserTokenString(tokenFuture.get().v1());
+        String token = tokenService.getAccessTokenAsString(tokenFuture.get().v1());
         when(client.prepareMultiGet()).thenReturn(new MultiGetRequestBuilder(client, MultiGetAction.INSTANCE));
         mockGetTokenFromId(tokenFuture.get().v1(), false, client);
         when(securityIndex.isAvailable()).thenReturn(true);
@@ -1192,7 +1192,7 @@ public class AuthenticationServiceTests extends ESTestCase {
             Authentication originatingAuth = new Authentication(new User("creator"), new RealmRef("test", "test", "test"), null);
             tokenService.createUserToken(expected, originatingAuth, tokenFuture, Collections.emptyMap(), true);
         }
-        String token = tokenService.getUserTokenString(tokenFuture.get().v1());
+        String token = tokenService.getAccessTokenAsString(tokenFuture.get().v1());
         mockGetTokenFromId(tokenFuture.get().v1(), true, client);
         doAnswer(invocationOnMock -> {
             ((Runnable) invocationOnMock.getArguments()[1]).run();

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java

@@ -5,11 +5,14 @@
  */
 package org.elasticsearch.xpack.security.authc;
 
+import org.apache.directory.api.util.Strings;
 import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.common.settings.SecureString;
@@ -23,6 +26,7 @@ import org.elasticsearch.test.SecuritySettingsSource;
 import org.elasticsearch.test.SecuritySettingsSourceField;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.security.action.token.CreateTokenRequest;
 import org.elasticsearch.xpack.core.security.action.token.CreateTokenResponse;
 import org.elasticsearch.xpack.core.security.action.token.InvalidateTokenRequest;
 import org.elasticsearch.xpack.core.security.action.token.InvalidateTokenResponse;
@@ -38,7 +42,13 @@ import org.junit.Before;
 
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
@@ -330,7 +340,7 @@ public class TokenAuthIntegTests extends SecurityIntegTestCase {
         assertEquals("token has been invalidated", e.getHeader("error_description").get(0));
     }
 
-    public void testRefreshingMultipleTimes() {
+    public void testRefreshingMultipleTimesFails() throws Exception {
         Client client = client().filterWithHeader(Collections.singletonMap("Authorization",
                 UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
                         SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
@@ -343,12 +353,101 @@ public class TokenAuthIntegTests extends SecurityIntegTestCase {
         assertNotNull(createTokenResponse.getRefreshToken());
         CreateTokenResponse refreshResponse = securityClient.prepareRefreshToken(createTokenResponse.getRefreshToken()).get();
         assertNotNull(refreshResponse);
+        // We now have two documents, the original(now refreshed) token doc and the new one with the new access doc
+        AtomicReference<String> docId = new AtomicReference<>();
+        assertBusy(() -> {
+            SearchResponse searchResponse = client.prepareSearch(SecurityIndexManager.SECURITY_INDEX_NAME)
+                .setSource(SearchSourceBuilder.searchSource()
+                    .query(QueryBuilders.boolQuery()
+                        .must(QueryBuilders.termQuery("doc_type", "token"))
+                        .must(QueryBuilders.termQuery("refresh_token.refreshed", "true"))))
+                .setSize(1)
+                .setTerminateAfter(1)
+                .get();
+            assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L));
+            docId.set(searchResponse.getHits().getAt(0).getId());
+        });
 
+        // hack doc to modify the refresh time to 50 seconds ago so that we don't hit the lenient refresh case
+        Instant refreshed = Instant.now();
+        Instant aWhileAgo = refreshed.minus(50L, ChronoUnit.SECONDS);
+        assertTrue(Instant.now().isAfter(aWhileAgo));
+        UpdateResponse updateResponse = client.prepareUpdate(SecurityIndexManager.SECURITY_INDEX_NAME, "doc", docId.get())
+            .setDoc("refresh_token", Collections.singletonMap("refresh_time", aWhileAgo.toEpochMilli()))
+            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+            .setFetchSource("refresh_token", Strings.EMPTY_STRING)
+            .get();
+        assertNotNull(updateResponse);
+        Map<String, Object> refreshTokenMap = (Map<String, Object>) updateResponse.getGetResult().sourceAsMap().get("refresh_token");
+        assertTrue(
+            Instant.ofEpochMilli((long) refreshTokenMap.get("refresh_time")).isBefore(Instant.now().minus(30L, ChronoUnit.SECONDS)));
         ElasticsearchSecurityException e = expectThrows(ElasticsearchSecurityException.class,
                 () -> securityClient.prepareRefreshToken(createTokenResponse.getRefreshToken()).get());
         assertEquals("invalid_grant", e.getMessage());
         assertEquals(RestStatus.BAD_REQUEST, e.status());
-        assertEquals("token has already been refreshed", e.getHeader("error_description").get(0));
+        assertEquals("token has already been refreshed more than 30 seconds in the past", e.getHeader("error_description").get(0));
+    }
+
+    public void testRefreshingMultipleTimesWithinWindowSucceeds() throws Exception {
+        Client client = client().filterWithHeader(Collections.singletonMap("Authorization",
+            UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
+                SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
+        SecurityClient securityClient = new SecurityClient(client);
+        Set<String> refreshTokens = new HashSet<>();
+        Set<String> accessTokens = new HashSet<>();
+        CreateTokenResponse createTokenResponse = securityClient.prepareCreateToken()
+            .setGrantType("password")
+            .setUsername(SecuritySettingsSource.TEST_USER_NAME)
+            .setPassword(new SecureString(SecuritySettingsSourceField.TEST_PASSWORD.toCharArray()))
+            .get();
+        assertNotNull(createTokenResponse.getRefreshToken());
+        final int numberOfProcessors = Runtime.getRuntime().availableProcessors();
+        final int numberOfThreads = scaledRandomIntBetween((numberOfProcessors + 1) / 2, numberOfProcessors * 3);
+        List<Thread> threads = new ArrayList<>(numberOfThreads);
+        final CountDownLatch readyLatch = new CountDownLatch(numberOfThreads + 1);
+        final CountDownLatch completedLatch = new CountDownLatch(numberOfThreads);
+        AtomicBoolean failed = new AtomicBoolean();
+        for (int i = 0; i < numberOfThreads; i++) {
+            threads.add(new Thread(() -> {
+                // Each thread gets its own client so that more than one nodes will be hit
+                Client threadClient = client().filterWithHeader(Collections.singletonMap("Authorization",
+                    UsernamePasswordToken.basicAuthHeaderValue(SecuritySettingsSource.TEST_USER_NAME,
+                        SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING)));
+                SecurityClient threadSecurityClient = new SecurityClient(threadClient);
+                CreateTokenRequest refreshRequest =
+                    threadSecurityClient.prepareRefreshToken(createTokenResponse.getRefreshToken()).request();
+                readyLatch.countDown();
+                try {
+                    readyLatch.await();
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                    completedLatch.countDown();
+                    return;
+                }
+                threadSecurityClient.refreshToken(refreshRequest, ActionListener.wrap(result -> {
+                    accessTokens.add(result.getTokenString());
+                    refreshTokens.add(result.getRefreshToken());
+                    logger.info("received access token [{}] and refresh token [{}]", result.getTokenString(), result.getRefreshToken());
+                    completedLatch.countDown();
+                }, e -> {
+                    failed.set(true);
+                    completedLatch.countDown();
+                    logger.error("caught exception", e);
+                }));
+            }));
+        }
+        for (Thread thread : threads) {
+            thread.start();
+        }
+        readyLatch.countDown();
+        readyLatch.await();
+        for (Thread thread : threads) {
+            thread.join();
+        }
+        completedLatch.await();
+        assertThat(failed.get(), equalTo(false));
+        assertThat(accessTokens.size(), equalTo(1));
+        assertThat(refreshTokens.size(), equalTo(1));
     }
 
     public void testRefreshAsDifferentUser() {

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.security.authc;
 
 import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.NoShardAvailableActionException;
 import org.elasticsearch.action.get.GetAction;
@@ -23,6 +24,8 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -51,7 +54,11 @@ import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.security.GeneralSecurityException;
 import java.time.Clock;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
@@ -61,6 +68,7 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Consumer;
 
+import javax.crypto.CipherOutputStream;
 import javax.crypto.SecretKey;
 
 import static java.time.Clock.systemUTC;
@@ -151,7 +159,7 @@ public class TokenServiceTests extends ESTestCase {
         authentication = token.getAuthentication();
 
         ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
-        requestContext.putHeader("Authorization", randomFrom("Bearer ", "BEARER ", "bearer ") + tokenService.getUserTokenString(token));
+        requestContext.putHeader("Authorization", randomFrom("Bearer ", "BEARER ", "bearer ") + tokenService.getAccessTokenAsString(token));
 
         try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
             PlainActionFuture<UserToken> future = new PlainActionFuture<>();
@@ -198,7 +206,7 @@ public class TokenServiceTests extends ESTestCase {
         authentication = token.getAuthentication();
 
         ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
-        requestContext.putHeader("Authorization", "Bearer " + tokenService.getUserTokenString(token));
+        requestContext.putHeader("Authorization", "Bearer " + getDeprecatedAccessTokenString(tokenService, token));
 
         try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
             PlainActionFuture<UserToken> future = new PlainActionFuture<>();
@@ -219,10 +227,10 @@ public class TokenServiceTests extends ESTestCase {
         tokenService.createUserToken(authentication, authentication, newTokenFuture, Collections.emptyMap(), true);
         final UserToken newToken = newTokenFuture.get().v1();
         assertNotNull(newToken);
-        assertNotEquals(tokenService.getUserTokenString(newToken), tokenService.getUserTokenString(token));
+        assertNotEquals(getDeprecatedAccessTokenString(tokenService, newToken), getDeprecatedAccessTokenString(tokenService, token));
 
         requestContext = new ThreadContext(Settings.EMPTY);
-        requestContext.putHeader("Authorization", "Bearer " + tokenService.getUserTokenString(newToken));
+        requestContext.putHeader("Authorization", "Bearer " + getDeprecatedAccessTokenString(tokenService, newToken));
         mockGetTokenFromId(newToken, false);
 
         try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
@@ -247,7 +255,7 @@ public class TokenServiceTests extends ESTestCase {
             rotateKeys(tokenService);
         }
         TokenService otherTokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex,
-                clusterService);
+            clusterService);
         otherTokenService.refreshMetaData(tokenService.getTokenMetaData());
         Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null);
         PlainActionFuture<Tuple<UserToken, String>> tokenFuture = new PlainActionFuture<>();
@@ -258,7 +266,7 @@ public class TokenServiceTests extends ESTestCase {
         authentication = token.getAuthentication();
 
         ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
-        requestContext.putHeader("Authorization", "Bearer " + tokenService.getUserTokenString(token));
+        requestContext.putHeader("Authorization", "Bearer " + getDeprecatedAccessTokenString(tokenService, token));
         try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
             PlainActionFuture<UserToken> future = new PlainActionFuture<>();
             otherTokenService.getAndValidateToken(requestContext, future);
@@ -289,7 +297,7 @@ public class TokenServiceTests extends ESTestCase {
         authentication = token.getAuthentication();
 
         ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
-        requestContext.putHeader("Authorization", "Bearer " + tokenService.getUserTokenString(token));
+        requestContext.putHeader("Authorization", "Bearer " + getDeprecatedAccessTokenString(tokenService, token));
 
         try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
             PlainActionFuture<UserToken> future = new PlainActionFuture<>();
@@ -316,7 +324,7 @@ public class TokenServiceTests extends ESTestCase {
         tokenService.createUserToken(authentication, authentication, newTokenFuture, Collections.emptyMap(), true);
         final UserToken newToken = newTokenFuture.get().v1();
         assertNotNull(newToken);
-        assertNotEquals(tokenService.getUserTokenString(newToken), tokenService.getUserTokenString(token));
+        assertNotEquals(getDeprecatedAccessTokenString(tokenService, newToken), getDeprecatedAccessTokenString(tokenService, token));
 
         metaData = tokenService.pruneKeys(1);
         tokenService.refreshMetaData(metaData);
@@ -329,7 +337,7 @@ public class TokenServiceTests extends ESTestCase {
         }
 
         requestContext = new ThreadContext(Settings.EMPTY);
-        requestContext.putHeader("Authorization", "Bearer " + tokenService.getUserTokenString(newToken));
+        requestContext.putHeader("Authorization", "Bearer " + getDeprecatedAccessTokenString(tokenService, newToken));
         mockGetTokenFromId(newToken, false);
         try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
             PlainActionFuture<UserToken> future = new PlainActionFuture<>();
@@ -351,7 +359,7 @@ public class TokenServiceTests extends ESTestCase {
         authentication = token.getAuthentication();
 
         ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
-        requestContext.putHeader("Authorization", "Bearer " + tokenService.getUserTokenString(token));
+        requestContext.putHeader("Authorization", "Bearer " + getDeprecatedAccessTokenString(tokenService, token));
 
         try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
             PlainActionFuture<UserToken> future = new PlainActionFuture<>();
@@ -362,8 +370,8 @@ public class TokenServiceTests extends ESTestCase {
 
         try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
             // verify a second separate token service with its own passphrase cannot verify
-            TokenService anotherService = new TokenService(Settings.EMPTY, systemUTC(), client, securityIndex,
+            TokenService anotherService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex,
-                    clusterService);
+                clusterService);
             PlainActionFuture<UserToken> future = new PlainActionFuture<>();
             anotherService.getAndValidateToken(requestContext, future);
             assertNull(future.get());
@@ -377,10 +385,10 @@ public class TokenServiceTests extends ESTestCase {
         PlainActionFuture<Tuple<UserToken, String>> tokenFuture = new PlainActionFuture<>();
         tokenService.createUserToken(authentication, authentication, tokenFuture, Collections.emptyMap(), true);
         UserToken token = tokenFuture.get().v1();
-        assertThat(tokenService.getUserTokenString(token), notNullValue());
+        assertThat(getDeprecatedAccessTokenString(tokenService, token), notNullValue());
 
         tokenService.clearActiveKeyCache();
-        assertThat(tokenService.getUserTokenString(token), notNullValue());
+        assertThat(getDeprecatedAccessTokenString(tokenService, token), notNullValue());
     }
 
     public void testInvalidatedToken() throws Exception {
@@ -395,7 +403,7 @@ public class TokenServiceTests extends ESTestCase {
         mockGetTokenFromId(token, true);
 
         ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
-        requestContext.putHeader("Authorization", "Bearer " + tokenService.getUserTokenString(token));
+        requestContext.putHeader("Authorization", "Bearer " + tokenService.getAccessTokenAsString(token));
 
         try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
             PlainActionFuture<UserToken> future = new PlainActionFuture<>();
@@ -449,7 +457,7 @@ public class TokenServiceTests extends ESTestCase {
         authentication = token.getAuthentication();
 
         ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
-        requestContext.putHeader("Authorization", "Bearer " + tokenService.getUserTokenString(token));
+        requestContext.putHeader("Authorization", "Bearer " + tokenService.getAccessTokenAsString(token));
 
         try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) {
             // the clock is still frozen, so the cookie should be valid
@@ -559,7 +567,7 @@ public class TokenServiceTests extends ESTestCase {
         //mockGetTokenFromId(token, false);
 
         ThreadContext requestContext = new ThreadContext(Settings.EMPTY);
-        requestContext.putHeader("Authorization", "Bearer " + tokenService.getUserTokenString(token));
+        requestContext.putHeader("Authorization", "Bearer " + tokenService.getAccessTokenAsString(token));
 
         doAnswer(invocationOnMock -> {
             ActionListener<GetResponse> listener = (ActionListener<GetResponse>) invocationOnMock.getArguments()[1];
@@ -598,7 +606,7 @@ public class TokenServiceTests extends ESTestCase {
         Authentication authentication = new Authentication(new User("joe", "admin"), new RealmRef("native_realm", "native", "node1"), null);
         UserToken expired = new UserToken(authentication, Instant.now().minus(3L, ChronoUnit.DAYS));
         mockGetTokenFromId(expired, false);
-        String userTokenString = tokenService.getUserTokenString(expired);
+        String userTokenString = tokenService.getAccessTokenAsString(expired);
         PlainActionFuture<Tuple<Authentication, Map<String, Object>>> authFuture = new PlainActionFuture<>();
         tokenService.getAuthenticationAndMetaData(userTokenString, authFuture);
         Authentication retrievedAuth = authFuture.actionGet().v1();
@@ -639,4 +647,28 @@ public class TokenServiceTests extends ESTestCase {
         assertEquals(expected.getMetadata(), result.getMetadata());
         assertEquals(AuthenticationType.TOKEN, result.getAuthenticationType());
     }
+
+    protected String getDeprecatedAccessTokenString(TokenService tokenService, UserToken userToken) throws IOException,
+        GeneralSecurityException {
+        try (ByteArrayOutputStream os = new ByteArrayOutputStream(TokenService.MINIMUM_BASE64_BYTES);
+             OutputStream base64 = Base64.getEncoder().wrap(os);
+             StreamOutput out = new OutputStreamStreamOutput(base64)) {
+            out.setVersion(Version.V_7_0_0);
+            TokenService.KeyAndCache keyAndCache = tokenService.getActiveKeyCache();
+            Version.writeVersion(Version.V_7_0_0, out);
+            out.writeByteArray(keyAndCache.getSalt().bytes);
+            out.writeByteArray(keyAndCache.getKeyHash().bytes);
+            final byte[] initializationVector = tokenService.getNewInitializationVector();
+            out.writeByteArray(initializationVector);
+            try (CipherOutputStream encryptedOutput =
+                     new CipherOutputStream(out, tokenService.getEncryptionCipher(initializationVector, keyAndCache, Version.V_7_0_0));
+                 StreamOutput encryptedStreamOutput = new OutputStreamStreamOutput(encryptedOutput)) {
+                encryptedStreamOutput.setVersion(Version.V_7_0_0);
+                encryptedStreamOutput.writeString(userToken.getId());
+                encryptedStreamOutput.close();
+                return new String(os.toByteArray(), StandardCharsets.UTF_8);
+            }
+        }
+    }
+
 }

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/TokensInvalidationResultTests.java

@@ -25,8 +25,7 @@ public class TokensInvalidationResultTests extends ESTestCase {
         TokensInvalidationResult result = new TokensInvalidationResult(Arrays.asList("token1", "token2"),
             Arrays.asList("token3", "token4"),
             Arrays.asList(new ElasticsearchException("foo", new IllegalStateException("bar")),
-                new ElasticsearchException("boo", new IllegalStateException("far"))),
+                new ElasticsearchException("boo", new IllegalStateException("far"))));
-            randomIntBetween(0, 5));
 
         try (XContentBuilder builder = JsonXContent.contentBuilder()) {
             result.toXContent(builder, ToXContent.EMPTY_PARAMS);
@@ -56,9 +55,8 @@ public class TokensInvalidationResultTests extends ESTestCase {
     }
 
     public void testToXcontentWithNoErrors() throws Exception{
-        TokensInvalidationResult result = new TokensInvalidationResult(Arrays.asList("token1", "token2"),
+        TokensInvalidationResult result = new TokensInvalidationResult(Arrays.asList("token1", "token2"), Collections.emptyList(),
-            Collections.emptyList(),
+            Collections.emptyList());
-            Collections.emptyList(), randomIntBetween(0, 5));
         try (XContentBuilder builder = JsonXContent.contentBuilder()) {
             result.toXContent(builder, ToXContent.EMPTY_PARAMS);
             assertThat(Strings.toString(builder),