Test: wait for all token expiration jobs to finish

This adds a best effort wait for the token expiration jobs to finish to hopefully address the
issues in tests with on going requests and locked shards.

Relates elastic/x-pack-elasticsearch#1220

Original commit: elastic/x-pack-elasticsearch@ae789c1e58

plugin/src/main/java/org/elasticsearch/xpack/security/authc/ExpiredTokenRemover.java

@@ -68,6 +68,10 @@ final class ExpiredTokenRemover extends AbstractRunnable {
         }
     }
 
+    boolean isExpirationInProgress() {
+        return inProgress.get();
+    }
+
     @Override
     public void onFailure(Exception e) {
         logger.error("failed to delete expired tokens", e);

plugin/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java

@@ -467,6 +467,10 @@ public final class TokenService extends AbstractComponent {
         return headers != null && headers.stream().anyMatch(EXPIRED_TOKEN_WWW_AUTH_VALUE::equals);
     }
 
+    boolean isExpirationInProgress() {
+        return expiredTokenRemover.isExpirationInProgress();
+    }
+
     private class KeyComputingRunnable extends AbstractRunnable {
 
         private final StreamInput in;

plugin/src/test/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java

@@ -99,7 +99,13 @@ public class TokenAuthIntegTests extends SecurityIntegTestCase {
     }
 
     @After
-    public void wipeSecurityIndex() {
+    public void wipeSecurityIndex() throws InterruptedException {
+        // get the token service and wait until token expiration is not in progress!
+        for (TokenService tokenService : internalCluster().getInstances(TokenService.class)) {
+            final boolean done = awaitBusy(() -> tokenService.isExpirationInProgress() == false);
+            assertTrue(done);
+        }
+
         try {
             // this is a hack to clean up the .security index since only superusers can delete it and the default test user is not a
             // superuser since the role used there is a file based role since we cannot guarantee the superuser role is always available