Simplifies security index state changes and template/mapping updates (elastic/x-pack-elasticsearch#551)

Currently, both the NativeUsersStore and NativeRolesStore can undergo
multiple state transitions. This is done primarily to check if the
security index is usable before it proceeds. However, such checks are
only needed for the tests, because if the security index is unavailable
when it is needed, the downstream actions invoked by the
NativeUsersStore and NativeRolesStore will throw the appropriate
exceptions notifying of that condition. In addition, both the
NativeUsersStore and NativeRolesStore had much duplicate code that
listened for cluster state changes and made the exact same state
transitions.

This commit removes the complicated state transitions in both classes
and enables both classes to use the SecurityTemplateService to monitor
all of the security index lifecycle changes they need to be aware of.

This commit also moves the logic for determining if the security index
needs template and/or mapping updates to the SecurityLifecycleService,
and makes the NativeRealmMigrator solely responsible for applying the
updates.

Original commit: elastic/x-pack-elasticsearch@b31d144597

plugin/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -14,7 +14,6 @@ import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.bootstrap.BootstrapCheck;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.bootstrap.BootstrapCheck;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Nullable;
@@ -256,25 +255,6 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin {
         securityContext.set(new SecurityContext(settings, threadPool.getThreadContext()));
         components.add(securityContext.get());
 
-        // realms construction
-        final NativeUsersStore nativeUsersStore = new NativeUsersStore(settings, client);
-        final AnonymousUser anonymousUser = new AnonymousUser(settings);
-        final ReservedRealm reservedRealm = new ReservedRealm(env, settings, nativeUsersStore, anonymousUser);
-        Map<String, Realm.Factory> realmFactories = new HashMap<>();
-        realmFactories.putAll(InternalRealms.getFactories(threadPool, resourceWatcherService, sslService, nativeUsersStore));
-        for (XPackExtension extension : extensions) {
-            Map<String, Realm.Factory> newRealms = extension.getRealms(resourceWatcherService);
-            for (Map.Entry<String, Realm.Factory> entry : newRealms.entrySet()) {
-                if (realmFactories.put(entry.getKey(), entry.getValue()) != null) {
-                    throw new IllegalArgumentException("Realm type [" + entry.getKey() + "] is already registered");
-                }
-            }
-        }
-        final Realms realms = new Realms(settings, env, realmFactories, licenseState, reservedRealm);
-        components.add(nativeUsersStore);
-        components.add(realms);
-        components.add(reservedRealm);
-
         // audit trails construction
         IndexAuditTrail indexAuditTrail = null;
         Set<AuditTrail> auditTrails = new LinkedHashSet<>();
@@ -303,6 +283,28 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin {
             new AuditTrailService(settings, auditTrails.stream().collect(Collectors.toList()), licenseState);
         components.add(auditTrailService);
 
+        SecurityLifecycleService securityLifecycleService =
+            new SecurityLifecycleService(settings, clusterService, threadPool, client, licenseState, indexAuditTrail);
+
+        // realms construction
+        final NativeUsersStore nativeUsersStore = new NativeUsersStore(settings, client, securityLifecycleService);
+        final AnonymousUser anonymousUser = new AnonymousUser(settings);
+        final ReservedRealm reservedRealm = new ReservedRealm(env, settings, nativeUsersStore, anonymousUser, securityLifecycleService);
+        Map<String, Realm.Factory> realmFactories = new HashMap<>();
+        realmFactories.putAll(InternalRealms.getFactories(threadPool, resourceWatcherService, sslService, nativeUsersStore));
+        for (XPackExtension extension : extensions) {
+            Map<String, Realm.Factory> newRealms = extension.getRealms(resourceWatcherService);
+            for (Map.Entry<String, Realm.Factory> entry : newRealms.entrySet()) {
+                if (realmFactories.put(entry.getKey(), entry.getValue()) != null) {
+                    throw new IllegalArgumentException("Realm type [" + entry.getKey() + "] is already registered");
+                }
+            }
+        }
+        final Realms realms = new Realms(settings, env, realmFactories, licenseState, reservedRealm);
+        components.add(nativeUsersStore);
+        components.add(realms);
+        components.add(reservedRealm);
+
         AuthenticationFailureHandler failureHandler = null;
         String extensionName = null;
         for (XPackExtension extension : extensions) {
@@ -325,7 +327,7 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin {
         components.add(authcService.get());
 
         final FileRolesStore fileRolesStore = new FileRolesStore(settings, env, resourceWatcherService, licenseState);
-        final NativeRolesStore nativeRolesStore = new NativeRolesStore(settings, client, licenseState);
+        final NativeRolesStore nativeRolesStore = new NativeRolesStore(settings, client, licenseState, securityLifecycleService);
         final ReservedRolesStore reservedRolesStore = new ReservedRolesStore();
         final CompositeRolesStore allRolesStore =
                 new CompositeRolesStore(settings, fileRolesStore, nativeRolesStore, reservedRolesStore, licenseState);
@@ -339,8 +341,7 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin {
         components.add(allRolesStore); // for SecurityFeatureSet and clear roles cache
         components.add(authzService);
 
-        components.add(new SecurityLifecycleService(settings, clusterService, threadPool, indexAuditTrail,
+        components.add(securityLifecycleService);
-            nativeUsersStore, nativeRolesStore, licenseState, client));
 
         ipFilter.set(new IPFilter(settings, auditTrailService, clusterService.getClusterSettings(), licenseState));
         components.add(ipFilter.get());
@@ -653,7 +654,7 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin {
         final String auditIndex = indexAuditingEnabled ? "," + IndexAuditTrail.INDEX_NAME_PREFIX + "*" : "";
         String errorMessage = LoggerMessageFormat.format("the [action.auto_create_index] setting value [{}] is too" +
                 " restrictive. disable [action.auto_create_index] or set it to " +
-                "[{}{}]", (Object) value, SecurityTemplateService.SECURITY_INDEX_NAME, auditIndex);
+                "[{}{}]", (Object) value, SecurityLifecycleService.SECURITY_INDEX_NAME, auditIndex);
         if (Booleans.isFalse(value)) {
             throw new IllegalArgumentException(errorMessage);
         }
@@ -664,7 +665,7 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin {
 
         String[] matches = Strings.commaDelimitedListToStringArray(value);
         List<String> indices = new ArrayList<>();
-        indices.add(SecurityTemplateService.SECURITY_INDEX_NAME);
+        indices.add(SecurityLifecycleService.SECURITY_INDEX_NAME);
         if (indexAuditingEnabled) {
             DateTime now = new DateTime(DateTimeZone.UTC);
             // just use daily rollover

plugin/src/main/java/org/elasticsearch/xpack/security/SecurityLifecycleService.java

@@ -5,20 +5,54 @@
  */
 package org.elasticsearch.xpack.security;
 
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.logging.log4j.util.Supplier;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
 import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.component.LifecycleListener;
+import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.inject.internal.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.gateway.GatewayService;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail;
 import org.elasticsearch.xpack.security.authc.esnative.NativeRealmMigrator;
-import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
+import org.elasticsearch.xpack.template.TemplateUtils;
-import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Predicate;
+import java.util.regex.Pattern;
 
 /**
  * This class is used to provide a lifecycle for services that is based on the cluster's state
@@ -33,31 +67,46 @@ import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
  */
 public class SecurityLifecycleService extends AbstractComponent implements ClusterStateListener {
 
+    public static final String SECURITY_INDEX_NAME = ".security";
+    public static final String SECURITY_TEMPLATE_NAME = "security-index-template";
+    private static final String SECURITY_VERSION_STRING = "security-version";
+    private static final Version MIN_READ_VERSION = Version.V_5_0_0;
+    static final String SECURITY_INDEX_TEMPLATE_VERSION_PATTERN = Pattern.quote("${security.template.version}");
+
     private final Settings settings;
     private final ThreadPool threadPool;
+    private final InternalClient client;
     private final IndexAuditTrail indexAuditTrail;
-    private final NativeUsersStore nativeUserStore;
+    private final NativeRealmMigrator nativeRealmMigrator;
-    private final NativeRolesStore nativeRolesStore;
+    final AtomicBoolean templateCreationPending = new AtomicBoolean(false);
+    final AtomicBoolean updateMappingPending = new AtomicBoolean(false);
+    final AtomicReference<UpgradeState> upgradeDataState = new AtomicReference<>(UpgradeState.NOT_STARTED);
+    private volatile boolean securityIndexExists;
+    private volatile boolean securityIndexAvailable;
+    private volatile boolean canWriteToSecurityIndex;
+    private volatile Version mappingVersion;
 
-    public SecurityLifecycleService(Settings settings, ClusterService clusterService, ThreadPool threadPool,
+    enum UpgradeState {
-                                    @Nullable IndexAuditTrail indexAuditTrail, NativeUsersStore nativeUserStore,
+        NOT_STARTED, IN_PROGRESS, COMPLETE, FAILED
-                                    NativeRolesStore nativeRolesStore, XPackLicenseState licenseState, InternalClient client) {
+    }
+
+
+    public SecurityLifecycleService(Settings settings, ClusterService clusterService, ThreadPool threadPool, InternalClient client,
+                                    XPackLicenseState licenseState, @Nullable IndexAuditTrail indexAuditTrail) {
+        this(settings, clusterService, threadPool, client, new NativeRealmMigrator(settings, licenseState, client), indexAuditTrail);
+    }
+
+    // package private for testing
+    SecurityLifecycleService(Settings settings, ClusterService clusterService, ThreadPool threadPool, InternalClient client,
+                             NativeRealmMigrator migrator, @Nullable IndexAuditTrail indexAuditTrail) {
         super(settings);
         this.settings = settings;
         this.threadPool = threadPool;
+        this.client = client;
         this.indexAuditTrail = indexAuditTrail;
-        this.nativeUserStore = nativeUserStore;
+        this.nativeRealmMigrator = migrator;
-        this.nativeRolesStore = nativeRolesStore;
-        // TODO: define a common interface for these and delegate from one place. nativeUserStore store is it's on
-        // cluster
-        // state listener , but is also activated from this clusterChanged method
         clusterService.addListener(this);
-        clusterService.addListener(nativeUserStore);
-        clusterService.addListener(nativeRolesStore);
-        final NativeRealmMigrator nativeRealmMigrator = new NativeRealmMigrator(settings, licenseState, client);
-        clusterService.addListener(new SecurityTemplateService(settings, client, nativeRealmMigrator));
         clusterService.addLifecycleListener(new LifecycleListener() {
-
             @Override
             public void beforeStop() {
                 stop();
@@ -67,45 +116,31 @@ public class SecurityLifecycleService extends AbstractComponent implements Clust
 
     @Override
     public void clusterChanged(ClusterChangedEvent event) {
+        final ClusterState state = event.state();
+        if (state.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
+            // wait until the gateway has recovered from disk, otherwise we think we don't have the
+            // .security index but they may not have been restored from the cluster state on disk
+            logger.debug("lifecycle service waiting until state has been recovered");
+            return;
+        }
+
+        securityIndexExists = event.state().metaData().indices().get(SECURITY_INDEX_NAME) != null;
+        securityIndexAvailable = securityIndexAvailable(state, logger);
+        final boolean securityTemplateUpToDate = securityTemplateExistsAndIsUpToDate(state, logger);
+        final boolean securityMappingUpToDate = securityIndexMappingUpToDate(state, logger);
+        canWriteToSecurityIndex = securityTemplateUpToDate && securityMappingUpToDate;
+        mappingVersion = oldestSecurityIndexMappingVersion(event.state(), logger);
+
+        if (event.localNodeMaster()) {
+            if (securityTemplateUpToDate == false) {
+                updateSecurityTemplate();
+            }
+            if (securityIndexAvailable && securityMappingUpToDate == false) {
+                upgradeSecurityData(state, this::updateSecurityMapping);
+            }
+        }
+
         final boolean master = event.localNodeMaster();
-        try {
-            if (nativeUserStore.canStart(event.state(), master)) {
-                threadPool.generic().execute(new AbstractRunnable() {
-                    @Override
-                    public void onFailure(Exception throwable) {
-                        logger.error("failed to start native user store service", throwable);
-                        assert false : "security lifecycle services startup failed";
-                    }
-
-                    @Override
-                    public void doRun() {
-                        nativeUserStore.start();
-                    }
-                });
-            }
-        } catch (Exception e) {
-            logger.error("failed to start native user store", e);
-        }
-
-        try {
-            if (nativeRolesStore.canStart(event.state(), master)) {
-                threadPool.generic().execute(new AbstractRunnable() {
-                    @Override
-                    public void onFailure(Exception throwable) {
-                        logger.error("failed to start native roles store services", throwable);
-                        assert false : "security lifecycle services startup failed";
-                    }
-
-                    @Override
-                    public void doRun() {
-                        nativeRolesStore.start();
-                    }
-                });
-            }
-        } catch (Exception e) {
-            logger.error("failed to start native roles store", e);
-        }
-
         try {
             if (Security.indexAuditLoggingEnabled(settings) &&
                     indexAuditTrail.state() == IndexAuditTrail.State.INITIALIZED) {
@@ -128,20 +163,287 @@ public class SecurityLifecycleService extends AbstractComponent implements Clust
         } catch (Exception e) {
             logger.error("failed to start index audit trail", e);
         }
+    }
 
+    public boolean securityIndexExists() {
+        return securityIndexExists;
+    }
+
+    public boolean securityIndexAvailable() {
+        return securityIndexAvailable;
+    }
+
+    public boolean canWriteToSecurityIndex() {
+        return canWriteToSecurityIndex;
+    }
+
+    private boolean securityIndexAvailable(ClusterState state, Logger logger) {
+        final IndexRoutingTable routingTable = getSecurityIndexRoutingTable(state);
+        if (routingTable != null && routingTable.allPrimaryShardsActive()) {
+            return true;
+        }
+        logger.debug("Security index is not yet active");
+        return false;
+    }
+
+    /**
+     * Returns the routing-table for the security index, or <code>null</code> if the security index does not exist.
+     */
+    public static IndexRoutingTable getSecurityIndexRoutingTable(ClusterState clusterState) {
+        IndexMetaData metaData = clusterState.metaData().index(SECURITY_INDEX_NAME);
+        if (metaData == null) {
+            return null;
+        } else {
+            return clusterState.routingTable().index(SECURITY_INDEX_NAME);
+        }
+    }
+
+    public static boolean securityIndexMappingAndTemplateUpToDate(ClusterState clusterState, Logger logger) {
+        if (securityTemplateExistsAndIsUpToDate(clusterState, logger) == false) {
+            logger.debug("security template [{}] does not exist or is not up to date, so security module is not ready for use",
+                         SECURITY_TEMPLATE_NAME);
+            return false;
+        }
+        if (securityIndexMappingUpToDate(clusterState, logger) == false) {
+            logger.debug("mapping for the security index is not up to date, so security module is not ready for use");
+            return false;
+        }
+        return true;
+    }
+
+    public static boolean securityIndexMappingAndTemplateSufficientToRead(ClusterState clusterState, Logger logger) {
+        if (securityTemplateExistsAndVersionMatches(clusterState, logger, MIN_READ_VERSION::onOrBefore) == false) {
+            logger.debug("security template [{}] does not exist or is not up to date, so security module is not ready for use",
+                         SECURITY_TEMPLATE_NAME);
+            return false;
+        }
+        if (securityIndexMappingVersionMatches(clusterState, logger, MIN_READ_VERSION::onOrBefore) == false) {
+            logger.debug("mapping for the security index is not up to date, so security module is not ready for use");
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * Test whether the effective (active) version of the security mapping meets the <code>requiredVersion</code>.
+     *
+     * @return <code>true</code> if the effective version passes the predicate, or the security mapping does not exist (<code>null</code>
+     * version). Otherwise, <code>false</code>.
+     */
+    public boolean checkMappingVersion(Predicate<Version> requiredVersion) {
+        return this.mappingVersion == null || requiredVersion.test(this.mappingVersion);
+    }
+
+    static boolean securityIndexMappingUpToDate(ClusterState clusterState, Logger logger) {
+        return securityIndexMappingVersionMatches(clusterState, logger, Version.CURRENT::equals);
+    }
+
+    static boolean securityIndexMappingVersionMatches(ClusterState clusterState, Logger logger, Predicate<Version> predicate) {
+        return securityIndexMappingVersions(clusterState, logger).stream().allMatch(predicate);
+    }
+
+    private static Set<Version> securityIndexMappingVersions(ClusterState clusterState, Logger logger) {
+        Set<Version> versions = new HashSet<>();
+        IndexMetaData indexMetaData = clusterState.metaData().getIndices().get(SECURITY_INDEX_NAME);
+        if (indexMetaData != null) {
+            for (Object object : indexMetaData.getMappings().values().toArray()) {
+                MappingMetaData mappingMetaData = (MappingMetaData) object;
+                if (mappingMetaData.type().equals(MapperService.DEFAULT_MAPPING)) {
+                    continue;
+                }
+                versions.add(readMappingVersion(mappingMetaData, logger));
+            }
+        }
+        return versions;
+    }
+
+    private static Version readMappingVersion(MappingMetaData mappingMetaData, Logger logger) {
+        try {
+            Map<String, Object> meta = (Map<String, Object>) mappingMetaData.sourceAsMap().get("_meta");
+            if (meta == null) {
+                // something pre-5.0, but we don't know what. Use 2.3.0 as a placeholder for "old"
+                return Version.V_2_3_0;
+            }
+            return Version.fromString((String) meta.get(SECURITY_VERSION_STRING));
+        } catch (IOException e) {
+            logger.error("Cannot parse the mapping for security index.", e);
+            throw new ElasticsearchException("Cannot parse the mapping for security index.", e);
+        }
+    }
+
+    static boolean securityTemplateExistsAndIsUpToDate(ClusterState state, Logger logger) {
+        return securityTemplateExistsAndVersionMatches(state, logger, Version.CURRENT::equals);
+    }
+
+    static boolean securityTemplateExistsAndVersionMatches(ClusterState state, Logger logger, Predicate<Version> predicate) {
+        IndexTemplateMetaData templateMeta = state.metaData().templates().get(SECURITY_TEMPLATE_NAME);
+        if (templateMeta == null) {
+            return false;
+        }
+        ImmutableOpenMap<String, CompressedXContent> mappings = templateMeta.getMappings();
+        // check all mappings contain correct version in _meta
+        // we have to parse the source here which is annoying
+        for (Object typeMapping : mappings.values().toArray()) {
+            CompressedXContent typeMappingXContent = (CompressedXContent) typeMapping;
+            try  {
+                Map<String, Object> typeMappingMap =
+                XContentHelper.convertToMap(new BytesArray(typeMappingXContent.uncompressed()), false, XContentType.JSON).v2();
+                // should always contain one entry with key = typename
+                assert (typeMappingMap.size() == 1);
+                String key = typeMappingMap.keySet().iterator().next();
+                // get the actual mapping entries
+                @SuppressWarnings("unchecked")
+                Map<String, Object> mappingMap = (Map<String, Object>) typeMappingMap.get(key);
+                if (containsCorrectVersion(mappingMap, predicate) == false) {
+                    return false;
+                }
+            } catch (ElasticsearchParseException e) {
+                logger.error("Cannot parse the template for security index.", e);
+                throw new IllegalStateException("Cannot parse the template for security index.", e);
+            }
+        }
+        return true;
+    }
+
+    private static boolean containsCorrectVersion(Map<String, Object> typeMappingMap, Predicate<Version> predicate) {
+        @SuppressWarnings("unchecked")
+        Map<String, Object> meta = (Map<String, Object>) typeMappingMap.get("_meta");
+        if (meta == null) {
+            // pre 5.0, cannot be up to date
+            return false;
+        }
+        return predicate.test(Version.fromString((String) meta.get(SECURITY_VERSION_STRING)));
+    }
+
+    public static Version oldestSecurityIndexMappingVersion(ClusterState clusterState, Logger logger) {
+        final Set<Version> versions = securityIndexMappingVersions(clusterState, logger);
+        return versions.stream().min(Version::compareTo).orElse(null);
+    }
+
+    private void updateSecurityTemplate() {
+        // only put the template if this is not already in progress
+        if (templateCreationPending.compareAndSet(false, true)) {
+            putSecurityTemplate();
+        }
+    }
+
+    private boolean upgradeSecurityData(ClusterState state, Runnable andThen) {
+        // only update the data if this is not already in progress
+        if (upgradeDataState.compareAndSet(UpgradeState.NOT_STARTED, UpgradeState.IN_PROGRESS) ) {
+            final Version previousVersion = oldestSecurityIndexMappingVersion(state, logger);
+            nativeRealmMigrator.performUpgrade(previousVersion, new ActionListener<Boolean>() {
+
+                @Override
+                public void onResponse(Boolean upgraded) {
+                    upgradeDataState.set(UpgradeState.COMPLETE);
+                    andThen.run();
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    upgradeDataState.set(UpgradeState.FAILED);
+                    logger.error((Supplier<?>) () -> new ParameterizedMessage("failed to upgrade security data from version [{}] ",
+                                                                              previousVersion), e);
+                }
+            });
+            return true;
+        } else {
+            if (upgradeDataState.get() == UpgradeState.COMPLETE) {
+                andThen.run();
+            }
+            return false;
+        }
+    }
+
+    private void updateSecurityMapping() {
+        // only update the mapping if this is not already in progress
+        if (updateMappingPending.compareAndSet(false, true)) {
+            putSecurityMappings();
+        }
+    }
+
+    private void putSecurityMappings() {
+        String template = TemplateUtils.loadTemplate("/" + SECURITY_TEMPLATE_NAME + ".json", Version.CURRENT.toString(),
+                                                     SECURITY_INDEX_TEMPLATE_VERSION_PATTERN);
+        Map<String, Object> typeMappingMap;
+        try {
+            typeMappingMap = XContentHelper.convertToMap(JsonXContent.jsonXContent, template, false);
+        } catch (ElasticsearchParseException e) {
+            updateMappingPending.set(false);
+            logger.error("failed to parse the security index template", e);
+            throw new ElasticsearchException("failed to parse the security index template", e);
+        }
+
+        // here go over all types found in the template and update them
+        // we need to wait for all types
+        final Map<String, PutMappingResponse> updateResults = ConcurrentCollections.newConcurrentMap();
+        @SuppressWarnings("unchecked")
+        Map<String, Object> typeMappings = (Map<String, Object>) typeMappingMap.get("mappings");
+        int expectedResults = typeMappings.size();
+        for (String type : typeMappings.keySet()) {
+            // get the mappings from the template definition
+            @SuppressWarnings("unchecked")
+            Map<String, Object> typeMapping = (Map<String, Object>) typeMappings.get(type);
+            // update the mapping
+            putSecurityMapping(updateResults, expectedResults, type, typeMapping);
+        }
+    }
+
+    private void putSecurityMapping(final Map<String, PutMappingResponse> updateResults, int expectedResults,
+                                    final String type, Map<String, Object> typeMapping) {
+        logger.debug("updating mapping of the security index for type [{}]", type);
+        PutMappingRequest putMappingRequest = client.admin().indices()
+            .preparePutMapping(SECURITY_INDEX_NAME).setSource(typeMapping).setType(type).request();
+        client.admin().indices().putMapping(putMappingRequest, new ActionListener<PutMappingResponse>() {
+            @Override
+            public void onResponse(PutMappingResponse putMappingResponse) {
+                if (putMappingResponse.isAcknowledged() == false) {
+                    updateMappingPending.set(false);
+                    throw new ElasticsearchException("update mapping for [{}] security index " +
+                                                     "was not acknowledged", type);
+                } else {
+                    updateResults.put(type, putMappingResponse);
+                    if (updateResults.size() == expectedResults) {
+                        updateMappingPending.set(false);
+                    }
+                }
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                updateMappingPending.set(false);
+                logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to update mapping for [{}] on security index", type), e);
+            }
+        });
+    }
+
+    private void putSecurityTemplate() {
+        logger.debug("putting the security index template");
+        String template = TemplateUtils.loadTemplate("/" + SECURITY_TEMPLATE_NAME + ".json", Version.CURRENT.toString(),
+                                                     SECURITY_INDEX_TEMPLATE_VERSION_PATTERN);
+
+        PutIndexTemplateRequest putTemplateRequest = client.admin().indices()
+            .preparePutTemplate(SECURITY_TEMPLATE_NAME)
+            .setSource(new BytesArray(template.getBytes(StandardCharsets.UTF_8)), XContentType.JSON)
+            .request();
+        client.admin().indices().putTemplate(putTemplateRequest, new ActionListener<PutIndexTemplateResponse>() {
+            @Override
+            public void onResponse(PutIndexTemplateResponse putIndexTemplateResponse) {
+                templateCreationPending.set(false);
+                if (putIndexTemplateResponse.isAcknowledged() == false) {
+                    throw new ElasticsearchException("put template for security index was not acknowledged");
+                }
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                templateCreationPending.set(false);
+                logger.warn("failed to put security index template", e);
+            }
+        });
     }
 
     public void stop() {
-        try {
-            nativeUserStore.stop();
-        } catch (Exception e) {
-            logger.error("failed to stop native user module", e);
-        }
-        try {
-            nativeRolesStore.stop();
-        } catch (Exception e) {
-            logger.error("failed to stop native roles module", e);
-        }
         if (indexAuditTrail != null) {
             try {
                 indexAuditTrail.stop();

plugin/src/main/java/org/elasticsearch/xpack/security/SecurityTemplateService.java

@@ -1,361 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.security;
-
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.logging.log4j.util.Supplier;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
-import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
-import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
-import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateListener;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.cluster.routing.IndexRoutingTable;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.gateway.GatewayService;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.xpack.security.authc.esnative.NativeRealmMigrator;
-import org.elasticsearch.xpack.template.TemplateUtils;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Predicate;
-import java.util.regex.Pattern;
-
-/**
- * SecurityTemplateService is responsible for adding the template needed for the
- * {@code .security} administrative index.
- */
-public class SecurityTemplateService extends AbstractComponent implements ClusterStateListener {
-
-    public static final String SECURITY_INDEX_NAME = ".security";
-    public static final String SECURITY_TEMPLATE_NAME = "security-index-template";
-    private static final String SECURITY_VERSION_STRING = "security-version";
-    static final String SECURITY_INDEX_TEMPLATE_VERSION_PATTERN = Pattern.quote("${security.template.version}");
-    static final Version MIN_READ_VERSION = Version.V_5_0_0;
-
-    enum UpgradeState {
-        NOT_STARTED, IN_PROGRESS, COMPLETE, FAILED
-    }
-
-    private final InternalClient client;
-    final AtomicBoolean templateCreationPending = new AtomicBoolean(false);
-    final AtomicBoolean updateMappingPending = new AtomicBoolean(false);
-    final AtomicReference upgradeDataState = new AtomicReference<>(UpgradeState.NOT_STARTED);
-    private final NativeRealmMigrator nativeRealmMigrator;
-
-    public SecurityTemplateService(Settings settings, InternalClient client, NativeRealmMigrator nativeRealmMigrator) {
-        super(settings);
-        this.client = client;
-        this.nativeRealmMigrator = nativeRealmMigrator;
-    }
-
-    @Override
-    public void clusterChanged(ClusterChangedEvent event) {
-        if (event.localNodeMaster() == false) {
-            return;
-        }
-        ClusterState state = event.state();
-        if (state.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
-            // wait until the gateway has recovered from disk, otherwise we think may not have .security-audit-
-            // but they may not have been restored from the cluster state on disk
-            logger.debug("template service waiting until state has been recovered");
-            return;
-        }
-        if (securityTemplateExistsAndIsUpToDate(state, logger) == false) {
-            updateSecurityTemplate();
-        }
-        // make sure mapping is up to date
-        if (state.metaData().getIndices() != null) {
-            if (securityIndexMappingUpToDate(state, logger) == false) {
-                if (securityIndexAvailable(state, logger)) {
-                    upgradeSecurityData(state, this::updateSecurityMapping);
-                }
-            }
-        }
-    }
-
-    private boolean securityIndexAvailable(ClusterState state, Logger logger) {
-        final IndexRoutingTable routingTable = getSecurityIndexRoutingTable(state);
-        if (routingTable == null) {
-            throw new IllegalStateException("Security index does not exist");
-        }
-        if (routingTable.allPrimaryShardsActive() == false) {
-            logger.debug("Security index is not yet active");
-            return false;
-        }
-        return true;
-    }
-
-    private void updateSecurityTemplate() {
-        // only put the template if this is not already in progress
-        if (templateCreationPending.compareAndSet(false, true)) {
-            putSecurityTemplate();
-        }
-    }
-
-    private boolean upgradeSecurityData(ClusterState state, Runnable andThen) {
-        // only update the data if this is not already in progress
-        if (upgradeDataState.compareAndSet(UpgradeState.NOT_STARTED, UpgradeState.IN_PROGRESS) ) {
-            final Version previousVersion = oldestSecurityIndexMappingVersion(state, logger);
-            nativeRealmMigrator.performUpgrade(previousVersion, new ActionListener<Boolean>() {
-
-                @Override
-                public void onResponse(Boolean upgraded) {
-                    upgradeDataState.set(UpgradeState.COMPLETE);
-                    andThen.run();
-                }
-
-                @Override
-                public void onFailure(Exception e) {
-                    upgradeDataState.set(UpgradeState.FAILED);
-                    logger.error((Supplier<?>) () -> new ParameterizedMessage("failed to upgrade security data from version [{}] ",
-                            previousVersion), e);
-                }
-            });
-            return true;
-        } else {
-            if (upgradeDataState.get() == UpgradeState.COMPLETE) {
-                andThen.run();
-            }
-            return false;
-        }
-    }
-
-    private void updateSecurityMapping() {
-        // only update the mapping if this is not already in progress
-        if (updateMappingPending.compareAndSet(false, true) ) {
-            putSecurityMappings();
-        }
-    }
-
-    private void putSecurityMappings() {
-        String template = TemplateUtils.loadTemplate("/" + SECURITY_TEMPLATE_NAME + ".json", Version.CURRENT.toString()
-                , SECURITY_INDEX_TEMPLATE_VERSION_PATTERN);
-        Map<String, Object> typeMappingMap;
-        try {
-            typeMappingMap = XContentHelper.convertToMap(JsonXContent.jsonXContent, template, false);
-        } catch (ElasticsearchParseException e) {
-            updateMappingPending.set(false);
-            logger.error("failed to parse the security index template", e);
-            throw new ElasticsearchException("failed to parse the security index template", e);
-        }
-
-        // here go over all types found in the template and update them
-        // we need to wait for all types
-        final Map<String, PutMappingResponse> updateResults = ConcurrentCollections.newConcurrentMap();
-        @SuppressWarnings("unchecked")
-        Map<String, Object> typeMappings = (Map<String, Object>) typeMappingMap.get("mappings");
-        int expectedResults = typeMappings.size();
-        for (String type : typeMappings.keySet()) {
-            // get the mappings from the template definition
-            @SuppressWarnings("unchecked")
-            Map<String, Object> typeMapping = (Map<String, Object>) typeMappings.get(type);
-            // update the mapping
-            putSecurityMapping(updateResults, expectedResults, type, typeMapping);
-        }
-    }
-
-    private void putSecurityMapping(final Map<String, PutMappingResponse> updateResults, int expectedResults,
-                                    final String type, Map<String, Object> typeMapping) {
-        logger.debug("updating mapping of the security index for type [{}]", type);
-        PutMappingRequest putMappingRequest = client.admin().indices()
-                .preparePutMapping(SECURITY_INDEX_NAME).setSource(typeMapping).setType(type).request();
-        client.admin().indices().putMapping(putMappingRequest, new ActionListener<PutMappingResponse>() {
-            @Override
-            public void onResponse(PutMappingResponse putMappingResponse) {
-                if (putMappingResponse.isAcknowledged() == false) {
-                    updateMappingPending.set(false);
-                    throw new ElasticsearchException("update mapping for [{}] security index " +
-                            "was not acknowledged", type);
-                } else {
-                    updateResults.put(type, putMappingResponse);
-                    if (updateResults.size() == expectedResults) {
-                        updateMappingPending.set(false);
-                    }
-                }
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                updateMappingPending.set(false);
-                logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to update mapping for [{}] on security index", type), e);
-            }
-        });
-    }
-
-    private void putSecurityTemplate() {
-        logger.debug("putting the security index template");
-        String template = TemplateUtils.loadTemplate("/" + SECURITY_TEMPLATE_NAME + ".json", Version.CURRENT.toString()
-                , SECURITY_INDEX_TEMPLATE_VERSION_PATTERN);
-
-        PutIndexTemplateRequest putTemplateRequest = client.admin().indices()
-                .preparePutTemplate(SECURITY_TEMPLATE_NAME)
-                .setSource(new BytesArray(template.getBytes(StandardCharsets.UTF_8)), XContentType.JSON)
-                .request();
-        client.admin().indices().putTemplate(putTemplateRequest, new ActionListener<PutIndexTemplateResponse>() {
-            @Override
-            public void onResponse(PutIndexTemplateResponse putIndexTemplateResponse) {
-                templateCreationPending.set(false);
-                if (putIndexTemplateResponse.isAcknowledged() == false) {
-                    throw new ElasticsearchException("put template for security index was not acknowledged");
-                }
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                templateCreationPending.set(false);
-                logger.warn("failed to put security index template", e);
-            }
-        });
-    }
-
-    static boolean securityIndexMappingUpToDate(ClusterState clusterState, Logger logger) {
-        return securityIndexMappingVersionMatches(clusterState, logger, Version.CURRENT::equals);
-    }
-
-    static boolean securityIndexMappingVersionMatches(ClusterState clusterState, Logger logger, Predicate<Version> predicate) {
-        return securityIndexMappingVersions(clusterState, logger).stream().allMatch(predicate);
-    }
-
-    public static Version oldestSecurityIndexMappingVersion(ClusterState clusterState, Logger logger) {
-        final Set<Version> versions = securityIndexMappingVersions(clusterState, logger);
-        return versions.stream().min(Version::compareTo).orElse(null);
-    }
-
-    private static Set<Version> securityIndexMappingVersions(ClusterState clusterState, Logger logger) {
-        Set<Version> versions = new HashSet<>();
-        IndexMetaData indexMetaData = clusterState.metaData().getIndices().get(SECURITY_INDEX_NAME);
-        if (indexMetaData != null) {
-            for (Object object : indexMetaData.getMappings().values().toArray()) {
-                MappingMetaData mappingMetaData = (MappingMetaData) object;
-                if (mappingMetaData.type().equals(MapperService.DEFAULT_MAPPING)) {
-                    continue;
-                }
-                versions.add(readMappingVersion(mappingMetaData, logger));
-            }
-        }
-        return versions;
-    }
-
-    private static Version readMappingVersion(MappingMetaData mappingMetaData, Logger logger) {
-        try {
-            Map<String, Object> meta = (Map<String, Object>) mappingMetaData.sourceAsMap().get("_meta");
-            if (meta == null) {
-                // something pre-5.0, but we don't know what. Use 2.3.0 as a placeholder for "old"
-                return Version.V_2_3_0;
-            }
-            return Version.fromString((String) meta.get(SECURITY_VERSION_STRING));
-        } catch (IOException e) {
-            logger.error("Cannot parse the mapping for security index.", e);
-            throw new ElasticsearchException("Cannot parse the mapping for security index.", e);
-        }
-    }
-
-    static boolean securityTemplateExistsAndIsUpToDate(ClusterState state, Logger logger) {
-        return securityTemplateExistsAndVersionMatches(state, logger, Version.CURRENT::equals);
-    }
-
-    static boolean securityTemplateExistsAndVersionMatches(ClusterState state, Logger logger, Predicate<Version> predicate) {
-        IndexTemplateMetaData templateMeta = state.metaData().templates().get(SECURITY_TEMPLATE_NAME);
-        if (templateMeta == null) {
-            return false;
-        }
-        ImmutableOpenMap<String, CompressedXContent> mappings = templateMeta.getMappings();
-        // check all mappings contain correct version in _meta
-        // we have to parse the source here which is annoying
-        for (Object typeMapping : mappings.values().toArray()) {
-            CompressedXContent typeMappingXContent = (CompressedXContent) typeMapping;
-            try  {
-                Map<String, Object> typeMappingMap =
-                        XContentHelper.convertToMap(new BytesArray(typeMappingXContent.uncompressed()), false, XContentType.JSON).v2();
-                // should always contain one entry with key = typename
-                assert (typeMappingMap.size() == 1);
-                String key = typeMappingMap.keySet().iterator().next();
-                // get the actual mapping entries
-                @SuppressWarnings("unchecked")
-                Map<String, Object> mappingMap = (Map<String, Object>) typeMappingMap.get(key);
-                if (containsCorrectVersion(mappingMap, predicate) == false) {
-                    return false;
-                }
-            } catch (ElasticsearchParseException e) {
-                logger.error("Cannot parse the template for security index.", e);
-                throw new IllegalStateException("Cannot parse the template for security index.", e);
-            }
-        }
-        return true;
-    }
-
-    private static boolean containsCorrectVersion(Map<String, Object> typeMappingMap, Predicate<Version> predicate) {
-        @SuppressWarnings("unchecked")
-        Map<String, Object> meta = (Map<String, Object>) typeMappingMap.get("_meta");
-        if (meta == null) {
-            // pre 5.0, cannot be up to date
-            return false;
-        }
-        return predicate.test(Version.fromString((String) meta.get(SECURITY_VERSION_STRING)));
-    }
-
-    /**
-     * Returns the routing-table for the security index, or <code>null</code> if the security index does not exist.
-     */
-    public static IndexRoutingTable getSecurityIndexRoutingTable(ClusterState clusterState) {
-        IndexMetaData metaData = clusterState.metaData().index(SECURITY_INDEX_NAME);
-        if (metaData == null) {
-            return null;
-        } else {
-            return clusterState.routingTable().index(SECURITY_INDEX_NAME);
-        }
-    }
-
-    public static boolean securityIndexMappingAndTemplateUpToDate(ClusterState clusterState, Logger logger) {
-        if (securityTemplateExistsAndIsUpToDate(clusterState, logger) == false) {
-            logger.debug("security template [{}] does not exist or is not up to date, so service cannot start",
-                    SecurityTemplateService.SECURITY_TEMPLATE_NAME);
-            return false;
-        }
-        if (SecurityTemplateService.securityIndexMappingUpToDate(clusterState, logger) == false) {
-            logger.debug("mapping for security index not up to date, so service cannot start");
-            return false;
-        }
-        return true;
-    }
-
-    public static boolean securityIndexMappingAndTemplateSufficientToRead(ClusterState clusterState, Logger logger) {
-        if (securityTemplateExistsAndVersionMatches(clusterState, logger, MIN_READ_VERSION::onOrBefore) == false) {
-            logger.debug("security template [{}] does not exist or is not up to date, so service cannot start",
-                    SecurityTemplateService.SECURITY_TEMPLATE_NAME);
-            return false;
-        }
-        if (securityIndexMappingVersionMatches(clusterState, logger, MIN_READ_VERSION::onOrBefore) == false) {
-            logger.debug("mapping for security index not up to date, so service cannot start");
-            return false;
-        }
-        return true;
-    }
-}

plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmMigrator.java

@@ -5,50 +5,48 @@
  */
 package org.elasticsearch.xpack.security.authc.esnative;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.function.BiConsumer;
-
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.update.UpdateResponse;
-import org.elasticsearch.client.Client;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.inject.internal.Nullable;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.xpack.common.GroupedActionListener;
-import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.security.InternalClient;
-import org.elasticsearch.xpack.security.SecurityTemplateService;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.authc.support.Hasher;
 import org.elasticsearch.xpack.security.authc.support.SecuredString;
 import org.elasticsearch.xpack.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.user.LogstashSystemUser;
 import org.elasticsearch.xpack.security.user.User;
-import org.elasticsearch.xpack.security.user.User.Fields;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.BiConsumer;
+
+import static java.util.Collections.emptyList;
+import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME;
 
 /**
  * Performs migration steps for the {@link NativeRealm} and {@link ReservedRealm}.
  * When upgrading an Elasticsearch/X-Pack installation from a previous version, this class is responsible for ensuring that user/role
  * data stored in the security index is converted to a format that is appropriate for the newly installed version.
- *
- * @see SecurityTemplateService
  */
 public class NativeRealmMigrator {
 
     private final XPackLicenseState licenseState;
     private final Logger logger;
-    private Client client;
+    private InternalClient client;
 
     public NativeRealmMigrator(Settings settings, XPackLicenseState licenseState, InternalClient internalClient) {
         this.licenseState = licenseState;
@@ -64,9 +62,9 @@ public class NativeRealmMigrator {
      * @param listener A listener for the results of the upgrade. Calls {@link ActionListener#onFailure(Exception)} if a problem occurs,
      *                 {@link ActionListener#onResponse(Object) onResponse(true)} if an upgrade is performed, or
      *                 {@link ActionListener#onResponse(Object) onResponse(false)} if no upgrade was required.
-     * @see SecurityTemplateService#securityIndexMappingAndTemplateSufficientToRead(ClusterState, Logger)
+     * @see SecurityLifecycleService#securityIndexMappingAndTemplateSufficientToRead(ClusterState, Logger)
-     * @see NativeUsersStore#canWrite
+     * @see SecurityLifecycleService#canWriteToSecurityIndex
-     * @see NativeUsersStore#mappingVersion
+     * @see SecurityLifecycleService#mappingVersion
      */
     public void performUpgrade(@Nullable Version previousVersion, ActionListener<Boolean> listener) {
         try {
@@ -75,9 +73,7 @@ public class NativeRealmMigrator {
                 listener.onResponse(false);
             } else {
                 final GroupedActionListener<Void> countDownListener = new GroupedActionListener<>(
-                        ActionListener.wrap(r -> listener.onResponse(true), listener::onFailure),
+                    ActionListener.wrap(r -> listener.onResponse(true), listener::onFailure), tasks.size(), emptyList()
-                        tasks.size(),
-                        Collections.emptyList()
                 );
                 logger.info("Performing {} security migration task(s)", tasks.size());
                 tasks.forEach(t -> t.accept(previousVersion, countDownListener));
@@ -100,8 +96,8 @@ public class NativeRealmMigrator {
 
     /**
      * If we're upgrading from a security version where the {@link LogstashSystemUser} did not exist, then we mark the user as disabled.
-     * Otherwise the user will exist with a default password, which is desirable for an <em>out-of-the-box</em> experience in fresh installs
+     * Otherwise the user will exist with a default password, which is desirable for an <em>out-of-the-box</em> experience in fresh
-     * but problematic for already-locked-down upgrades.
+     * installs but problematic for already-locked-down upgrades.
      */
     private boolean shouldDisableLogstashUser(@Nullable Version previousVersion) {
         return previousVersion != null && previousVersion.before(LogstashSystemUser.DEFINED_SINCE);
@@ -109,40 +105,38 @@ public class NativeRealmMigrator {
 
     private void createLogstashUserAsDisabled(@Nullable Version previousVersion, ActionListener<Void> listener) {
         logger.info("Upgrading security from version [{}] - new reserved user [{}] will default to disabled",
-                previousVersion, LogstashSystemUser.NAME);
+        previousVersion, LogstashSystemUser.NAME);
         // Only clear the cache is authentication is allowed by the current license
         // otherwise the license management checks will prevent it from completing successfully.
         final boolean clearCache = licenseState.isAuthAllowed();
-        client.prepareGet(SecurityTemplateService.SECURITY_INDEX_NAME, NativeUsersStore.RESERVED_USER_DOC_TYPE, LogstashSystemUser.NAME)
+        client.prepareGet(SECURITY_INDEX_NAME, NativeUsersStore.RESERVED_USER_DOC_TYPE, LogstashSystemUser.NAME).execute(
-                .execute(ActionListener.wrap(getResponse -> {
+        ActionListener.wrap(getResponse -> {
-                    if (getResponse.isExists()) {
+            if (getResponse.isExists()) {
-                        // the document exists - we shouldn't do anything
+                // the document exists - we shouldn't do anything
-                        listener.onResponse(null);
+                listener.onResponse(null);
-                    } else {
+            } else {
-                        client.prepareIndex(SecurityTemplateService.SECURITY_INDEX_NAME, NativeUsersStore.RESERVED_USER_DOC_TYPE,
+                client.prepareIndex(SECURITY_INDEX_NAME, NativeUsersStore.RESERVED_USER_DOC_TYPE, LogstashSystemUser.NAME)
-                                LogstashSystemUser.NAME)
+                      .setSource(Requests.INDEX_CONTENT_TYPE, User.Fields.ENABLED.getPreferredName(), false,
-                                .setSource(Requests.INDEX_CONTENT_TYPE,
+                                 User.Fields.PASSWORD.getPreferredName(), "")
-                                        User.Fields.ENABLED.getPreferredName(), false,
+                      .setCreate(true)
-                                        User.Fields.PASSWORD.getPreferredName(), "")
+                      .execute(ActionListener.wrap(r -> {
-                                .setCreate(true)
+                          if (clearCache) {
-                                .execute(ActionListener.wrap(r -> {
+                              new SecurityClient(client).prepareClearRealmCache()
-                                    if (clearCache) {
+                                    .usernames(LogstashSystemUser.NAME)
-                                        new SecurityClient(client).prepareClearRealmCache()
+                                    .execute(ActionListener.wrap(re -> listener.onResponse(null), listener::onFailure));
-                                                .usernames(LogstashSystemUser.NAME)
+                          } else {
-                                                .execute(ActionListener.wrap(re -> listener.onResponse(null), listener::onFailure));
+                              listener.onResponse(null);
-                                    } else {
+                          }
-                                        listener.onResponse(null);
+                      }, listener::onFailure));
-                                    }
+            }
-                                }, listener::onFailure));
+        }, listener::onFailure));
-                    }
-                }, listener::onFailure));
     }
 
     /**
-     * Old versions of X-Pack security would assign the default password content to a user if it was enabled/disabled before the password
+     * Old versions of X-Pack security would assign the default password content to a user if it was enabled/disabled before the
-     * was explicitly set to another value. If upgrading from one of those versions, then we want to change those users to be flagged as
+     * password was explicitly set to another value. If upgrading from one of those versions, then we want to change those users to be
-     * having a "default password" (which is stored as blank) so that {@link ReservedRealm#ACCEPT_DEFAULT_PASSWORD_SETTING} does the
+     * flagged as having a "default password" (which is stored as blank) so that {@link ReservedRealm#ACCEPT_DEFAULT_PASSWORD_SETTING}
-     * right thing.
+     * does the right thing.
      */
     private boolean shouldConvertDefaultPasswords(@Nullable Version previousVersion) {
         return previousVersion != null && previousVersion.before(Version.V_6_0_0_alpha1_UNRELEASED);
@@ -150,40 +144,37 @@ public class NativeRealmMigrator {
 
     @SuppressWarnings("unused")
     private void doConvertDefaultPasswords(@Nullable Version previousVersion, ActionListener<Void> listener) {
-        client.prepareSearch(SecurityTemplateService.SECURITY_INDEX_NAME)
+        client.prepareSearch(SECURITY_INDEX_NAME)
-                .setTypes(NativeUsersStore.RESERVED_USER_DOC_TYPE)
+        .setTypes(NativeUsersStore.RESERVED_USER_DOC_TYPE)
-                .setQuery(QueryBuilders.matchAllQuery())
+        .setQuery(QueryBuilders.matchAllQuery())
-                .setFetchSource(true)
+        .setFetchSource(true)
-                .execute(ActionListener.wrap(searchResponse -> {
+        .execute(ActionListener.wrap(searchResponse -> {
-                    assert searchResponse.getHits().getTotalHits() <= 10 : "there are more than 10 reserved users we need to change " +
+            assert searchResponse.getHits().getTotalHits() <= 10 :
-                            "this to retrieve them all!";
+            "there are more than 10 reserved users we need to change this to retrieve them all!";
-                    Set<String> toConvert = new HashSet<>();
+            Set<String> toConvert = new HashSet<>();
-                    for (SearchHit searchHit : searchResponse.getHits()) {
+            for (SearchHit searchHit : searchResponse.getHits()) {
-                        Map<String, Object> sourceMap = searchHit.getSourceAsMap();
+                Map<String, Object> sourceMap = searchHit.getSourceAsMap();
-                        if (hasOldStyleDefaultPassword(sourceMap)) {
+                if (hasOldStyleDefaultPassword(sourceMap)) {
-                            toConvert.add(searchHit.getId());
+                    toConvert.add(searchHit.getId());
-                        }
+                }
-                    }
+            }
 
-                    if (toConvert.isEmpty()) {
+            if (toConvert.isEmpty()) {
-                        listener.onResponse(null);
+                listener.onResponse(null);
-                    } else {
+            } else {
-                        GroupedActionListener<UpdateResponse> countDownListener = new GroupedActionListener<>(
+                GroupedActionListener<UpdateResponse> countDownListener = new GroupedActionListener<>(
-                                ActionListener.wrap((r) -> listener.onResponse(null), listener::onFailure),
+                    ActionListener.wrap((r) -> listener.onResponse(null), listener::onFailure), toConvert.size(), emptyList()
-                                toConvert.size(), Collections.emptyList()
+                );
-                        );
+                toConvert.forEach(username -> {
-                        toConvert.forEach(username -> {
+                    logger.debug("Upgrading security from version [{}] - marking reserved user [{}] as having default password",
-                            logger.debug(
+                                 previousVersion, username);
-                                    "Upgrading security from version [{}] - marking reserved user [{}] as having default password",
+                    client.prepareUpdate(SECURITY_INDEX_NAME, NativeUsersStore.RESERVED_USER_DOC_TYPE, username)
-                                    previousVersion, username);
+                          .setDoc(User.Fields.PASSWORD.getPreferredName(), "")
-                            client.prepareUpdate(
+                          .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
-                                    SecurityTemplateService.SECURITY_INDEX_NAME,NativeUsersStore.RESERVED_USER_DOC_TYPE, username)
+                          .execute(countDownListener);
-                                    .setDoc(Fields.PASSWORD.getPreferredName(), "")
+                });
-                                    .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+            }
-                                    .execute(countDownListener);
+        }, listener::onFailure));
-                        });
-                    }
-                }, listener::onFailure));
     }
 
     /**

plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java

@@ -9,7 +9,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.delete.DeleteRequest;
@@ -22,10 +21,6 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.client.Requests;
-import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateListener;
-import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.ValidationException;
@@ -33,14 +28,13 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.engine.DocumentMissingException;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.xpack.security.InternalClient;
-import org.elasticsearch.xpack.security.SecurityTemplateService;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.action.realm.ClearRealmCacheRequest;
 import org.elasticsearch.xpack.security.action.realm.ClearRealmCacheResponse;
 import org.elasticsearch.xpack.security.action.user.ChangePasswordRequest;
@@ -60,13 +54,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
-import java.util.function.Predicate;
-
-import static org.elasticsearch.xpack.security.SecurityTemplateService.oldestSecurityIndexMappingVersion;
-import static org.elasticsearch.xpack.security.SecurityTemplateService.securityIndexMappingAndTemplateSufficientToRead;
-import static org.elasticsearch.xpack.security.SecurityTemplateService.securityIndexMappingAndTemplateUpToDate;
 
 /**
  * NativeUsersStore is a store for users that reads from an Elasticsearch index. This store is responsible for fetching the full
@@ -75,58 +63,37 @@ import static org.elasticsearch.xpack.security.SecurityTemplateService.securityI
  * No caching is done by this class, it is handled at a higher level and no polling for changes is done by this class. Modification
  * operations make a best effort attempt to clear the cache on all nodes for the user that was modified.
  */
-public class NativeUsersStore extends AbstractComponent implements ClusterStateListener {
+public class NativeUsersStore extends AbstractComponent {
-
-    public enum State {
-        INITIALIZED,
-        STARTING,
-        STARTED,
-        STOPPING,
-        STOPPED,
-        FAILED
-    }
 
     private static final String USER_DOC_TYPE = "user";
-    static final String RESERVED_USER_DOC_TYPE = "reserved-user";
+    public static final String RESERVED_USER_DOC_TYPE = "reserved-user";
 
     private final Hasher hasher = Hasher.BCRYPT;
-    private final AtomicReference<State> state = new AtomicReference<>(State.INITIALIZED);
     private final InternalClient client;
     private final boolean isTribeNode;
 
-    private volatile boolean securityIndexExists = false;
+    private volatile SecurityLifecycleService securityLifecycleService;
-    private volatile boolean canWrite = false;
-    private volatile Version mappingVersion = null;
 
-    public NativeUsersStore(Settings settings, InternalClient client) {
+    public NativeUsersStore(Settings settings, InternalClient client, SecurityLifecycleService securityLifecycleService) {
         super(settings);
         this.client = client;
         this.isTribeNode = settings.getGroups("tribe", true).isEmpty() == false;
+        this.securityLifecycleService = securityLifecycleService;
     }
 
     /**
      * Blocking version of {@code getUser} that blocks until the User is returned
      */
     public void getUser(String username, ActionListener<User> listener) {
-        if (state() != State.STARTED) {
+        getUserAndPassword(username, ActionListener.wrap((uap) -> {
-            logger.trace("attempted to get user [{}] before service was started", username);
+            listener.onResponse(uap == null ? null : uap.user());
-            listener.onResponse(null);
+        }, listener::onFailure));
-        } else {
-            getUserAndPassword(username, ActionListener.wrap((uap) -> {
-                listener.onResponse(uap == null ? null : uap.user());
-            }, listener::onFailure));
-        }
     }
 
     /**
      * Retrieve a list of users, if userNames is null or empty, fetch all users
      */
     public void getUsers(String[] userNames, final ActionListener<Collection<User>> listener) {
-        if (state() != State.STARTED) {
-            logger.trace("attempted to get users before service was started");
-            listener.onFailure(new IllegalStateException("users cannot be retrieved as native user service has not been started"));
-            return;
-        }
         final Consumer<Exception> handleException = (t) -> {
             if (t instanceof IndexNotFoundException) {
                 logger.trace("could not retrieve users because security index does not exist");
@@ -149,7 +116,7 @@ public class NativeUsersStore extends AbstractComponent implements ClusterStateL
                 } else {
                     query = QueryBuilders.boolQuery().filter(QueryBuilders.idsQuery(USER_DOC_TYPE).addIds(userNames));
                 }
-                SearchRequest request = client.prepareSearch(SecurityTemplateService.SECURITY_INDEX_NAME)
+                SearchRequest request = client.prepareSearch(SecurityLifecycleService.SECURITY_INDEX_NAME)
                         .setScroll(TimeValue.timeValueSeconds(10L))
                         .setTypes(USER_DOC_TYPE)
                         .setQuery(query)
@@ -173,7 +140,7 @@ public class NativeUsersStore extends AbstractComponent implements ClusterStateL
      */
     private void getUserAndPassword(final String user, final ActionListener<UserAndPassword> listener) {
         try {
-            GetRequest request = client.prepareGet(SecurityTemplateService.SECURITY_INDEX_NAME, USER_DOC_TYPE, user).request();
+            GetRequest request = client.prepareGet(SecurityLifecycleService.SECURITY_INDEX_NAME, USER_DOC_TYPE, user).request();
             client.get(request, new ActionListener<GetResponse>() {
                 @Override
                 public void onResponse(GetResponse response) {
@@ -210,13 +177,10 @@ public class NativeUsersStore extends AbstractComponent implements ClusterStateL
     public void changePassword(final ChangePasswordRequest request, final ActionListener<Void> listener) {
         final String username = request.username();
         assert SystemUser.NAME.equals(username) == false && XPackUser.NAME.equals(username) == false : username + "is internal!";
-        if (state() != State.STARTED) {
+        if (isTribeNode) {
-            listener.onFailure(new IllegalStateException("password cannot be changed as user service has not been started"));
-            return;
-        } else if (isTribeNode) {
             listener.onFailure(new UnsupportedOperationException("users may not be created or modified using a tribe node"));
             return;
-        } else if (canWrite == false) {
+        } else if (securityLifecycleService.canWriteToSecurityIndex() == false) {
             listener.onFailure(new IllegalStateException("password cannot be changed as user service cannot write until template and " +
                     "mappings are up to date"));
             return;
@@ -229,7 +193,7 @@ public class NativeUsersStore extends AbstractComponent implements ClusterStateL
             docType = USER_DOC_TYPE;
         }
 
-        client.prepareUpdate(SecurityTemplateService.SECURITY_INDEX_NAME, docType, username)
+        client.prepareUpdate(SecurityLifecycleService.SECURITY_INDEX_NAME, docType, username)
                 .setDoc(Requests.INDEX_CONTENT_TYPE, Fields.PASSWORD.getPreferredName(), String.valueOf(request.passwordHash()))
                 .setRefreshPolicy(request.getRefreshPolicy())
                 .execute(new ActionListener<UpdateResponse>() {
@@ -263,7 +227,7 @@ public class NativeUsersStore extends AbstractComponent implements ClusterStateL
      * has been indexed
      */
     private void createReservedUser(String username, char[] passwordHash, RefreshPolicy refresh, ActionListener<Void> listener) {
-        client.prepareIndex(SecurityTemplateService.SECURITY_INDEX_NAME, RESERVED_USER_DOC_TYPE, username)
+        client.prepareIndex(SecurityLifecycleService.SECURITY_INDEX_NAME, RESERVED_USER_DOC_TYPE, username)
                 .setSource(Fields.PASSWORD.getPreferredName(), String.valueOf(passwordHash), Fields.ENABLED.getPreferredName(), true)
                 .setRefreshPolicy(refresh)
                 .execute(new ActionListener<IndexResponse>() {
@@ -286,13 +250,10 @@ public class NativeUsersStore extends AbstractComponent implements ClusterStateL
      * method will not modify the enabled value.
      */
     public void putUser(final PutUserRequest request, final ActionListener<Boolean> listener) {
-        if (state() != State.STARTED) {
+        if (isTribeNode) {
-            listener.onFailure(new IllegalStateException("user cannot be added as native user service has not been started"));
-            return;
-        } else if (isTribeNode) {
             listener.onFailure(new UnsupportedOperationException("users may not be created or modified using a tribe node"));
             return;
-        }  else if (canWrite == false) {
+        }  else if (securityLifecycleService.canWriteToSecurityIndex() == false) {
             listener.onFailure(new IllegalStateException("user cannot be created or changed as the user service cannot write until " +
                     "template and mappings are up to date"));
             return;
@@ -316,7 +277,7 @@ public class NativeUsersStore extends AbstractComponent implements ClusterStateL
     private void updateUserWithoutPassword(final PutUserRequest putUserRequest, final ActionListener<Boolean> listener) {
         assert putUserRequest.passwordHash() == null;
         // We must have an existing document
-        client.prepareUpdate(SecurityTemplateService.SECURITY_INDEX_NAME, USER_DOC_TYPE, putUserRequest.username())
+        client.prepareUpdate(SecurityLifecycleService.SECURITY_INDEX_NAME, USER_DOC_TYPE, putUserRequest.username())
                 .setDoc(Requests.INDEX_CONTENT_TYPE,
                         User.Fields.USERNAME.getPreferredName(), putUserRequest.username(),
                         User.Fields.ROLES.getPreferredName(), putUserRequest.roles(),
@@ -351,7 +312,7 @@ public class NativeUsersStore extends AbstractComponent implements ClusterStateL
 
     private void indexUser(final PutUserRequest putUserRequest, final ActionListener<Boolean> listener) {
         assert putUserRequest.passwordHash() != null;
-        client.prepareIndex(SecurityTemplateService.SECURITY_INDEX_NAME,
+        client.prepareIndex(SecurityLifecycleService.SECURITY_INDEX_NAME,
                 USER_DOC_TYPE, putUserRequest.username())
                 .setSource(User.Fields.USERNAME.getPreferredName(), putUserRequest.username(),
                         User.Fields.PASSWORD.getPreferredName(), String.valueOf(putUserRequest.passwordHash()),
@@ -380,13 +341,10 @@ public class NativeUsersStore extends AbstractComponent implements ClusterStateL
      */
     public void setEnabled(final String username, final boolean enabled, final RefreshPolicy refreshPolicy,
                            final ActionListener<Void> listener) {
-        if (state() != State.STARTED) {
+        if (isTribeNode) {
-            listener.onFailure(new IllegalStateException("enabled status cannot be changed as native user service has not been started"));
-            return;
-        } else if (isTribeNode) {
             listener.onFailure(new UnsupportedOperationException("users may not be created or modified using a tribe node"));
             return;
-        }  else if (canWrite == false) {
+        }  else if (securityLifecycleService.canWriteToSecurityIndex() == false) {
             listener.onFailure(new IllegalStateException("enabled status cannot be changed as user service cannot write until template " +
                     "and mappings are up to date"));
             return;
@@ -402,7 +360,7 @@ public class NativeUsersStore extends AbstractComponent implements ClusterStateL
     private void setRegularUserEnabled(final String username, final boolean enabled, final RefreshPolicy refreshPolicy,
                             final ActionListener<Void> listener) {
         try {
-            client.prepareUpdate(SecurityTemplateService.SECURITY_INDEX_NAME, USER_DOC_TYPE, username)
+            client.prepareUpdate(SecurityLifecycleService.SECURITY_INDEX_NAME, USER_DOC_TYPE, username)
                     .setDoc(Requests.INDEX_CONTENT_TYPE, User.Fields.ENABLED.getPreferredName(), enabled)
                     .setRefreshPolicy(refreshPolicy)
                     .execute(new ActionListener<UpdateResponse>() {
@@ -434,7 +392,7 @@ public class NativeUsersStore extends AbstractComponent implements ClusterStateL
     private void setReservedUserEnabled(final String username, final boolean enabled, final RefreshPolicy refreshPolicy,
                                         boolean clearCache, final ActionListener<Void> listener) {
         try {
-            client.prepareUpdate(SecurityTemplateService.SECURITY_INDEX_NAME, RESERVED_USER_DOC_TYPE, username)
+            client.prepareUpdate(SecurityLifecycleService.SECURITY_INDEX_NAME, RESERVED_USER_DOC_TYPE, username)
                     .setDoc(Requests.INDEX_CONTENT_TYPE, User.Fields.ENABLED.getPreferredName(), enabled)
                     .setUpsert(XContentType.JSON,
                             User.Fields.PASSWORD.getPreferredName(), "",
@@ -461,20 +419,17 @@ public class NativeUsersStore extends AbstractComponent implements ClusterStateL
     }
 
     public void deleteUser(final DeleteUserRequest deleteUserRequest, final ActionListener<Boolean> listener) {
-        if (state() != State.STARTED) {
+        if (isTribeNode) {
-            listener.onFailure(new IllegalStateException("user cannot be deleted as native user service has not been started"));
-            return;
-        } else if (isTribeNode) {
             listener.onFailure(new UnsupportedOperationException("users may not be deleted using a tribe node"));
             return;
-        }  else if (canWrite == false) {
+        }  else if (securityLifecycleService.canWriteToSecurityIndex() == false) {
             listener.onFailure(new IllegalStateException("user cannot be deleted as user service cannot write until template and " +
                     "mappings are up to date"));
             return;
         }
 
         try {
-            DeleteRequest request = client.prepareDelete(SecurityTemplateService.SECURITY_INDEX_NAME,
+            DeleteRequest request = client.prepareDelete(SecurityLifecycleService.SECURITY_INDEX_NAME,
                     USER_DOC_TYPE, deleteUserRequest.username()).request();
             request.indicesOptions().ignoreUnavailable();
             request.setRefreshPolicy(deleteUserRequest.getRefreshPolicy());
@@ -496,64 +451,6 @@ public class NativeUsersStore extends AbstractComponent implements ClusterStateL
         }
     }
 
-    public boolean canStart(ClusterState clusterState, boolean master) {
-        if (state() != State.INITIALIZED) {
-            return false;
-        }
-
-        if (clusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
-            // wait until the gateway has recovered from disk, otherwise we
-            // think may not have the .security index but they it may not have
-            // been restored from the cluster state on disk yet
-            logger.debug("native users store waiting until gateway has recovered from disk");
-            return false;
-        }
-
-        if (isTribeNode) {
-            return true;
-        }
-
-        if (securityIndexMappingAndTemplateUpToDate(clusterState, logger)) {
-            canWrite = true;
-        } else if (securityIndexMappingAndTemplateSufficientToRead(clusterState, logger)) {
-            mappingVersion = oldestSecurityIndexMappingVersion(clusterState, logger);
-            canWrite = false;
-        } else {
-            canWrite = false;
-            return false;
-        }
-
-        final IndexRoutingTable routingTable = SecurityTemplateService.getSecurityIndexRoutingTable(clusterState);
-        if (routingTable == null) {
-            logger.debug("security index [{}] does not exist, so service can start", SecurityTemplateService.SECURITY_INDEX_NAME);
-            return true;
-        }
-        if (routingTable.allPrimaryShardsActive()) {
-            logger.debug("security index [{}] all primary shards started, so service can start",
-                    SecurityTemplateService.SECURITY_INDEX_NAME);
-            securityIndexExists = true;
-            return true;
-        }
-        return false;
-    }
-
-    public void start() {
-        try {
-            if (state.compareAndSet(State.INITIALIZED, State.STARTING)) {
-                state.set(State.STARTED);
-            }
-        } catch (Exception e) {
-            logger.error("failed to start native user store", e);
-            state.set(State.FAILED);
-        }
-    }
-
-    public void stop() {
-        if (state.compareAndSet(State.STARTED, State.STOPPING)) {
-            state.set(State.STOPPED);
-        }
-    }
-
     /**
      * This method is used to verify the username and credentials against those stored in the system.
      *
@@ -561,46 +458,23 @@ public class NativeUsersStore extends AbstractComponent implements ClusterStateL
      * @param password the plaintext password to verify
      */
     void verifyPassword(String username, final SecuredString password, ActionListener<User> listener) {
-        if (state() != State.STARTED) {
+        getUserAndPassword(username, ActionListener.wrap((userAndPassword) -> {
-            logger.trace("attempted to verify user credentials for [{}] but service was not started", username);
+            if (userAndPassword == null || userAndPassword.passwordHash() == null) {
-            listener.onResponse(null);
+                listener.onResponse(null);
-        } else {
+            } else if (hasher.verify(password, userAndPassword.passwordHash())) {
-            getUserAndPassword(username, ActionListener.wrap((userAndPassword) -> {
+                listener.onResponse(userAndPassword.user());
-                if (userAndPassword == null || userAndPassword.passwordHash() == null) {
+            } else {
-                    listener.onResponse(null);
+                listener.onResponse(null);
-                } else if (hasher.verify(password, userAndPassword.passwordHash())) {
+            }
-                    listener.onResponse(userAndPassword.user());
+        }, listener::onFailure));
-                } else {
-                    listener.onResponse(null);
-                }
-            }, listener::onFailure));
-        }
-    }
-
-    public boolean started() {
-        return state() == State.STARTED;
-    }
-
-    boolean securityIndexExists() {
-        return securityIndexExists;
-    }
-
-    /**
-     * Test whether the effective (active) version of the security mapping meets the <code>requiredVersion</code>.
-     *
-     * @return <code>true</code> if the effective version passes the predicate, or the security mapping does not exist (<code>null</code>
-     * version). Otherwise, <code>false</code>.
-     */
-    public boolean checkMappingVersion(Predicate<Version> requiredVersion) {
-        return this.mappingVersion == null || requiredVersion.test(this.mappingVersion);
     }
 
     void getReservedUserInfo(String username, ActionListener<ReservedUserInfo> listener) {
-        if (!started() && !securityIndexExists()) {
+        if (!securityLifecycleService.securityIndexExists()) {
-            listener.onFailure(new IllegalStateException("Attempt to get reserved user info - started=" + started() +
+            listener.onFailure(new IllegalStateException("Attempt to get reserved user info but the security index does not exist"));
-                    " index-exists=" + securityIndexExists()));
+            return;
         }
-        client.prepareGet(SecurityTemplateService.SECURITY_INDEX_NAME, RESERVED_USER_DOC_TYPE, username)
+        client.prepareGet(SecurityLifecycleService.SECURITY_INDEX_NAME, RESERVED_USER_DOC_TYPE, username)
                 .execute(new ActionListener<GetResponse>() {
                     @Override
                     public void onResponse(GetResponse getResponse) {
@@ -639,8 +513,7 @@ public class NativeUsersStore extends AbstractComponent implements ClusterStateL
     }
 
     void getAllReservedUserInfo(ActionListener<Map<String, ReservedUserInfo>> listener) {
-        assert started();
+        client.prepareSearch(SecurityLifecycleService.SECURITY_INDEX_NAME)
-        client.prepareSearch(SecurityTemplateService.SECURITY_INDEX_NAME)
                 .setTypes(RESERVED_USER_DOC_TYPE)
                 .setQuery(QueryBuilders.matchAllQuery())
                 .setFetchSource(true)
@@ -702,29 +575,6 @@ public class NativeUsersStore extends AbstractComponent implements ClusterStateL
         });
     }
 
-    @Override
-    public void clusterChanged(ClusterChangedEvent event) {
-        securityIndexExists = event.state().metaData().indices().get(SecurityTemplateService.SECURITY_INDEX_NAME) != null;
-        canWrite = securityIndexMappingAndTemplateUpToDate(event.state(), logger);
-        mappingVersion = oldestSecurityIndexMappingVersion(event.state(), logger);
-    }
-
-    public State state() {
-        return state.get();
-    }
-
-    // FIXME hack for testing
-    public void reset() {
-        final State state = state();
-        if (state != State.STOPPED && state != State.FAILED) {
-            throw new IllegalStateException("can only reset if stopped!!!");
-        }
-        this.securityIndexExists = false;
-        this.canWrite = false;
-        this.mappingVersion = null;
-        this.state.set(State.INITIALIZED);
-    }
-
     @Nullable
     private UserAndPassword transformUser(String username, Map<String, Object> sourceMap) {
         if (sourceMap == null) {
@@ -760,9 +610,9 @@ public class NativeUsersStore extends AbstractComponent implements ClusterStateL
 
     static class ReservedUserInfo {
 
-        final char[] passwordHash;
+        public final char[] passwordHash;
-        final boolean enabled;
+        public final boolean enabled;
-        final boolean hasDefaultPassword;
+        public final boolean hasDefaultPassword;
 
         ReservedUserInfo(char[] passwordHash, boolean enabled, boolean hasDefaultPassword) {
             this.passwordHash = passwordHash;

plugin/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java

@@ -14,6 +14,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.xpack.XPackSettings;
 import org.elasticsearch.xpack.security.Security;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.authc.RealmConfig;
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore.ReservedUserInfo;
 import org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm;
@@ -41,7 +42,7 @@ public class ReservedRealm extends CachingUsernamePasswordRealm {
 
     public static final String TYPE = "reserved";
 
-    static final SecuredString DEFAULT_PASSWORD_TEXT = new SecuredString("changeme".toCharArray());
+    public static final SecuredString DEFAULT_PASSWORD_TEXT = new SecuredString("changeme".toCharArray());
     static final char[] DEFAULT_PASSWORD_HASH = Hasher.BCRYPT.hash(DEFAULT_PASSWORD_TEXT);
 
     private static final ReservedUserInfo DEFAULT_USER_INFO = new ReservedUserInfo(DEFAULT_PASSWORD_HASH, true, true);
@@ -55,14 +56,17 @@ public class ReservedRealm extends CachingUsernamePasswordRealm {
     private final boolean realmEnabled;
     private final boolean anonymousEnabled;
     private final boolean defaultPasswordEnabled;
+    private final SecurityLifecycleService securityLifecycleService;
 
-    public ReservedRealm(Environment env, Settings settings, NativeUsersStore nativeUsersStore, AnonymousUser anonymousUser) {
+    public ReservedRealm(Environment env, Settings settings, NativeUsersStore nativeUsersStore, AnonymousUser anonymousUser,
+                         SecurityLifecycleService securityLifecycleService) {
         super(TYPE, new RealmConfig(TYPE, Settings.EMPTY, settings, env));
         this.nativeUsersStore = nativeUsersStore;
         this.realmEnabled = XPackSettings.RESERVED_REALM_ENABLED_SETTING.get(settings);
         this.anonymousUser = anonymousUser;
         this.anonymousEnabled = AnonymousUser.isAnonymousEnabled(settings);
         this.defaultPasswordEnabled = ACCEPT_DEFAULT_PASSWORD_SETTING.get(settings);
+        this.securityLifecycleService = securityLifecycleService;
     }
 
     @Override
@@ -162,7 +166,7 @@ public class ReservedRealm extends CachingUsernamePasswordRealm {
 
 
     public void users(ActionListener<Collection<User>> listener) {
-        if (nativeUsersStore.started() == false || realmEnabled == false) {
+        if (realmEnabled == false) {
             listener.onResponse(anonymousEnabled ? Collections.singletonList(anonymousUser) : Collections.emptyList());
         } else {
             nativeUsersStore.getAllReservedUserInfo(ActionListener.wrap((reservedUserInfos) -> {
@@ -190,13 +194,10 @@ public class ReservedRealm extends CachingUsernamePasswordRealm {
     }
 
     private void getUserInfo(final String username, ActionListener<ReservedUserInfo> listener) {
-        if (nativeUsersStore.started() == false) {
+        if (userIsDefinedForCurrentSecurityMapping(username) == false) {
-            // we need to be able to check for the user store being started...
-            listener.onResponse(null);
-        } else if (userIsDefinedForCurrentSecurityMapping(username) == false) {
             logger.debug("Marking user [{}] as disabled because the security mapping is not at the required version", username);
             listener.onResponse(DISABLED_USER_INFO);
-        } else if (nativeUsersStore.securityIndexExists() == false) {
+        } else if (securityLifecycleService.securityIndexExists() == false) {
             listener.onResponse(DEFAULT_USER_INFO);
         } else {
             nativeUsersStore.getReservedUserInfo(username, ActionListener.wrap((userInfo) -> {
@@ -215,7 +216,7 @@ public class ReservedRealm extends CachingUsernamePasswordRealm {
 
     private boolean userIsDefinedForCurrentSecurityMapping(String username) {
         final Version requiredVersion = getDefinedVersion(username);
-        return nativeUsersStore.checkMappingVersion(requiredVersion::onOrBefore);
+        return securityLifecycleService.checkMappingVersion(requiredVersion::onOrBefore);
     }
 
     private Version getDefinedVersion(String username) {

plugin/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java

@@ -33,7 +33,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.xpack.security.SecurityTemplateService;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.action.user.AuthenticateAction;
 import org.elasticsearch.xpack.security.action.user.ChangePasswordAction;
 import org.elasticsearch.xpack.security.action.user.UserRequest;
@@ -234,15 +234,15 @@ public class AuthorizationService extends AbstractComponent {
         IndicesAccessControl indicesAccessControl = permission.authorize(action, indexNames, metaData, fieldPermissionsCache);
         if (!indicesAccessControl.isGranted()) {
             throw denial(authentication, action, request);
-        } else if (indicesAccessControl.getIndexPermissions(SecurityTemplateService.SECURITY_INDEX_NAME) != null
+        } else if (indicesAccessControl.getIndexPermissions(SecurityLifecycleService.SECURITY_INDEX_NAME) != null
-                && indicesAccessControl.getIndexPermissions(SecurityTemplateService.SECURITY_INDEX_NAME).isGranted()
+                && indicesAccessControl.getIndexPermissions(SecurityLifecycleService.SECURITY_INDEX_NAME).isGranted()
                 && XPackUser.is(authentication.getRunAsUser()) == false
                 && MONITOR_INDEX_PREDICATE.test(action) == false
                 && Arrays.binarySearch(authentication.getRunAsUser().roles(), ReservedRolesStore.SUPERUSER_ROLE.name()) < 0) {
             // only the XPackUser is allowed to work with this index, but we should allow indices monitoring actions through for debugging
             // purposes. These monitor requests also sometimes resolve indices concretely and then requests them
             logger.debug("user [{}] attempted to directly perform [{}] against the security index [{}]",
-                    authentication.getRunAsUser().principal(), action, SecurityTemplateService.SECURITY_INDEX_NAME);
+                    authentication.getRunAsUser().principal(), action, SecurityLifecycleService.SECURITY_INDEX_NAME);
             throw denial(authentication, action, request);
         } else {
             setIndicesAccessControl(indicesAccessControl);

plugin/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizedIndices.java

@@ -7,7 +7,7 @@ package org.elasticsearch.xpack.security.authz;
 
 import org.elasticsearch.cluster.metadata.AliasOrIndex;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.xpack.security.SecurityTemplateService;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.authz.permission.Role;
 import org.elasticsearch.xpack.security.authz.store.ReservedRolesStore;
 import org.elasticsearch.xpack.security.user.User;
@@ -59,7 +59,7 @@ class AuthorizedIndices {
 
         if (XPackUser.is(user) == false && Arrays.binarySearch(user.roles(), ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR.getName()) < 0) {
             // we should filter out the .security index from wildcards
-            indicesAndAliases.remove(SecurityTemplateService.SECURITY_INDEX_NAME);
+            indicesAndAliases.remove(SecurityLifecycleService.SECURITY_INDEX_NAME);
         }
         return Collections.unmodifiableList(indicesAndAliases);
     }

plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java

@@ -21,10 +21,6 @@ import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.MultiSearchResponse.Item;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.TransportActions;
-import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateListener;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.component.AbstractComponent;
@@ -34,7 +30,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -42,7 +37,7 @@ import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.security.InternalClient;
-import org.elasticsearch.xpack.security.SecurityTemplateService;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.action.role.ClearRolesCacheRequest;
 import org.elasticsearch.xpack.security.action.role.ClearRolesCacheResponse;
 import org.elasticsearch.xpack.security.action.role.DeleteRoleRequest;
@@ -59,13 +54,10 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.existsQuery;
 import static org.elasticsearch.xpack.security.Security.setting;
-import static org.elasticsearch.xpack.security.SecurityTemplateService.securityIndexMappingAndTemplateSufficientToRead;
-import static org.elasticsearch.xpack.security.SecurityTemplateService.securityIndexMappingAndTemplateUpToDate;
 
 /**
  * NativeRolesStore is a {@code RolesStore} that, instead of reading from a
@@ -75,16 +67,7 @@ import static org.elasticsearch.xpack.security.SecurityTemplateService.securityI
  *
  * No caching is done by this class, it is handled at a higher level
  */
-public class NativeRolesStore extends AbstractComponent implements ClusterStateListener {
+public class NativeRolesStore extends AbstractComponent {
-
-    public enum State {
-        INITIALIZED,
-        STARTING,
-        STARTED,
-        STOPPING,
-        STOPPED,
-        FAILED
-    }
 
     // these are no longer used, but leave them around for users upgrading
     private static final Setting<Integer> CACHE_SIZE_SETTING =
@@ -96,89 +79,25 @@ public class NativeRolesStore extends AbstractComponent implements ClusterStateL
 
     private final InternalClient client;
     private final XPackLicenseState licenseState;
-    private final AtomicReference<State> state = new AtomicReference<>(State.INITIALIZED);
     private final boolean isTribeNode;
 
     private SecurityClient securityClient;
+    private final SecurityLifecycleService securityLifecycleService;
 
-    private volatile boolean securityIndexExists = false;
+    public NativeRolesStore(Settings settings, InternalClient client, XPackLicenseState licenseState,
-    private volatile boolean canWrite = false;
+                            SecurityLifecycleService securityLifecycleService) {
-
-    public NativeRolesStore(Settings settings, InternalClient client, XPackLicenseState licenseState) {
         super(settings);
         this.client = client;
         this.isTribeNode = settings.getGroups("tribe", true).isEmpty() == false;
         this.securityClient = new SecurityClient(client);
         this.licenseState = licenseState;
-    }
+        this.securityLifecycleService = securityLifecycleService;
-
-    public boolean canStart(ClusterState clusterState, boolean master) {
-        if (state() != NativeRolesStore.State.INITIALIZED) {
-            return false;
-        }
-
-        if (clusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK)) {
-            // wait until the gateway has recovered from disk, otherwise we
-            // think may not have the security index but it may not have
-            // been restored from the cluster state on disk yet
-            logger.debug("native roles store waiting until gateway has recovered from disk");
-            return false;
-        }
-
-        if (isTribeNode) {
-            return true;
-        }
-
-        if (securityIndexMappingAndTemplateUpToDate(clusterState, logger)) {
-            canWrite = true;
-        } else if (securityIndexMappingAndTemplateSufficientToRead(clusterState, logger)) {
-            canWrite = false;
-        } else {
-            canWrite = false;
-            return false;
-        }
-
-        IndexMetaData metaData = clusterState.metaData().index(SecurityTemplateService.SECURITY_INDEX_NAME);
-        if (metaData == null) {
-            logger.debug("security index [{}] does not exist, so service can start", SecurityTemplateService.SECURITY_INDEX_NAME);
-            return true;
-        }
-
-        if (clusterState.routingTable().index(SecurityTemplateService.SECURITY_INDEX_NAME).allPrimaryShardsActive()) {
-            logger.debug("security index [{}] all primary shards started, so service can start",
-                    SecurityTemplateService.SECURITY_INDEX_NAME);
-            securityIndexExists = true;
-            return true;
-        }
-        return false;
-    }
-
-    public void start() {
-        try {
-            if (state.compareAndSet(State.INITIALIZED, State.STARTING)) {
-                state.set(State.STARTED);
-            }
-        } catch (Exception e) {
-            logger.error("failed to start ESNativeRolesStore", e);
-            state.set(State.FAILED);
-        }
-    }
-
-    public void stop() {
-        if (state.compareAndSet(State.STARTED, State.STOPPING)) {
-            state.set(State.STOPPED);
-        }
     }
 
     /**
      * Retrieve a list of roles, if rolesToGet is null or empty, fetch all roles
      */
     public void getRoleDescriptors(String[] names, final ActionListener<Collection<RoleDescriptor>> listener) {
-        if (state() != State.STARTED) {
-            logger.trace("attempted to get roles before service was started");
-            listener.onResponse(Collections.emptySet());
-            return;
-        }
         if (names != null && names.length == 1) {
             getRoleDescriptor(Objects.requireNonNull(names[0]), ActionListener.wrap(roleDescriptor ->
                     listener.onResponse(roleDescriptor == null ? Collections.emptyList() : Collections.singletonList(roleDescriptor)),
@@ -191,7 +110,7 @@ public class NativeRolesStore extends AbstractComponent implements ClusterStateL
                 } else {
                     query = QueryBuilders.boolQuery().filter(QueryBuilders.idsQuery(ROLE_DOC_TYPE).addIds(names));
                 }
-                SearchRequest request = client.prepareSearch(SecurityTemplateService.SECURITY_INDEX_NAME)
+                SearchRequest request = client.prepareSearch(SecurityLifecycleService.SECURITY_INDEX_NAME)
                         .setTypes(ROLE_DOC_TYPE)
                         .setScroll(TimeValue.timeValueSeconds(10L))
                         .setQuery(query)
@@ -209,20 +128,17 @@ public class NativeRolesStore extends AbstractComponent implements ClusterStateL
     }
 
     public void deleteRole(final DeleteRoleRequest deleteRoleRequest, final ActionListener<Boolean> listener) {
-        if (state() != State.STARTED) {
+        if (isTribeNode) {
-            logger.trace("attempted to delete role [{}] before service was started", deleteRoleRequest.name());
-            listener.onResponse(false);
-        } else if (isTribeNode) {
             listener.onFailure(new UnsupportedOperationException("roles may not be deleted using a tribe node"));
             return;
-        } else if (canWrite == false) {
+        } else if (securityLifecycleService.canWriteToSecurityIndex() == false) {
             listener.onFailure(new IllegalStateException("role cannot be deleted as service cannot write until template and " +
                     "mappings are up to date"));
             return;
         }
 
         try {
-            DeleteRequest request = client.prepareDelete(SecurityTemplateService.SECURITY_INDEX_NAME,
+            DeleteRequest request = client.prepareDelete(SecurityLifecycleService.SECURITY_INDEX_NAME,
                     ROLE_DOC_TYPE, deleteRoleRequest.name()).request();
             request.setRefreshPolicy(deleteRoleRequest.getRefreshPolicy());
             client.delete(request, new ActionListener<DeleteResponse>() {
@@ -247,12 +163,9 @@ public class NativeRolesStore extends AbstractComponent implements ClusterStateL
     }
 
     public void putRole(final PutRoleRequest request, final RoleDescriptor role, final ActionListener<Boolean> listener) {
-        if (state() != State.STARTED) {
+        if (isTribeNode) {
-            logger.trace("attempted to put role [{}] before service was started", request.name());
-            listener.onResponse(false);
-        } else if (isTribeNode) {
             listener.onFailure(new UnsupportedOperationException("roles may not be created or modified using a tribe node"));
-        } else if (canWrite == false) {
+        } else if (securityLifecycleService.canWriteToSecurityIndex() == false) {
             listener.onFailure(new IllegalStateException("role cannot be created or modified as service cannot write until template and " +
                     "mappings are up to date"));
         } else if (licenseState.isDocumentAndFieldLevelSecurityAllowed()) {
@@ -267,7 +180,7 @@ public class NativeRolesStore extends AbstractComponent implements ClusterStateL
     // pkg-private for testing
     void innerPutRole(final PutRoleRequest request, final RoleDescriptor role, final ActionListener<Boolean> listener) {
         try {
-            client.prepareIndex(SecurityTemplateService.SECURITY_INDEX_NAME, ROLE_DOC_TYPE, role.getName())
+            client.prepareIndex(SecurityLifecycleService.SECURITY_INDEX_NAME, ROLE_DOC_TYPE, role.getName())
                     .setSource(role.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS, false))
                     .setRefreshPolicy(request.getRefreshPolicy())
                     .execute(new ActionListener<IndexResponse>() {
@@ -290,14 +203,10 @@ public class NativeRolesStore extends AbstractComponent implements ClusterStateL
     }
 
     public Map<String, Object> usageStats() {
-        if (state() != State.STARTED) {
-            return Collections.emptyMap();
-        }
-
         boolean dls = false;
         boolean fls = false;
         Map<String, Object> usageStats = new HashMap<>();
-        if (securityIndexExists == false) {
+        if (securityLifecycleService.securityIndexExists() == false) {
             usageStats.put("size", 0L);
             usageStats.put("fls", fls);
             usageStats.put("dls", dls);
@@ -309,13 +218,13 @@ public class NativeRolesStore extends AbstractComponent implements ClusterStateL
         // query for necessary information
         if (fls == false || dls == false) {
             MultiSearchRequestBuilder builder = client.prepareMultiSearch()
-                    .add(client.prepareSearch(SecurityTemplateService.SECURITY_INDEX_NAME)
+                    .add(client.prepareSearch(SecurityLifecycleService.SECURITY_INDEX_NAME)
                             .setTypes(ROLE_DOC_TYPE)
                             .setQuery(QueryBuilders.matchAllQuery())
                             .setSize(0));
 
             if (fls == false) {
-                builder.add(client.prepareSearch(SecurityTemplateService.SECURITY_INDEX_NAME)
+                builder.add(client.prepareSearch(SecurityLifecycleService.SECURITY_INDEX_NAME)
                         .setTypes(ROLE_DOC_TYPE)
                         .setQuery(QueryBuilders.boolQuery()
                                 .should(existsQuery("indices.field_security.grant"))
@@ -327,7 +236,7 @@ public class NativeRolesStore extends AbstractComponent implements ClusterStateL
             }
 
             if (dls == false) {
-                builder.add(client.prepareSearch(SecurityTemplateService.SECURITY_INDEX_NAME)
+                builder.add(client.prepareSearch(SecurityLifecycleService.SECURITY_INDEX_NAME)
                         .setTypes(ROLE_DOC_TYPE)
                         .setQuery(existsQuery("indices.query"))
                         .setSize(0)
@@ -361,7 +270,7 @@ public class NativeRolesStore extends AbstractComponent implements ClusterStateL
     }
 
     private void getRoleDescriptor(final String roleId, ActionListener<RoleDescriptor> roleActionListener) {
-        if (securityIndexExists == false) {
+        if (securityLifecycleService.securityIndexExists() == false) {
             roleActionListener.onResponse(null);
         } else {
             executeGetRoleRequest(roleId, new ActionListener<GetResponse>() {
@@ -389,32 +298,20 @@ public class NativeRolesStore extends AbstractComponent implements ClusterStateL
 
     private void executeGetRoleRequest(String role, ActionListener<GetResponse> listener) {
         try {
-            GetRequest request = client.prepareGet(SecurityTemplateService.SECURITY_INDEX_NAME, ROLE_DOC_TYPE, role).request();
+            GetRequest request = client.prepareGet(SecurityLifecycleService.SECURITY_INDEX_NAME, ROLE_DOC_TYPE, role).request();
             client.get(request, listener);
         } catch (IndexNotFoundException e) {
             logger.trace(
                     (Supplier<?>) () -> new ParameterizedMessage(
                             "unable to retrieve role [{}] since security index does not exist", role), e);
             listener.onResponse(new GetResponse(
-                    new GetResult(SecurityTemplateService.SECURITY_INDEX_NAME, ROLE_DOC_TYPE, role, -1, false, null, null)));
+                    new GetResult(SecurityLifecycleService.SECURITY_INDEX_NAME, ROLE_DOC_TYPE, role, -1, false, null, null)));
         } catch (Exception e) {
             logger.error("unable to retrieve role", e);
             listener.onFailure(e);
         }
     }
 
-
-    // FIXME hack for testing
-    public void reset() {
-        final State state = state();
-        if (state != State.STOPPED && state != State.FAILED) {
-            throw new IllegalStateException("can only reset if stopped!!!");
-        }
-        this.securityIndexExists = false;
-        this.canWrite = false;
-        this.state.set(State.INITIALIZED);
-    }
-
     private <Response> void clearRoleCache(final String role, ActionListener<Response> listener, Response response) {
         ClearRolesCacheRequest request = new ClearRolesCacheRequest().names(role);
         securityClient.clearRolesCache(request, new ActionListener<ClearRolesCacheResponse>() {
@@ -433,17 +330,6 @@ public class NativeRolesStore extends AbstractComponent implements ClusterStateL
         });
     }
 
-    // TODO abstract this code rather than duplicating...
-    @Override
-    public void clusterChanged(ClusterChangedEvent event) {
-        securityIndexExists = event.state().metaData().indices().get(SecurityTemplateService.SECURITY_INDEX_NAME) != null;
-        canWrite = securityIndexMappingAndTemplateUpToDate(event.state(), logger);
-    }
-
-    public State state() {
-        return state.get();
-    }
-
     @Nullable
     private RoleDescriptor transformRole(GetResponse response) {
         if (response.isExists() == false) {

plugin/src/test/java/org/elasticsearch/AbstractOldXPackIndicesBackwardsCompatibilityTestCase.java

@@ -117,6 +117,7 @@ public abstract class AbstractOldXPackIndicesBackwardsCompatibilityTestCase exte
     }
 
     public void testOldIndexes() throws Exception {
+        assertSecurityIndexWriteable();
         Collections.shuffle(dataFiles, random());
         for (String dataFile : dataFiles) {
             Version version = Version.fromString(dataFile.replace("x-pack-", "").replace(".zip", ""));

plugin/src/test/java/org/elasticsearch/OldSecurityIndexBackwardsCompatibilityTests.java

@@ -23,11 +23,9 @@ import org.elasticsearch.xpack.security.action.role.GetRolesResponse;
 import org.elasticsearch.xpack.security.action.role.PutRoleResponse;
 import org.elasticsearch.xpack.security.action.user.GetUsersResponse;
 import org.elasticsearch.xpack.security.action.user.PutUserResponse;
-import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
 import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.security.authz.permission.FieldPermissions;
-import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
 import org.elasticsearch.xpack.security.client.SecurityClient;
 import org.elasticsearch.xpack.security.user.User;
 
@@ -77,11 +75,9 @@ public class OldSecurityIndexBackwardsCompatibilityTests extends AbstractOldXPac
     }
 
     protected void checkVersion(Version version) throws Exception {
-       // wait for service to start
+        // wait for service to start
         SecurityClient securityClient = new SecurityClient(client());
-        assertBusy(() -> {
+        assertSecurityIndexActive();
-            assertEquals(NativeRolesStore.State.STARTED, internalCluster().getInstance(NativeRolesStore.class).state());
-        });
 
         // make sure usage stats are still working even with old fls format
         ClearRolesCacheResponse clearResponse = new ClearRolesCacheRequestBuilder(client()).get();
@@ -124,9 +120,7 @@ public class OldSecurityIndexBackwardsCompatibilityTests extends AbstractOldXPac
         assertThat(builder.string(), containsString("\"field_security\":{\"grant\":[\"title\",\"body\"]}"));
 
         logger.info("Getting users...");
-        assertBusy(() -> {
+        assertSecurityIndexActive();
-            assertEquals(NativeUsersStore.State.STARTED, internalCluster().getInstance(NativeUsersStore.class).state());
-        });
         GetUsersResponse getUsersResponse = securityClient.prepareGetUsers("bwc_test_user").get();
         assertThat(getUsersResponse.users(), arrayWithSize(1));
         User user = getUsersResponse.users()[0];

plugin/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java

@@ -10,7 +10,7 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.NativeRealmIntegTestCase;
-import org.elasticsearch.xpack.security.SecurityTemplateService;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.action.role.DeleteRoleResponse;
 import org.elasticsearch.xpack.security.action.role.GetRolesResponse;
 import org.elasticsearch.xpack.security.action.role.PutRoleResponse;
@@ -57,7 +57,7 @@ public class ClearRolesCacheTests extends NativeRealmIntegTestCase {
             logger.debug("--> created role [{}]", role);
         }
 
-        ensureGreen(SecurityTemplateService.SECURITY_INDEX_NAME);
+        ensureGreen(SecurityLifecycleService.SECURITY_INDEX_NAME);
 
         // warm up the caches on every node
         for (NativeRolesStore rolesStore : internalCluster().getInstances(NativeRolesStore.class)) {

plugin/src/test/java/org/elasticsearch/license/LicensingTests.java

@@ -8,15 +8,12 @@ package org.elasticsearch.license;
 import org.apache.http.message.BasicHeader;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.Version;
-import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsIndices;
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
@@ -28,9 +25,6 @@ import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRespo
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
@@ -53,7 +47,7 @@ import org.elasticsearch.transport.Transport;
 import org.elasticsearch.xpack.TestXPackTransportClient;
 import org.elasticsearch.xpack.XPackPlugin;
 import org.elasticsearch.xpack.security.Security;
-import org.elasticsearch.xpack.security.SecurityTemplateService;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.action.user.GetUsersResponse;
 import org.elasticsearch.xpack.security.authc.support.SecuredString;
 import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
@@ -275,11 +269,11 @@ public class LicensingTests extends SecurityIntegTestCase {
         final String expectedVersionAfterMigration = Version.CURRENT.toString();
 
         final Client client = internalCluster().transportClient();
-        final String template = TemplateUtils.loadTemplate("/" + SecurityTemplateService.SECURITY_TEMPLATE_NAME + ".json",
+        final String template = TemplateUtils.loadTemplate("/" + SecurityLifecycleService.SECURITY_TEMPLATE_NAME + ".json",
                 oldVersionThatRequiresMigration, Pattern.quote("${security.template.version}"));
 
         PutIndexTemplateRequest putTemplateRequest = client.admin().indices()
-                .preparePutTemplate(SecurityTemplateService.SECURITY_TEMPLATE_NAME)
+                .preparePutTemplate(SecurityLifecycleService.SECURITY_TEMPLATE_NAME)
                 .setSource(new BytesArray(template.getBytes(StandardCharsets.UTF_8)), XContentType.JSON)
                 .request();
         final PutIndexTemplateResponse putTemplateResponse = client.admin().indices().putTemplate(putTemplateRequest).actionGet();

plugin/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java

@@ -6,16 +6,11 @@
 package org.elasticsearch.test;
 
 import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.xpack.security.SecurityTemplateService;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
-import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
-import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
 import org.elasticsearch.xpack.security.client.SecurityClient;
 import org.junit.After;
 import org.junit.Before;
 
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.isOneOf;
-
 /**
  * Test case with method to handle the starting and stopping the stores for native users and roles
  */
@@ -23,54 +18,14 @@ public abstract class NativeRealmIntegTestCase extends SecurityIntegTestCase {
 
     @Before
     public void ensureNativeStoresStarted() throws Exception {
-        for (NativeUsersStore store : internalCluster().getInstances(NativeUsersStore.class)) {
+        assertSecurityIndexActive();
-            assertBusy(new Runnable() {
-                @Override
-                public void run() {
-                    assertThat(store.state(), is(NativeUsersStore.State.STARTED));
-                }
-            });
-        }
-
-        for (NativeRolesStore store : internalCluster().getInstances(NativeRolesStore.class)) {
-            assertBusy(new Runnable() {
-                @Override
-                public void run() {
-                    assertThat(store.state(), is(NativeRolesStore.State.STARTED));
-                }
-            });
-        }
     }
 
     @After
     public void stopESNativeStores() throws Exception {
-        for (NativeUsersStore store : internalCluster().getInstances(NativeUsersStore.class)) {
-            store.stop();
-            // the store may already be stopping so wait until it is stopped
-            assertBusy(new Runnable() {
-                @Override
-                public void run() {
-                    assertThat(store.state(), isOneOf(NativeUsersStore.State.STOPPED, NativeUsersStore.State.FAILED));
-                }
-            });
-            store.reset();
-        }
-
-        for (NativeRolesStore store : internalCluster().getInstances(NativeRolesStore.class)) {
-            store.stop();
-            // the store may already be stopping so wait until it is stopped
-            assertBusy(new Runnable() {
-                @Override
-                public void run() {
-                    assertThat(store.state(), isOneOf(NativeRolesStore.State.STOPPED, NativeRolesStore.State.FAILED));
-                }
-            });
-            store.reset();
-        }
-
         try {
             // this is a hack to clean up the .security index since only the XPack user can delete it
-            internalClient().admin().indices().prepareDelete(SecurityTemplateService.SECURITY_INDEX_NAME).get();
+            internalClient().admin().indices().prepareDelete(SecurityLifecycleService.SECURITY_INDEX_NAME).get();
         } catch (IndexNotFoundException e) {
             // ignore it since not all tests create this index...
         }

plugin/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java

@@ -12,16 +12,21 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.xpack.XPackClient;
 import org.elasticsearch.xpack.XPackPlugin;
 import org.elasticsearch.xpack.XPackSettings;
 import org.elasticsearch.xpack.security.InternalClient;
 import org.elasticsearch.xpack.security.Security;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.authc.support.SecuredString;
 import org.elasticsearch.xpack.security.client.SecurityClient;
 import org.junit.AfterClass;
@@ -41,6 +46,8 @@ import java.util.stream.Collectors;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout;
+import static org.elasticsearch.xpack.security.SecurityLifecycleService.securityIndexMappingAndTemplateSufficientToRead;
+import static org.elasticsearch.xpack.security.SecurityLifecycleService.securityIndexMappingAndTemplateUpToDate;
 import static org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.core.IsCollectionContaining.hasItem;
@@ -424,4 +431,32 @@ public abstract class SecurityIntegTestCase extends ESIntegTestCase {
         InetSocketAddress address = publishAddress.address();
         return (useSSL ? "https://" : "http://") + NetworkAddress.format(address.getAddress()) + ":" + address.getPort();
     }
+
+    public void assertSecurityIndexActive() throws Exception {
+        for (ClusterService clusterService : internalCluster().getInstances(ClusterService.class)) {
+            assertBusy(() -> {
+                ClusterState clusterState = clusterService.state();
+                assertFalse(clusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK));
+                assertTrue(securityIndexMappingAndTemplateSufficientToRead(clusterState, logger));
+                IndexRoutingTable indexRoutingTable = clusterState.routingTable().index(SecurityLifecycleService.SECURITY_INDEX_NAME);
+                if (indexRoutingTable != null) {
+                    assertTrue(indexRoutingTable.allPrimaryShardsActive());
+                }
+            });
+        }
+    }
+
+    public void assertSecurityIndexWriteable() throws Exception {
+        for (ClusterService clusterService : internalCluster().getInstances(ClusterService.class)) {
+            assertBusy(() -> {
+                ClusterState clusterState = clusterService.state();
+                assertFalse(clusterState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK));
+                assertTrue(securityIndexMappingAndTemplateUpToDate(clusterState, logger));
+                IndexRoutingTable indexRoutingTable = clusterState.routingTable().index(SecurityLifecycleService.SECURITY_INDEX_NAME);
+                if (indexRoutingTable != null) {
+                    assertTrue(indexRoutingTable.allPrimaryShardsActive());
+                }
+            });
+        }
+    }
 }

plugin/src/test/java/org/elasticsearch/xpack/security/SecurityLifecycleServiceTests.java

@@ -38,6 +38,8 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.MockTransportClient;
+import org.elasticsearch.xpack.security.SecurityLifecycleService.UpgradeState;
+import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail;
 import org.elasticsearch.xpack.security.authc.esnative.NativeRealmMigrator;
 import org.elasticsearch.xpack.security.test.SecurityTestUtils;
 import org.elasticsearch.xpack.template.TemplateUtils;
@@ -45,12 +47,11 @@ import org.junit.After;
 import org.junit.Before;
 import org.mockito.Mockito;
 
-import static org.elasticsearch.xpack.security.SecurityTemplateService.SECURITY_INDEX_NAME;
+import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_NAME;
-import static org.elasticsearch.xpack.security.SecurityTemplateService.SECURITY_INDEX_TEMPLATE_VERSION_PATTERN;
+import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_INDEX_TEMPLATE_VERSION_PATTERN;
-import static org.elasticsearch.xpack.security.SecurityTemplateService.SECURITY_TEMPLATE_NAME;
+import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_TEMPLATE_NAME;
-import static org.elasticsearch.xpack.security.SecurityTemplateService.UpgradeState;
+import static org.elasticsearch.xpack.security.SecurityLifecycleService.securityIndexMappingVersionMatches;
-import static org.elasticsearch.xpack.security.SecurityTemplateService.securityIndexMappingVersionMatches;
+import static org.elasticsearch.xpack.security.SecurityLifecycleService.securityTemplateExistsAndVersionMatches;
-import static org.elasticsearch.xpack.security.SecurityTemplateService.securityTemplateExistsAndVersionMatches;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
@@ -58,13 +59,13 @@ import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-public class SecurityTemplateServiceTests extends ESTestCase {
+public class SecurityLifecycleServiceTests extends ESTestCase {
     private InternalClient client;
     private TransportClient transportClient;
     private ThreadPool threadPool;
     private ClusterService clusterService;
     private NativeRealmMigrator nativeRealmMigrator;
-    SecurityTemplateService securityTemplateService;
+    SecurityLifecycleService securityLifecycleService;
     private static final ClusterState EMPTY_CLUSTER_STATE =
             new ClusterState.Builder(new ClusterName("test-cluster")).build();
 
@@ -101,7 +102,8 @@ public class SecurityTemplateServiceTests extends ESTestCase {
         }).when(nativeRealmMigrator).performUpgrade(any(Version.class), any(ActionListener.class));
 
         client = new IClient(transportClient);
-        securityTemplateService = new SecurityTemplateService(Settings.EMPTY, client, nativeRealmMigrator);
+        securityLifecycleService = new SecurityLifecycleService(Settings.EMPTY, clusterService, threadPool,
+                                                                client, nativeRealmMigrator, mock(IndexAuditTrail.class));
         listeners = new CopyOnWriteArrayList<>();
     }
 
@@ -116,8 +118,8 @@ public class SecurityTemplateServiceTests extends ESTestCase {
     public void testIndexTemplateIsIdentifiedAsUpToDate() throws IOException {
         String templateString = "/" + SECURITY_TEMPLATE_NAME + ".json";
         ClusterState.Builder clusterStateBuilder = createClusterStateWithTemplate(templateString);
-        assertTrue(SecurityTemplateService.securityTemplateExistsAndIsUpToDate(clusterStateBuilder.build(), logger));
+        assertTrue(SecurityLifecycleService.securityTemplateExistsAndIsUpToDate(clusterStateBuilder.build(), logger));
-        securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build()
+        securityLifecycleService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build()
                 , EMPTY_CLUSTER_STATE));
         assertThat(listeners.size(), equalTo(0));
     }
@@ -125,7 +127,7 @@ public class SecurityTemplateServiceTests extends ESTestCase {
     public void testFaultyIndexTemplateIsIdentifiedAsNotUpToDate() throws IOException {
         String templateString = "/wrong-version-" + SECURITY_TEMPLATE_NAME + ".json";
         ClusterState.Builder clusterStateBuilder = createClusterStateWithTemplate(templateString);
-        assertFalse(SecurityTemplateService.securityTemplateExistsAndIsUpToDate(clusterStateBuilder.build(), logger));
+        assertFalse(SecurityLifecycleService.securityTemplateExistsAndIsUpToDate(clusterStateBuilder.build(), logger));
         checkTemplateUpdateWorkCorrectly(clusterStateBuilder);
     }
 
@@ -137,28 +139,28 @@ public class SecurityTemplateServiceTests extends ESTestCase {
     }
 
     private void checkTemplateUpdateWorkCorrectly(ClusterState.Builder clusterStateBuilder) throws IOException {
-        securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build()
+        securityLifecycleService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build()
                 , EMPTY_CLUSTER_STATE));
         assertThat(listeners.size(), equalTo(1));
-        assertTrue(securityTemplateService.templateCreationPending.get());
+        assertTrue(securityLifecycleService.templateCreationPending.get());
 
         // if we do it again this should not send an update
         ActionListener listener = listeners.get(0);
         listeners.clear();
-        securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build()
+        securityLifecycleService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build()
                 , EMPTY_CLUSTER_STATE));
         assertThat(listeners.size(), equalTo(0));
-        assertTrue(securityTemplateService.templateCreationPending.get());
+        assertTrue(securityLifecycleService.templateCreationPending.get());
 
         // if we now simulate an error...
         listener.onFailure(new Exception());
-        assertFalse(securityTemplateService.templateCreationPending.get());
+        assertFalse(securityLifecycleService.templateCreationPending.get());
 
         // ... we should be able to send a new update
-        securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build()
+        securityLifecycleService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build()
                 , EMPTY_CLUSTER_STATE));
         assertThat(listeners.size(), equalTo(1));
-        assertTrue(securityTemplateService.templateCreationPending.get());
+        assertTrue(securityLifecycleService.templateCreationPending.get());
 
         // now check what happens if we get back an unacknowledged response
         try {
@@ -166,16 +168,16 @@ public class SecurityTemplateServiceTests extends ESTestCase {
             fail("this should have failed because request was not acknowledged");
         } catch (ElasticsearchException e) {
         }
-        assertFalse(securityTemplateService.updateMappingPending.get());
+        assertFalse(securityLifecycleService.updateMappingPending.get());
 
         // and now let's see what happens if we get back a response
         listeners.clear();
-        securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build()
+        securityLifecycleService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build()
                 , EMPTY_CLUSTER_STATE));
-        assertTrue(securityTemplateService.templateCreationPending.get());
+        assertTrue(securityLifecycleService.templateCreationPending.get());
         assertThat(listeners.size(), equalTo(1));
         listeners.get(0).onResponse(new TestPutIndexTemplateResponse(true));
-        assertFalse(securityTemplateService.templateCreationPending.get());
+        assertFalse(securityLifecycleService.templateCreationPending.get());
     }
 
     public void testMissingIndexTemplateIsIdentifiedAsMissing() throws IOException {
@@ -186,14 +188,14 @@ public class SecurityTemplateServiceTests extends ESTestCase {
         MetaData.Builder builder = new MetaData.Builder(clusterStateBuilder.build().getMetaData());
         builder.put(indexMeta);
         clusterStateBuilder.metaData(builder);
-        assertFalse(SecurityTemplateService.securityTemplateExistsAndIsUpToDate(clusterStateBuilder.build(), logger));
+        assertFalse(SecurityLifecycleService.securityTemplateExistsAndIsUpToDate(clusterStateBuilder.build(), logger));
         checkTemplateUpdateWorkCorrectly(clusterStateBuilder);
     }
 
     public void testMissingVersionIndexTemplateIsIdentifiedAsNotUpToDate() throws IOException {
         String templateString = "/missing-version-" + SECURITY_TEMPLATE_NAME + ".json";
         ClusterState.Builder clusterStateBuilder = createClusterStateWithTemplate(templateString);
-        assertFalse(SecurityTemplateService.securityTemplateExistsAndIsUpToDate(clusterStateBuilder.build(), logger));
+        assertFalse(SecurityLifecycleService.securityTemplateExistsAndIsUpToDate(clusterStateBuilder.build(), logger));
         checkTemplateUpdateWorkCorrectly(clusterStateBuilder);
     }
 
@@ -201,8 +203,8 @@ public class SecurityTemplateServiceTests extends ESTestCase {
         String templateString = "/wrong-version-" + SECURITY_TEMPLATE_NAME + ".json";
         final Version wrongVersion = Version.fromString("4.0.0");
         ClusterState.Builder clusterStateBuilder = createClusterStateWithMapping(templateString);
-        assertFalse(SecurityTemplateService.securityIndexMappingUpToDate(clusterStateBuilder.build(), logger));
+        assertFalse(SecurityLifecycleService.securityIndexMappingUpToDate(clusterStateBuilder.build(), logger));
-        assertThat(SecurityTemplateService.oldestSecurityIndexMappingVersion(clusterStateBuilder.build(), logger), equalTo(wrongVersion));
+        assertThat(SecurityLifecycleService.oldestSecurityIndexMappingVersion(clusterStateBuilder.build(), logger), equalTo(wrongVersion));
         checkMappingUpdateWorkCorrectly(clusterStateBuilder, wrongVersion);
     }
 
@@ -215,40 +217,40 @@ public class SecurityTemplateServiceTests extends ESTestCase {
             return null;
         }).when(nativeRealmMigrator).performUpgrade(any(Version.class), any(ActionListener.class));
 
-        assertThat(securityTemplateService.upgradeDataState.get(), equalTo(UpgradeState.NOT_STARTED));
+        assertThat(securityLifecycleService.upgradeDataState.get(), equalTo(UpgradeState.NOT_STARTED));
 
-        securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build(),
+        securityLifecycleService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build(),
                 EMPTY_CLUSTER_STATE));
 
         assertThat(migratorVersionRef.get(), equalTo(expectedOldVersion));
         assertThat(migratorListenerRef.get(), notNullValue());
         assertThat(listeners.size(), equalTo(0)); // migrator has not responded yet
-        assertThat(securityTemplateService.updateMappingPending.get(), equalTo(false));
+        assertThat(securityLifecycleService.updateMappingPending.get(), equalTo(false));
-        assertThat(securityTemplateService.upgradeDataState.get(), equalTo(UpgradeState.IN_PROGRESS));
+        assertThat(securityLifecycleService.upgradeDataState.get(), equalTo(UpgradeState.IN_PROGRESS));
 
         migratorListenerRef.get().onResponse(true);
 
         assertThat(listeners.size(), equalTo(3)); // we have three types in the mapping
-        assertTrue(securityTemplateService.updateMappingPending.get());
+        assertTrue(securityLifecycleService.updateMappingPending.get());
-        assertThat(securityTemplateService.upgradeDataState.get(), equalTo(UpgradeState.COMPLETE));
+        assertThat(securityLifecycleService.upgradeDataState.get(), equalTo(UpgradeState.COMPLETE));
 
         // if we do it again this should not send an update
         ActionListener listener = listeners.get(0);
         listeners.clear();
-        securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build()
+        securityLifecycleService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build()
                 , EMPTY_CLUSTER_STATE));
         assertThat(listeners.size(), equalTo(0));
-        assertTrue(securityTemplateService.updateMappingPending.get());
+        assertTrue(securityLifecycleService.updateMappingPending.get());
 
         // if we now simulate an error...
         listener.onFailure(new Exception("Testing failure handling"));
-        assertFalse(securityTemplateService.updateMappingPending.get());
+        assertFalse(securityLifecycleService.updateMappingPending.get());
 
         // ... we should be able to send a new update
-        securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build()
+        securityLifecycleService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build()
                 , EMPTY_CLUSTER_STATE));
         assertThat(listeners.size(), equalTo(3));
-        assertTrue(securityTemplateService.updateMappingPending.get());
+        assertTrue(securityLifecycleService.updateMappingPending.get());
 
         // now check what happens if we get back an unacknowledged response
         try {
@@ -256,20 +258,20 @@ public class SecurityTemplateServiceTests extends ESTestCase {
             fail("this hould have failed because request was not acknowledged");
         } catch (ElasticsearchException e) {
         }
-        assertFalse(securityTemplateService.updateMappingPending.get());
+        assertFalse(securityLifecycleService.updateMappingPending.get());
 
         // and now check what happens if we get back an acknowledged response
         listeners.clear();
-        securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build()
+        securityLifecycleService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build()
                 , EMPTY_CLUSTER_STATE));
         assertThat(listeners.size(), equalTo(3)); // we have three types in the mapping
         int counter = 0;
         for (ActionListener actionListener : listeners) {
             actionListener.onResponse(new TestPutMappingResponse(true));
             if (counter++ < 2) {
-                assertTrue(securityTemplateService.updateMappingPending.get());
+                assertTrue(securityLifecycleService.updateMappingPending.get());
             } else {
-                assertFalse(securityTemplateService.updateMappingPending.get());
+                assertFalse(securityLifecycleService.updateMappingPending.get());
             }
         }
     }
@@ -277,8 +279,8 @@ public class SecurityTemplateServiceTests extends ESTestCase {
     public void testUpToDateMappingIsIdentifiedAstUpToDate() throws IOException {
         String templateString = "/" + SECURITY_TEMPLATE_NAME + ".json";
         ClusterState.Builder clusterStateBuilder = createClusterStateWithMapping(templateString);
-        assertTrue(SecurityTemplateService.securityIndexMappingUpToDate(clusterStateBuilder.build(), logger));
+        assertTrue(SecurityLifecycleService.securityIndexMappingUpToDate(clusterStateBuilder.build(), logger));
-        securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build()
+        securityLifecycleService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build()
                 , EMPTY_CLUSTER_STATE));
         assertThat(listeners.size(), equalTo(0));
     }
@@ -293,8 +295,8 @@ public class SecurityTemplateServiceTests extends ESTestCase {
     public void testMissingVersionMappingIsIdentifiedAsNotUpToDate() throws IOException {
         String templateString = "/missing-version-" + SECURITY_TEMPLATE_NAME + ".json";
         ClusterState.Builder clusterStateBuilder = createClusterStateWithMapping(templateString);
-        assertFalse(SecurityTemplateService.securityIndexMappingUpToDate(clusterStateBuilder.build(), logger));
+        assertFalse(SecurityLifecycleService.securityIndexMappingUpToDate(clusterStateBuilder.build(), logger));
-        assertThat(SecurityTemplateService.oldestSecurityIndexMappingVersion(clusterStateBuilder.build(), logger),
+        assertThat(SecurityLifecycleService.oldestSecurityIndexMappingVersion(clusterStateBuilder.build(), logger),
                 equalTo(Version.V_2_3_0));
         checkMappingUpdateWorkCorrectly(clusterStateBuilder, Version.V_2_3_0);
     }
@@ -306,10 +308,10 @@ public class SecurityTemplateServiceTests extends ESTestCase {
         MetaData.Builder builder = new MetaData.Builder(clusterStateBuilder.build().getMetaData());
         builder.put(templateMeta);
         clusterStateBuilder.metaData(builder);
-        assertTrue(SecurityTemplateService.securityIndexMappingUpToDate(clusterStateBuilder.build(), logger));
+        assertTrue(SecurityLifecycleService.securityIndexMappingUpToDate(clusterStateBuilder.build(), logger));
-        securityTemplateService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build()
+        securityLifecycleService.clusterChanged(new ClusterChangedEvent("test-event", clusterStateBuilder.build()
                 , EMPTY_CLUSTER_STATE));
-        assertThat(SecurityTemplateService.oldestSecurityIndexMappingVersion(clusterStateBuilder.build(), logger), nullValue());
+        assertThat(SecurityLifecycleService.oldestSecurityIndexMappingVersion(clusterStateBuilder.build(), logger), nullValue());
         assertThat(listeners.size(), equalTo(0));
     }
 
@@ -328,8 +330,7 @@ public class SecurityTemplateServiceTests extends ESTestCase {
     }
 
     private IndexMetaData.Builder createIndexMetadata(String templateString) throws IOException {
-        String template = TemplateUtils.loadTemplate(templateString, Version.CURRENT.toString()
+        String template = TemplateUtils.loadTemplate(templateString, Version.CURRENT.toString(), SECURITY_INDEX_TEMPLATE_VERSION_PATTERN);
-                , SECURITY_INDEX_TEMPLATE_VERSION_PATTERN);
         PutIndexTemplateRequest request = new PutIndexTemplateRequest();
         request.source(template, XContentType.JSON);
         IndexMetaData.Builder indexMetaData = IndexMetaData.builder(SECURITY_INDEX_NAME);

plugin/src/test/java/org/elasticsearch/xpack/security/SecuritySettingsTests.java

@@ -145,7 +145,7 @@ public class SecuritySettingsTests extends ESTestCase {
             Security.validateAutoCreateIndex(Settings.builder().put("action.auto_create_index", false).build());
             fail("IllegalArgumentException expected");
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString(SecurityTemplateService.SECURITY_INDEX_NAME));
+            assertThat(e.getMessage(), containsString(SecurityLifecycleService.SECURITY_INDEX_NAME));
             assertThat(e.getMessage(), not(containsString(IndexAuditTrail.INDEX_NAME_PREFIX)));
         }
 
@@ -157,7 +157,7 @@ public class SecuritySettingsTests extends ESTestCase {
             Security.validateAutoCreateIndex(Settings.builder().put("action.auto_create_index", "foo").build());
             fail("IllegalArgumentException expected");
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString(SecurityTemplateService.SECURITY_INDEX_NAME));
+            assertThat(e.getMessage(), containsString(SecurityLifecycleService.SECURITY_INDEX_NAME));
             assertThat(e.getMessage(), not(containsString(IndexAuditTrail.INDEX_NAME_PREFIX)));
         }
 
@@ -165,7 +165,7 @@ public class SecuritySettingsTests extends ESTestCase {
             Security.validateAutoCreateIndex(Settings.builder().put("action.auto_create_index", ".security_audit_log*").build());
             fail("IllegalArgumentException expected");
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString(SecurityTemplateService.SECURITY_INDEX_NAME));
+            assertThat(e.getMessage(), containsString(SecurityLifecycleService.SECURITY_INDEX_NAME));
         }
 
         Security.validateAutoCreateIndex(Settings.builder()
@@ -181,7 +181,7 @@ public class SecuritySettingsTests extends ESTestCase {
                     .build());
             fail("IllegalArgumentException expected");
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString(SecurityTemplateService.SECURITY_INDEX_NAME));
+            assertThat(e.getMessage(), containsString(SecurityLifecycleService.SECURITY_INDEX_NAME));
             assertThat(e.getMessage(), containsString(IndexAuditTrail.INDEX_NAME_PREFIX));
         }
 

plugin/src/test/java/org/elasticsearch/xpack/security/SecurityTribeIT.java

@@ -109,7 +109,7 @@ public class SecurityTribeIT extends NativeRealmIntegTestCase {
                 try {
                     // this is a hack to clean up the .security index since only the XPack user or superusers can delete it
                     cluster2.getInstance(InternalClient.class)
-                            .admin().indices().prepareDelete(SecurityTemplateService.SECURITY_INDEX_NAME).get();
+                            .admin().indices().prepareDelete(SecurityLifecycleService.SECURITY_INDEX_NAME).get();
                 } catch (IndexNotFoundException e) {
                     // ignore it since not all tests create this index...
                 }
@@ -241,9 +241,9 @@ public class SecurityTribeIT extends NativeRealmIntegTestCase {
         List<String> shouldFailUsers = new ArrayList<>();
         final Client preferredClient = "t1".equals(preferredTribe) ? cluster1Client : cluster2Client;
         // always ensure the index exists on all of the clusters in this test
-        assertAcked(internalClient().admin().indices().prepareCreate(SecurityTemplateService.SECURITY_INDEX_NAME).get());
+        assertAcked(internalClient().admin().indices().prepareCreate(SecurityLifecycleService.SECURITY_INDEX_NAME).get());
         assertAcked(cluster2.getInstance(InternalClient.class).admin().indices()
-                .prepareCreate(SecurityTemplateService.SECURITY_INDEX_NAME).get());
+                .prepareCreate(SecurityLifecycleService.SECURITY_INDEX_NAME).get());
         for (int i = 0; i < randomUsers; i++) {
             final String username = "user" + i;
             Client clusterClient = randomBoolean() ? cluster1Client : cluster2Client;
@@ -329,9 +329,9 @@ public class SecurityTribeIT extends NativeRealmIntegTestCase {
         List<String> shouldFailRoles = new ArrayList<>();
         final Client preferredClient = "t1".equals(preferredTribe) ? cluster1Client : cluster2Client;
         // always ensure the index exists on all of the clusters in this test
-        assertAcked(internalClient().admin().indices().prepareCreate(SecurityTemplateService.SECURITY_INDEX_NAME).get());
+        assertAcked(internalClient().admin().indices().prepareCreate(SecurityLifecycleService.SECURITY_INDEX_NAME).get());
         assertAcked(cluster2.getInstance(InternalClient.class).admin().indices()
-                .prepareCreate(SecurityTemplateService.SECURITY_INDEX_NAME).get());
+                .prepareCreate(SecurityLifecycleService.SECURITY_INDEX_NAME).get());
 
         for (int i = 0; i < randomRoles; i++) {
             final String rolename = "role" + i;

plugin/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java

@@ -14,6 +14,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealmTests;
@@ -71,9 +72,11 @@ public class TransportGetUsersActionTests extends ESTestCase {
 
     public void testAnonymousUser() {
         NativeUsersStore usersStore = mock(NativeUsersStore.class);
-        when(usersStore.started()).thenReturn(true);
+        SecurityLifecycleService securityLifecycleService = mock(SecurityLifecycleService.class);
+        when(securityLifecycleService.securityIndexAvailable()).thenReturn(true);
         AnonymousUser anonymousUser = new AnonymousUser(settings);
-        ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, anonymousUser);
+        ReservedRealm reservedRealm =
+            new ReservedRealm(mock(Environment.class), settings, usersStore, anonymousUser, securityLifecycleService);
         TransportService transportService = new TransportService(Settings.EMPTY, null, null, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
                 x -> null, null);
         TransportGetUsersAction action = new TransportGetUsersAction(Settings.EMPTY, mock(ThreadPool.class), mock(ActionFilters.class),
@@ -139,11 +142,13 @@ public class TransportGetUsersActionTests extends ESTestCase {
 
     public void testReservedUsersOnly() {
         NativeUsersStore usersStore = mock(NativeUsersStore.class);
-        when(usersStore.started()).thenReturn(true);
+        SecurityLifecycleService securityLifecycleService = mock(SecurityLifecycleService.class);
-        when(usersStore.checkMappingVersion(any())).thenReturn(true);
+        when(securityLifecycleService.securityIndexAvailable()).thenReturn(true);
+        when(securityLifecycleService.checkMappingVersion(any())).thenReturn(true);
 
         ReservedRealmTests.mockGetAllReservedUserInfo(usersStore, Collections.emptyMap());
-        ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(settings));
+        ReservedRealm reservedRealm =
+            new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(settings), securityLifecycleService);
         PlainActionFuture<Collection<User>> userFuture = new PlainActionFuture<>();
         reservedRealm.users(userFuture);
         final Collection<User> allReservedUsers = userFuture.actionGet();
@@ -183,9 +188,11 @@ public class TransportGetUsersActionTests extends ESTestCase {
         final List<User> storeUsers = randomFrom(Collections.<User>emptyList(), Collections.singletonList(new User("joe")),
                 Arrays.asList(new User("jane"), new User("fred")), randomUsers());
         NativeUsersStore usersStore = mock(NativeUsersStore.class);
-        when(usersStore.started()).thenReturn(true);
+        SecurityLifecycleService securityLifecycleService = mock(SecurityLifecycleService.class);
+        when(securityLifecycleService.securityIndexAvailable()).thenReturn(true);
         ReservedRealmTests.mockGetAllReservedUserInfo(usersStore, Collections.emptyMap());
-        ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(settings));
+        ReservedRealm reservedRealm =
+            new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(settings), securityLifecycleService);
         TransportService transportService = new TransportService(Settings.EMPTY, null, null, TransportService.NOOP_TRANSPORT_INTERCEPTOR,
                 x -> null, null);
         TransportGetUsersAction action = new TransportGetUsersAction(Settings.EMPTY, mock(ThreadPool.class), mock(ActionFilters.class),

plugin/src/test/java/org/elasticsearch/xpack/security/action/user/TransportPutUserActionTests.java

@@ -13,6 +13,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm;
 import org.elasticsearch.xpack.security.authc.esnative.ReservedRealmTests;
@@ -113,10 +114,12 @@ public class TransportPutUserActionTests extends ESTestCase {
 
     public void testReservedUser() {
         NativeUsersStore usersStore = mock(NativeUsersStore.class);
-        when(usersStore.started()).thenReturn(true);
+        SecurityLifecycleService securityLifecycleService = mock(SecurityLifecycleService.class);
+        when(securityLifecycleService.securityIndexAvailable()).thenReturn(true);
         ReservedRealmTests.mockGetAllReservedUserInfo(usersStore, Collections.emptyMap());
         Settings settings = Settings.builder().put("path.home", createTempDir()).build();
-        ReservedRealm reservedRealm = new ReservedRealm(new Environment(settings), settings, usersStore, new AnonymousUser(settings));
+        ReservedRealm reservedRealm = new ReservedRealm(new Environment(settings), settings, usersStore,
+                                                        new AnonymousUser(settings), securityLifecycleService);
         PlainActionFuture<Collection<User>> userFuture = new PlainActionFuture<>();
         reservedRealm.users(userFuture);
         final User reserved = randomFrom(userFuture.actionGet().toArray(new User[0]));
@@ -145,7 +148,6 @@ public class TransportPutUserActionTests extends ESTestCase {
         assertThat(responseRef.get(), is(nullValue()));
         assertThat(throwableRef.get(), instanceOf(IllegalArgumentException.class));
         assertThat(throwableRef.get().getMessage(), containsString("is reserved and only the password"));
-        verify(usersStore).started();
     }
 
     public void testValidUser() {

plugin/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ESNativeMigrateToolTests.java

@@ -14,7 +14,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.NativeRealmIntegTestCase;
 import org.elasticsearch.test.SecuritySettingsSource;
-import org.elasticsearch.xpack.security.SecurityTemplateService;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.client.SecurityClient;
 import org.junit.BeforeClass;
 
@@ -72,7 +72,7 @@ public class ESNativeMigrateToolTests extends NativeRealmIntegTestCase {
             addedUsers.add(uname);
         }
         logger.error("--> waiting for .security index");
-        ensureGreen(SecurityTemplateService.SECURITY_INDEX_NAME);
+        ensureGreen(SecurityLifecycleService.SECURITY_INDEX_NAME);
 
         MockTerminal t = new MockTerminal();
         String username = nodeClientUsername();
@@ -117,7 +117,7 @@ public class ESNativeMigrateToolTests extends NativeRealmIntegTestCase {
             addedRoles.add(rname);
         }
         logger.error("--> waiting for .security index");
-        ensureGreen(SecurityTemplateService.SECURITY_INDEX_NAME);
+        ensureGreen(SecurityLifecycleService.SECURITY_INDEX_NAME);
 
         MockTerminal t = new MockTerminal();
         String username = nodeClientUsername();

plugin/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java

@@ -17,7 +17,7 @@ import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.xpack.security.SecurityTemplateService;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.action.role.DeleteRoleResponse;
 import org.elasticsearch.xpack.security.action.role.GetRolesResponse;
 import org.elasticsearch.xpack.security.action.role.PutRoleResponse;
@@ -123,7 +123,7 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
         logger.error("--> creating user");
         c.preparePutUser("joe", "s3kirt".toCharArray(), "role1", "user").get();
         logger.error("--> waiting for .security index");
-        ensureGreen(SecurityTemplateService.SECURITY_INDEX_NAME);
+        ensureGreen(SecurityLifecycleService.SECURITY_INDEX_NAME);
         logger.info("--> retrieving user");
         GetUsersResponse resp = c.prepareGetUsers("joe").get();
         assertTrue("user should exist", resp.hasUsers());
@@ -178,7 +178,7 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
                 .metadata(metadata)
                 .get();
         logger.error("--> waiting for .security index");
-        ensureGreen(SecurityTemplateService.SECURITY_INDEX_NAME);
+        ensureGreen(SecurityLifecycleService.SECURITY_INDEX_NAME);
         logger.info("--> retrieving role");
         GetRolesResponse resp = c.prepareGetRoles().names("test_role").get();
         assertTrue("role should exist", resp.hasRoles());
@@ -229,7 +229,7 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
         logger.error("--> creating user");
         c.preparePutUser("joe", "s3krit".toCharArray(), "test_role").get();
         logger.error("--> waiting for .security index");
-        ensureGreen(SecurityTemplateService.SECURITY_INDEX_NAME);
+        ensureGreen(SecurityLifecycleService.SECURITY_INDEX_NAME);
         logger.info("--> retrieving user");
         GetUsersResponse resp = c.prepareGetUsers("joe").get();
         assertTrue("user should exist", resp.hasUsers());
@@ -250,7 +250,7 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
         logger.error("--> creating user");
         c.preparePutUser("joe", "s3krit".toCharArray(), SecuritySettingsSource.DEFAULT_ROLE).get();
         logger.error("--> waiting for .security index");
-        ensureGreen(SecurityTemplateService.SECURITY_INDEX_NAME);
+        ensureGreen(SecurityLifecycleService.SECURITY_INDEX_NAME);
         logger.info("--> retrieving user");
         GetUsersResponse resp = c.prepareGetUsers("joe").get();
         assertTrue("user should exist", resp.hasUsers());
@@ -285,7 +285,7 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
         logger.error("--> creating user");
         c.preparePutUser("joe", "s3krit".toCharArray(), SecuritySettingsSource.DEFAULT_ROLE).get();
         logger.error("--> waiting for .security index");
-        ensureGreen(SecurityTemplateService.SECURITY_INDEX_NAME);
+        ensureGreen(SecurityLifecycleService.SECURITY_INDEX_NAME);
         logger.info("--> retrieving user");
         GetUsersResponse resp = c.prepareGetUsers("joe").get();
         assertTrue("user should exist", resp.hasUsers());
@@ -323,7 +323,7 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
         logger.error("--> creating user");
         c.preparePutUser("joe", "s3krit".toCharArray(), "test_role").get();
         logger.error("--> waiting for .security index");
-        ensureGreen(SecurityTemplateService.SECURITY_INDEX_NAME);
+        ensureGreen(SecurityLifecycleService.SECURITY_INDEX_NAME);
 
         if (authenticate) {
             final String token = basicAuthHeaderValue("joe", new SecuredString("s3krit".toCharArray()));
@@ -372,7 +372,7 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
                 .get();
         c.preparePutUser("joe", "s3krit".toCharArray(), "test_role").get();
         logger.error("--> waiting for .security index");
-        ensureGreen(SecurityTemplateService.SECURITY_INDEX_NAME);
+        ensureGreen(SecurityLifecycleService.SECURITY_INDEX_NAME);
 
         final String token = basicAuthHeaderValue("joe", new SecuredString("s3krit".toCharArray()));
         ClusterHealthResponse response = client().filterWithHeader(Collections.singletonMap("Authorization", token)).admin().cluster()
@@ -498,12 +498,12 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
                     .get();
         }
 
-        IndicesStatsResponse response = client().admin().indices().prepareStats("foo", SecurityTemplateService.SECURITY_INDEX_NAME).get();
+        IndicesStatsResponse response = client().admin().indices().prepareStats("foo", SecurityLifecycleService.SECURITY_INDEX_NAME).get();
         assertThat(response.getFailedShards(), is(0));
         assertThat(response.getIndices().size(), is(2));
-        assertThat(response.getIndices().get(SecurityTemplateService.SECURITY_INDEX_NAME), notNullValue());
+        assertThat(response.getIndices().get(SecurityLifecycleService.SECURITY_INDEX_NAME), notNullValue());
-        assertThat(response.getIndices().get(SecurityTemplateService.SECURITY_INDEX_NAME).getIndex(),
+        assertThat(response.getIndices().get(SecurityLifecycleService.SECURITY_INDEX_NAME).getIndex(),
-                is(SecurityTemplateService.SECURITY_INDEX_NAME));
+                is(SecurityLifecycleService.SECURITY_INDEX_NAME));
     }
 
     public void testOperationsOnReservedUsers() throws Exception {

plugin/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmMigratorTests.java

@@ -40,7 +40,7 @@ import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.security.InternalClient;
-import org.elasticsearch.xpack.security.SecurityTemplateService;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.action.realm.ClearRealmCacheAction;
 import org.elasticsearch.xpack.security.action.realm.ClearRealmCacheRequest;
 import org.elasticsearch.xpack.security.authc.support.Hasher;
@@ -84,7 +84,7 @@ public class NativeRealmMigratorTests extends ESTestCase {
         doAnswer(invocationOnMock -> {
             SearchRequest request = (SearchRequest) invocationOnMock.getArguments()[1];
             ActionListener listener = (ActionListener) invocationOnMock.getArguments()[2];
-            if (request.indices().length == 1 && request.indices()[0].equals(SecurityTemplateService.SECURITY_INDEX_NAME)) {
+            if (request.indices().length == 1 && request.indices()[0].equals(SecurityLifecycleService.SECURITY_INDEX_NAME)) {
                 SearchResponse response = new SearchResponse() {
                     @Override
                     public SearchHits getHits() {
@@ -111,10 +111,10 @@ public class NativeRealmMigratorTests extends ESTestCase {
         doAnswer(invocationOnMock -> {
             GetRequest request = (GetRequest) invocationOnMock.getArguments()[1];
             ActionListener listener = (ActionListener) invocationOnMock.getArguments()[2];
-            if (request.indices().length == 1 && request.indices()[0].equals(SecurityTemplateService.SECURITY_INDEX_NAME)
+            if (request.indices().length == 1 && request.indices()[0].equals(SecurityLifecycleService.SECURITY_INDEX_NAME)
                     && request.type().equals(NativeUsersStore.RESERVED_USER_DOC_TYPE)) {
                 final boolean exists = reservedUsers.get(request.id()) != null;
-                GetResult getResult = new GetResult(SecurityTemplateService.SECURITY_INDEX_NAME, NativeUsersStore.RESERVED_USER_DOC_TYPE,
+                GetResult getResult = new GetResult(SecurityLifecycleService.SECURITY_INDEX_NAME, NativeUsersStore.RESERVED_USER_DOC_TYPE,
                         request.id(), randomLong(), exists, JsonXContent.contentBuilder().map(reservedUsers.get(request.id())).bytes(),
                         emptyMap());
                 listener.onResponse(new GetResponse(getResult));

plugin/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStoreTests.java

@@ -26,8 +26,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.security.InternalClient;
-import org.elasticsearch.xpack.security.SecurityTemplateService;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
-import org.elasticsearch.xpack.security.test.SecurityTestUtils;
 import org.elasticsearch.xpack.security.user.ElasticUser;
 import org.elasticsearch.xpack.security.user.KibanaUser;
 import org.elasticsearch.xpack.security.user.LogstashSystemUser;
@@ -37,6 +36,8 @@ import org.junit.Before;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.nullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class NativeUsersStoreTests extends ESTestCase {
 
@@ -92,7 +93,7 @@ public class NativeUsersStoreTests extends ESTestCase {
         values.put(PASSWORD_FIELD, BLANK_PASSWORD);
 
         final GetResult result = new GetResult(
-                SecurityTemplateService.SECURITY_INDEX_NAME,
+                SecurityLifecycleService.SECURITY_INDEX_NAME,
                 NativeUsersStore.RESERVED_USER_DOC_TYPE,
                 randomAsciiOfLength(12),
                 1L,
@@ -127,10 +128,11 @@ public class NativeUsersStoreTests extends ESTestCase {
     }
 
     private NativeUsersStore startNativeUsersStore() {
-        final NativeUsersStore nativeUsersStore = new NativeUsersStore(Settings.EMPTY, internalClient);
+        SecurityLifecycleService securityLifecycleService = mock(SecurityLifecycleService.class);
-        assertTrue(nativeUsersStore + " should be ready to start",
+        when(securityLifecycleService.securityIndexAvailable()).thenReturn(true);
-                nativeUsersStore.canStart(SecurityTestUtils.getClusterStateWithSecurityIndex(), true));
+        when(securityLifecycleService.securityIndexExists()).thenReturn(true);
-        nativeUsersStore.start();
+        when(securityLifecycleService.canWriteToSecurityIndex()).thenReturn(true);
+        final NativeUsersStore nativeUsersStore = new NativeUsersStore(Settings.EMPTY, internalClient, securityLifecycleService);
         return nativeUsersStore;
     }
 

plugin/src/test/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmTests.java

@@ -12,6 +12,7 @@ import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.xpack.XPackSettings;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore.ReservedUserInfo;
 import org.elasticsearch.xpack.security.authc.support.Hasher;
 import org.elasticsearch.xpack.security.authc.support.SecuredString;
@@ -58,33 +59,22 @@ public class ReservedRealmTests extends ESTestCase {
     private static final SecuredString DEFAULT_PASSWORD = new SecuredString("changeme".toCharArray());
     public static final String ACCEPT_DEFAULT_PASSWORDS = ReservedRealm.ACCEPT_DEFAULT_PASSWORD_SETTING.getKey();
     private NativeUsersStore usersStore;
+    private SecurityLifecycleService securityLifecycleService;
 
     @Before
-    public void setupMocks() {
+    public void setupMocks() throws Exception {
         usersStore = mock(NativeUsersStore.class);
-        when(usersStore.started()).thenReturn(true);
+        securityLifecycleService = mock(SecurityLifecycleService.class);
-        when(usersStore.checkMappingVersion(any())).thenReturn(true);
+        when(securityLifecycleService.securityIndexAvailable()).thenReturn(true);
+        when(securityLifecycleService.checkMappingVersion(any())).thenReturn(true);
         mockGetAllReservedUserInfo(usersStore, Collections.emptyMap());
     }
 
-    public void testUserStoreNotStarted() {
-        when(usersStore.started()).thenReturn(false);
-        final ReservedRealm reservedRealm =
-                new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, new AnonymousUser(Settings.EMPTY));
-        final String principal = randomFrom(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME);
-
-        PlainActionFuture<User> listener = new PlainActionFuture<>();
-        reservedRealm.doAuthenticate(new UsernamePasswordToken(principal, DEFAULT_PASSWORD), listener);
-        ElasticsearchSecurityException expected = expectThrows(ElasticsearchSecurityException.class, listener::actionGet);
-        assertThat(expected.getMessage(), containsString("failed to authenticate user [" + principal));
-        verify(usersStore).started();
-        verifyNoMoreInteractions(usersStore);
-    }
-
     public void testMappingVersionFromBeforeUserExisted() throws ExecutionException, InterruptedException {
-        when(usersStore.checkMappingVersion(any())).thenReturn(false);
+        when(securityLifecycleService.checkMappingVersion(any())).thenReturn(false);
         final ReservedRealm reservedRealm =
-                new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, new AnonymousUser(Settings.EMPTY));
+            new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore,
+                              new AnonymousUser(Settings.EMPTY), securityLifecycleService);
         final String principal = randomFrom(ElasticUser.NAME, KibanaUser.NAME, LogstashSystemUser.NAME);
 
         PlainActionFuture<User> future = new PlainActionFuture<>();
@@ -97,7 +87,7 @@ public class ReservedRealmTests extends ESTestCase {
         final String principal = expected.principal();
         final boolean securityIndexExists = randomBoolean();
         if (securityIndexExists) {
-            when(usersStore.securityIndexExists()).thenReturn(true);
+            when(securityLifecycleService.securityIndexExists()).thenReturn(true);
             doAnswer((i) -> {
                 ActionListener listener = (ActionListener) i.getArguments()[1];
                 listener.onResponse(null);
@@ -105,19 +95,19 @@ public class ReservedRealmTests extends ESTestCase {
             }).when(usersStore).getReservedUserInfo(eq(principal), any(ActionListener.class));
         }
         final ReservedRealm reservedRealm =
-                new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, new AnonymousUser(Settings.EMPTY));
+            new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore,
+                              new AnonymousUser(Settings.EMPTY), securityLifecycleService);
 
         PlainActionFuture<User> listener = new PlainActionFuture<>();
         reservedRealm.doAuthenticate(new UsernamePasswordToken(principal, DEFAULT_PASSWORD), listener);
         final User authenticated = listener.actionGet();
         assertEquals(expected, authenticated);
-        verify(usersStore).started();
+        verify(securityLifecycleService).securityIndexExists();
-        verify(usersStore).securityIndexExists();
         if (securityIndexExists) {
             verify(usersStore).getReservedUserInfo(eq(principal), any(ActionListener.class));
         }
         final ArgumentCaptor<Predicate> predicateCaptor = ArgumentCaptor.forClass(Predicate.class);
-        verify(usersStore).checkMappingVersion(predicateCaptor.capture());
+        verify(securityLifecycleService).checkMappingVersion(predicateCaptor.capture());
         verifyVersionPredicate(principal, predicateCaptor.getValue());
         verifyNoMoreInteractions(usersStore);
     }
@@ -128,7 +118,7 @@ public class ReservedRealmTests extends ESTestCase {
         final Environment environment = mock(Environment.class);
         final AnonymousUser anonymousUser = new AnonymousUser(Settings.EMPTY);
         final Settings settings = Settings.builder().put(ACCEPT_DEFAULT_PASSWORDS, false).build();
-        final ReservedRealm reservedRealm = new ReservedRealm(environment, settings, usersStore, anonymousUser);
+        final ReservedRealm reservedRealm = new ReservedRealm(environment, settings, usersStore, anonymousUser, securityLifecycleService);
 
         final ActionListener<User> listener = new ActionListener<User>() {
             @Override
@@ -149,9 +139,11 @@ public class ReservedRealmTests extends ESTestCase {
         Settings settings = Settings.builder().put(XPackSettings.RESERVED_REALM_ENABLED_SETTING.getKey(), false).build();
         final boolean securityIndexExists = randomBoolean();
         if (securityIndexExists) {
-            when(usersStore.securityIndexExists()).thenReturn(true);
+            when(securityLifecycleService.securityIndexExists()).thenReturn(true);
         }
-        final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(settings));
+        final ReservedRealm reservedRealm =
+            new ReservedRealm(mock(Environment.class), settings, usersStore,
+                              new AnonymousUser(settings), securityLifecycleService);
         final User expected = randomFrom(new ElasticUser(true), new KibanaUser(true), new LogstashSystemUser(true));
         final String principal = expected.principal();
 
@@ -172,11 +164,12 @@ public class ReservedRealmTests extends ESTestCase {
 
     private void verifySuccessfulAuthentication(boolean enabled) {
         final Settings settings = Settings.builder().put(ACCEPT_DEFAULT_PASSWORDS, randomBoolean()).build();
-        final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(settings));
+        final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore,
+                                                              new AnonymousUser(settings), securityLifecycleService);
         final User expectedUser = randomFrom(new ElasticUser(enabled), new KibanaUser(enabled), new LogstashSystemUser(enabled));
         final String principal = expectedUser.principal();
         final SecuredString newPassword = new SecuredString("foobar".toCharArray());
-        when(usersStore.securityIndexExists()).thenReturn(true);
+        when(securityLifecycleService.securityIndexExists()).thenReturn(true);
         doAnswer((i) -> {
             ActionListener callback = (ActionListener) i.getArguments()[1];
             callback.onResponse(new ReservedUserInfo(Hasher.BCRYPT.hash(newPassword), enabled, false));
@@ -203,18 +196,18 @@ public class ReservedRealmTests extends ESTestCase {
         assertEquals(expectedUser, authenticated);
         assertThat(expectedUser.enabled(), is(enabled));
 
-        verify(usersStore, times(2)).started();
+        verify(securityLifecycleService, times(2)).securityIndexExists();
-        verify(usersStore, times(2)).securityIndexExists();
         verify(usersStore, times(2)).getReservedUserInfo(eq(principal), any(ActionListener.class));
         final ArgumentCaptor<Predicate> predicateCaptor = ArgumentCaptor.forClass(Predicate.class);
-        verify(usersStore, times(2)).checkMappingVersion(predicateCaptor.capture());
+        verify(securityLifecycleService, times(2)).checkMappingVersion(predicateCaptor.capture());
         verifyVersionPredicate(principal, predicateCaptor.getValue());
         verifyNoMoreInteractions(usersStore);
     }
 
     public void testLookup() throws Exception {
         final ReservedRealm reservedRealm =
-                new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, new AnonymousUser(Settings.EMPTY));
+            new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore,
+                              new AnonymousUser(Settings.EMPTY), securityLifecycleService);
         final User expectedUser = randomFrom(new ElasticUser(true), new KibanaUser(true), new LogstashSystemUser(true));
         final String principal = expectedUser.principal();
 
@@ -222,11 +215,10 @@ public class ReservedRealmTests extends ESTestCase {
         reservedRealm.doLookupUser(principal, listener);
         final User user = listener.actionGet();
         assertEquals(expectedUser, user);
-        verify(usersStore).started();
+        verify(securityLifecycleService).securityIndexExists();
-        verify(usersStore).securityIndexExists();
 
         final ArgumentCaptor<Predicate> predicateCaptor = ArgumentCaptor.forClass(Predicate.class);
-        verify(usersStore).checkMappingVersion(predicateCaptor.capture());
+        verify(securityLifecycleService).checkMappingVersion(predicateCaptor.capture());
         verifyVersionPredicate(principal, predicateCaptor.getValue());
 
         PlainActionFuture<User> future = new PlainActionFuture<>();
@@ -239,7 +231,7 @@ public class ReservedRealmTests extends ESTestCase {
     public void testLookupDisabled() throws Exception {
         Settings settings = Settings.builder().put(XPackSettings.RESERVED_REALM_ENABLED_SETTING.getKey(), false).build();
         final ReservedRealm reservedRealm =
-                new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(settings));
+            new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(settings), securityLifecycleService);
         final User expectedUser = randomFrom(new ElasticUser(true), new KibanaUser(true), new LogstashSystemUser(true));
         final String principal = expectedUser.principal();
 
@@ -252,10 +244,11 @@ public class ReservedRealmTests extends ESTestCase {
 
     public void testLookupThrows() throws Exception {
         final ReservedRealm reservedRealm =
-                new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, new AnonymousUser(Settings.EMPTY));
+            new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore,
+                              new AnonymousUser(Settings.EMPTY), securityLifecycleService);
         final User expectedUser = randomFrom(new ElasticUser(true), new KibanaUser(true), new LogstashSystemUser(true));
         final String principal = expectedUser.principal();
-        when(usersStore.securityIndexExists()).thenReturn(true);
+        when(securityLifecycleService.securityIndexExists()).thenReturn(true);
         final RuntimeException e = new RuntimeException("store threw");
         doAnswer((i) -> {
             ActionListener callback = (ActionListener) i.getArguments()[1];
@@ -268,12 +261,11 @@ public class ReservedRealmTests extends ESTestCase {
         ElasticsearchSecurityException securityException = expectThrows(ElasticsearchSecurityException.class, future::actionGet);
         assertThat(securityException.getMessage(), containsString("failed to lookup"));
 
-        verify(usersStore).started();
+        verify(securityLifecycleService).securityIndexExists();
-        verify(usersStore).securityIndexExists();
         verify(usersStore).getReservedUserInfo(eq(principal), any(ActionListener.class));
 
         final ArgumentCaptor<Predicate> predicateCaptor = ArgumentCaptor.forClass(Predicate.class);
-        verify(usersStore).checkMappingVersion(predicateCaptor.capture());
+        verify(securityLifecycleService).checkMappingVersion(predicateCaptor.capture());
         verifyVersionPredicate(principal, predicateCaptor.getValue());
 
         verifyNoMoreInteractions(usersStore);
@@ -300,7 +292,8 @@ public class ReservedRealmTests extends ESTestCase {
 
     public void testGetUsers() {
         final ReservedRealm reservedRealm =
-                new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, new AnonymousUser(Settings.EMPTY));
+            new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore,
+                              new AnonymousUser(Settings.EMPTY), securityLifecycleService);
         PlainActionFuture<Collection<User>> userFuture = new PlainActionFuture<>();
         reservedRealm.users(userFuture);
         assertThat(userFuture.actionGet(), containsInAnyOrder(new ElasticUser(true), new KibanaUser(true), new LogstashSystemUser(true)));
@@ -313,7 +306,8 @@ public class ReservedRealmTests extends ESTestCase {
                 .put(AnonymousUser.ROLES_SETTING.getKey(), anonymousEnabled ? "user" : "")
                 .build();
         final AnonymousUser anonymousUser = new AnonymousUser(settings);
-        final ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, anonymousUser);
+        final ReservedRealm reservedRealm =
+            new ReservedRealm(mock(Environment.class), settings, usersStore, anonymousUser, securityLifecycleService);
         PlainActionFuture<Collection<User>> userFuture = new PlainActionFuture<>();
         reservedRealm.users(userFuture);
         if (anonymousEnabled) {
@@ -325,7 +319,8 @@ public class ReservedRealmTests extends ESTestCase {
 
     public void testFailedAuthentication() {
         final ReservedRealm reservedRealm =
-                new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore, new AnonymousUser(Settings.EMPTY));
+            new ReservedRealm(mock(Environment.class), Settings.EMPTY, usersStore,
+                              new AnonymousUser(Settings.EMPTY), securityLifecycleService);
         // maybe cache a successful auth
         if (randomBoolean()) {
             PlainActionFuture<User> future = new PlainActionFuture<>();

plugin/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java

@@ -79,7 +79,7 @@ import org.elasticsearch.license.GetLicenseAction;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.xpack.security.SecurityTemplateService;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.action.user.AuthenticateAction;
 import org.elasticsearch.xpack.security.action.user.AuthenticateRequest;
 import org.elasticsearch.xpack.security.action.user.AuthenticateRequestBuilder;
@@ -551,25 +551,25 @@ public class AuthorizationServiceTests extends ESTestCase {
         ClusterState state = mock(ClusterState.class);
         when(clusterService.state()).thenReturn(state);
         when(state.metaData()).thenReturn(MetaData.builder()
-                .put(new IndexMetaData.Builder(SecurityTemplateService.SECURITY_INDEX_NAME)
+                .put(new IndexMetaData.Builder(SecurityLifecycleService.SECURITY_INDEX_NAME)
                         .settings(Settings.builder().put("index.version.created", Version.CURRENT).build())
                         .numberOfShards(1).numberOfReplicas(0).build(), true)
                 .build());
 
         List<Tuple<String, TransportRequest>> requests = new ArrayList<>();
-        requests.add(new Tuple<>(BulkAction.NAME + "[s]", new DeleteRequest(SecurityTemplateService.SECURITY_INDEX_NAME, "type", "id")));
+        requests.add(new Tuple<>(BulkAction.NAME + "[s]", new DeleteRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id")));
-        requests.add(new Tuple<>(UpdateAction.NAME, new UpdateRequest(SecurityTemplateService.SECURITY_INDEX_NAME, "type", "id")));
+        requests.add(new Tuple<>(UpdateAction.NAME, new UpdateRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id")));
-        requests.add(new Tuple<>(BulkAction.NAME + "[s]", new IndexRequest(SecurityTemplateService.SECURITY_INDEX_NAME, "type", "id")));
+        requests.add(new Tuple<>(BulkAction.NAME + "[s]", new IndexRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id")));
-        requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(SecurityTemplateService.SECURITY_INDEX_NAME)));
+        requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(SecurityLifecycleService.SECURITY_INDEX_NAME)));
         requests.add(new Tuple<>(TermVectorsAction.NAME,
-                new TermVectorsRequest(SecurityTemplateService.SECURITY_INDEX_NAME, "type", "id")));
+                new TermVectorsRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id")));
-        requests.add(new Tuple<>(GetAction.NAME, new GetRequest(SecurityTemplateService.SECURITY_INDEX_NAME, "type", "id")));
+        requests.add(new Tuple<>(GetAction.NAME, new GetRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id")));
         requests.add(new Tuple<>(TermVectorsAction.NAME,
-                new TermVectorsRequest(SecurityTemplateService.SECURITY_INDEX_NAME, "type", "id")));
+                new TermVectorsRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id")));
         requests.add(new Tuple<>(IndicesAliasesAction.NAME, new IndicesAliasesRequest()
-                .addAliasAction(AliasActions.add().alias("security_alias").index(SecurityTemplateService.SECURITY_INDEX_NAME))));
+                .addAliasAction(AliasActions.add().alias("security_alias").index(SecurityLifecycleService.SECURITY_INDEX_NAME))));
         requests.add(
-                new Tuple<>(UpdateSettingsAction.NAME, new UpdateSettingsRequest().indices(SecurityTemplateService.SECURITY_INDEX_NAME)));
+                new Tuple<>(UpdateSettingsAction.NAME, new UpdateSettingsRequest().indices(SecurityLifecycleService.SECURITY_INDEX_NAME)));
 
         for (Tuple<String, TransportRequest> requestTuple : requests) {
             String action = requestTuple.v1();
@@ -582,12 +582,12 @@ public class AuthorizationServiceTests extends ESTestCase {
         }
 
         // we should allow waiting for the health of the index or any index if the user has this permission
-        ClusterHealthRequest request = new ClusterHealthRequest(SecurityTemplateService.SECURITY_INDEX_NAME);
+        ClusterHealthRequest request = new ClusterHealthRequest(SecurityLifecycleService.SECURITY_INDEX_NAME);
         authorize(createAuthentication(user), ClusterHealthAction.NAME, request);
         verify(auditTrail).accessGranted(user, ClusterHealthAction.NAME, request);
 
         // multiple indices
-        request = new ClusterHealthRequest(SecurityTemplateService.SECURITY_INDEX_NAME, "foo", "bar");
+        request = new ClusterHealthRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "foo", "bar");
         authorize(createAuthentication(user), ClusterHealthAction.NAME, request);
         verify(auditTrail).accessGranted(user, ClusterHealthAction.NAME, request);
 
@@ -604,21 +604,21 @@ public class AuthorizationServiceTests extends ESTestCase {
         ClusterState state = mock(ClusterState.class);
         when(clusterService.state()).thenReturn(state);
         when(state.metaData()).thenReturn(MetaData.builder()
-                .put(new IndexMetaData.Builder(SecurityTemplateService.SECURITY_INDEX_NAME)
+                .put(new IndexMetaData.Builder(SecurityLifecycleService.SECURITY_INDEX_NAME)
                         .settings(Settings.builder().put("index.version.created", Version.CURRENT).build())
                         .numberOfShards(1).numberOfReplicas(0).build(), true)
                 .build());
 
         List<Tuple<String, ? extends TransportRequest>> requests = new ArrayList<>();
-        requests.add(new Tuple<>(IndicesStatsAction.NAME, new IndicesStatsRequest().indices(SecurityTemplateService.SECURITY_INDEX_NAME)));
+        requests.add(new Tuple<>(IndicesStatsAction.NAME, new IndicesStatsRequest().indices(SecurityLifecycleService.SECURITY_INDEX_NAME)));
-        requests.add(new Tuple<>(RecoveryAction.NAME, new RecoveryRequest().indices(SecurityTemplateService.SECURITY_INDEX_NAME)));
+        requests.add(new Tuple<>(RecoveryAction.NAME, new RecoveryRequest().indices(SecurityLifecycleService.SECURITY_INDEX_NAME)));
         requests.add(new Tuple<>(IndicesSegmentsAction.NAME,
-                new IndicesSegmentsRequest().indices(SecurityTemplateService.SECURITY_INDEX_NAME)));
+                new IndicesSegmentsRequest().indices(SecurityLifecycleService.SECURITY_INDEX_NAME)));
-        requests.add(new Tuple<>(GetSettingsAction.NAME, new GetSettingsRequest().indices(SecurityTemplateService.SECURITY_INDEX_NAME)));
+        requests.add(new Tuple<>(GetSettingsAction.NAME, new GetSettingsRequest().indices(SecurityLifecycleService.SECURITY_INDEX_NAME)));
         requests.add(new Tuple<>(IndicesShardStoresAction.NAME,
-                new IndicesShardStoresRequest().indices(SecurityTemplateService.SECURITY_INDEX_NAME)));
+                new IndicesShardStoresRequest().indices(SecurityLifecycleService.SECURITY_INDEX_NAME)));
         requests.add(new Tuple<>(UpgradeStatusAction.NAME,
-                new UpgradeStatusRequest().indices(SecurityTemplateService.SECURITY_INDEX_NAME)));
+                new UpgradeStatusRequest().indices(SecurityLifecycleService.SECURITY_INDEX_NAME)));
 
         for (Tuple<String, ? extends TransportRequest> requestTuple : requests) {
             String action = requestTuple.v1();
@@ -634,30 +634,31 @@ public class AuthorizationServiceTests extends ESTestCase {
         ClusterState state = mock(ClusterState.class);
         when(clusterService.state()).thenReturn(state);
         when(state.metaData()).thenReturn(MetaData.builder()
-                .put(new IndexMetaData.Builder(SecurityTemplateService.SECURITY_INDEX_NAME)
+                .put(new IndexMetaData.Builder(SecurityLifecycleService.SECURITY_INDEX_NAME)
                         .settings(Settings.builder().put("index.version.created", Version.CURRENT).build())
                         .numberOfShards(1).numberOfReplicas(0).build(), true)
                 .build());
 
         for (User user : Arrays.asList(XPackUser.INSTANCE, superuser)) {
             List<Tuple<String, TransportRequest>> requests = new ArrayList<>();
-            requests.add(new Tuple<>(DeleteAction.NAME, new DeleteRequest(SecurityTemplateService.SECURITY_INDEX_NAME, "type", "id")));
+            requests.add(new Tuple<>(DeleteAction.NAME, new DeleteRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id")));
             requests.add(new Tuple<>(BulkAction.NAME + "[s]",
-                    new DeleteRequest(SecurityTemplateService.SECURITY_INDEX_NAME, "type", "id")));
+                    new DeleteRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id")));
-            requests.add(new Tuple<>(UpdateAction.NAME, new UpdateRequest(SecurityTemplateService.SECURITY_INDEX_NAME, "type", "id")));
+            requests.add(new Tuple<>(UpdateAction.NAME, new UpdateRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id")));
-            requests.add(new Tuple<>(IndexAction.NAME, new IndexRequest(SecurityTemplateService.SECURITY_INDEX_NAME, "type", "id")));
+            requests.add(new Tuple<>(IndexAction.NAME, new IndexRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id")));
-            requests.add(new Tuple<>(BulkAction.NAME + "[s]", new IndexRequest(SecurityTemplateService.SECURITY_INDEX_NAME, "type", "id")));
+            requests.add(new Tuple<>(BulkAction.NAME + "[s]", new IndexRequest(SecurityLifecycleService.SECURITY_INDEX_NAME,
-            requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(SecurityTemplateService.SECURITY_INDEX_NAME)));
+                                                                               "type", "id")));
+            requests.add(new Tuple<>(SearchAction.NAME, new SearchRequest(SecurityLifecycleService.SECURITY_INDEX_NAME)));
             requests.add(new Tuple<>(TermVectorsAction.NAME,
-                    new TermVectorsRequest(SecurityTemplateService.SECURITY_INDEX_NAME, "type", "id")));
+                    new TermVectorsRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id")));
-            requests.add(new Tuple<>(GetAction.NAME, new GetRequest(SecurityTemplateService.SECURITY_INDEX_NAME, "type", "id")));
+            requests.add(new Tuple<>(GetAction.NAME, new GetRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id")));
             requests.add(new Tuple<>(TermVectorsAction.NAME,
-                    new TermVectorsRequest(SecurityTemplateService.SECURITY_INDEX_NAME, "type", "id")));
+                    new TermVectorsRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "type", "id")));
             requests.add(new Tuple<>(IndicesAliasesAction.NAME, new IndicesAliasesRequest()
-                    .addAliasAction(AliasActions.add().alias("security_alias").index(SecurityTemplateService.SECURITY_INDEX_NAME))));
+                    .addAliasAction(AliasActions.add().alias("security_alias").index(SecurityLifecycleService.SECURITY_INDEX_NAME))));
-            requests.add(new Tuple<>(ClusterHealthAction.NAME, new ClusterHealthRequest(SecurityTemplateService.SECURITY_INDEX_NAME)));
+            requests.add(new Tuple<>(ClusterHealthAction.NAME, new ClusterHealthRequest(SecurityLifecycleService.SECURITY_INDEX_NAME)));
             requests.add(new Tuple<>(ClusterHealthAction.NAME,
-                    new ClusterHealthRequest(SecurityTemplateService.SECURITY_INDEX_NAME, "foo", "bar")));
+                    new ClusterHealthRequest(SecurityLifecycleService.SECURITY_INDEX_NAME, "foo", "bar")));
 
             for (Tuple<String, TransportRequest> requestTuple : requests) {
                 String action = requestTuple.v1();
@@ -674,7 +675,7 @@ public class AuthorizationServiceTests extends ESTestCase {
         ClusterState state = mock(ClusterState.class);
         when(clusterService.state()).thenReturn(state);
         when(state.metaData()).thenReturn(MetaData.builder()
-                .put(new IndexMetaData.Builder(SecurityTemplateService.SECURITY_INDEX_NAME)
+                .put(new IndexMetaData.Builder(SecurityLifecycleService.SECURITY_INDEX_NAME)
                         .settings(Settings.builder().put("index.version.created", Version.CURRENT).build())
                         .numberOfShards(1).numberOfReplicas(0).build(), true)
                 .build());

plugin/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java

@@ -38,7 +38,7 @@ import org.elasticsearch.search.internal.ShardSearchTransportRequest;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.xpack.security.SecurityTemplateService;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.elasticsearch.xpack.security.authc.DefaultAuthenticationFailureHandler;
 import org.elasticsearch.xpack.security.authz.RoleDescriptor.IndicesPrivileges;
@@ -104,7 +104,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
                 .put(indexBuilder("-index11").settings(settings))
                 .put(indexBuilder("-index20").settings(settings))
                 .put(indexBuilder("-index21").settings(settings))
-                .put(indexBuilder(SecurityTemplateService.SECURITY_INDEX_NAME).settings(settings)).build();
+                .put(indexBuilder(SecurityLifecycleService.SECURITY_INDEX_NAME).settings(settings)).build();
 
         user = new User("user", "role");
         userDashIndices = new User("dash", "dash");
@@ -1053,14 +1053,14 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
         {
             Set<String> indices = defaultIndicesResolver.resolve(request,
                     metaData, buildAuthorizedIndices(XPackUser.INSTANCE, SearchAction.NAME));
-            assertThat(indices, hasItem(SecurityTemplateService.SECURITY_INDEX_NAME));    
+            assertThat(indices, hasItem(SecurityLifecycleService.SECURITY_INDEX_NAME));
         }
         {
             IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest();
             aliasesRequest.addAliasAction(AliasActions.add().alias("security_alias").index("*"));
             Set<String> indices = defaultIndicesResolver.resolve(aliasesRequest,
                     metaData, buildAuthorizedIndices(XPackUser.INSTANCE, IndicesAliasesAction.NAME));
-            assertThat(indices, hasItem(SecurityTemplateService.SECURITY_INDEX_NAME));
+            assertThat(indices, hasItem(SecurityLifecycleService.SECURITY_INDEX_NAME));
         }
     }
 
@@ -1073,7 +1073,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
             SearchRequest request = new SearchRequest();
             Set<String> indices = defaultIndicesResolver.resolve(request,
                     metaData, buildAuthorizedIndices(allAccessUser, SearchAction.NAME));
-            assertThat(indices, not(hasItem(SecurityTemplateService.SECURITY_INDEX_NAME)));    
+            assertThat(indices, not(hasItem(SecurityLifecycleService.SECURITY_INDEX_NAME)));
         }
         
         {
@@ -1081,7 +1081,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
             aliasesRequest.addAliasAction(AliasActions.add().alias("security_alias1").index("*"));
             Set<String> indices = defaultIndicesResolver.resolve(aliasesRequest,
                     metaData, buildAuthorizedIndices(allAccessUser, IndicesAliasesAction.NAME));
-            assertThat(indices, not(hasItem(SecurityTemplateService.SECURITY_INDEX_NAME)));
+            assertThat(indices, not(hasItem(SecurityLifecycleService.SECURITY_INDEX_NAME)));
         }
     }
 

plugin/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java

@@ -21,6 +21,7 @@ import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.UnassignedInfo.Reason;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -33,11 +34,16 @@ import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.security.InternalClient;
-import org.elasticsearch.xpack.security.SecurityTemplateService;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.action.role.PutRoleRequest;
+import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail;
 import org.elasticsearch.xpack.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.security.authz.RoleDescriptor.IndicesPrivileges;
+import org.junit.After;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.nio.charset.Charset;
@@ -58,6 +64,18 @@ import static org.hamcrest.Matchers.arrayContaining;
 
 public class NativeRolesStoreTests extends ESTestCase {
 
+    private ThreadPool threadPool;
+
+    @Before
+    public void createThreadPool() {
+        threadPool = new TestThreadPool("index audit trail update mapping tests");
+    }
+
+    @After
+    public void terminateThreadPool() throws Exception {
+        terminate(threadPool);
+    }
+
     // test that we can read a role where field permissions are stored in 2.x format (fields:...)
     public void testBWCFieldPermissions() throws IOException {
         Path path = getDataPath("roles2xformat.json");
@@ -161,14 +179,13 @@ public class NativeRolesStoreTests extends ESTestCase {
 
     public void testPutOfRoleWithFlsDlsUnlicensed() {
         final InternalClient internalClient = mock(InternalClient.class);
+        final ClusterService clusterService = mock(ClusterService.class);
         final XPackLicenseState licenseState = mock(XPackLicenseState.class);
         final AtomicBoolean methodCalled = new AtomicBoolean(false);
-        final NativeRolesStore rolesStore = new NativeRolesStore(Settings.EMPTY, internalClient, licenseState) {
+        final SecurityLifecycleService securityLifecycleService =
-            @Override
+            new SecurityLifecycleService(Settings.EMPTY, clusterService, threadPool, internalClient,
-            public State state() {
+                                         licenseState, mock(IndexAuditTrail.class));
-                return State.STARTED;
+        final NativeRolesStore rolesStore = new NativeRolesStore(Settings.EMPTY, internalClient, licenseState, securityLifecycleService) {
-            }
-
             @Override
             void innerPutRole(final PutRoleRequest request, final RoleDescriptor role, final ActionListener<Boolean> listener) {
                 if (methodCalled.compareAndSet(false, true)) {
@@ -179,7 +196,8 @@ public class NativeRolesStoreTests extends ESTestCase {
             }
         };
         // setup the roles store so the security index exists
-        rolesStore.clusterChanged(new ClusterChangedEvent("fls_dls_license", getClusterStateWithSecurityIndex(), getEmptyClusterState()));
+        securityLifecycleService.clusterChanged(new ClusterChangedEvent(
+            "fls_dls_license", getClusterStateWithSecurityIndex(), getEmptyClusterState()));
 
         PutRoleRequest putRoleRequest = new PutRoleRequest();
         RoleDescriptor flsRole = new RoleDescriptor("fls", null,
@@ -230,12 +248,12 @@ public class NativeRolesStoreTests extends ESTestCase {
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                 .build();
         MetaData metaData = MetaData.builder()
-                .put(IndexMetaData.builder(SecurityTemplateService.SECURITY_INDEX_NAME).settings(settings))
+                .put(IndexMetaData.builder(SecurityLifecycleService.SECURITY_INDEX_NAME).settings(settings))
-                .put(new IndexTemplateMetaData(SecurityTemplateService.SECURITY_TEMPLATE_NAME, 0, 0,
+                .put(new IndexTemplateMetaData(SecurityLifecycleService.SECURITY_TEMPLATE_NAME, 0, 0,
-                        Collections.singletonList(SecurityTemplateService.SECURITY_INDEX_NAME), Settings.EMPTY, ImmutableOpenMap.of(),
+                        Collections.singletonList(SecurityLifecycleService.SECURITY_INDEX_NAME), Settings.EMPTY, ImmutableOpenMap.of(),
                         ImmutableOpenMap.of(), ImmutableOpenMap.of()))
                 .build();
-        Index index = new Index(SecurityTemplateService.SECURITY_INDEX_NAME, UUID.randomUUID().toString());
+        Index index = new Index(SecurityLifecycleService.SECURITY_INDEX_NAME, UUID.randomUUID().toString());
         ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId(index, 0), true, EXISTING_STORE_INSTANCE,
                 new UnassignedInfo(Reason.INDEX_CREATED, ""));
         IndexShardRoutingTable table = new IndexShardRoutingTable.Builder(new ShardId(index, 0))

plugin/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java

@@ -24,7 +24,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.security.SecurityTemplateService;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.elasticsearch.xpack.security.authz.store.NativeRolesStoreTests;
 
 import java.io.IOException;
@@ -79,9 +79,9 @@ public class SecurityTestUtils {
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                 .build();
         MetaData metaData = MetaData.builder()
-                .put(IndexMetaData.builder(SecurityTemplateService.SECURITY_INDEX_NAME).settings(settings))
+                .put(IndexMetaData.builder(SecurityLifecycleService.SECURITY_INDEX_NAME).settings(settings))
-                .put(new IndexTemplateMetaData(SecurityTemplateService.SECURITY_TEMPLATE_NAME, 0, 0,
+                .put(new IndexTemplateMetaData(SecurityLifecycleService.SECURITY_TEMPLATE_NAME, 0, 0,
-                        Collections.singletonList(SecurityTemplateService.SECURITY_INDEX_NAME), Settings.EMPTY, ImmutableOpenMap.of(),
+                        Collections.singletonList(SecurityLifecycleService.SECURITY_INDEX_NAME), Settings.EMPTY, ImmutableOpenMap.of(),
                         ImmutableOpenMap.of(), ImmutableOpenMap.of()))
                 .build();
         RoutingTable routingTable = buildSecurityIndexRoutingTable();
@@ -93,7 +93,7 @@ public class SecurityTestUtils {
     }
 
     public static RoutingTable buildSecurityIndexRoutingTable() {
-        Index index = new Index(SecurityTemplateService.SECURITY_INDEX_NAME, UUID.randomUUID().toString());
+        Index index = new Index(SecurityLifecycleService.SECURITY_INDEX_NAME, UUID.randomUUID().toString());
         ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId(index, 0), true, EXISTING_STORE_INSTANCE,
                 new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, ""));
         String nodeId = ESTestCase.randomAsciiOfLength(8);

plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java

@@ -9,7 +9,7 @@ import org.apache.http.HttpStatus;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
 import org.elasticsearch.xpack.ml.integration.MlRestTestStateCleaner;
-import org.elasticsearch.xpack.security.SecurityTemplateService;
+import org.elasticsearch.xpack.security.SecurityLifecycleService;
 import org.junit.After;
 import org.junit.Before;
 
@@ -34,12 +34,12 @@ public class XPackRestIT extends XPackRestTestCase {
     }
 
     /**
-     * Waits for the Security template to be created by the {@link SecurityTemplateService}.
+     * Waits for the Security template to be created by the {@link SecurityLifecycleService}.
      */
     @Before
     public void waitForSecurityTemplate() throws Exception {
         String templateApi = "indices.exists_template";
-        Map<String, String> params = singletonMap("name", SecurityTemplateService.SECURITY_TEMPLATE_NAME);
+        Map<String, String> params = singletonMap("name", SecurityLifecycleService.SECURITY_TEMPLATE_NAME);
 
         AtomicReference<IOException> exceptionHolder = new AtomicReference<>();
         awaitBusy(() -> {

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java

@@ -8,19 +8,84 @@ package org.elasticsearch.upgrades;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 
+import org.apache.http.util.EntityUtils;
 import org.apache.lucene.util.TimeUnits;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+import org.elasticsearch.test.rest.yaml.ObjectPath;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Base64;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.security.SecurityLifecycleService.SECURITY_TEMPLATE_NAME;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 @TimeoutSuite(millis = 5 * TimeUnits.MINUTE) // to account for slow as hell VMs
 public class UpgradeClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
 
+    @Before
+    public void waitForSecuritySetup() throws Exception {
+        String masterNode = null;
+        String catNodesResponse = EntityUtils.toString(
+            client().performRequest("GET", "/_cat/nodes?h=id,master").getEntity(), StandardCharsets.UTF_8
+        );
+        for (String line : catNodesResponse.split("\n")) {
+            int indexOfStar = line.indexOf('*'); // * in the node's output denotes it is master
+            if (indexOfStar != -1) {
+               masterNode = line.substring(0, indexOfStar).trim();
+               break;
+            }
+        }
+        assertNotNull(masterNode);
+        final String masterNodeId = masterNode;
+
+        assertBusy(() -> {
+            try {
+                Response nodeDetailsResponse = client().performRequest("GET", "/_nodes");
+                ObjectPath path = objectPath(nodeDetailsResponse);
+                Map<String, Object> nodes = path.evaluate("nodes");
+                assertThat(nodes.size(), greaterThanOrEqualTo(2));
+                String masterVersion = null;
+                for (String key : nodes.keySet()) {
+                    // get the ES version number master is on
+                    if (key.startsWith(masterNodeId)) {
+                        masterVersion = path.evaluate("nodes." + key + ".version");
+                        break;
+                    }
+                }
+                assertNotNull(masterVersion);
+                final String masterTemplateVersion = masterVersion;
+
+                Response response = client().performRequest("GET", "/_cluster/state/metadata");
+                ObjectPath objectPath = objectPath(response);
+                final String mappingsPath = "metadata.templates." + SECURITY_TEMPLATE_NAME + ".mappings";
+                Map<String, Object> mappings = objectPath.evaluate(mappingsPath);
+                assertNotNull(mappings);
+                assertThat(mappings.size(), greaterThanOrEqualTo(1));
+                for (String key : mappings.keySet()) {
+                    String templateVersion = objectPath.evaluate(mappingsPath + "." + key + "._meta.security-version");
+                    assertEquals(masterTemplateVersion, templateVersion);
+                }
+            } catch (Exception e) {
+                throw new AssertionError("failed to get cluster state", e);
+            }
+        });
+    }
+
+    private ObjectPath objectPath(Response response) throws IOException {
+        String body = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
+        String contentType = response.getHeader("Content-Type");
+        XContentType xContentType = XContentType.fromMediaTypeOrFormat(contentType);
+        return ObjectPath.createFromXContent(xContentType.xContent(), body);
+    }
+
     @Override
     protected boolean preserveIndicesUponCompletion() {
         return true;