Make XPackFeatureSet#usage calls asynchronous (elastic/x-pack-elasticsearch#738)

This commit makes the XPackFeatureSet#usage calls asynchronous. Previously these were synchronous
calls that would execute a multi-search request from the currently elected master node in a blocking
fashion. The multi-search request is now executed asynchronously.

relates elastic/x-pack-elasticsearch#213

Original commit: elastic/x-pack-elasticsearch@a0cb988442

plugin/src/main/java/org/elasticsearch/xpack/XPackFeatureSet.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack;
 
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -26,7 +27,7 @@ public interface XPackFeatureSet {
 
     Map<String, Object> nativeCodeInfo();
 
-    Usage usage();
+    void usage(ActionListener<Usage> listener);
 
     abstract class Usage implements ToXContentObject, NamedWriteable {
 

plugin/src/main/java/org/elasticsearch/xpack/action/TransportXPackUsageAction.java

@@ -17,14 +17,20 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.XPackFeatureSet;
+import org.elasticsearch.xpack.XPackFeatureSet.Usage;
+import org.elasticsearch.xpack.common.IteratingActionListener;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Set;
-import java.util.stream.Collectors;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReferenceArray;
+import java.util.function.BiConsumer;
 
 public class TransportXPackUsageAction extends TransportMasterNodeAction<XPackUsageRequest, XPackUsageResponse> {
 
-    private final Set<XPackFeatureSet> featureSets;
+    private final List<XPackFeatureSet> featureSets;
 
     @Inject
     public TransportXPackUsageAction(Settings settings, ThreadPool threadPool, TransportService transportService,
@@ -32,7 +38,7 @@ public class TransportXPackUsageAction extends TransportMasterNodeAction<XPackUs
                                      IndexNameExpressionResolver indexNameExpressionResolver, Set<XPackFeatureSet> featureSets) {
         super(settings, XPackUsageAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver,
                 XPackUsageRequest::new);
-        this.featureSets = featureSets;
+        this.featureSets = Collections.unmodifiableList(new ArrayList<>(featureSets));
     }
 
     @Override
@@ -48,10 +54,45 @@ public class TransportXPackUsageAction extends TransportMasterNodeAction<XPackUs
     @Override
     protected void masterOperation(XPackUsageRequest request, ClusterState state, ActionListener<XPackUsageResponse> listener)
             throws Exception {
-        List<XPackFeatureSet.Usage> usages = featureSets.stream().map(XPackFeatureSet::usage).collect(Collectors.toList());
+        final ActionListener<List<XPackFeatureSet.Usage>> usageActionListener = new ActionListener<List<Usage>>() {
+            @Override
+            public void onResponse(List<Usage> usages) {
                 listener.onResponse(new XPackUsageResponse(usages));
             }
 
+            @Override
+            public void onFailure(Exception e) {
+                listener.onFailure(e);
+            }
+        };
+        final AtomicReferenceArray<Usage> featureSetUsages = new AtomicReferenceArray<>(featureSets.size());
+        final AtomicInteger position = new AtomicInteger(0);
+        final BiConsumer<XPackFeatureSet, ActionListener<List<Usage>>> consumer = (featureSet, iteratingListener) -> {
+            featureSet.usage(new ActionListener<Usage>() {
+                @Override
+                public void onResponse(Usage usage) {
+                    featureSetUsages.set(position.getAndIncrement(), usage);
+                    iteratingListener.onResponse(null); // just send null back and keep iterating
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    iteratingListener.onFailure(e);
+                }
+            });
+        };
+        IteratingActionListener<List<XPackFeatureSet.Usage>, XPackFeatureSet> iteratingActionListener =
+                new IteratingActionListener<>(usageActionListener, consumer, featureSets,
+                        threadPool.getThreadContext(), () -> {
+                    final List<Usage> usageList = new ArrayList<>(featureSetUsages.length());
+                    for (int i = 0; i < featureSetUsages.length(); i++) {
+                        usageList.add(featureSetUsages.get(i));
+                    }
+                    return usageList;
+                });
+        iteratingActionListener.run();
+    }
+
     @Override
     protected ClusterBlockException checkBlock(XPackUsageRequest request, ClusterState state) {
         return null;

plugin/src/main/java/org/elasticsearch/xpack/graph/GraphFeatureSet.java

@@ -8,6 +8,7 @@ package org.elasticsearch.xpack.graph;
 import java.io.IOException;
 import java.util.Map;
 
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -54,8 +55,8 @@ public class GraphFeatureSet implements XPackFeatureSet {
     }
 
     @Override
-    public XPackFeatureSet.Usage usage() {
+    public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
-        return new Usage(available(), enabled());
+        listener.onResponse(new Usage(available(), enabled()));
     }
 
     public static class Usage extends XPackFeatureSet.Usage {

plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSet.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.ml;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -77,8 +78,8 @@ public class MachineLearningFeatureSet implements XPackFeatureSet {
     }
 
     @Override
-    public XPackFeatureSet.Usage usage() {
+    public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
-        return new Usage(available(), enabled());
+        listener.onResponse(new Usage(available(), enabled()));
     }
 
     public static class Usage extends XPackFeatureSet.Usage {

plugin/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringFeatureSet.java

@@ -9,6 +9,7 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -60,8 +61,8 @@ public class MonitoringFeatureSet implements XPackFeatureSet {
     }
 
     @Override
-    public XPackFeatureSet.Usage usage() {
+    public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
-        return new Usage(available(), enabled(), exportersUsage(exporters));
+        listener.onResponse(new Usage(available(), enabled(), exportersUsage(exporters)));
     }
 
     static Map<String, Object> exportersUsage(Exporters exporters) {

plugin/src/main/java/org/elasticsearch/xpack/security/SecurityFeatureSet.java

@@ -6,20 +6,23 @@
 package org.elasticsearch.xpack.security;
 
 import java.io.IOException;
+import java.nio.file.Files;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
 
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.XPackFeatureSet;
 import org.elasticsearch.xpack.XPackPlugin;
 import org.elasticsearch.xpack.XPackSettings;
-import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.elasticsearch.xpack.security.authc.Realms;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
 import org.elasticsearch.xpack.security.crypto.CryptoService;
@@ -33,8 +36,6 @@ import static org.elasticsearch.xpack.XPackSettings.HTTP_SSL_ENABLED;
  */
 public class SecurityFeatureSet implements XPackFeatureSet {
 
-    private static final Map<String, Object> DISABLED_FEATURE_MAP = Collections.singletonMap("enabled", false);
-
     private final Settings settings;
     private final boolean enabled;
     private final XPackLicenseState licenseState;
@@ -44,23 +45,19 @@ public class SecurityFeatureSet implements XPackFeatureSet {
     private final CompositeRolesStore rolesStore;
     @Nullable
     private final IPFilter ipFilter;
-    @Nullable
+    private final boolean systemKeyUsed;
-    private final AuditTrailService auditTrailService;
-    @Nullable
-    private final CryptoService cryptoService;
 
     @Inject
     public SecurityFeatureSet(Settings settings, @Nullable XPackLicenseState licenseState, @Nullable Realms realms,
                               @Nullable CompositeRolesStore rolesStore, @Nullable IPFilter ipFilter,
-                              @Nullable AuditTrailService auditTrailService, @Nullable CryptoService cryptoService) {
+                              Environment environment) {
         this.enabled = XPackSettings.SECURITY_ENABLED.get(settings);
         this.licenseState = licenseState;
         this.realms = realms;
         this.rolesStore = rolesStore;
         this.settings = settings;
         this.ipFilter = ipFilter;
-        this.auditTrailService = auditTrailService;
+        this.systemKeyUsed = enabled && Files.exists(CryptoService.resolveSystemKey(environment));
-        this.cryptoService = cryptoService;
     }
 
     @Override
@@ -89,16 +86,21 @@ public class SecurityFeatureSet implements XPackFeatureSet {
     }
 
     @Override
-    public XPackFeatureSet.Usage usage() {
+    public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
         Map<String, Object> realmsUsage = buildRealmsUsage(realms);
-        Map<String, Object> rolesStoreUsage = rolesStore == null ? Collections.emptyMap() : rolesStore.usageStats();
         Map<String, Object> sslUsage = sslUsage(settings);
-        Map<String, Object> auditUsage = auditUsage(auditTrailService);
+        Map<String, Object> auditUsage = auditUsage(settings);
         Map<String, Object> ipFilterUsage = ipFilterUsage(ipFilter);
-        Map<String, Object> systemKeyUsage = systemKeyUsage(cryptoService);
+        Map<String, Object> systemKeyUsage = systemKeyUsage();
         Map<String, Object> anonymousUsage = Collections.singletonMap("enabled", AnonymousUser.isAnonymousEnabled(settings));
-        return new Usage(available(), enabled(), realmsUsage, rolesStoreUsage, sslUsage, auditUsage, ipFilterUsage, systemKeyUsage,
+        final ActionListener<Map<String, Object>> rolesStoreUsageListener =
-                anonymousUsage);
+                ActionListener.wrap(rolesStoreUsage -> listener.onResponse(new Usage(available(), enabled(), realmsUsage, rolesStoreUsage,
+                        sslUsage, auditUsage, ipFilterUsage, systemKeyUsage, anonymousUsage)),listener::onFailure);
+        if (rolesStore == null) {
+            rolesStoreUsageListener.onResponse(Collections.emptyMap());
+        } else {
+            rolesStore.usageStats(rolesStoreUsageListener);
+        }
     }
 
     static Map<String, Object> buildRealmsUsage(Realms realms) {
@@ -112,11 +114,11 @@ public class SecurityFeatureSet implements XPackFeatureSet {
         return Collections.singletonMap("http", Collections.singletonMap("enabled", HTTP_SSL_ENABLED.get(settings)));
     }
 
-    static Map<String, Object> auditUsage(@Nullable AuditTrailService auditTrailService) {
+    static Map<String, Object> auditUsage(Settings settings) {
-        if (auditTrailService == null) {
+        Map<String, Object> map = new HashMap<>(2);
-            return DISABLED_FEATURE_MAP;
+        map.put("enabled", XPackSettings.AUDIT_ENABLED.get(settings));
-        }
+        map.put("outputs", Security.AUDIT_OUTPUTS_SETTING.get(settings));
-        return auditTrailService.usageStats();
+        return map;
     }
 
     static Map<String, Object> ipFilterUsage(@Nullable IPFilter ipFilter) {
@@ -126,9 +128,9 @@ public class SecurityFeatureSet implements XPackFeatureSet {
         return ipFilter.usageStats();
     }
 
-    static Map<String, Object> systemKeyUsage(CryptoService cryptoService) {
+    Map<String, Object> systemKeyUsage() {
         // we can piggy back on the encryption enabled method as it is only enabled if there is a system key
-        return Collections.singletonMap("enabled", cryptoService != null && cryptoService.isEncryptionEnabled());
+        return Collections.singletonMap("enabled", systemKeyUsed);
     }
 
     public static class Usage extends XPackFeatureSet.Usage {

plugin/src/main/java/org/elasticsearch/xpack/security/audit/AuditTrailService.java

@@ -7,17 +7,13 @@ package org.elasticsearch.xpack.security.audit;
 
 import java.net.InetAddress;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.transport.TransportMessage;
-import org.elasticsearch.xpack.XPackSettings;
-import org.elasticsearch.xpack.security.Security;
 import org.elasticsearch.xpack.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.security.transport.filter.SecurityIpFilterRule;
 import org.elasticsearch.xpack.security.user.User;
@@ -220,11 +216,4 @@ public class AuditTrailService extends AbstractComponent implements AuditTrail {
             }
         }
     }
-
-    public Map<String, Object> usageStats() {
-        Map<String, Object> map = new HashMap<>(2);
-        map.put("enabled", XPackSettings.AUDIT_ENABLED.get(settings));
-        map.put("outputs", Security.AUDIT_OUTPUTS_SETTING.get(settings));
-        return map;
-    }
 }

plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java

@@ -278,11 +278,13 @@ public class CompositeRolesStore extends AbstractComponent {
         negativeLookupCache.remove(role);
     }
 
-    public Map<String, Object> usageStats() {
+    public void usageStats(ActionListener<Map<String, Object>> listener) {
-        Map<String, Object> usage = new HashMap<>(2);
+        final Map<String, Object> usage = new HashMap<>(2);
         usage.put("file", fileRolesStore.usageStats());
-        usage.put("native", nativeRolesStore.usageStats());
+        nativeRolesStore.usageStats(ActionListener.wrap(map -> {
-        return usage;
+            usage.put("native", map);
+            listener.onResponse(usage);
+        }, listener::onFailure));
     }
 
     /**

plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java

@@ -16,7 +16,6 @@ import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.action.search.MultiSearchRequestBuilder;
 import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.MultiSearchResponse.Item;
 import org.elasticsearch.action.search.SearchRequest;
@@ -202,29 +201,20 @@ public class NativeRolesStore extends AbstractComponent {
         }
     }
 
-    public Map<String, Object> usageStats() {
+    public void usageStats(ActionListener<Map<String, Object>> listener) {
-        boolean dls = false;
-        boolean fls = false;
         Map<String, Object> usageStats = new HashMap<>();
         if (securityLifecycleService.securityIndexExists() == false) {
             usageStats.put("size", 0L);
-            usageStats.put("fls", fls);
+            usageStats.put("fls", false);
-            usageStats.put("dls", dls);
+            usageStats.put("dls", false);
-            return usageStats;
+            listener.onResponse(usageStats);
-        }
+        } else {
-
+            client.prepareMultiSearch()
-        // FIXME this needs to be async
-        long count = 0L;
-        // query for necessary information
-        if (fls == false || dls == false) {
-            MultiSearchRequestBuilder builder = client.prepareMultiSearch()
                     .add(client.prepareSearch(SecurityLifecycleService.SECURITY_INDEX_NAME)
                             .setTypes(ROLE_DOC_TYPE)
                             .setQuery(QueryBuilders.matchAllQuery())
-                            .setSize(0));
+                            .setSize(0))
-
+                    .add(client.prepareSearch(SecurityLifecycleService.SECURITY_INDEX_NAME)
-            if (fls == false) {
-                builder.add(client.prepareSearch(SecurityLifecycleService.SECURITY_INDEX_NAME)
                         .setTypes(ROLE_DOC_TYPE)
                         .setQuery(QueryBuilders.boolQuery()
                                 .should(existsQuery("indices.field_security.grant"))
@@ -232,43 +222,44 @@ public class NativeRolesStore extends AbstractComponent {
                                 // for backwardscompat with 2.x
                                 .should(existsQuery("indices.fields")))
                         .setSize(0)
-                        .setTerminateAfter(1));
+                        .setTerminateAfter(1))
-            }
+                    .add(client.prepareSearch(SecurityLifecycleService.SECURITY_INDEX_NAME)
-
-            if (dls == false) {
-                builder.add(client.prepareSearch(SecurityLifecycleService.SECURITY_INDEX_NAME)
                         .setTypes(ROLE_DOC_TYPE)
                         .setQuery(existsQuery("indices.query"))
                         .setSize(0)
-                        .setTerminateAfter(1));
+                        .setTerminateAfter(1))
+                    .execute(new ActionListener<MultiSearchResponse>() {
+                        @Override
+                        public void onResponse(MultiSearchResponse items) {
+                            Item[] responses = items.getResponses();
+                            if (responses[0].isFailure()) {
+                                usageStats.put("size", 0);
+                            } else {
+                                usageStats.put("size", responses[0].getResponse().getHits().getTotalHits());
                             }
 
-            MultiSearchResponse multiSearchResponse = builder.get();
+                            if (responses[1].isFailure()) {
-            int pos = 0;
+                                usageStats.put("fls", false);
-            Item[] responses = multiSearchResponse.getResponses();
+                            } else {
-            if (responses[pos].isFailure() == false) {
+                                usageStats.put("fls", responses[1].getResponse().getHits().getTotalHits() > 0L);
-                count = responses[pos].getResponse().getHits().getTotalHits();
                             }
 
-            if (fls == false) {
+                            if (responses[2].isFailure()) {
-                if (responses[++pos].isFailure() == false) {
+                                usageStats.put("dls", false);
-                    fls = responses[pos].getResponse().getHits().getTotalHits() > 0L;
+                            } else {
+                                usageStats.put("dls", responses[2].getResponse().getHits().getTotalHits() > 0L);
                             }
+                            listener.onResponse(usageStats);
                         }
 
-            if (dls == false) {
+                        @Override
-                if (responses[++pos].isFailure() == false) {
+                        public void onFailure(Exception e) {
-                    dls = responses[pos].getResponse().getHits().getTotalHits() > 0L;
+                            listener.onFailure(e);
                         }
+                    });
         }
     }
 
-        usageStats.put("size", count);
-        usageStats.put("fls", fls);
-        usageStats.put("dls", dls);
-        return usageStats;
-    }
-
     private void getRoleDescriptor(final String roleId, ActionListener<RoleDescriptor> roleActionListener) {
         if (securityLifecycleService.securityIndexExists() == false) {
             roleActionListener.onResponse(null);

plugin/src/main/java/org/elasticsearch/xpack/watcher/WatcherFeatureSet.java

@@ -9,6 +9,7 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
 
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -17,6 +18,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.XPackFeatureSet;
+import org.elasticsearch.xpack.XPackFeatureSet.Usage;
 import org.elasticsearch.xpack.XPackPlugin;
 import org.elasticsearch.xpack.XPackSettings;
 
@@ -59,8 +61,9 @@ public class WatcherFeatureSet implements XPackFeatureSet {
     }
 
     @Override
-    public XPackFeatureSet.Usage usage() {
+    public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
-        return new Usage(available(), enabled(), watcherService != null ? watcherService.usageStats() : Collections.emptyMap());
+        listener.onResponse(
+                new Usage(available(), enabled(), watcherService != null ? watcherService.usageStats() : Collections.emptyMap()));
     }
 
     public static class Usage extends XPackFeatureSet.Usage {

plugin/src/test/java/org/elasticsearch/xpack/graph/GraphFeatureSetTests.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.graph;
 
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
@@ -30,10 +31,13 @@ public class GraphFeatureSetTests extends ESTestCase {
         boolean available = randomBoolean();
         when(licenseState.isGraphAllowed()).thenReturn(available);
         assertThat(featureSet.available(), is(available));
-        assertThat(featureSet.usage().available(), is(available));
+        PlainActionFuture<XPackFeatureSet.Usage> future = new PlainActionFuture<>();
+        featureSet.usage(future);
+        XPackFeatureSet.Usage usage = future.get();
+        assertThat(usage.available(), is(available));
 
         BytesStreamOutput out = new BytesStreamOutput();
-        featureSet.usage().writeTo(out);
+        usage.writeTo(out);
         XPackFeatureSet.Usage serializedUsage = new GraphFeatureSet.Usage(out.bytes().streamInput());
         assertThat(serializedUsage.available(), is(available));
     }
@@ -50,10 +54,13 @@ public class GraphFeatureSetTests extends ESTestCase {
         }
         GraphFeatureSet featureSet = new GraphFeatureSet(settings.build(), licenseState);
         assertThat(featureSet.enabled(), is(enabled));
-        assertThat(featureSet.usage().enabled(), is(enabled));
+        PlainActionFuture<XPackFeatureSet.Usage> future = new PlainActionFuture<>();
+        featureSet.usage(future);
+        XPackFeatureSet.Usage usage = future.get();
+        assertThat(usage.enabled(), is(enabled));
 
         BytesStreamOutput out = new BytesStreamOutput();
-        featureSet.usage().writeTo(out);
+        usage.writeTo(out);
         XPackFeatureSet.Usage serializedUsage = new GraphFeatureSet.Usage(out.bytes().streamInput());
         assertThat(serializedUsage.enabled(), is(enabled));
     }

plugin/src/test/java/org/elasticsearch/xpack/ml/MachineLearningFeatureSetTests.java

@@ -5,11 +5,13 @@
  */
 package org.elasticsearch.xpack.ml;
 
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.XPackFeatureSet;
+import org.elasticsearch.xpack.XPackFeatureSet.Usage;
 import org.junit.Before;
 
 import static org.hamcrest.core.Is.is;
@@ -30,10 +32,13 @@ public class MachineLearningFeatureSetTests extends ESTestCase {
         boolean available = randomBoolean();
         when(licenseState.isMachineLearningAllowed()).thenReturn(available);
         assertThat(featureSet.available(), is(available));
-        assertThat(featureSet.usage().available(), is(available));
+        PlainActionFuture<Usage> future = new PlainActionFuture<>();
+        featureSet.usage(future);
+        XPackFeatureSet.Usage usage = future.get();
+        assertThat(usage.available(), is(available));
 
         BytesStreamOutput out = new BytesStreamOutput();
-        featureSet.usage().writeTo(out);
+        usage.writeTo(out);
         XPackFeatureSet.Usage serializedUsage = new MachineLearningFeatureSet.Usage(out.bytes().streamInput());
         assertThat(serializedUsage.available(), is(available));
     }
@@ -49,10 +54,13 @@ public class MachineLearningFeatureSetTests extends ESTestCase {
         boolean expected = enabled || useDefault;
         MachineLearningFeatureSet featureSet = new MachineLearningFeatureSet(settings.build(), licenseState);
         assertThat(featureSet.enabled(), is(expected));
-        assertThat(featureSet.usage().enabled(), is(expected));
+        PlainActionFuture<Usage> future = new PlainActionFuture<>();
+        featureSet.usage(future);
+        XPackFeatureSet.Usage usage = future.get();
+        assertThat(usage.enabled(), is(expected));
 
         BytesStreamOutput out = new BytesStreamOutput();
-        featureSet.usage().writeTo(out);
+        usage.writeTo(out);
         XPackFeatureSet.Usage serializedUsage = new MachineLearningFeatureSet.Usage(out.bytes().streamInput());
         assertThat(serializedUsage.enabled(), is(expected));
     }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/MonitoringFeatureSetTests.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.monitoring;
 
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -12,6 +13,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.XPackFeatureSet;
+import org.elasticsearch.xpack.XPackFeatureSet.Usage;
 import org.elasticsearch.xpack.monitoring.exporter.Exporter;
 import org.elasticsearch.xpack.monitoring.exporter.Exporters;
 import org.elasticsearch.xpack.monitoring.exporter.http.HttpExporter;
@@ -95,7 +97,9 @@ public class MonitoringFeatureSetTests extends ESTestCase {
         when(exporters.iterator()).thenReturn(exporterList.iterator());
 
         MonitoringFeatureSet featureSet = new MonitoringFeatureSet(Settings.EMPTY, licenseState, exporters);
-        XPackFeatureSet.Usage monitoringUsage = featureSet.usage();
+        PlainActionFuture<Usage> future = new PlainActionFuture<>();
+        featureSet.usage(future);
+        XPackFeatureSet.Usage monitoringUsage = future.get();
         BytesStreamOutput out = new BytesStreamOutput();
         monitoringUsage.writeTo(out);
         XPackFeatureSet.Usage serializedUsage = new MonitoringFeatureSet.Usage(out.bytes().streamInput());

plugin/src/test/java/org/elasticsearch/xpack/security/SecurityFeatureSetTests.java

@@ -5,16 +5,20 @@
  */
 package org.elasticsearch.xpack.security;
 
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.XPackFeatureSet;
 import org.elasticsearch.xpack.XPackPlugin;
+import org.elasticsearch.xpack.XPackSettings;
 import org.elasticsearch.xpack.security.audit.AuditTrailService;
 import org.elasticsearch.xpack.security.authc.Realms;
 import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
@@ -24,6 +28,10 @@ import org.elasticsearch.xpack.security.user.AnonymousUser;
 import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource;
 import org.junit.Before;
 
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -34,12 +42,15 @@ import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.core.Is.is;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 public class SecurityFeatureSetTests extends ESTestCase {
 
     private Settings settings;
+    private Environment environment;
     private XPackLicenseState licenseState;
     private Realms realms;
     private IPFilter ipFilter;
@@ -50,17 +61,16 @@ public class SecurityFeatureSetTests extends ESTestCase {
     @Before
     public void init() throws Exception {
         settings = Settings.builder().put("path.home", createTempDir()).build();
+        environment = new Environment(settings);
         licenseState = mock(XPackLicenseState.class);
         realms = mock(Realms.class);
         ipFilter = mock(IPFilter.class);
         rolesStore = mock(CompositeRolesStore.class);
-        auditTrail = mock(AuditTrailService.class);
-        cryptoService = mock(CryptoService.class);
     }
 
     public void testAvailable() throws Exception {
         SecurityFeatureSet featureSet = new SecurityFeatureSet(settings, licenseState, realms, rolesStore,
-                ipFilter, auditTrail, cryptoService);
+                ipFilter, environment);
         boolean available = randomBoolean();
         when(licenseState.isAuthAllowed()).thenReturn(available);
         assertThat(featureSet.available(), is(available));
@@ -73,26 +83,26 @@ public class SecurityFeatureSetTests extends ESTestCase {
                 .put("xpack.security.enabled", enabled)
                 .build();
         SecurityFeatureSet featureSet = new SecurityFeatureSet(settings, licenseState, realms, rolesStore,
-                ipFilter, auditTrail, cryptoService);
+                ipFilter, environment);
         assertThat(featureSet.enabled(), is(enabled));
     }
 
     public void testEnabledDefault() throws Exception {
         SecurityFeatureSet featureSet = new SecurityFeatureSet(settings, licenseState, realms, rolesStore,
-                        ipFilter, auditTrail, cryptoService);
+                        ipFilter, environment);
         assertThat(featureSet.enabled(), is(true));
     }
 
-    public void testSystemKeyUsageEnabledByCryptoService() {
+    public void testSystemKeyUsageEnabledByCryptoService() throws IOException {
         final boolean enabled = randomBoolean();
-
+        if (enabled) {
-        when(cryptoService.isEncryptionEnabled()).thenReturn(enabled);
+            Path path = CryptoService.resolveSystemKey(environment);
-
+            Files.createDirectories(path.getParent());
-        assertThat(SecurityFeatureSet.systemKeyUsage(cryptoService), hasEntry("enabled", enabled));
+            Files.write(path, new byte[0]);
         }
-
+        SecurityFeatureSet featureSet = new SecurityFeatureSet(settings, licenseState, realms, rolesStore,
-    public void testSystemKeyUsageNotEnabledIfNull() {
+                ipFilter, environment);
-        assertThat(SecurityFeatureSet.systemKeyUsage(null), hasEntry("enabled", false));
+        assertThat(featureSet.systemKeyUsage(), hasEntry("enabled", enabled));
     }
 
     public void testUsage() throws Exception {
@@ -108,13 +118,9 @@ public class SecurityFeatureSetTests extends ESTestCase {
         final boolean httpSSLEnabled = randomBoolean();
         settings.put("xpack.security.http.ssl.enabled", httpSSLEnabled);
         final boolean auditingEnabled = randomBoolean();
+        settings.put(XPackSettings.AUDIT_ENABLED.getKey(), auditingEnabled);
         final String[] auditOutputs = randomFrom(new String[] {"logfile"}, new String[] {"index"}, new String[] {"logfile", "index"});
-        when(auditTrail.usageStats())
+        settings.putArray(Security.AUDIT_OUTPUTS_SETTING.getKey(), auditOutputs);
-                .thenReturn(MapBuilder.<String, Object>newMapBuilder()
-                        .put("enabled", auditingEnabled)
-                        .put("outputs", auditOutputs)
-                        .map());
-
         final boolean httpIpFilterEnabled = randomBoolean();
         final boolean transportIPFilterEnabled = randomBoolean();
         when(ipFilter.usageStats())
@@ -125,13 +131,21 @@ public class SecurityFeatureSetTests extends ESTestCase {
 
 
         final boolean rolesStoreEnabled = randomBoolean();
+        doAnswer(invocationOnMock -> {
+            ActionListener<Map<String, Object>> listener = (ActionListener<Map<String, Object>>) invocationOnMock.getArguments()[0];
             if (rolesStoreEnabled) {
-            when(rolesStore.usageStats()).thenReturn(Collections.singletonMap("count", 1));
+                listener.onResponse(Collections.singletonMap("count", 1));
             } else {
-            when(rolesStore.usageStats()).thenReturn(Collections.emptyMap());
+                listener.onResponse(Collections.emptyMap());
             }
+            return Void.TYPE;
+        }).when(rolesStore).usageStats(any(ActionListener.class));
         final boolean useSystemKey = randomBoolean();
-        when(cryptoService.isEncryptionEnabled()).thenReturn(useSystemKey);
+        if (useSystemKey) {
+            Path path = CryptoService.resolveSystemKey(environment);
+            Files.createDirectories(path.getParent());
+            Files.write(path, new byte[0], StandardOpenOption.CREATE_NEW);
+        }
 
         Map<String, Object> realmsUsageStats = new HashMap<>();
         for (int i = 0; i < 5; i++) {
@@ -148,9 +162,10 @@ public class SecurityFeatureSetTests extends ESTestCase {
             settings.put(AnonymousUser.ROLES_SETTING.getKey(), "foo");
         }
 
-        SecurityFeatureSet featureSet = new SecurityFeatureSet(settings.build(), licenseState, realms, rolesStore,
+        SecurityFeatureSet featureSet = new SecurityFeatureSet(settings.build(), licenseState, realms, rolesStore, ipFilter, environment);
-                ipFilter, auditTrail, cryptoService);
+        PlainActionFuture<XPackFeatureSet.Usage> future = new PlainActionFuture<>();
-        XPackFeatureSet.Usage securityUsage = featureSet.usage();
+        featureSet.usage(future);
+        XPackFeatureSet.Usage securityUsage = future.get();
         BytesStreamOutput out = new BytesStreamOutput();
         securityUsage.writeTo(out);
         XPackFeatureSet.Usage serializedUsage = new SecurityFeatureSet.Usage(out.bytes().streamInput());

plugin/src/test/java/org/elasticsearch/xpack/security/authc/esnative/NativeRealmIntegTests.java

@@ -10,6 +10,7 @@ import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.ValidationException;
@@ -599,7 +600,9 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
     public void testRolesUsageStats() throws Exception {
         NativeRolesStore rolesStore = internalCluster().getInstance(NativeRolesStore.class);
         long roles = anonymousEnabled && roleExists ? 1L: 0L;
-        Map<String, Object> usage = rolesStore.usageStats();
+        PlainActionFuture<Map<String, Object>> future = new PlainActionFuture<>();
+        rolesStore.usageStats(future);
+        Map<String, Object> usage = future.get();
         assertEquals(roles, usage.get("size"));
         assertThat(usage.get("fls"), is(false));
         assertThat(usage.get("dls"), is(false));
@@ -644,7 +647,9 @@ public class NativeRealmIntegTests extends NativeRealmIntegTestCase {
 
         client.prepareClearRolesCache().get();
 
-        usage = rolesStore.usageStats();
+        future = new PlainActionFuture<>();
+        rolesStore.usageStats(future);
+        usage = future.get();
         assertThat(usage.get("size"), is(roles));
         assertThat(usage.get("fls"), is(fls));
         assertThat(usage.get("dls"), is(dls));

plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherFeatureSetTests.java

@@ -9,6 +9,7 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -63,7 +64,9 @@ public class WatcherFeatureSetTests extends ESTestCase {
         statsMap.put("foo", "bar");
         when(watcherService.usageStats()).thenReturn(statsMap);
 
-        XPackFeatureSet.Usage watcherUsage = new WatcherFeatureSet(Settings.EMPTY, licenseState, watcherService).usage();
+        PlainActionFuture<XPackFeatureSet.Usage> future = new PlainActionFuture<>();
+        new WatcherFeatureSet(Settings.EMPTY, licenseState, watcherService).usage(future);
+        XPackFeatureSet.Usage watcherUsage = future.get();
         BytesStreamOutput out = new BytesStreamOutput();
         watcherUsage.writeTo(out);
         XPackFeatureSet.Usage serializedUsage = new WatcherFeatureSet.Usage(out.bytes().streamInput());