Merge branch 'master' into java9

Original commit: elastic/x-pack-elasticsearch@8a5736fcd5

elasticsearch/license/base/src/main/java/org/elasticsearch/license/core/License.java

@@ -77,6 +77,7 @@ public class License implements ToXContent {
      * Note: The mode indicates features that should be made available, but it does not indicate whether the license is active!
      */
     public enum OperationMode {
+        MISSING,
         TRIAL,
         BASIC,
         STANDARD,
@@ -85,6 +86,8 @@ public class License implements ToXContent {
 
         public static OperationMode resolve(String type) {
             switch (type.toLowerCase(Locale.ROOT)) {
+                case "missing":
+                    return MISSING;
                 case "trial":
                 case "none": // bwc for 1.x subscription_type field
                 case "dev": // bwc for 1.x subscription_type field

elasticsearch/license/licensor/bin/key-pair-generator

@@ -34,25 +34,5 @@ else
     JAVA=`which java`
 fi
 
-
+exec "$JAVA" $JAVA_OPTS -Xmx64m -Xms16m -cp "$LICENSE_CLASSPATH" -Des.path.home="`pwd`" org.elasticsearch.license.licensor.tools.KeyPairGeneratorTool "$@"
-# Parse any long getopt options and put them into properties before calling getopt below
-# Be dash compatible to make sure running under ubuntu works
-ARGCOUNT=$#
-COUNT=0
-while [ $COUNT -lt $ARGCOUNT ]
-do
-    case $1 in
-      -D*=*)
-           properties="$properties $1"
-           shift 1; COUNT=$(($COUNT+1))
-           ;;
-      -D*) 
-           properties="$properties $1=$2"
-           shift ; shift; COUNT=$(($COUNT+2))
-           ;;
-      *) set -- "$@" "$1"; shift; COUNT=$(($COUNT+1))
-    esac
-done
-
-exec "$JAVA" $JAVA_OPTS -Xmx64m -Xms16m $properties -cp "$LICENSE_CLASSPATH" -Des.path.home="`pwd`" org.elasticsearch.license.licensor.tools.KeyPairGeneratorTool "$@"
 

elasticsearch/license/licensor/bin/license-generator

@@ -34,23 +34,4 @@ else
     JAVA=`which java`
 fi
 
-# Parse any long getopt options and put them into properties before calling getopt below
+exec "$JAVA" $JAVA_OPTS -Xmx64m -Xms16m -cp "$LICENSE_CLASSPATH" -Des.path.home="`pwd`" org.elasticsearch.license.licensor.tools.LicenseGeneratorTool "$@"
-# Be dash compatible to make sure running under ubuntu works
-ARGCOUNT=$#
-COUNT=0
-while [ $COUNT -lt $ARGCOUNT ]
-do
-    case $1 in
-      -D*=*)
-           properties="$properties $1"
-           shift 1; COUNT=$(($COUNT+1))
-           ;;
-      -D*)
-           properties="$properties $1=$2"
-           shift ; shift; COUNT=$(($COUNT+2))
-           ;;
-      *) set -- "$@" "$1"; shift; COUNT=$(($COUNT+1))
-    esac
-done
-
-exec "$JAVA" $JAVA_OPTS -Xmx64m -Xms16m $properties  -cp "$LICENSE_CLASSPATH" -Des.path.home="`pwd`" org.elasticsearch.license.licensor.tools.LicenseGeneratorTool "$@"

elasticsearch/license/licensor/bin/verify-license

@@ -34,24 +34,5 @@ else
     JAVA=`which java`
 fi
 
-# Parse any long getopt options and put them into properties before calling getopt below
+exec "$JAVA" $JAVA_OPTS -Xmx64m -Xms16m -cp "$LICENSE_CLASSPATH" -Des.path.home="`pwd`" org.elasticsearch.license.licensor.tools.LicenseVerificationTool "$@"
-# Be dash compatible to make sure running under ubuntu works
-ARGCOUNT=$#
-COUNT=0
-while [ $COUNT -lt $ARGCOUNT ]
-do
-    case $1 in
-      -D*=*)
-           properties="$properties $1"
-           shift 1; COUNT=$(($COUNT+1))
-           ;;
-      -D*)
-           properties="$properties $1=$2"
-           shift ; shift; COUNT=$(($COUNT+2))
-           ;;
-      *) set -- "$@" "$1"; shift; COUNT=$(($COUNT+1))
-    esac
-done
-
-exec "$JAVA" $JAVA_OPTS -Xmx64m -Xms16m $properties -cp "$LICENSE_CLASSPATH" -Des.path.home="`pwd`" org.elasticsearch.license.licensor.tools.LicenseVerificationTool "$@"
 

elasticsearch/license/licensor/src/main/java/org/elasticsearch/license/licensor/tools/KeyPairGeneratorTool.java

@@ -5,22 +5,21 @@
  */
 package org.elasticsearch.license.licensor.tools;
 
-import java.io.File;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.elasticsearch.cli.Command;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserError;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.io.PathUtils;
+
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.KeyPair;
 import java.security.KeyPairGenerator;
 import java.security.SecureRandom;
 
-import joptsimple.OptionSet;
-import joptsimple.OptionSpec;
-import org.elasticsearch.cli.Command;
-import org.elasticsearch.cli.ExitCodes;
-import org.elasticsearch.cli.UserError;
-import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.io.PathUtils;
-
 import static org.elasticsearch.license.core.CryptUtils.writeEncryptedPrivateKey;
 import static org.elasticsearch.license.core.CryptUtils.writeEncryptedPublicKey;
 

elasticsearch/qa/build.gradle

@@ -1,5 +1,5 @@
 subprojects {
-  project.afterEvaluate {
+  tasks.withType(org.elasticsearch.gradle.precommit.LicenseHeadersTask) {
     // someone figure out what the x-plugins logic should be
     project.licenseHeaders.enabled = false
   }

elasticsearch/qa/smoke-test-monitoring-with-shield/build.gradle

@@ -1,41 +1,50 @@
-apply plugin: 'elasticsearch.rest-test'
+subprojects {
+  apply plugin: 'elasticsearch.rest-test'
 
-dependencies {
+  dependencies {
-  testCompile project(path: ':x-plugins:elasticsearch:x-pack', configuration: 'runtime')
+    testCompile project(path: ':x-plugins:elasticsearch:x-pack', configuration: 'runtime')
-}
+  }
 
-// bring in monitoring rest test suite
+  // bring in monitoring rest test suite
-task copyMonitoringRestTests(type: Copy) {
+  task copyMonitoringRestTests(type: Copy) {
-  into project.sourceSets.test.output.resourcesDir
+    into project.sourceSets.test.output.resourcesDir
-  from project(':x-plugins:elasticsearch:x-pack').sourceSets.test.resources.srcDirs
+    from project(':x-plugins:elasticsearch:x-pack').sourceSets.test.resources.srcDirs
-  include 'rest-api-spec/test/monitoring/**'
+    include 'rest-api-spec/test/monitoring/**'
-}
+  }
 
-integTest {
+  integTest {
-  dependsOn copyMonitoringRestTests
+    dependsOn copyMonitoringRestTests
 
-  cluster {
+    cluster {
-    systemProperty 'es.logger.level', 'TRACE'
+      systemProperty 'es.logger.level', 'TRACE'
-    plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack')
+      plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack')
-    setting 'xpack.monitoring.agent.interval', '3s'
+      setting 'xpack.monitoring.agent.interval', '3s'
-    extraConfigFile 'x-pack/roles.yml', 'roles.yml'
+      extraConfigFile 'x-pack/roles.yml', '../roles.yml'
-    setupCommand 'setupTestAdminUser',
+      setupCommand 'setupTestAdminUser',
-                 'bin/x-pack/users', 'useradd', 'test_admin', '-p', 'changeme', '-r', 'superuser'
+                   'bin/x-pack/users', 'useradd', 'test_admin', '-p', 'changeme', '-r', 'superuser'
-    setupCommand 'setupMonitoredSystemUser',
+      setupCommand 'setupMonitoredSystemUser',
-                 'bin/x-pack/users', 'useradd', 'monitored_system', '-p', 'changeme', '-r', 'monitored_system,required_for_test'
+                   'bin/x-pack/users', 'useradd', 'monitoring_system', '-p', 'changeme', '-r', 'monitoring_system,monitoring_without_bulk'
-    setupCommand 'setupPowerlessUser',
+      setupCommand 'setupPowerlessUser',
-                 'bin/x-pack/users', 'useradd', 'no_monitored_system', '-p', 'changeme', '-r', 'required_for_test'
+                   'bin/x-pack/users', 'useradd', 'not_monitoring_system', '-p', 'changeme', '-r', 'monitoring_without_bulk'
 
-    waitCondition = { node, ant ->
+      waitCondition = { node, ant ->
-      File tmpFile = new File(node.cwd, 'wait.success')
+        File tmpFile = new File(node.cwd, 'wait.success')
-      ant.get(src: "http://${node.httpUri()}",
+        ant.get(src: "http://${node.httpUri()}",
-              dest: tmpFile.toString(),
+                dest: tmpFile.toString(),
-              username: 'test_admin',
+                username: 'test_admin',
-              password: 'changeme',
+                password: 'changeme',
-              ignoreerrors: true,
+                ignoreerrors: true,
-              retries: 10)
+                retries: 10)
-      return tmpFile.exists()
+        return tmpFile.exists()
+      }
     }
   }
 }
 
+/**
+ * Allow {@code integTest} to be invoked on this project to run both Monitoring+Security smoke tests.
+ */
+task integTest {
+  dependsOn subprojects.integTest
+}
+

elasticsearch/qa/smoke-test-monitoring-with-shield/insufficient-rights/src/test/java/org/elasticsearch/smoketest/MonitoringWithShieldInsufficientRoleIT.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.smoketest;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.shield.authc.support.SecuredString;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.RestTestCandidate;
+import org.elasticsearch.test.rest.parser.RestTestParseException;
+
+import java.io.IOException;
+
+import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+import static org.hamcrest.Matchers.containsString;
+
+public class MonitoringWithShieldInsufficientRoleIT extends ESRestTestCase {
+
+    public MonitoringWithShieldInsufficientRoleIT(@Name("yaml") RestTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws IOException, RestTestParseException {
+        return ESRestTestCase.createParameters(0, 1);
+    }
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = basicAuthHeaderValue("not_monitoring_system", new SecuredString("changeme".toCharArray()));
+        return Settings.builder()
+                .put(ThreadContext.PREFIX + ".Authorization", token)
+                .build();
+    }
+
+    @Override
+    protected Settings restAdminSettings() {
+        String token = basicAuthHeaderValue("test_admin", new SecuredString("changeme".toCharArray()));
+        return Settings.builder()
+                .put(ThreadContext.PREFIX + ".Authorization", token)
+                .build();
+    }
+
+    @Override
+    public void test() throws IOException {
+        try {
+            super.test();
+            fail("should have failed because of missing role");
+        } catch(AssertionError ae) {
+            assertThat(ae.getMessage(), containsString("action [cluster:admin/xpack/monitoring/bulk]"));
+            assertThat(ae.getMessage(), containsString("returned [403 Forbidden]"));
+            assertThat(ae.getMessage(), containsString("is unauthorized for user [not_monitoring_system]"));
+        }
+    }
+
+}

elasticsearch/qa/smoke-test-monitoring-with-shield/roles.yml

@@ -1,15 +1,7 @@
-admin:
+monitoring_system:
-  cluster:
-    - all
-  indices:
-    - names: '*'
-      privileges:
-        - all
-
-monitored_system:
   cluster: [ 'cluster:admin/xpack/monitoring/bulk' ]
 
-required_for_test:
+monitoring_without_bulk:
   cluster: [ 'monitor' ]
   indices:
     - names: '.monitoring-*'

elasticsearch/qa/smoke-test-monitoring-with-shield/src/test/java/org/elasticsearch/smoketest/MonitoringWithShieldInsufficientRoleIT.java

@@ -1,36 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.smoketest;
-
-import com.carrotsearch.randomizedtesting.annotations.Name;
-import org.elasticsearch.test.rest.RestTestCandidate;
-
-import java.io.IOException;
-
-import static org.hamcrest.Matchers.containsString;
-
-public class MonitoringWithShieldInsufficientRoleIT extends MonitoringWithShieldIT {
-
-    public MonitoringWithShieldInsufficientRoleIT(@Name("yaml") RestTestCandidate testCandidate) {
-        super(testCandidate);
-    }
-
-    public void test() throws IOException {
-        try {
-            super.test();
-            fail("should have failed because of missing role");
-        } catch(AssertionError ae) {
-            assertThat(ae.getMessage(), containsString("action [cluster:admin/xpack/monitoring/bulk]"));
-            assertThat(ae.getMessage(), containsString("returned [403 Forbidden]"));
-            assertThat(ae.getMessage(), containsString("is unauthorized for user [no_monitored_system]"));
-        }
-    }
-
-    @Override
-    protected String[] getCredentials() {
-        return new String[]{"no_monitored_system", "changeme"};
-    }
-}

elasticsearch/qa/smoke-test-monitoring-with-shield/sufficient-rights/src/test/java/org/elasticsearch/smoketest/MonitoringWithShieldIT.java

@@ -20,9 +20,6 @@ import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basic
 
 public class MonitoringWithShieldIT extends ESRestTestCase {
 
-    private final static String TEST_ADMIN_USERNAME = "test_admin";
-    private final static String TEST_ADMIN_PASSWORD = "changeme";
-
     public MonitoringWithShieldIT(@Name("yaml") RestTestCandidate testCandidate) {
         super(testCandidate);
     }
@@ -32,14 +29,9 @@ public class MonitoringWithShieldIT extends ESRestTestCase {
         return ESRestTestCase.createParameters(0, 1);
     }
 
-    protected String[] getCredentials() {
-        return new String[]{"monitored_system", "changeme"};
-    }
-
     @Override
     protected Settings restClientSettings() {
-        String[] creds = getCredentials();
+        String token = basicAuthHeaderValue("monitoring_system", new SecuredString("changeme".toCharArray()));
-        String token = basicAuthHeaderValue(creds[0], new SecuredString(creds[1].toCharArray()));
         return Settings.builder()
                 .put(ThreadContext.PREFIX + ".Authorization", token)
                 .build();
@@ -47,7 +39,7 @@ public class MonitoringWithShieldIT extends ESRestTestCase {
 
     @Override
     protected Settings restAdminSettings() {
-        String token = basicAuthHeaderValue(TEST_ADMIN_USERNAME, new SecuredString(TEST_ADMIN_PASSWORD.toCharArray()));
+        String token = basicAuthHeaderValue("test_admin", new SecuredString("changeme".toCharArray()));
         return Settings.builder()
                 .put(ThreadContext.PREFIX + ".Authorization", token)
                 .build();

elasticsearch/qa/smoke-test-plugins-ssl/build.gradle

@@ -177,6 +177,9 @@ integTest {
     setupCommand 'setupMarvelUser',
                  'bin/x-pack/users', 'useradd', 'monitoring_agent', '-p', 'changeme', '-r', 'remote_monitoring_agent'
 
+    // Required to detect that the monitoring agent service has started
+    setting 'logger.level', 'DEBUG'
+
     waitCondition = { node, ant ->
       // HTTPS check is tricky to do, so we wait for the log file to indicate that the node is started
       String waitForNodeStartProp = "waitForNodeStart${name}"

elasticsearch/x-dev-tools/smoke_test_xpack_rc.py

@@ -112,7 +112,7 @@ def smoke_test_release(release, files, expected_hash):
     
     print('  Starting elasticsearch daemon from [%s]' % es_dir)
     try:
-      run('%s; %s -Ees.node.name=smoke_tester -Ees.cluster.name=prepare_release -Ees.script.inline=true -Ees.script.stored=true -Ees.repositories.url.allowed_urls=http://snapshot.test* %s -Ees.pidfile=%s -Ees.node.portsfile=true'
+      run('%s; %s -Enode.name=smoke_tester -Ecluster.name=prepare_release -Escript.inline=true -Escript.stored=true -Erepositories.url.allowed_urls=http://snapshot.test* %s -Epidfile=%s -Enode.portsfile=true'
           % (java_exe(), es_run_path, '-d', os.path.join(es_dir, 'es-smoke.pid')))
       if not wait_for_node_startup(es_dir, headers=headers):
         print("elasticsearch logs:")

elasticsearch/x-pack/bin/x-pack/extension

@@ -70,38 +70,6 @@ elif [ -f "/etc/default/elasticsearch" ]; then
    . "/etc/default/elasticsearch"
 fi
 
-# Parse any long getopt options and put them into properties before calling getopt below
-# Be dash compatible to make sure running under ubuntu works
-ARGCOUNT=$#
-COUNT=0
-while [ $COUNT -lt $ARGCOUNT ]
-do
-    case $1 in
-      --*=*) properties="$properties -Des.${1#--}"
-           shift 1; COUNT=$(($COUNT+1))
-           ;;
-      --*) properties="$properties -Des.${1#--}=$2"
-           shift ; shift; COUNT=$(($COUNT+2))
-           ;;
-      *) set -- "$@" "$1"; shift; COUNT=$(($COUNT+1))
-    esac
-done
-
-# check if properties already has a config file or config dir
-if [ -e "$CONF_DIR" ]; then
-    case "$properties" in
-        *-Des.default.path.conf=*) ;;
-        *)
-            if [ ! -d "$CONF_DIR/x-pack" ]; then
-                echo "ERROR: The configuration directory [$CONF_DIR/x-pack] does not exist. The extension tool expects security configuration files in that location."
-                echo "The plugin may not have been installed with the correct configuration path. If [$ES_HOME/config/x-pack] exists, please copy the 'x-pack' directory to [$CONF_DIR]"
-                exit 1
-            fi
-            properties="$properties -Des.default.path.conf=$CONF_DIR"
-            ;;
-        esac
-fi
-
 export HOSTNAME=`hostname -s`
 
 # include x-pack jars in classpath
@@ -113,8 +81,14 @@ if [ ! -z "$CONF_FILE" ]; then
     exit 1
 fi
 
+declare -a args=("$@")
+
+if [ -e "$CONF_DIR" ]; then
+  args=("${args[@]}" -Edefault.path.conf="$CONF_DIR")
+fi
+
 cd "$ES_HOME" > /dev/null
-"$JAVA" $ES_JAVA_OPTS -cp "$ES_CLASSPATH" -Des.path.home="$ES_HOME" $properties org.elasticsearch.xpack.extensions.XPackExtensionCli "$@"
+"$JAVA" $ES_JAVA_OPTS -cp "$ES_CLASSPATH" -Des.path.home="$ES_HOME" org.elasticsearch.xpack.extensions.XPackExtensionCli "${args[@]}"
 status=$?
 cd - > /dev/null
 exit $status

elasticsearch/x-pack/graph/src/main/java/org/elasticsearch/graph/GraphFeatureSet.java

@@ -7,9 +7,13 @@ package org.elasticsearch.graph;
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.XPackFeatureSet;
 
+import java.io.IOException;
+
 /**
  *
  */
@@ -19,9 +23,10 @@ public class GraphFeatureSet implements XPackFeatureSet {
     private final GraphLicensee licensee;
 
     @Inject
-    public GraphFeatureSet(Settings settings, @Nullable  GraphLicensee licensee) {
+    public GraphFeatureSet(Settings settings, @Nullable GraphLicensee licensee, NamedWriteableRegistry namedWriteableRegistry) {
         this.enabled = Graph.enabled(settings);
         this.licensee = licensee;
+        namedWriteableRegistry.register(Usage.class, Usage.writeableName(Graph.NAME), Usage::new);
     }
 
     @Override
@@ -43,4 +48,20 @@ public class GraphFeatureSet implements XPackFeatureSet {
     public boolean enabled() {
         return enabled;
     }
+
+    @Override
+    public XPackFeatureSet.Usage usage() {
+        return new Usage(available(), enabled());
+    }
+
+    static class Usage extends XPackFeatureSet.Usage {
+
+        public Usage(StreamInput input) throws IOException {
+            super(input);
+        }
+
+        public Usage(boolean available, boolean enabled) {
+            super(Graph.NAME, available, enabled);
+        }
+    }
 }

elasticsearch/x-pack/graph/src/main/java/org/elasticsearch/graph/action/GraphExploreRequest.java

@@ -15,7 +15,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator;
 
@@ -39,7 +39,7 @@ public class GraphExploreRequest extends ActionRequest<GraphExploreRequest> impl
     private String routing;
     private TimeValue timeout;
 
-    private int sampleSize = SamplerAggregatorBuilder.DEFAULT_SHARD_SAMPLE_SIZE;
+    private int sampleSize = SamplerAggregationBuilder.DEFAULT_SHARD_SAMPLE_SIZE;
     private String sampleDiversityField;
     private int maxDocsPerDiversityValue;
     private boolean useSignificance = true;
@@ -199,7 +199,7 @@ public class GraphExploreRequest extends ActionRequest<GraphExploreRequest> impl
 
     /**
      * The number of top-matching documents that are considered during each hop (default is 
-     * {@link SamplerAggregatorBuilder#DEFAULT_SHARD_SAMPLE_SIZE}
+     * {@link SamplerAggregationBuilder#DEFAULT_SHARD_SAMPLE_SIZE}
      * Very small values (less than 50) may not provide sufficient weight-of-evidence to identify
      * significant connections between terms. 
      * <p> Very large values (many thousands) are not recommended with loosely defined queries (fuzzy queries or those 

elasticsearch/x-pack/graph/src/main/java/org/elasticsearch/graph/action/GraphExploreRequestBuilder.java

@@ -11,7 +11,7 @@ import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator;
 
@@ -138,7 +138,7 @@ public class GraphExploreRequestBuilder extends ActionRequestBuilder<GraphExplor
     
     /**
      * The number of top-matching documents that are considered during each hop (default is 
-     * {@link SamplerAggregatorBuilder#DEFAULT_SHARD_SAMPLE_SIZE}
+     * {@link SamplerAggregationBuilder#DEFAULT_SHARD_SAMPLE_SIZE}
      * Very small values (less than 50) may not provide sufficient weight-of-evidence to identify
      * significant connections between terms. 
      * <p> Very large values (many thousands) are not recommended with loosely defined queries (fuzzy queries or 

elasticsearch/x-pack/graph/src/main/java/org/elasticsearch/graph/action/TransportGraphExploreAction.java

@@ -29,14 +29,14 @@ import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.license.plugin.core.LicenseUtils;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
-import org.elasticsearch.search.aggregations.AggregatorBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.sampler.Sampler;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms.Bucket;
-import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -189,9 +189,9 @@ public class TransportGraphExploreAction extends HandledTransportAction<GraphExp
             // each had non-overlapping sets of terms that needed frequencies looking up for significant terms.
             // A common sample pool reduces the specialization that can be given to each root term but
             // ultimately is much faster to run because of the shared vocabulary in a single sample set. 
-            AggregatorBuilder sampleAgg = null;
+            AggregationBuilder sampleAgg = null;
             if (request.sampleDiversityField() != null) {
-                DiversifiedAggregatorBuilder diversifiedSampleAgg = AggregationBuilders.diversifiedSampler("sample")
+                DiversifiedAggregationBuilder diversifiedSampleAgg = AggregationBuilders.diversifiedSampler("sample")
                         .shardSize(request.sampleSize());
                 diversifiedSampleAgg.field(request.sampleDiversityField());
                 diversifiedSampleAgg.maxDocsPerValue(request.maxDocsPerDiversityValue());
@@ -227,7 +227,7 @@ public class TransportGraphExploreAction extends HandledTransportAction<GraphExp
                 for (Vertex v : lastWaveVerticesForField) {
                     terms[i++] = v.term;
                 }
-                TermsAggregatorBuilder lastWaveTermsAgg = AggregationBuilders.terms("field" + fieldNum)
+                TermsAggregationBuilder lastWaveTermsAgg = AggregationBuilders.terms("field" + fieldNum)
                         .includeExclude(new IncludeExclude(terms, null))
                         .shardMinDocCount(1)
                         .field(lastVr.fieldName()).minDocCount(1)
@@ -246,7 +246,7 @@ public class TransportGraphExploreAction extends HandledTransportAction<GraphExp
                         size++;
                     }
                     if (request.useSignificance()) {
-                        SignificantTermsAggregatorBuilder nextWaveSigTerms = AggregationBuilders.significantTerms("field" + f)
+                        SignificantTermsAggregationBuilder nextWaveSigTerms = AggregationBuilders.significantTerms("field" + f)
                                 .field(vr.fieldName())
                                 .minDocCount(vr.minDocCount()).shardMinDocCount(vr.shardMinDocCount()).executionHint("map").size(size);
 //                        nextWaveSigTerms.significanceHeuristic(new PercentageScore.PercentageScoreBuilder());
@@ -277,7 +277,7 @@ public class TransportGraphExploreAction extends HandledTransportAction<GraphExp
                         }
                         lastWaveTermsAgg.subAggregation(nextWaveSigTerms);
                     } else {
-                        TermsAggregatorBuilder nextWavePopularTerms = AggregationBuilders.terms("field" + f).field(vr.fieldName())
+                        TermsAggregationBuilder nextWavePopularTerms = AggregationBuilders.terms("field" + f).field(vr.fieldName())
                                 .minDocCount(vr.minDocCount()).shardMinDocCount(vr.shardMinDocCount())
                                 // Map execution mode used because Sampler agg keeps us
                                 // focused on smaller sets of high quality docs and therefore
@@ -567,9 +567,9 @@ public class TransportGraphExploreAction extends HandledTransportAction<GraphExp
 
                 BoolQueryBuilder rootBool = QueryBuilders.boolQuery();
                 
-                AggregatorBuilder rootSampleAgg = null;
+                AggregationBuilder rootSampleAgg = null;
                 if (request.sampleDiversityField() != null) {
-                    DiversifiedAggregatorBuilder diversifiedRootSampleAgg = AggregationBuilders.diversifiedSampler("sample")
+                    DiversifiedAggregationBuilder diversifiedRootSampleAgg = AggregationBuilders.diversifiedSampler("sample")
                             .shardSize(request.sampleSize());
                     diversifiedRootSampleAgg.field(request.sampleDiversityField());
                     diversifiedRootSampleAgg.maxDocsPerValue(request.maxDocsPerDiversityValue());
@@ -600,7 +600,7 @@ public class TransportGraphExploreAction extends HandledTransportAction<GraphExp
                 for (int i = 0; i < rootHop.getNumberVertexRequests(); i++) {
                     VertexRequest vr = rootHop.getVertexRequest(i);
                     if (request.useSignificance()) {
-                        SignificantTermsAggregatorBuilder sigBuilder = AggregationBuilders.significantTerms("field" + i);
+                        SignificantTermsAggregationBuilder sigBuilder = AggregationBuilders.significantTerms("field" + i);
                         sigBuilder.field(vr.fieldName()).shardMinDocCount(vr.shardMinDocCount()).minDocCount(vr.minDocCount())
                                 // Map execution mode used because Sampler agg
                                 // keeps us focused on smaller sets of high quality 
@@ -621,7 +621,7 @@ public class TransportGraphExploreAction extends HandledTransportAction<GraphExp
                         }
                         rootSampleAgg.subAggregation(sigBuilder);
                     } else {
-                        TermsAggregatorBuilder termsBuilder = AggregationBuilders.terms("field" + i);
+                        TermsAggregationBuilder termsBuilder = AggregationBuilders.terms("field" + i);
                         // Min doc count etc really only applies when we are
                         // thinking about certainty of significance scores -
                         // perhaps less necessary when considering popularity

elasticsearch/x-pack/graph/src/test/java/org/elasticsearch/graph/GraphFeatureSetTests.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.graph;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import static org.hamcrest.core.Is.is;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ *
+ */
+public class GraphFeatureSetTests extends ESTestCase {
+
+    private GraphLicensee licensee;
+    private NamedWriteableRegistry namedWriteableRegistry;
+
+    @Before
+    public void init() throws Exception {
+        licensee = mock(GraphLicensee.class);
+        namedWriteableRegistry = mock(NamedWriteableRegistry.class);
+    }
+
+    public void testWritableRegistration() throws Exception {
+        new GraphFeatureSet(Settings.EMPTY, licensee, namedWriteableRegistry);
+        verify(namedWriteableRegistry).register(eq(GraphFeatureSet.Usage.class), eq("xpack.usage.graph"), anyObject());
+    }
+
+    public void testAvailable() throws Exception {
+        GraphFeatureSet featureSet = new GraphFeatureSet(Settings.EMPTY, licensee, namedWriteableRegistry);
+        boolean available = randomBoolean();
+        when(licensee.isAvailable()).thenReturn(available);
+        assertThat(featureSet.available(), is(available));
+    }
+
+    public void testEnabled() throws Exception {
+        boolean enabled = randomBoolean();
+        Settings.Builder settings = Settings.builder();
+        if (enabled) {
+            if (randomBoolean()) {
+                settings.put("xpack.graph.enabled", enabled);
+            }
+        } else {
+            settings.put("xpack.graph.enabled", enabled);
+        }
+        GraphFeatureSet featureSet = new GraphFeatureSet(settings.build(), licensee, namedWriteableRegistry);
+        assertThat(featureSet.enabled(), is(enabled));
+    }
+
+}

elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/core/Licensee.java

@@ -53,6 +53,7 @@ public interface Licensee {
     class Status {
 
         public static Status ENABLED = new Status(OperationMode.TRIAL, LicenseState.ENABLED);
+        public static Status MISSING = new Status(OperationMode.MISSING, LicenseState.DISABLED);
 
         private final OperationMode mode;
         private final LicenseState licenseState;

elasticsearch/x-pack/license-plugin/src/main/java/org/elasticsearch/license/plugin/core/LicensesService.java

@@ -262,25 +262,26 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
             if (!request.acknowledged()) {
                 final LicensesMetaData currentMetaData = clusterService.state().metaData().custom(LicensesMetaData.TYPE);
                 final License currentLicense = getLicense(currentMetaData);
-                Map<String, String[]> acknowledgeMessages = new HashMap<>(registeredLicensees.size() + 1);
+                if (currentLicense != null && currentLicense != LicensesMetaData.LICENSE_TOMBSTONE) {
-                if (currentLicense != null && !License.isAutoGeneratedLicense(currentLicense.signature()) // when current license is not
+                    Map<String, String[]> acknowledgeMessages = new HashMap<>(registeredLicensees.size() + 1);
-                        // an auto-generated license
+                    if (!License.isAutoGeneratedLicense(currentLicense.signature()) // current license is not auto-generated
-                        && currentLicense.issueDate() > newLicense.issueDate()) { // and has a later issue date
+                            && currentLicense.issueDate() > newLicense.issueDate()) { // and has a later issue date
-                    acknowledgeMessages.put("license",
+                        acknowledgeMessages.put("license",
-                            new String[]{"The new license is older than the currently installed license. Are you sure you want to " +
+                                new String[]{"The new license is older than the currently installed license. Are you sure you want to " +
-                                    "override the current license?"});
+                                        "override the current license?"});
-                }
+                    }
-                for (InternalLicensee licensee : registeredLicensees) {
+                    for (InternalLicensee licensee : registeredLicensees) {
-                    String[] listenerAcknowledgeMessages = licensee.acknowledgmentMessages(currentLicense, newLicense);
+                        String[] listenerAcknowledgeMessages = licensee.acknowledgmentMessages(currentLicense, newLicense);
-                    if (listenerAcknowledgeMessages.length > 0) {
+                        if (listenerAcknowledgeMessages.length > 0) {
-                        acknowledgeMessages.put(licensee.id(), listenerAcknowledgeMessages);
+                            acknowledgeMessages.put(licensee.id(), listenerAcknowledgeMessages);
+                        }
+                    }
+                    if (!acknowledgeMessages.isEmpty()) {
+                        // needs acknowledgement
+                        listener.onResponse(new LicensesUpdateResponse(false, LicensesStatus.VALID, ACKNOWLEDGEMENT_HEADER,
+                                acknowledgeMessages));
+                        return;
                     }
-                }
-                if (!acknowledgeMessages.isEmpty()) {
-                    // needs acknowledgement
-                    listener.onResponse(new LicensesUpdateResponse(false, LicensesStatus.VALID, ACKNOWLEDGEMENT_HEADER,
-                            acknowledgeMessages));
-                    return;
                 }
             }
             clusterService.submitStateUpdateTask("register license [" + newLicense.uid() + "]", new
@@ -385,8 +386,8 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
 
     @Override
     public License getLicense() {
-        final LicensesMetaData metaData = clusterService.state().metaData().custom(LicensesMetaData.TYPE);
+        final License license = getLicense(clusterService.state().metaData().custom(LicensesMetaData.TYPE));
-        return getLicense(metaData);
+        return license == LicensesMetaData.LICENSE_TOMBSTONE ? null : license;
     }
 
     /**
@@ -511,6 +512,14 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
      */
     private void notifyAndSchedule(final LicensesMetaData currentLicensesMetaData) {
         final License license = getLicense(currentLicensesMetaData);
+        if (license == LicensesMetaData.LICENSE_TOMBSTONE) {
+            // implies license has been explicitly deleted
+            // update licensee states
+            registeredLicensees.forEach(InternalLicensee::onRemove);
+            return;
+        }
+        // license can be null if the trial license is yet to be auto-generated
+        // in this case, it is a no-op
         if (license != null) {
             logger.debug("notifying [{}] listeners", registeredLicensees.size());
             long now = System.currentTimeMillis();
@@ -770,10 +779,12 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
                 REGISTER_TRIAL_LICENSE_ACTION_NAME, TransportRequest.Empty.INSTANCE, EmptyTransportResponseHandler.INSTANCE_SAME);
     }
 
-    public License getLicense(final LicensesMetaData metaData) {
+    License getLicense(final LicensesMetaData metaData) {
         if (metaData != null) {
             License license = metaData.getLicense();
-            if (license != LicensesMetaData.LICENSE_TOMBSTONE) {
+            if (license == LicensesMetaData.LICENSE_TOMBSTONE) {
+                return license;
+            } else {
                 boolean autoGeneratedLicense = License.isAutoGeneratedLicense(license.signature());
                 if ((autoGeneratedLicense && TrialLicense.verify(license))
                         || (!autoGeneratedLicense && verifyLicense(license))) {
@@ -845,6 +856,16 @@ public class LicensesService extends AbstractLifecycleComponent<LicensesService>
                 }
             }
         }
+
+        public void onRemove() {
+            synchronized (this) {
+                if (currentLicense != null || currentLicenseState != LicenseState.DISABLED) {
+                    currentLicense = null;
+                    currentLicenseState = LicenseState.DISABLED;
+                    licensee.onChange(Licensee.Status.MISSING);
+                }
+            }
+        }
     }
 
     /**

elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/LicensesServiceNodeTests.java

@@ -23,6 +23,7 @@ import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
  */
 @ESIntegTestCase.ClusterScope(scope = TEST, numDataNodes = 10, numClientNodes = 0)
 public class LicensesServiceNodeTests extends AbstractLicensesIntegrationTestCase {
+
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()

elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/TestUtils.java

@@ -23,7 +23,6 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.license.core.License;
 import org.elasticsearch.license.licensor.LicenseSigner;
 import org.elasticsearch.license.plugin.action.put.PutLicenseRequest;
-import org.elasticsearch.license.plugin.core.LicenseState;
 import org.elasticsearch.license.plugin.core.Licensee;
 import org.elasticsearch.license.plugin.core.LicensesService;
 import org.elasticsearch.license.plugin.core.LicensesStatus;
@@ -162,7 +161,7 @@ public class TestUtils {
     public static class AssertingLicensee implements Licensee {
         public final ESLogger logger;
         public final String id;
-        public final List<LicenseState> licenseStates = new CopyOnWriteArrayList<>();
+        public final List<Licensee.Status> statuses = new CopyOnWriteArrayList<>();
         public final AtomicInteger expirationMessagesCalled = new AtomicInteger(0);
         public final List<Tuple<License, License>> acknowledgementRequested = new CopyOnWriteArrayList<>();
 
@@ -196,7 +195,7 @@ public class TestUtils {
         @Override
         public void onChange(Status status) {
             assertNotNull(status);
-            licenseStates.add(status.getLicenseState());
+            statuses.add(status);
         }
     }
 }

elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesExpiryNotificationTests.java

@@ -41,10 +41,10 @@ public class LicensesExpiryNotificationTests extends ESSingleNodeTestCase {
         awaitNoBlock(client());
         licensesService.register(licensee);
         awaitNoPendingTasks(client());
-        boolean success = awaitBusy(() -> licensee.licenseStates.size() == 3);
+        boolean success = awaitBusy(() -> licensee.statuses.size() == 3);
         // trail license: enable, grace, disabled
         assertLicenseStates(licensee, LicenseState.ENABLED, LicenseState.GRACE_PERIOD, LicenseState.DISABLED);
-        assertTrue(dumpLicensingStates(licensee.licenseStates), success);
+        assertTrue(dumpLicensingStates(licensee.statuses), success);
         licensesService.stop();
     }
 
@@ -61,10 +61,10 @@ public class LicensesExpiryNotificationTests extends ESSingleNodeTestCase {
         licensesService.register(licensee1);
         licensesService.register(licensee2);
         awaitNoPendingTasks(client());
-        boolean success = awaitBusy(() -> licensee1.licenseStates.size() == 3);
+        boolean success = awaitBusy(() -> licensee1.statuses.size() == 3);
-        assertTrue(dumpLicensingStates(licensee1.licenseStates), success);
+        assertTrue(dumpLicensingStates(licensee1.statuses), success);
-        success = awaitBusy(() -> licensee2.licenseStates.size() == 3);
+        success = awaitBusy(() -> licensee2.statuses.size() == 3);
-        assertTrue(dumpLicensingStates(licensee2.licenseStates), success);
+        assertTrue(dumpLicensingStates(licensee2.statuses), success);
         // trail license: enable, grace, disabled
         assertLicenseStates(licensee1, LicenseState.ENABLED, LicenseState.GRACE_PERIOD, LicenseState.DISABLED);
         assertLicenseStates(licensee2, LicenseState.ENABLED, LicenseState.GRACE_PERIOD, LicenseState.DISABLED);
@@ -81,13 +81,13 @@ public class LicensesExpiryNotificationTests extends ESSingleNodeTestCase {
         awaitNoBlock(client());
         licensesService.register(licensee);
         awaitNoPendingTasks(client());
-        boolean success = awaitBusy(() -> licensee.licenseStates.size() == 1);
+        boolean success = awaitBusy(() -> licensee.statuses.size() == 1);
-        assertTrue(dumpLicensingStates(licensee.licenseStates), success);
+        assertTrue(dumpLicensingStates(licensee.statuses), success);
         registerAndAckSignedLicenses(licensesService, generateSignedLicense(TimeValue.timeValueSeconds(4)), LicensesStatus.VALID);
-        success = awaitBusy(() -> licensee.licenseStates.size() == 4);
+        success = awaitBusy(() -> licensee.statuses.size() == 4);
         // trial: enable, signed: enable, signed: grace, signed: disabled
         assertLicenseStates(licensee, LicenseState.ENABLED, LicenseState.ENABLED, LicenseState.GRACE_PERIOD, LicenseState.DISABLED);
-        assertTrue(dumpLicensingStates(licensee.licenseStates), success);
+        assertTrue(dumpLicensingStates(licensee.statuses), success);
         licensesService.stop();
     }
 
@@ -102,10 +102,10 @@ public class LicensesExpiryNotificationTests extends ESSingleNodeTestCase {
         registerAndAckSignedLicenses(licensesService, generateSignedLicense(TimeValue.timeValueSeconds(2)), LicensesStatus.VALID);
         licensesService.register(licensee);
         awaitNoPendingTasks(client());
-        boolean success = awaitBusy(() -> licensee.licenseStates.size() == 3);
+        boolean success = awaitBusy(() -> licensee.statuses.size() == 3);
         // signed: enable, signed: grace, signed: disabled
         assertLicenseStates(licensee, LicenseState.ENABLED, LicenseState.GRACE_PERIOD, LicenseState.DISABLED);
-        assertTrue(dumpLicensingStates(licensee.licenseStates), success);
+        assertTrue(dumpLicensingStates(licensee.statuses), success);
         licensesService.stop();
     }
 
@@ -123,10 +123,10 @@ public class LicensesExpiryNotificationTests extends ESSingleNodeTestCase {
         licensesService.register(licensee1);
         licensesService.register(licensee2);
         awaitNoPendingTasks(client());
-        boolean success = awaitBusy(() -> licensee1.licenseStates.size() == 3);
+        boolean success = awaitBusy(() -> licensee1.statuses.size() == 3);
-        assertTrue(dumpLicensingStates(licensee1.licenseStates), success);
+        assertTrue(dumpLicensingStates(licensee1.statuses), success);
-        success = awaitBusy(() -> licensee2.licenseStates.size() == 3);
+        success = awaitBusy(() -> licensee2.statuses.size() == 3);
-        assertTrue(dumpLicensingStates(licensee2.licenseStates), success);
+        assertTrue(dumpLicensingStates(licensee2.statuses), success);
         // signed license: enable, grace, disabled
         assertLicenseStates(licensee1, LicenseState.ENABLED, LicenseState.GRACE_PERIOD, LicenseState.DISABLED);
         assertLicenseStates(licensee2, LicenseState.ENABLED, LicenseState.GRACE_PERIOD, LicenseState.DISABLED);
@@ -145,18 +145,18 @@ public class LicensesExpiryNotificationTests extends ESSingleNodeTestCase {
         licensesService.register(licensee);
         awaitNoPendingTasks(client());
         // trial license enabled
-        boolean success = awaitBusy(() -> licensee.licenseStates.size() == 1);
+        boolean success = awaitBusy(() -> licensee.statuses.size() == 1);
-        assertTrue(dumpLicensingStates(licensee.licenseStates), success);
+        assertTrue(dumpLicensingStates(licensee.statuses), success);
         registerAndAckSignedLicenses(licensesService, generateSignedLicense("basic", TimeValue.timeValueSeconds(3)), LicensesStatus.VALID);
         // signed license enabled
-        success = awaitBusy(() -> licensee.licenseStates.size() == 2);
+        success = awaitBusy(() -> licensee.statuses.size() == 2);
-        assertTrue(dumpLicensingStates(licensee.licenseStates), success);
+        assertTrue(dumpLicensingStates(licensee.statuses), success);
         registerAndAckSignedLicenses(licensesService, generateSignedLicense("gold", TimeValue.timeValueSeconds(2)), LicensesStatus.VALID);
         // second signed license enabled, grace and expired
-        success = awaitBusy(() ->licensee.licenseStates.size() == 5);
+        success = awaitBusy(() ->licensee.statuses.size() == 5);
         assertLicenseStates(licensee, LicenseState.ENABLED, LicenseState.ENABLED, LicenseState.ENABLED, LicenseState.GRACE_PERIOD,
                 LicenseState.DISABLED);
-        assertTrue(dumpLicensingStates(licensee.licenseStates), success);
+        assertTrue(dumpLicensingStates(licensee.statuses), success);
         licensesService.stop();
     }
 
@@ -171,33 +171,41 @@ public class LicensesExpiryNotificationTests extends ESSingleNodeTestCase {
         awaitNoBlock(client());
         licensesService.register(licensee);
         // trial license: enabled, grace, disabled
-        boolean success = awaitBusy(() -> licensee.licenseStates.size() == 3);
+        boolean success = awaitBusy(() -> licensee.statuses.size() == 3);
 
-        assertTrue(dumpLicensingStates(licensee.licenseStates), success);
+        assertTrue(dumpLicensingStates(licensee.statuses), success);
         // install license
         registerAndAckSignedLicenses(licensesService, generateSignedLicense("basic", TimeValue.timeValueSeconds(2)), LicensesStatus.VALID);
         // trial license: enabled, grace, disabled, signed license: enabled, grace, disabled
-        success = awaitBusy(() -> licensee.licenseStates.size() == 6);
+        success = awaitBusy(() -> licensee.statuses.size() == 6);
         assertLicenseStates(licensee, LicenseState.ENABLED, LicenseState.GRACE_PERIOD, LicenseState.DISABLED, LicenseState.ENABLED,
                 LicenseState.GRACE_PERIOD, LicenseState.DISABLED);
-        assertTrue(dumpLicensingStates(licensee.licenseStates), success);
+        assertTrue(dumpLicensingStates(licensee.statuses), success);
         licensesService.stop();
     }
 
     private void assertLicenseStates(AssertingLicensee licensee, LicenseState... states) {
         StringBuilder msg = new StringBuilder();
         msg.append("Actual: ");
-        msg.append(dumpLicensingStates(licensee.licenseStates));
+        msg.append(dumpLicensingStates(licensee.statuses));
         msg.append(" Expected: ");
         msg.append(dumpLicensingStates(states));
-        assertThat(msg.toString(), licensee.licenseStates.size(), equalTo(states.length));
+        assertThat(msg.toString(), licensee.statuses.size(), equalTo(states.length));
         for (int i = 0; i < states.length; i++) {
-            assertThat(msg.toString(), licensee.licenseStates.get(i), equalTo(states[i]));
+            assertThat(msg.toString(), licensee.statuses.get(i).getLicenseState(), equalTo(states[i]));
         }
     }
 
-    private String dumpLicensingStates(List<LicenseState> states) {
+    private String dumpLicensingStates(List<Licensee.Status> statuses) {
-        return dumpLicensingStates(states.toArray(new LicenseState[states.size()]));
+        return dumpLicensingStates(statuses.toArray(new Licensee.Status[statuses.size()]));
+    }
+
+    private String dumpLicensingStates(Licensee.Status... statuses) {
+        LicenseState[] states = new LicenseState[statuses.length];
+        for (int i = 0; i < statuses.length; i++) {
+            states[i] = statuses[i].getLicenseState();
+        }
+        return dumpLicensingStates(states);
     }
 
     private String dumpLicensingStates(LicenseState... states) {

elasticsearch/x-pack/license-plugin/src/test/java/org/elasticsearch/license/plugin/core/LicensesManagerServiceTests.java

@@ -6,9 +6,9 @@
 package org.elasticsearch.license.plugin.core;
 
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.license.core.License;
 import org.elasticsearch.license.plugin.TestUtils;
@@ -20,9 +20,12 @@ import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.elasticsearch.license.plugin.TestUtils.generateSignedLicense;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 
 public class LicensesManagerServiceTests extends ESSingleNodeTestCase {
+
     static {
         MetaData.registerPrototype(LicensesMetaData.TYPE, LicensesMetaData.PROTO);
     }
@@ -111,6 +114,53 @@ public class LicensesManagerServiceTests extends ESSingleNodeTestCase {
         assertThat(licensesMetaData.getLicense(), equalTo(LicensesMetaData.LICENSE_TOMBSTONE));
     }
 
+    public void testRemoveLicensesAndLicenseeNotification() throws Exception {
+        LicensesService licensesService = getInstanceFromNode(LicensesService.class);
+        licensesService.start();
+        ClusterService clusterService = getInstanceFromNode(ClusterService.class);
+
+        // generate a trial license for one feature
+        TestUtils.AssertingLicensee licensee = new TestUtils.AssertingLicensee("", logger);
+        licensesService.register(licensee);
+
+        // we should get a trial license to begin with
+        assertBusy(new Runnable() {
+            @Override
+            public void run() {
+                assertThat(licensee.statuses, hasSize(1));
+                assertThat(licensee.statuses.get(0).getMode(), is(License.OperationMode.TRIAL));
+                assertThat(licensee.statuses.get(0).getLicenseState(), is(LicenseState.ENABLED));
+            }
+        });
+
+
+        // generate signed licenses
+        License license = generateSignedLicense("gold", TimeValue.timeValueHours(1));
+        TestUtils.registerAndAckSignedLicenses(licensesService, license, LicensesStatus.VALID);
+        assertBusy(new Runnable() {
+            @Override
+            public void run() {
+                assertThat(licensee.statuses, hasSize(2));
+                assertThat(licensee.statuses.get(1).getMode(), not(License.OperationMode.TRIAL));
+                assertThat(licensee.statuses.get(1).getLicenseState(), is(LicenseState.ENABLED));
+            }
+        });
+
+        // remove signed licenses
+        removeAndAckSignedLicenses(licensesService);
+        assertBusy(new Runnable() {
+            @Override
+            public void run() {
+                assertThat(licensee.statuses, hasSize(3));
+            }
+        });
+        LicensesMetaData licensesMetaData = clusterService.state().metaData().custom(LicensesMetaData.TYPE);
+        assertThat(licensesMetaData.getLicense(), is(LicensesMetaData.LICENSE_TOMBSTONE));
+        assertThat(licensee.statuses, hasSize(3));
+        assertThat(licensee.statuses.get(2).getLicenseState(), is(LicenseState.DISABLED));
+        assertThat(licensee.statuses.get(2).getMode(), is(License.OperationMode.MISSING));
+    }
+
     private void removeAndAckSignedLicenses(final LicensesService licensesService) {
         final CountDownLatch latch = new CountDownLatch(1);
         final AtomicBoolean success = new AtomicBoolean(false);

elasticsearch/x-pack/license-plugin/src/test/resources/rest-api-spec/test/license/20_put_license.yaml

@@ -4,6 +4,7 @@
     ## current license version
     - do:
       license.post:
+        acknowledge: true
         body: |
           {"license":{"uid":"893361dc-9749-4997-93cb-802e3d7fa4a8","type":"basic","issue_date_in_millis":1411948800000,"expiry_date_in_millis":1914278399999,"max_nodes":1,"issued_to":"issuedTo","issuer":"issuer","signature":"AAAAAgAAAA0lKPZ0a7aZquUltho/AAABmC9ZN0hjZDBGYnVyRXpCOW5Bb3FjZDAxOWpSbTVoMVZwUzRxVk1PSmkxakxZdW5IMlhlTHNoN1N2MXMvRFk4d3JTZEx3R3RRZ0pzU3lobWJKZnQvSEFva0ppTHBkWkprZWZSQi9iNmRQNkw1SlpLN0lDalZCS095MXRGN1lIZlpYcVVTTnFrcTE2dzhJZmZrdFQrN3JQeGwxb0U0MXZ0dDJHSERiZTVLOHNzSDByWnpoZEphZHBEZjUrTVBxRENNSXNsWWJjZllaODdzVmEzUjNiWktNWGM5TUhQV2plaUo4Q1JOUml4MXNuL0pSOEhQaVB2azhmUk9QVzhFeTFoM1Q0RnJXSG53MWk2K055c28zSmRnVkF1b2JSQkFLV2VXUmVHNDZ2R3o2VE1qbVNQS2lxOHN5bUErZlNIWkZSVmZIWEtaSU9wTTJENDVvT1NCYklacUYyK2FwRW9xa0t6dldMbmMzSGtQc3FWOTgzZ3ZUcXMvQkt2RUZwMFJnZzlvL2d2bDRWUzh6UG5pdENGWFRreXNKNkE9PQAAAQAALuQ44S3IG6SzolcXVJ6Z4CIXORDrYQ+wdLCeey0XdujTslAOj+k+vNgo6wauc7Uswi01esHu4lb5IgpvKy7RRCbh5bj/z2ubu2qMJqopp9BQyD7VQjVfqmG6seUMJwJ1a5Avvm9r41YPSPcrii3bKK2e1l6jK6N8ibCvnTyY/XkYGCJrBWTSJePDbg6ErbyodrZ37x1StLbPWcNAkmweyHjDJnvYnbeZZO7A3NmubXZjW7Ttf8/YwQyE00PqMcl7fVPY3hkKpAeHf8aaJbqkKYbqZuER3EWJX7ZvLVb1dNdNg8aXRn7YrkQcYwWgptYQpfV+D7yEJ4j5muAEoler"}}
 
@@ -18,6 +19,7 @@
     ## bwc for licenses format
     - do:
       license.post:
+        acknowledge: true
         body: |
           {"licenses":[{"uid":"893361dc-9749-4997-93cb-802e3d7fa4a8","type":"basic","issue_date_in_millis":1411948800000,"expiry_date_in_millis":1914278399999,"max_nodes":1,"issued_to":"issuedTo","issuer":"issuer","signature":"AAAAAgAAAA0lKPZ0a7aZquUltho/AAABmC9ZN0hjZDBGYnVyRXpCOW5Bb3FjZDAxOWpSbTVoMVZwUzRxVk1PSmkxakxZdW5IMlhlTHNoN1N2MXMvRFk4d3JTZEx3R3RRZ0pzU3lobWJKZnQvSEFva0ppTHBkWkprZWZSQi9iNmRQNkw1SlpLN0lDalZCS095MXRGN1lIZlpYcVVTTnFrcTE2dzhJZmZrdFQrN3JQeGwxb0U0MXZ0dDJHSERiZTVLOHNzSDByWnpoZEphZHBEZjUrTVBxRENNSXNsWWJjZllaODdzVmEzUjNiWktNWGM5TUhQV2plaUo4Q1JOUml4MXNuL0pSOEhQaVB2azhmUk9QVzhFeTFoM1Q0RnJXSG53MWk2K055c28zSmRnVkF1b2JSQkFLV2VXUmVHNDZ2R3o2VE1qbVNQS2lxOHN5bUErZlNIWkZSVmZIWEtaSU9wTTJENDVvT1NCYklacUYyK2FwRW9xa0t6dldMbmMzSGtQc3FWOTgzZ3ZUcXMvQkt2RUZwMFJnZzlvL2d2bDRWUzh6UG5pdENGWFRreXNKNkE9PQAAAQAALuQ44S3IG6SzolcXVJ6Z4CIXORDrYQ+wdLCeey0XdujTslAOj+k+vNgo6wauc7Uswi01esHu4lb5IgpvKy7RRCbh5bj/z2ubu2qMJqopp9BQyD7VQjVfqmG6seUMJwJ1a5Avvm9r41YPSPcrii3bKK2e1l6jK6N8ibCvnTyY/XkYGCJrBWTSJePDbg6ErbyodrZ37x1StLbPWcNAkmweyHjDJnvYnbeZZO7A3NmubXZjW7Ttf8/YwQyE00PqMcl7fVPY3hkKpAeHf8aaJbqkKYbqZuER3EWJX7ZvLVb1dNdNg8aXRn7YrkQcYwWgptYQpfV+D7yEJ4j5muAEoler"}]}
 
@@ -31,6 +33,7 @@
     ## license version: 1.x
     - do:
       license.post:
+        acknowledge: true
         body: |
           {"licenses":[{"uid":"893361dc-9749-4997-93cb-802e3d7fa4a8","type":"subscription","subscription_type":"gold","issue_date_in_millis":1411948800000,"feature":"shield","expiry_date_in_millis":1914278399999,"max_nodes":1,"issued_to":"issuedTo","issuer":"issuer","signature":"AAAAAQAAAA0LVAywwpSH94cyXr4zAAABmC9ZN0hjZDBGYnVyRXpCOW5Bb3FjZDAxOWpSbTVoMVZwUzRxVk1PSmkxakxZdW5IMlhlTHNoN1N2MXMvRFk4d3JTZEx3R3RRZ0pzU3lobWJKZnQvSEFva0ppTHBkWkprZWZSQi9iNmRQNkw1SlpLN0lDalZCS095MXRGN1lIZlpYcVVTTnFrcTE2dzhJZmZrdFQrN3JQeGwxb0U0MXZ0dDJHSERiZTVLOHNzSDByWnpoZEphZHBEZjUrTVBxRENNSXNsWWJjZllaODdzVmEzUjNiWktNWGM5TUhQV2plaUo4Q1JOUml4MXNuL0pSOEhQaVB2azhmUk9QVzhFeTFoM1Q0RnJXSG53MWk2K055c28zSmRnVkF1b2JSQkFLV2VXUmVHNDZ2R3o2VE1qbVNQS2lxOHN5bUErZlNIWkZSVmZIWEtaSU9wTTJENDVvT1NCYklacUYyK2FwRW9xa0t6dldMbmMzSGtQc3FWOTgzZ3ZUcXMvQkt2RUZwMFJnZzlvL2d2bDRWUzh6UG5pdENGWFRreXNKNkE9PQAAAQA4qscc/URRZVdFoLwgy9dqybYEQLW8YLkiAyPV5XHHHdtk+dtZIepiNEDkUXhSX2waVJlsNRF8/4kqplDfwNoD2TUM8fTgiIfiSiZYGDTGST+yW/5eAveEU5J5v1liBN27bwkqL+V4YAa0Tcm7NKKwjScWKAHiTU3vF8chPkGfCHE0kQgVwPC9RE82pTw0s6/uR4PfLGNFfqPM0uiE5nucfVrtj89JQiO/KA/7ZyFbo7VTNXxZQt7T7rZWBCP9KIjptXzcWuk08Q5S+rSoJNYbFo3HGKtrCVsRz/55rceNtdwKKXu1IwnSeir4I1/KLduQTtFLy0+1th87VS8T88UT"}]}
 
@@ -44,6 +47,7 @@
     ## multiple licenses version: 1.x
     - do:
       license.post:
+        acknowledge: true
         body: |
           {"licenses":[{"uid":"893361dc-9749-4997-93cb-802e3d7fa4a8","type":"internal","subscription_type":"none","issue_date_in_millis":1411948800000,"feature":"shield","expiry_date_in_millis":1440892799999,"max_nodes":1,"issued_to":"issuedTo","issuer":"issuer","signature":"AAAAAQAAAA04Q4ky3rFyyWLFkytEAAABmC9ZN0hjZDBGYnVyRXpCOW5Bb3FjZDAxOWpSbTVoMVZwUzRxVk1PSmkxakxZdW5IMlhlTHNoN1N2MXMvRFk4d3JTZEx3R3RRZ0pzU3lobWJKZnQvSEFva0ppTHBkWkprZWZSQi9iNmRQNkw1SlpLN0lDalZCS095MXRGN1lIZlpYcVVTTnFrcTE2dzhJZmZrdFQrN3JQeGwxb0U0MXZ0dDJHSERiZTVLOHNzSDByWnpoZEphZHBEZjUrTVBxRENNSXNsWWJjZllaODdzVmEzUjNiWktNWGM5TUhQV2plaUo4Q1JOUml4MXNuL0pSOEhQaVB2azhmUk9QVzhFeTFoM1Q0RnJXSG53MWk2K055c28zSmRnVkF1b2JSQkFLV2VXUmVHNDZ2R3o2VE1qbVNQS2lxOHN5bUErZlNIWkZSVmZIWEtaSU9wTTJENDVvT1NCYklacUYyK2FwRW9xa0t6dldMbmMzSGtQc3FWOTgzZ3ZUcXMvQkt2RUZwMFJnZzlvL2d2bDRWUzh6UG5pdENGWFRreXNKNkE9PQAAAQBxMvUMn4h2E4R4TQMijahTxQj4LPQO4f1M79UxX/XkDlGcH+J5pRHx08OtTRPsFL1lED+h+PIXx307Vo+PNDsOxrWvoYZeYBkOLAO3ny9vhQga+52jYhMxIuFrT9xbcSCSNpMhGojgOIPU2WgiopVdVcimo1+Gk8VtklPB1wPwFzfOjOnPgp/Icx3WYpfkeAUUOyWUYiFIBAe4bnz84iF+xwLKbgYk6aHF25ECBtdb/Uruhcm9+jEFpoIEUtCouvvk9C+NJZ4OickV4xpRgaRG2x9PONH8ZN0QGhGYhJGbisoCxuDmlLsyVxqxfMu3n/r7/jdsEJScjAlSrsLDOu6H"},{"uid":"893361dc-9749-4997-93cb-802e3dofh7aa","type":"internal","subscription_type":"none","issue_date_in_millis":1443484800000,"feature":"watcher","expiry_date_in_millis":1914278399999,"max_nodes":1,"issued_to":"issuedTo","issuer":"issuer","signature":"AAAAAQAAAA0Sc90guRIaQEmgLvMnAAABmC9ZN0hjZDBGYnVyRXpCOW5Bb3FjZDAxOWpSbTVoMVZwUzRxVk1PSmkxakxZdW5IMlhlTHNoN1N2MXMvRFk4d3JTZEx3R3RRZ0pzU3lobWJKZnQvSEFva0ppTHBkWkprZWZSQi9iNmRQNkw1SlpLN0lDalZCS095MXRGN1lIZlpYcVVTTnFrcTE2dzhJZmZrdFQrN3JQeGwxb0U0MXZ0dDJHSERiZTVLOHNzSDByWnpoZEphZHBEZjUrTVBxRENNSXNsWWJjZllaODdzVmEzUjNiWktNWGM5TUhQV2plaUo4Q1JOUml4MXNuL0pSOEhQaVB2azhmUk9QVzhFeTFoM1Q0RnJXSG53MWk2K055c28zSmRnVkF1b2JSQkFLV2VXUmVHNDZ2R3o2VE1qbVNQS2lxOHN5bUErZlNIWkZSVmZIWEtaSU9wTTJENDVvT1NCYklacUYyK2FwRW9xa0t6dldMbmMzSGtQc3FWOTgzZ3ZUcXMvQkt2RUZwMFJnZzlvL2d2bDRWUzh6UG5pdENGWFRreXNKNkE9PQAAAQCQ94dju0pnDZR3Uuypi0ic3aQJ+nvVqe+U8u79Dga5n1qIjcHDh7HvIBJEkF+tnVPlo/PXV/x7BZSwVY1PVErit+6rYix1yuHEgqwxmx/VdRICjCaZM6tk0Ob4dZCPv6Ebn2Mmk89KHC/PwiLPqF6QfwV/Pkpa8k2A3ORJmvYSDvXhe6tCs8dqc4ebrsFxqrZjwWh5CZSpzqqZBFXlngDv2N0hHhpGlueRszD0JJ5dfEL5ZA1DDOrgO9OJVejSHyRqe1L5QRUNdXPVfS+EAG0Dd1cNdJ/sMpYCPnVjbw6iq2/YgM3cuztsXVBY7ij4WnoP3ce7Zjs9TwHn+IqzftC6"}]}
 

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/Monitoring.java

@@ -46,7 +46,7 @@ public class Monitoring {
 
     public Monitoring(Settings settings) {
         this.settings = settings;
-        this.enabled = MonitoringSettings.ENABLED.get(settings);
+        this.enabled = enabled(settings);
         this.transportClientMode = XPackPlugin.transportClientMode(settings);
         this.tribeNode = XPackPlugin.isTribeNode(settings);
     }
@@ -62,9 +62,9 @@ public class Monitoring {
     public Collection<Module> nodeModules() {
         List<Module> modules = new ArrayList<>();
         modules.add(new MonitoringModule(enabled, transportClientMode));
+        modules.add(new ExporterModule(settings));
         if (enabled && transportClientMode == false && tribeNode == false) {
             modules.add(new CollectorModule());
-            modules.add(new ExporterModule(settings));
             modules.add(new MonitoringClientModule());
         }
         return modules;
@@ -100,4 +100,8 @@ public class Monitoring {
             module.registerLazyInitializable(MonitoringClientProxy.class);
         }
     }
+
+    public static boolean enabled(Settings settings) {
+        return MonitoringSettings.ENABLED.get(settings);
+    }
 }

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MonitoringFeatureSet.java

@@ -7,9 +7,19 @@ package org.elasticsearch.marvel;
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.marvel.agent.exporter.Exporter;
+import org.elasticsearch.marvel.agent.exporter.Exporters;
 import org.elasticsearch.xpack.XPackFeatureSet;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
 /**
  *
  */
@@ -17,11 +27,15 @@ public class MonitoringFeatureSet implements XPackFeatureSet {
 
     private final boolean enabled;
     private final MonitoringLicensee licensee;
+    private final Exporters exporters;
 
     @Inject
-    public MonitoringFeatureSet(Settings settings, @Nullable MonitoringLicensee licensee) {
+    public MonitoringFeatureSet(Settings settings, @Nullable MonitoringLicensee licensee, @Nullable Exporters exporters,
+                                NamedWriteableRegistry namedWriteableRegistry) {
         this.enabled = MonitoringSettings.ENABLED.get(settings);
         this.licensee = licensee;
+        this.exporters = exporters;
+        namedWriteableRegistry.register(Usage.class, Usage.writeableName(Monitoring.NAME), Usage::new);
     }
 
     @Override
@@ -36,11 +50,63 @@ public class MonitoringFeatureSet implements XPackFeatureSet {
 
     @Override
     public boolean available() {
-        return licensee != null && licensee.available();
+        return licensee != null && licensee.isAvailable();
     }
 
     @Override
     public boolean enabled() {
         return enabled;
     }
+
+    @Override
+    public XPackFeatureSet.Usage usage() {
+        return new Usage(available(), enabled(), exportersUsage(exporters));
+    }
+
+    static Map<String, Object> exportersUsage(Exporters exporters) {
+        if (exporters == null) {
+            return null;
+        }
+        Map<String, Object> usage = new HashMap<>();
+        for (Exporter exporter : exporters) {
+            if (exporter.config().enabled()) {
+                String type = exporter.type();
+                int count = (Integer) usage.getOrDefault(type, 0);
+                usage.put(type, count + 1);
+            }
+        }
+        return usage;
+    }
+
+    static class Usage extends XPackFeatureSet.Usage {
+
+        private static final String ENABLED_EXPORTERS_XFIELD = "enabled_exporters";
+
+        private @Nullable Map<String, Object> exporters;
+
+        public Usage(StreamInput in) throws IOException {
+            super(in);
+            exporters = in.readMap();
+        }
+
+        public Usage(boolean available, boolean enabled, Map<String, Object> exporters) {
+            super(Monitoring.NAME, available, enabled);
+            this.exporters = exporters;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeMap(exporters);
+        }
+
+        @Override
+        protected void innerXContent(XContentBuilder builder, Params params) throws IOException {
+            super.innerXContent(builder, params);
+            if (exporters != null) {
+                builder.field(ENABLED_EXPORTERS_XFIELD, exporters);
+            }
+        }
+    }
+
 }

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/MonitoringLicensee.java

@@ -75,12 +75,12 @@ public class MonitoringLicensee extends AbstractLicenseeComponent<MonitoringLice
     }
 
     /**
-     * Monitoring is always available regardless of the license type (operation mode)
+     * Monitoring is always available as long as there is a valid license
      *
      * @return true
      */
-    public boolean available() {
+    public boolean isAvailable() {
-        return true;
+        return status.getLicenseState() != LicenseState.DISABLED;
     }
 
     /**
@@ -118,6 +118,7 @@ public class MonitoringLicensee extends AbstractLicenseeComponent<MonitoringLice
      * @return {@code true} if the user is allowed to modify the retention. Otherwise {@code false}.
      */
     public boolean allowUpdateRetention() {
-        return status.getMode() != OperationMode.BASIC;
+        final OperationMode mode = status.getMode();
+        return mode != OperationMode.BASIC && mode != OperationMode.MISSING;
     }
 }

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/AgentService.java

@@ -120,7 +120,8 @@ public class AgentService extends AbstractLifecycleComponent<AgentService> {
 
     @Override
     protected void doStart() {
-        logger.info("monitoring service started");
+        // Please don't remove this log message since it can be used in integration tests
+        logger.debug("monitoring service started");
 
         for (Collector collector : collectors) {
             collector.start();

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/Exporter.java

@@ -41,6 +41,10 @@ public abstract class Exporter implements AutoCloseable {
         return config.name;
     }
 
+    public Config config() {
+        return config;
+    }
+
     public boolean masterOnly() {
         return false;
     }

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/ExporterModule.java

@@ -7,18 +7,20 @@ package org.elasticsearch.marvel.agent.exporter;
 
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.multibindings.MapBinder;
+import org.elasticsearch.common.inject.util.Providers;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.marvel.Monitoring;
 import org.elasticsearch.marvel.agent.exporter.http.HttpExporter;
 import org.elasticsearch.marvel.agent.exporter.local.LocalExporter;
+import org.elasticsearch.xpack.XPackPlugin;
 
 import java.util.HashMap;
 import java.util.Map;
 
 public class ExporterModule extends AbstractModule {
 
-    private final Map<String, Class<? extends Exporter.Factory<? extends Exporter>>> exporterFactories = new HashMap<>();
-
     private final Settings settings;
+    private final Map<String, Class<? extends Exporter.Factory<? extends Exporter>>> exporterFactories = new HashMap<>();
 
     public ExporterModule(Settings settings) {
         this.settings = settings;
@@ -28,13 +30,17 @@ public class ExporterModule extends AbstractModule {
 
     @Override
     protected void configure() {
-        bind(Exporters.class).asEagerSingleton();
+        if (Monitoring.enabled(settings) && XPackPlugin.transportClientMode(settings) == false
-        MapBinder<String, Exporter.Factory> factoryBinder = MapBinder.newMapBinder(binder(), String.class, Exporter.Factory.class);
+                && XPackPlugin.isTribeNode(settings) == false) {
-        for (Map.Entry<String, Class<? extends Exporter.Factory<? extends Exporter>>> entry : exporterFactories.entrySet()) {
+            bind(Exporters.class).asEagerSingleton();
-            bind(entry.getValue()).asEagerSingleton();
+            MapBinder<String, Exporter.Factory> factoryBinder = MapBinder.newMapBinder(binder(), String.class, Exporter.Factory.class);
-            factoryBinder.addBinding(entry.getKey()).to(entry.getValue());
+            for (Map.Entry<String, Class<? extends Exporter.Factory<? extends Exporter>>> entry : exporterFactories.entrySet()) {
+                bind(entry.getValue()).asEagerSingleton();
+                factoryBinder.addBinding(entry.getKey()).to(entry.getValue());
+            }
+        } else {
+            bind(Exporters.class).toProvider(Providers.of(null));
         }
-
     }
 
     public void registerExporter(String type, Class<? extends Exporter.Factory<? extends Exporter>> factory) {

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/MonitoringFeatureSetTests.java

@@ -0,0 +1,136 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.marvel;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.marvel.agent.exporter.Exporter;
+import org.elasticsearch.marvel.agent.exporter.Exporters;
+import org.elasticsearch.marvel.agent.exporter.http.HttpExporter;
+import org.elasticsearch.marvel.agent.exporter.local.LocalExporter;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.XPackFeatureSet;
+import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.core.Is.is;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ *
+ */
+public class MonitoringFeatureSetTests extends ESTestCase {
+
+    private MonitoringLicensee licensee;
+    private NamedWriteableRegistry namedWriteableRegistry;
+    private Exporters exporters;
+
+    @Before
+    public void init() throws Exception {
+        licensee = mock(MonitoringLicensee.class);
+        exporters = mock(Exporters.class);
+        namedWriteableRegistry = mock(NamedWriteableRegistry.class);
+    }
+
+    public void testWritableRegistration() throws Exception {
+        new MonitoringFeatureSet(Settings.EMPTY, licensee, exporters, namedWriteableRegistry);
+        verify(namedWriteableRegistry).register(eq(MonitoringFeatureSet.Usage.class), eq("xpack.usage.monitoring"), anyObject());
+    }
+
+    public void testAvailable() throws Exception {
+        MonitoringFeatureSet featureSet = new MonitoringFeatureSet(Settings.EMPTY, licensee, exporters, namedWriteableRegistry);
+        boolean available = randomBoolean();
+        when(licensee.isAvailable()).thenReturn(available);
+        assertThat(featureSet.available(), is(available));
+    }
+
+    public void testEnabledSetting() throws Exception {
+        boolean enabled = randomBoolean();
+        Settings.Builder settings = Settings.builder();
+        settings.put("xpack.monitoring.enabled", enabled);
+        MonitoringFeatureSet featureSet = new MonitoringFeatureSet(settings.build(), licensee, exporters, namedWriteableRegistry);
+        assertThat(featureSet.enabled(), is(enabled));
+    }
+
+    public void testEnabledDefault() throws Exception {
+        MonitoringFeatureSet featureSet = new MonitoringFeatureSet(Settings.EMPTY, licensee, exporters, namedWriteableRegistry);
+        assertThat(featureSet.enabled(), is(true));
+    }
+
+    public void testUsage() throws Exception {
+
+        List<Exporter> exporterList = new ArrayList<>();
+        int localCount = randomIntBetween(0, 5);
+        for (int i = 0; i < localCount; i++) {
+            Exporter exporter = mockExporter(LocalExporter.TYPE, true);
+            exporterList.add(exporter);
+            if (randomBoolean()) {
+                exporter = mockExporter(LocalExporter.TYPE, false);
+                exporterList.add(exporter);
+            }
+        }
+        int httpCount = randomIntBetween(0, 5);
+        for (int i = 0; i < httpCount; i++) {
+            Exporter exporter = mockExporter(HttpExporter.TYPE, true);
+            exporterList.add(exporter);
+            if (randomBoolean()) {
+                exporter = mockExporter(HttpExporter.TYPE, false);
+                exporterList.add(exporter);
+            }
+        }
+        int xCount = randomIntBetween(0, 5);
+        String xType = randomAsciiOfLength(10);
+        for (int i = 0; i < xCount; i++) {
+            Exporter exporter = mockExporter(xType, true);
+            exporterList.add(exporter);
+            if (randomBoolean()) {
+                exporter = mockExporter(xType, false);
+                exporterList.add(exporter);
+            }
+        }
+        when(exporters.iterator()).thenReturn(exporterList.iterator());
+
+        MonitoringFeatureSet featureSet = new MonitoringFeatureSet(Settings.EMPTY, licensee, exporters, namedWriteableRegistry);
+        XPackFeatureSet.Usage usage = featureSet.usage();
+        assertThat(usage.name(), is(featureSet.name()));
+        assertThat(usage.enabled(), is(featureSet.enabled()));
+        XContentSource source = new XContentSource(usage);
+        assertThat(source.getValue("enabled_exporters"), is(notNullValue()));
+        if (localCount > 0) {
+            assertThat(source.getValue("enabled_exporters.local"), is(localCount));
+        } else {
+            assertThat(source.getValue("enabled_exporters.local"), is(nullValue()));
+        }
+        if (httpCount > 0) {
+            assertThat(source.getValue("enabled_exporters.http"), is(httpCount));
+        } else {
+            assertThat(source.getValue("enabled_exporters.http"), is(nullValue()));
+        }
+        if (xCount > 0) {
+            assertThat(source.getValue("enabled_exporters." + xType), is(xCount));
+        } else {
+            assertThat(source.getValue("enabled_exporters." + xType), is(nullValue()));
+        }
+    }
+
+    private Exporter mockExporter(String type, boolean enabled) {
+        Exporter exporter = mock(Exporter.class);
+        when(exporter.type()).thenReturn(type);
+        Exporter.Config enabledConfig = mock(Exporter.Config.class);
+        when(enabledConfig.enabled()).thenReturn(enabled);
+        when(exporter.config()).thenReturn(enabledConfig);
+        return exporter;
+    }
+}

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/license/MarvelLicenseeTests.java

@@ -39,7 +39,7 @@ public class MarvelLicenseeTests extends AbstractLicenseeTestCase {
     }
 
     public void testAcknowledgementMessagesToBasicFromNotBasicNotesLimits() {
-        OperationMode from = randomModeExcept(OperationMode.BASIC);
+        OperationMode from = randomFrom(OperationMode.STANDARD, OperationMode.GOLD, OperationMode.PLATINUM, OperationMode.TRIAL);
         OperationMode to = OperationMode.BASIC;
 
         String[] messages = ackLicenseChange(from, to, licensee);
@@ -65,13 +65,18 @@ public class MarvelLicenseeTests extends AbstractLicenseeTestCase {
     }
 
     public void testAllowUpdateRetentionIsTrueForNotBasic() {
-        assertEnabled(randomModeExcept(OperationMode.BASIC), MonitoringLicensee::allowUpdateRetention, true);
+        OperationMode mode = randomFrom(OperationMode.STANDARD, OperationMode.GOLD, OperationMode.PLATINUM, OperationMode.TRIAL);
+        assertEnabled(mode, MonitoringLicensee::allowUpdateRetention, true);
     }
 
     public void testAllowUpdateRetentionIsFalseForBasic() {
         assertEnabled(OperationMode.BASIC, MonitoringLicensee::allowUpdateRetention, false);
     }
 
+    public void testAllowUpdateRetentionIsFalseForMissing() {
+        assertEnabled(OperationMode.MISSING, MonitoringLicensee::allowUpdateRetention, false);
+    }
+
     /**
      * Assert that the {@link #licensee} is {@code predicate}d as {@code expected} when setting the {@code state}.
      *

elasticsearch/x-pack/shield/bin/x-pack/syskeygen

@@ -70,38 +70,6 @@ elif [ -f "/etc/default/elasticsearch" ]; then
    . "/etc/default/elasticsearch"
 fi
 
-# Parse any long getopt options and put them into properties before calling getopt below
-# Be dash compatible to make sure running under ubuntu works
-ARGCOUNT=$#
-COUNT=0
-while [ $COUNT -lt $ARGCOUNT ]
-do
-    case $1 in
-      --*=*) properties="$properties -Des.${1#--}"
-           shift 1; COUNT=$(($COUNT+1))
-           ;;
-      --*) properties="$properties -Des.${1#--}=$2"
-           shift ; shift; COUNT=$(($COUNT+2))
-           ;;
-      *) set -- "$@" "$1"; shift; COUNT=$(($COUNT+1))
-    esac
-done
-
-# check if properties already has a config file or config dir
-if [ -e "$CONF_DIR" ]; then
-    case "$properties" in
-        *-Des.default.path.conf=*) ;;
-        *)
-            if [ ! -d "$CONF_DIR/x-pack" ]; then
-                echo "ERROR: The configuration directory [$CONF_DIR/x-pack] does not exist. The syskeygen tool expects security configuration files in that location."
-                echo "The plugin may not have been installed with the correct configuration path. If [$ES_HOME/config/x-pack] exists, please copy the 'x-pack' directory to [$CONF_DIR]"
-                exit 1
-            fi
-            properties="$properties -Des.default.path.conf=$CONF_DIR"
-            ;;
-        esac
-fi
-
 export HOSTNAME=`hostname -s`
 
 # include shield jars in classpath
@@ -121,8 +89,14 @@ if [ ! -z "$CONF_FILE" ]; then
     exit 1
 fi
 
+declare -a args=("$@")
+
+if [ -e "$CONF_DIR" ]; then
+  args=("${args[@]}" -Edefault.path.conf="$CONF_DIR")
+fi
+
 cd "$ES_HOME" > /dev/null
-"$JAVA" $ES_JAVA_OPTS -cp "$ES_CLASSPATH" -Des.path.home="$ES_HOME" $properties org.elasticsearch.shield.crypto.tool.SystemKeyTool "$@"
+"$JAVA" $ES_JAVA_OPTS -Des.path.home="$ES_HOME" -cp "$ES_CLASSPATH" org.elasticsearch.shield.crypto.tool.SystemKeyTool $properties "{args[@]}"
 status=$?
 cd - > /dev/null
 exit $status

elasticsearch/x-pack/shield/bin/x-pack/users

@@ -70,38 +70,6 @@ elif [ -f "/etc/default/elasticsearch" ]; then
    . "/etc/default/elasticsearch"
 fi
 
-# Parse any long getopt options and put them into properties before calling getopt below
-# Be dash compatible to make sure running under ubuntu works
-ARGCOUNT=$#
-COUNT=0
-while [ $COUNT -lt $ARGCOUNT ]
-do
-    case $1 in
-      --*=*) properties="$properties -Des.${1#--}"
-           shift 1; COUNT=$(($COUNT+1))
-           ;;
-      --*) properties="$properties -Des.${1#--}=$2"
-           shift ; shift; COUNT=$(($COUNT+2))
-           ;;
-      *) set -- "$@" "$1"; shift; COUNT=$(($COUNT+1))
-    esac
-done
-
-# check if properties already has a config file or config dir
-if [ -e "$CONF_DIR" ]; then
-    case "$properties" in
-        *-Des.default.path.conf=*) ;;
-        *)
-            if [ ! -d "$CONF_DIR/x-pack" ]; then
-                echo "ERROR: The configuration directory [$CONF_DIR/x-pack] does not exist. The users tool expects security configuration files in that location."
-                echo "The plugin may not have been installed with the correct configuration path. If [$ES_HOME/config/x-pack] exists, please copy the 'x-pack' directory to [$CONF_DIR]"
-                exit 1
-            fi
-            properties="$properties -Des.default.path.conf=$CONF_DIR"
-            ;;
-        esac
-fi
-
 export HOSTNAME=`hostname -s`
 
 # include shield jars in classpath
@@ -121,8 +89,14 @@ if [ ! -z "$CONF_FILE" ]; then
     exit 1
 fi
 
+declare -a args=("$@")
+
+if [ -e "$CONF_DIR" ]; then
+  args=("${args[@]}" -Edefault.path.conf="$CONF_DIR")
+fi
+
 cd "$ES_HOME" > /dev/null
-"$JAVA" $ES_JAVA_OPTS -cp "$ES_CLASSPATH" -Des.path.home="$ES_HOME" $properties org.elasticsearch.shield.authc.file.tool.UsersTool "$@"
+"$JAVA" $ES_JAVA_OPTS -cp "$ES_CLASSPATH" -Des.path.home="$ES_HOME" org.elasticsearch.shield.authc.file.tool.UsersTool "${args[@]}"
 status=$?
 cd - > /dev/null
 exit $status

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/Security.java

@@ -73,8 +73,8 @@ import org.elasticsearch.shield.rest.action.user.RestChangePasswordAction;
 import org.elasticsearch.shield.rest.action.user.RestDeleteUserAction;
 import org.elasticsearch.shield.rest.action.user.RestGetUsersAction;
 import org.elasticsearch.shield.rest.action.user.RestPutUserAction;
-import org.elasticsearch.shield.ssl.SSLModule;
 import org.elasticsearch.shield.ssl.SSLConfiguration;
+import org.elasticsearch.shield.ssl.SSLModule;
 import org.elasticsearch.shield.support.OptionalSettings;
 import org.elasticsearch.shield.transport.ShieldClientTransportService;
 import org.elasticsearch.shield.transport.ShieldServerTransportService;
@@ -125,27 +125,28 @@ public class Security {
     public Collection<Module> nodeModules() {
         List<Module> modules = new ArrayList<>();
 
-        // we can't load that at construction time since the license plugin might not have been loaded at that point
+        if (transportClientMode) {
-        // which might not be the case during Plugin class instantiation. Once nodeModules are pulled
+            if (enabled == false) {
-        // everything should have been loaded
+                return modules;
-        if (enabled && transportClientMode == false) {
+            }
-            securityLicenseState = new SecurityLicenseState();
+            modules.add(new SecurityModule(settings, securityLicenseState));
-        }
-
-        modules.add(new SecurityModule(settings, securityLicenseState));
-
-        if (enabled == false) {
-            return modules;
-        }
-
-        if (transportClientMode == true) {
             modules.add(new ShieldTransportModule(settings));
             modules.add(new SSLModule(settings));
             return modules;
         }
 
-        modules.add(new CryptoModule(settings));
         modules.add(new AuthenticationModule(settings));
+        if (enabled == false) {
+            modules.add(new SecurityModule(settings, securityLicenseState));
+            return modules;
+        }
+
+        // we can't load that at construction time since the license plugin might not have been loaded at that point
+        // which might not be the case during Plugin class instantiation. Once nodeModules are pulled
+        // everything should have been loaded
+        securityLicenseState = new SecurityLicenseState();
+        modules.add(new SecurityModule(settings, securityLicenseState));
+        modules.add(new CryptoModule(settings));
         modules.add(new AuthorizationModule(settings));
         modules.add(new AuditTrailModule(settings));
         modules.add(new ShieldRestModule(settings));

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/SecurityFeatureSet.java

@@ -7,9 +7,24 @@ package org.elasticsearch.shield;
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.shield.authc.Realm;
+import org.elasticsearch.shield.authc.Realms;
+import org.elasticsearch.shield.authc.esnative.ReservedRealm;
 import org.elasticsearch.xpack.XPackFeatureSet;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
 /**
  *
  */
@@ -17,11 +32,15 @@ public class SecurityFeatureSet implements XPackFeatureSet {
 
     private final boolean enabled;
     private final SecurityLicenseState licenseState;
+    private final @Nullable Realms realms;
 
     @Inject
-    public SecurityFeatureSet(Settings settings, @Nullable SecurityLicenseState licenseState) {
+    public SecurityFeatureSet(Settings settings, @Nullable SecurityLicenseState licenseState,
+                              @Nullable Realms realms, NamedWriteableRegistry namedWriteableRegistry) {
         this.enabled = Security.enabled(settings);
         this.licenseState = licenseState;
+        this.realms = realms;
+        namedWriteableRegistry.register(Usage.class, Usage.writeableName(Security.NAME), Usage::new);
     }
 
     @Override
@@ -43,4 +62,56 @@ public class SecurityFeatureSet implements XPackFeatureSet {
     public boolean enabled() {
         return enabled;
     }
+
+    @Override
+    public XPackFeatureSet.Usage usage() {
+        List<Map<String, Object>> enabledRealms = buildEnabledRealms(realms);
+        return new Usage(available(), enabled(), enabledRealms);
+    }
+
+    static List<Map<String, Object>> buildEnabledRealms(Realms realms) {
+        if (realms == null) {
+            return Collections.emptyList();
+        }
+        List<Map<String, Object>> enabledRealms = new ArrayList<>();
+        for (Realm realm : realms) {
+            if (realm instanceof ReservedRealm) {
+                continue; // we don't need usage of this one
+            }
+            Map<String, Object> stats = realm.usageStats();
+            enabledRealms.add(stats);
+        }
+        return enabledRealms;
+    }
+
+    static class Usage extends XPackFeatureSet.Usage {
+
+        private static final String ENABLED_REALMS_XFIELD = "enabled_realms";
+
+        private List<Map<String, Object>> enabledRealms;
+
+        public Usage(StreamInput in) throws IOException {
+            super(in);
+            enabledRealms = in.readList(StreamInput::readMap);
+        }
+
+        public Usage(boolean available, boolean enabled, List<Map<String, Object>> enabledRealms) {
+            super(Security.NAME, available, enabled);
+            this.enabledRealms = enabledRealms;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeList(enabledRealms.stream().map((m) -> (Writeable) o -> o.writeMap(m)).collect(Collectors.toList()));
+        }
+
+        @Override
+        protected void innerXContent(XContentBuilder builder, Params params) throws IOException {
+            super.innerXContent(builder, params);
+            if (enabled) {
+                builder.field(ENABLED_REALMS_XFIELD, enabledRealms);
+            }
+        }
+    }
 }

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/AuthenticationModule.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.shield.authc;
 
 import org.elasticsearch.common.inject.multibindings.MapBinder;
+import org.elasticsearch.common.inject.util.Providers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.shield.authc.activedirectory.ActiveDirectoryRealm;
 import org.elasticsearch.shield.authc.esnative.NativeRealm;
@@ -42,6 +43,11 @@ public class AuthenticationModule extends AbstractShieldModule.Node {
 
     @Override
     protected void configureNode() {
+        if (!shieldEnabled) {
+            bind(Realms.class).toProvider(Providers.of(null));
+            return;
+        }
+
         AnonymousUser.initialize(settings);
         MapBinder<String, Realm.Factory> mapBinder = MapBinder.newMapBinder(binder(), String.class, Realm.Factory.class);
         mapBinder.addBinding(FileRealm.TYPE).to(FileRealm.Factory.class).asEagerSingleton();

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/Realm.java

@@ -9,6 +9,9 @@ import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.shield.user.User;
 
+import java.util.HashMap;
+import java.util.Map;
+
 /**
  * An authentication mechanism to which the default authentication {@link org.elasticsearch.shield.authc.AuthenticationService service}
  * delegates the authentication process. Different realms may be defined, each may be based on different
@@ -84,6 +87,14 @@ public abstract class Realm<T extends AuthenticationToken> implements Comparable
      */
     public abstract User lookupUser(String username);
 
+    public Map<String, Object> usageStats() {
+        Map<String, Object> stats = new HashMap<>();
+        stats.put("type", type);
+        stats.put("name", name());
+        stats.put("order", order());
+        return stats;
+    }
+
     /**
      * Indicates whether this realm supports user lookup.
      * @return true if the realm supports user lookup

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/file/FileRealm.java

@@ -9,14 +9,16 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.rest.RestController;
-import org.elasticsearch.shield.user.User;
 import org.elasticsearch.shield.authc.RealmConfig;
 import org.elasticsearch.shield.authc.support.CachingUsernamePasswordRealm;
 import org.elasticsearch.shield.authc.support.RefreshListener;
 import org.elasticsearch.shield.authc.support.UsernamePasswordRealm;
 import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
+import org.elasticsearch.shield.user.User;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
+import java.util.Map;
+
 /**
  *
  */
@@ -55,6 +57,14 @@ public class FileRealm extends CachingUsernamePasswordRealm {
         return null;
     }
 
+    @Override
+    public Map<String, Object> usageStats() {
+        Map<String, Object> stats = super.usageStats();
+        // here we can determine the size based on the in mem user store
+        stats.put("size", UserbaseSize.resolve(userPasswdStore.usersCount()));
+        return stats;
+    }
+
     @Override
     public boolean userLookupSupported() {
         return true;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/file/tool/UsersTool.java

@@ -5,26 +5,14 @@
  */
 package org.elasticsearch.shield.authc.file.tool;
 
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
-import org.elasticsearch.cli.Command;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.MultiCommand;
+import org.elasticsearch.cli.SettingCommand;
+import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserError;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.env.Environment;
@@ -40,32 +28,41 @@ import org.elasticsearch.shield.support.FileAttributesChecker;
 import org.elasticsearch.shield.support.Validation;
 import org.elasticsearch.shield.support.Validation.Users;
 
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
 public class UsersTool extends MultiCommand {
 
     public static void main(String[] args) throws Exception {
-        Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, Terminal.DEFAULT);
+        exit(new UsersTool().main(args, Terminal.DEFAULT));
-        exit(new UsersTool(env).main(args, Terminal.DEFAULT));
     }
 
-    UsersTool(Environment env) {
+    UsersTool() {
         super("Manages elasticsearch native users");
-        subcommands.put("useradd", new AddUserCommand(env));
+        subcommands.put("useradd", new AddUserCommand());
-        subcommands.put("userdel", new DeleteUserCommand(env));
+        subcommands.put("userdel", new DeleteUserCommand());
-        subcommands.put("passwd", new PasswordCommand(env));
+        subcommands.put("passwd", new PasswordCommand());
-        subcommands.put("roles", new RolesCommand(env));
+        subcommands.put("roles", new RolesCommand());
-        subcommands.put("list", new ListCommand(env));
+        subcommands.put("list", new ListCommand());
     }
 
-    static class AddUserCommand extends Command {
+    static class AddUserCommand extends SettingCommand {
 
-        private final Environment env;
         private final OptionSpec<String> passwordOption;
         private final OptionSpec<String> rolesOption;
         private final OptionSpec<String> arguments;
 
-        AddUserCommand(Environment env) {
+        AddUserCommand() {
             super("Adds a native user");
-            this.env = env;
             this.passwordOption = parser.acceptsAll(Arrays.asList("p", "password"),
                 "The user password")
                 .withRequiredArg();
@@ -87,7 +84,7 @@ public class UsersTool extends MultiCommand {
         }
 
         @Override
-        protected void execute(Terminal terminal, OptionSet options) throws Exception {
+        protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
             String username = parseUsername(arguments.values(options));
             Validation.Error validationError = Users.validateUsername(username);
             if (validationError != null) {
@@ -95,6 +92,7 @@ public class UsersTool extends MultiCommand {
             }
 
             char[] password = parsePassword(terminal, passwordOption.value(options));
+            Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, terminal, settings);
             String[] roles = parseRoles(terminal, env, rolesOption.value(options));
 
             Settings fileSettings = Realms.fileRealmSettings(env.settings());
@@ -120,14 +118,12 @@ public class UsersTool extends MultiCommand {
         }
     }
 
-    static class DeleteUserCommand extends Command {
+    static class DeleteUserCommand extends SettingCommand {
 
-        private final Environment env;
         private final OptionSpec<String> arguments;
 
-        DeleteUserCommand(Environment env) {
+        DeleteUserCommand() {
             super("Deletes a file based user");
-            this.env = env;
             this.arguments = parser.nonOptions("username");
         }
 
@@ -143,8 +139,9 @@ public class UsersTool extends MultiCommand {
         }
 
         @Override
-        protected void execute(Terminal terminal, OptionSet options) throws Exception {
+        protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
             String username = parseUsername(arguments.values(options));
+            Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, terminal, settings);
             Settings fileSettings = Realms.fileRealmSettings(env.settings());
             Path passwordFile = FileUserPasswdStore.resolveFile(fileSettings, env);
             Path rolesFile = FileUserRolesStore.resolveFile(fileSettings, env);
@@ -173,15 +170,13 @@ public class UsersTool extends MultiCommand {
         }
     }
 
-    static class PasswordCommand extends Command {
+    static class PasswordCommand extends SettingCommand {
 
-        private final Environment env;
         private final OptionSpec<String> passwordOption;
         private final OptionSpec<String> arguments;
 
-        PasswordCommand(Environment env) {
+        PasswordCommand() {
             super("Changes the password of an existing file based user");
-            this.env = env;
             this.passwordOption = parser.acceptsAll(Arrays.asList("p", "password"),
                 "The user password")
                 .withRequiredArg();
@@ -200,10 +195,11 @@ public class UsersTool extends MultiCommand {
         }
 
         @Override
-        protected void execute(Terminal terminal, OptionSet options) throws Exception {
+        protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
             String username = parseUsername(arguments.values(options));
             char[] password = parsePassword(terminal, passwordOption.value(options));
 
+            Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, terminal, settings);
             Settings fileSettings = Realms.fileRealmSettings(env.settings());
             Path file = FileUserPasswdStore.resolveFile(fileSettings, env);
             FileAttributesChecker attributesChecker = new FileAttributesChecker(file);
@@ -218,16 +214,14 @@ public class UsersTool extends MultiCommand {
         }
     }
 
-    static class RolesCommand extends Command {
+    static class RolesCommand extends SettingCommand {
 
-        private final Environment env;
         private final OptionSpec<String> addOption;
         private final OptionSpec<String> removeOption;
         private final OptionSpec<String> arguments;
 
-        RolesCommand(Environment env) {
+        RolesCommand() {
             super("Edit roles of an existing user");
-            this.env = env;
             this.addOption = parser.acceptsAll(Arrays.asList("a", "add"),
                 "Adds supplied roles to the specified user")
                 .withRequiredArg().defaultsTo("");
@@ -246,8 +240,9 @@ public class UsersTool extends MultiCommand {
         }
 
         @Override
-        protected void execute(Terminal terminal, OptionSet options) throws Exception {
+        protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
             String username = parseUsername(arguments.values(options));
+            Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, terminal, settings);
             String[] addRoles = parseRoles(terminal, env, addOption.value(options));
             String[] removeRoles = parseRoles(terminal, env, removeOption.value(options));
 
@@ -290,14 +285,12 @@ public class UsersTool extends MultiCommand {
         }
     }
 
-    static class ListCommand extends Command {
+    static class ListCommand extends SettingCommand {
 
-        private final Environment env;
         private final OptionSpec<String> arguments;
 
-        ListCommand(Environment env) {
+        ListCommand() {
             super("List existing file based users and their corresponding roles");
-            this.env = env;
             this.arguments = parser.nonOptions("username");
         }
 
@@ -308,11 +301,12 @@ public class UsersTool extends MultiCommand {
         }
 
         @Override
-        protected void execute(Terminal terminal, OptionSet options) throws Exception {
+        protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
             String username = null;
             if (options.has(arguments)) {
                 username = arguments.value(options);
             }
+            Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, terminal, settings);
             listUsersAndRoles(terminal, env, username);
         }
     }

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/ldap/LdapRealm.java

@@ -17,6 +17,7 @@ import org.elasticsearch.shield.ssl.ClientSSLService;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
 import java.io.IOException;
+import java.util.Map;
 
 /**
  * Authenticates username/password tokens against ldap, locates groups and maps them to roles.
@@ -29,6 +30,13 @@ public class LdapRealm extends AbstractLdapRealm {
         super(TYPE, config, ldap, roleMapper);
     }
 
+    @Override
+    public Map<String, Object> usageStats() {
+        Map<String, Object> stats = super.usageStats();
+        stats.put("user_search", Factory.userSearchSettings(config).isEmpty() == false);
+        return stats;
+    }
+
     public static class Factory extends AbstractLdapRealm.Factory<LdapRealm> {
 
         private final ResourceWatcherService watcherService;
@@ -53,7 +61,7 @@ public class LdapRealm extends AbstractLdapRealm {
         }
 
         static SessionFactory sessionFactory(RealmConfig config, ClientSSLService clientSSLService) throws IOException {
-            Settings searchSettings = config.settings().getAsSettings("user_search");
+            Settings searchSettings = userSearchSettings(config);
             if (!searchSettings.names().isEmpty()) {
                 if (config.settings().getAsArray(LdapSessionFactory.USER_DN_TEMPLATES_SETTING).length > 0) {
                     throw new IllegalArgumentException("settings were found for both user search and user template modes of operation. " +
@@ -64,5 +72,9 @@ public class LdapRealm extends AbstractLdapRealm {
             }
             return new LdapSessionFactory(config, clientSSLService).init();
         }
+
+        static Settings userSearchSettings(RealmConfig config) {
+            return config.settings().getAsSettings("user_search");
+        }
     }
 }

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/AbstractLdapRealm.java

@@ -15,6 +15,7 @@ import org.elasticsearch.shield.authc.support.UsernamePasswordRealm;
 import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
 
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 /**
@@ -65,6 +66,14 @@ public abstract class AbstractLdapRealm extends CachingUsernamePasswordRealm {
         return sessionFactory.supportsUnauthenticatedSession();
     }
 
+    @Override
+    public Map<String, Object> usageStats() {
+        Map<String, Object> usage = super.usageStats();
+        usage.put("load_balance_type", LdapLoadBalancing.resolve(config.settings()).toString());
+        usage.put("ssl", sessionFactory.sslUsed);
+        return usage;
+    }
+
     private void logException(String action, Exception e, String principal) {
         if (logger.isDebugEnabled()) {
             logger.debug("{} failed for user [{}]", e, action, principal);

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/LdapLoadBalancing.java

@@ -16,7 +16,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 
 import javax.net.SocketFactory;
-import java.util.Arrays;
 import java.util.Locale;
 
 /**
@@ -83,25 +82,23 @@ public enum LdapLoadBalancing {
 
     @Override
     public String toString() {
-        return name().toLowerCase(Locale.ENGLISH);
+        return name().toLowerCase(Locale.ROOT);
+    }
+
+    public static LdapLoadBalancing resolve(Settings settings) {
+        Settings loadBalanceSettings = settings.getAsSettings(LOAD_BALANCE_SETTINGS);
+        String type = loadBalanceSettings.get(LOAD_BALANCE_TYPE_SETTING, LOAD_BALANCE_TYPE_DEFAULT);
+        try {
+            return valueOf(type.toUpperCase(Locale.ROOT));
+        } catch (IllegalArgumentException ilae) {
+            throw new IllegalArgumentException("unknown load balance type [" + type + "]", ilae);
+        }
     }
 
     public static ServerSet serverSet(String[] addresses, int[] ports, Settings settings, @Nullable SocketFactory socketFactory,
                                       @Nullable LDAPConnectionOptions options) {
+        LdapLoadBalancing loadBalancing = resolve(settings);
         Settings loadBalanceSettings = settings.getAsSettings(LOAD_BALANCE_SETTINGS);
-        String type = loadBalanceSettings.get(LOAD_BALANCE_TYPE_SETTING, LOAD_BALANCE_TYPE_DEFAULT);
+        return loadBalancing.buildServerSet(addresses, ports, loadBalanceSettings, socketFactory, options);
-        switch (type.toLowerCase(Locale.ENGLISH)) {
-            case "failover":
-                return FAILOVER.buildServerSet(addresses, ports, loadBalanceSettings, socketFactory, options);
-            case "dns_failover":
-                return DNS_FAILOVER.buildServerSet(addresses, ports, loadBalanceSettings, socketFactory, options);
-            case "round_robin":
-                return ROUND_ROBIN.buildServerSet(addresses, ports, loadBalanceSettings, socketFactory, options);
-            case "dns_round_robin":
-                return DNS_ROUND_ROBIN.buildServerSet(addresses, ports, loadBalanceSettings, socketFactory, options);
-            default:
-                throw new IllegalArgumentException("unknown server set type [" + type + "]. value must be one of " +
-                        Arrays.toString(LdapLoadBalancing.values()));
-        }
     }
 }

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/ldap/support/SessionFactory.java

@@ -52,7 +52,9 @@ public abstract class SessionFactory {
     protected final RealmConfig config;
     protected final TimeValue timeout;
     protected final ClientSSLService sslService;
+
     protected ServerSet serverSet;
+    protected boolean sslUsed;
 
     protected SessionFactory(RealmConfig config, ClientSSLService sslService) {
         this.config = config;
@@ -118,7 +120,9 @@ public abstract class SessionFactory {
     }
 
     public <T extends SessionFactory> T init() {
-        this.serverSet = serverSet(config.settings(), sslService, ldapServers(config.settings()));
+        LDAPServers ldapServers = ldapServers(config.settings());
+        this.serverSet = serverSet(config.settings(), sslService, ldapServers);
+        this.sslUsed = ldapServers.ssl;
         return (T) this;
     }
 

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealm.java

@@ -14,6 +14,7 @@ import org.elasticsearch.shield.authc.RealmConfig;
 import org.elasticsearch.shield.support.Exceptions;
 import org.elasticsearch.shield.user.User;
 
+import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
@@ -164,6 +165,13 @@ public abstract class CachingUsernamePasswordRealm extends UsernamePasswordRealm
         }
     }
 
+    @Override
+    public Map<String, Object> usageStats() {
+        Map<String, Object> stats = super.usageStats();
+        stats.put("size", UserbaseSize.resolve(cache.count()).toString());
+        return stats;
+    }
+
     protected abstract User doAuthenticate(UsernamePasswordToken token);
 
     protected abstract User doLookupUser(String username);

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/support/UsernamePasswordRealm.java

@@ -11,6 +11,8 @@ import org.elasticsearch.shield.authc.AuthenticationToken;
 import org.elasticsearch.shield.authc.Realm;
 import org.elasticsearch.shield.authc.RealmConfig;
 
+import java.util.Locale;
+
 /**
  *
  */
@@ -36,4 +38,34 @@ public abstract class UsernamePasswordRealm extends Realm<UsernamePasswordToken>
             restController.registerRelevantHeaders(UsernamePasswordToken.BASIC_AUTH_HEADER);
         }
     }
+
+    public enum UserbaseSize {
+
+        TINY,
+        SMALL,
+        MEDIUM,
+        LARGE,
+        XLARGE;
+
+        public static UserbaseSize resolve(int count) {
+            if (count < 10) {
+                return TINY;
+            }
+            if (count < 100) {
+                return SMALL;
+            }
+            if (count < 500) {
+                return MEDIUM;
+            }
+            if (count < 1000) {
+                return LARGE;
+            }
+            return XLARGE;
+        }
+
+        @Override
+        public String toString() {
+            return this == XLARGE ? "x-large" : name().toLowerCase(Locale.ROOT);
+        }
+    }
 }

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/crypto/tool/SystemKeyTool.java

@@ -5,21 +5,14 @@
  */
 package org.elasticsearch.shield.crypto.tool;
 
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
-import java.nio.file.attribute.PosixFileAttributeView;
-import java.nio.file.attribute.PosixFilePermission;
-import java.util.List;
-import java.util.Locale;
-import java.util.Set;
-
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
+import joptsimple.util.KeyValuePair;
 import org.elasticsearch.cli.Command;
 import org.elasticsearch.cli.ExitCodes;
-import org.elasticsearch.cli.UserError;
+import org.elasticsearch.cli.SettingCommand;
 import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserError;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Settings;
@@ -28,28 +21,47 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
 import org.elasticsearch.shield.crypto.InternalCryptoService;
 
-public class SystemKeyTool extends Command {
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.nio.file.attribute.PosixFileAttributeView;
+import java.nio.file.attribute.PosixFilePermission;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+public class SystemKeyTool extends SettingCommand {
+
+    private final OptionSpec<String> arguments;
+
+    SystemKeyTool() {
+        super("system key tool");
+        arguments = parser.nonOptions("key path");
+    }
 
     public static final Set<PosixFilePermission> PERMISSION_OWNER_READ_WRITE = Sets.newHashSet(PosixFilePermission.OWNER_READ,
             PosixFilePermission.OWNER_WRITE);
 
     public static void main(String[] args) throws Exception {
-        Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, Terminal.DEFAULT);
+        final SystemKeyTool tool = new SystemKeyTool();
-        exit(new SystemKeyTool(env).main(args, Terminal.DEFAULT));
+        int status = main(tool, args, Terminal.DEFAULT);
+        if (status != ExitCodes.OK) {
+            exit(status);
+        }
     }
 
-    private final Environment env;
+    static int main(SystemKeyTool tool, String[] args, Terminal terminal) throws Exception {
-    private final OptionSpec<String> arguments;
+        return tool.main(args, terminal);
-
-    public SystemKeyTool(Environment env) {
-        super("Generates the system key");
-        this.env = env;
-        this.arguments = parser.nonOptions("key path");
     }
 
     @Override
-    protected void execute(Terminal terminal, OptionSet options) throws Exception {
+    protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
         final Path keyPath;
+
+        final Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, terminal, settings);
+
         if (options.hasArgument(arguments)) {
             List<String> args = arguments.values(options);
             if (args.size() > 1) {
@@ -79,4 +91,5 @@ public class SystemKeyTool extends Command {
     private static Path parsePath(String path) {
         return PathUtils.get(path);
     }
+
 }

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/SecurityFeatureSetTests.java

@@ -0,0 +1,115 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.shield.authc.Realm;
+import org.elasticsearch.shield.authc.Realms;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.XPackFeatureSet;
+import org.elasticsearch.xpack.watcher.support.xcontent.XContentSource;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.core.Is.is;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ *
+ */
+public class SecurityFeatureSetTests extends ESTestCase {
+
+    private SecurityLicenseState licenseState;
+    private Realms realms;
+    private NamedWriteableRegistry namedWriteableRegistry;
+
+    @Before
+    public void init() throws Exception {
+        licenseState = mock(SecurityLicenseState.class);
+        realms = mock(Realms.class);
+        namedWriteableRegistry = mock(NamedWriteableRegistry.class);
+    }
+
+    public void testWritableRegistration() throws Exception {
+        new SecurityFeatureSet(Settings.EMPTY, licenseState, realms, namedWriteableRegistry);
+        verify(namedWriteableRegistry).register(eq(SecurityFeatureSet.Usage.class), eq("xpack.usage.security"), anyObject());
+    }
+
+    public void testAvailable() throws Exception {
+        SecurityFeatureSet featureSet = new SecurityFeatureSet(Settings.EMPTY, licenseState, realms, namedWriteableRegistry);
+        boolean available = randomBoolean();
+        when(licenseState.authenticationAndAuthorizationEnabled()).thenReturn(available);
+        assertThat(featureSet.available(), is(available));
+    }
+
+    public void testEnabledSetting() throws Exception {
+        boolean enabled = randomBoolean();
+        Settings settings = Settings.builder()
+                .put("xpack.security.enabled", enabled)
+                .build();
+        SecurityFeatureSet featureSet = new SecurityFeatureSet(settings, licenseState, realms, namedWriteableRegistry);
+        assertThat(featureSet.enabled(), is(enabled));
+    }
+
+    public void testEnabledDefault() throws Exception {
+        SecurityFeatureSet featureSet = new SecurityFeatureSet(Settings.EMPTY, licenseState, realms, namedWriteableRegistry);
+        assertThat(featureSet.enabled(), is(true));
+    }
+
+    public void testUsage() throws Exception {
+
+        boolean available = randomBoolean();
+        when(licenseState.authenticationAndAuthorizationEnabled()).thenReturn(available);
+
+        Settings.Builder settings = Settings.builder();
+
+        boolean enabled = randomBoolean();
+        settings.put("xpack.security.enabled", enabled);
+
+        List<Realm> realmsList= new ArrayList<>();
+
+        for (int i = 0; i < 5; i++) {
+            Realm realm = mock(Realm.class);
+            when(realm.type()).thenReturn("type" + i);
+            realmsList.add(realm);
+            Map<String, Object> realmUsage = new HashMap<>();
+            realmUsage.put("key1", "value" + i);
+            realmUsage.put("key2", i);
+            realmUsage.put("key3", i % 2 == 0);
+            when(realm.usageStats()).thenReturn(realmUsage);
+        }
+        when(realms.iterator()).thenReturn(realmsList.iterator());
+
+        SecurityFeatureSet featureSet = new SecurityFeatureSet(settings.build(), licenseState, realms, namedWriteableRegistry);
+        XPackFeatureSet.Usage usage = featureSet.usage();
+        assertThat(usage, is(notNullValue()));
+        assertThat(usage.name(), is(Security.NAME));
+        assertThat(usage.enabled(), is(enabled));
+        assertThat(usage.available(), is(available));
+        XContentSource source = new XContentSource(usage);
+
+        if (enabled) {
+            for (int i = 0; i < 5; i++) {
+                assertThat(source.getValue("enabled_realms." + i + ".key1"), is("value" + i));
+                assertThat(source.getValue("enabled_realms." + i + ".key2"), is(i));
+                assertThat(source.getValue("enabled_realms." + i + ".key3"), is(i % 2 == 0));
+            }
+        } else {
+            assertThat(source.getValue("enabled_realms"), is(nullValue()));
+        }
+    }
+}

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/ShieldLicenseeTests.java

@@ -89,7 +89,8 @@ public class ShieldLicenseeTests extends AbstractLicenseeTestCase {
     }
 
     public void testAcknowledgementMessagesFromBasicStandardTrialOrPlatinumToGoldNotesLimits() {
-        String[] messages = ackLicenseChange(randomModeExcept(OperationMode.GOLD), OperationMode.GOLD, this::buildLicensee);
+        OperationMode from = randomFrom(OperationMode.BASIC, OperationMode.PLATINUM, OperationMode.TRIAL, OperationMode.STANDARD);
+        String[] messages = ackLicenseChange(from, OperationMode.GOLD, this::buildLicensee);
 
         // leaving messages up to inspection
         assertThat(messages.length, equalTo(2));

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/activedirectory/AbstractActiveDirectoryIntegTests.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authc.activedirectory;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.shield.authc.ldap.support.LdapSearchScope;
+import org.elasticsearch.shield.ssl.ClientSSLService;
+import org.elasticsearch.shield.ssl.SSLConfiguration.Global;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.junit.annotations.Network;
+import org.junit.Before;
+
+import java.nio.file.Path;
+
+@Network
+public class AbstractActiveDirectoryIntegTests extends ESTestCase {
+
+    public static final String AD_LDAP_URL = "ldaps://54.213.145.20:636";
+    public static final String PASSWORD = "NickFuryHeartsES";
+    public static final String AD_DOMAIN = "ad.test.elasticsearch.com";
+
+    protected ClientSSLService clientSSLService;
+    protected Settings globalSettings;
+    protected boolean useGlobalSSL;
+
+    @Before
+    public void initializeSslSocketFactory() throws Exception {
+        useGlobalSSL = randomBoolean();
+        Path keystore = getDataPath("../ldap/support/ldaptrust.jks");
+        /*
+         * Prior to each test we reinitialize the socket factory with a new SSLService so that we get a new SSLContext.
+         * If we re-use a SSLContext, previously connected sessions can get re-established which breaks hostname
+         * verification tests since a re-established connection does not perform hostname verification.
+         */
+        Settings.Builder builder = Settings.builder().put("path.home", createTempDir());
+        if (useGlobalSSL) {
+            builder.put("xpack.security.ssl.keystore.path", keystore)
+                    .put("xpack.security.ssl.keystore.password", "changeit");
+        } else {
+            builder.put(Global.AUTO_GENERATE_SSL_SETTING.getKey(), false);
+        }
+        globalSettings = builder.build();
+        Environment environment = new Environment(globalSettings);
+        clientSSLService = new ClientSSLService(globalSettings, new Global(globalSettings));
+        clientSSLService.setEnvironment(environment);
+    }
+
+    Settings buildAdSettings(String ldapUrl, String adDomainName, String userSearchDN, LdapSearchScope scope,
+                                           boolean hostnameVerification) {
+        Settings.Builder builder = Settings.builder()
+                .putArray(ActiveDirectorySessionFactory.URLS_SETTING, ldapUrl)
+                .put(ActiveDirectorySessionFactory.AD_DOMAIN_NAME_SETTING, adDomainName)
+                .put(ActiveDirectorySessionFactory.AD_USER_SEARCH_BASEDN_SETTING, userSearchDN)
+                .put(ActiveDirectorySessionFactory.AD_USER_SEARCH_SCOPE_SETTING, scope)
+                .put(ActiveDirectorySessionFactory.HOSTNAME_VERIFICATION_SETTING, hostnameVerification);
+        if (useGlobalSSL == false) {
+            builder.put("ssl.truststore.path", getDataPath("../ldap/support/ldaptrust.jks"))
+                    .put("ssl.truststore.password", "changeit");
+        }
+        return builder.build();
+    }
+}

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryRealmTests.java

@@ -57,6 +57,7 @@ import static org.mockito.Mockito.verify;
  * additional bind DN with a password in the test setup since it really is not a DN in the ldif file
  */
 public class ActiveDirectoryRealmTests extends ESTestCase {
+
     private static final String PASSWORD = "password";
 
     protected static int numberOfLdapServers;

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectoryRealmUsageTests.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authc.activedirectory;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.shield.authc.RealmConfig;
+import org.elasticsearch.shield.authc.ldap.support.LdapSearchScope;
+import org.elasticsearch.shield.authc.support.DnRoleMapper;
+import org.elasticsearch.test.junit.annotations.Network;
+
+import java.util.Map;
+
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.mockito.Mockito.mock;
+
+@Network
+public class ActiveDirectoryRealmUsageTests extends AbstractActiveDirectoryIntegTests {
+
+    public void testUsageStats() throws Exception {
+        String loadBalanceType = randomFrom("failover", "round_robin");
+        Settings settings = Settings.builder()
+                .put(buildAdSettings(AD_LDAP_URL, AD_DOMAIN, "CN=Bruce Banner, CN=Users,DC=ad,DC=test,DC=elasticsearch,DC=com",
+                        LdapSearchScope.BASE, false))
+                .put("load_balance.type", loadBalanceType)
+                .build();
+        RealmConfig config = new RealmConfig("ad-test", settings, globalSettings);
+        ActiveDirectorySessionFactory sessionFactory = new ActiveDirectorySessionFactory(config, clientSSLService).init();
+        ActiveDirectoryRealm realm = new ActiveDirectoryRealm(config, sessionFactory, mock(DnRoleMapper.class));
+
+        Map<String, Object> stats = realm.usageStats();
+        assertThat(stats, is(notNullValue()));
+        assertThat(stats, hasEntry("type", "active_directory"));
+        assertThat(stats, hasEntry("name", "ad-test"));
+        assertThat(stats, hasEntry("order", realm.order()));
+        assertThat(stats, hasEntry("size", "tiny"));
+        assertThat(stats, hasEntry("ssl", true));
+        assertThat(stats, hasEntry("load_balance_type", loadBalanceType));
+    }
+}

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/activedirectory/ActiveDirectorySessionFactoryTests.java

@@ -7,7 +7,6 @@ package org.elasticsearch.shield.authc.activedirectory;
 
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
 import org.elasticsearch.shield.authc.RealmConfig;
 import org.elasticsearch.shield.authc.ldap.LdapSessionFactory;
 import org.elasticsearch.shield.authc.ldap.support.LdapSearchScope;
@@ -15,14 +14,9 @@ import org.elasticsearch.shield.authc.ldap.support.LdapSession;
 import org.elasticsearch.shield.authc.ldap.support.LdapTestCase;
 import org.elasticsearch.shield.authc.ldap.support.SessionFactory;
 import org.elasticsearch.shield.authc.support.SecuredStringTests;
-import org.elasticsearch.shield.ssl.ClientSSLService;
-import org.elasticsearch.shield.ssl.SSLConfiguration.Global;
-import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.junit.annotations.Network;
-import org.junit.Before;
 
 import java.io.IOException;
-import java.nio.file.Path;
 import java.util.List;
 
 import static org.elasticsearch.test.ShieldTestsUtils.assertAuthenticationException;
@@ -32,36 +26,7 @@ import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.is;
 
 @Network
-public class ActiveDirectorySessionFactoryTests extends ESTestCase {
+public class ActiveDirectorySessionFactoryTests extends AbstractActiveDirectoryIntegTests {
-    public static final String AD_LDAP_URL = "ldaps://54.213.145.20:636";
-    public static final String PASSWORD = "NickFuryHeartsES";
-    public static final String AD_DOMAIN = "ad.test.elasticsearch.com";
-
-    private ClientSSLService clientSSLService;
-    private Settings globalSettings;
-    private boolean useGlobalSSL;
-
-    @Before
-    public void initializeSslSocketFactory() throws Exception {
-        useGlobalSSL = randomBoolean();
-        Path keystore = getDataPath("../ldap/support/ldaptrust.jks");
-        /*
-         * Prior to each test we reinitialize the socket factory with a new SSLService so that we get a new SSLContext.
-         * If we re-use a SSLContext, previously connected sessions can get re-established which breaks hostname
-         * verification tests since a re-established connection does not perform hostname verification.
-         */
-        Settings.Builder builder = Settings.builder().put("path.home", createTempDir());
-        if (useGlobalSSL) {
-            builder.put("xpack.security.ssl.keystore.path", keystore)
-                    .put("xpack.security.ssl.keystore.password", "changeit");
-        } else {
-            builder.put(Global.AUTO_GENERATE_SSL_SETTING.getKey(), false);
-        }
-        globalSettings = builder.build();
-        Environment environment = new Environment(globalSettings);
-        clientSSLService = new ClientSSLService(globalSettings, new Global(globalSettings));
-        clientSSLService.setEnvironment(environment);
-    }
 
     @SuppressWarnings("unchecked")
     public void testAdAuth() throws Exception {
@@ -333,19 +298,4 @@ public class ActiveDirectorySessionFactoryTests extends ESTestCase {
         }
         return builder.build();
     }
-
-    Settings buildAdSettings(String ldapUrl, String adDomainName, String userSearchDN, LdapSearchScope scope,
-                                           boolean hostnameVerification) {
-        Settings.Builder builder = Settings.builder()
-                .putArray(ActiveDirectorySessionFactory.URLS_SETTING, ldapUrl)
-                .put(ActiveDirectorySessionFactory.AD_DOMAIN_NAME_SETTING, adDomainName)
-                .put(ActiveDirectorySessionFactory.AD_USER_SEARCH_BASEDN_SETTING, userSearchDN)
-                .put(ActiveDirectorySessionFactory.AD_USER_SEARCH_SCOPE_SETTING, scope)
-                .put(ActiveDirectorySessionFactory.HOSTNAME_VERIFICATION_SETTING, hostnameVerification);
-        if (useGlobalSSL == false) {
-            builder.put("ssl.truststore.path", getDataPath("../ldap/support/ldaptrust.jks"))
-                    .put("ssl.truststore.password", "changeit");
-        }
-        return builder.build();
-    }
 }

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/esnative/ReservedRealmTests.java

@@ -120,7 +120,7 @@ public class ReservedRealmTests extends ESTestCase {
     }
 
     public void testHelperMethods() {
-        final User expectedUser = randomFrom((User) XPackUser.INSTANCE, (User) KibanaUser.INSTANCE);
+        final User expectedUser = randomFrom((User) XPackUser.INSTANCE, KibanaUser.INSTANCE);
         final String principal = expectedUser.principal();
         assertThat(ReservedRealm.isReserved(principal), is(true));
         assertThat(ReservedRealm.getUser(principal), sameInstance(expectedUser));

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/file/FileRealmTests.java

@@ -7,23 +7,25 @@ package org.elasticsearch.shield.authc.file;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.shield.user.User;
 import org.elasticsearch.shield.authc.RealmConfig;
 import org.elasticsearch.shield.authc.support.Hasher;
 import org.elasticsearch.shield.authc.support.SecuredStringTests;
+import org.elasticsearch.shield.authc.support.UsernamePasswordRealm;
 import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
+import org.elasticsearch.shield.user.User;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.junit.Before;
 
 import java.util.Locale;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
@@ -161,6 +163,27 @@ public class FileRealmTests extends ESTestCase {
         assertThat(user5, sameInstance(user6));
     }
 
+    public void testUsageStats() throws Exception {
+        int userCount = randomIntBetween(0, 1000);
+        when(userPasswdStore.usersCount()).thenReturn(userCount);
+
+        Settings.Builder settings = Settings.builder();
+
+        int order = randomIntBetween(0, 10);
+        settings.put("order", order);
+
+        RealmConfig config = new RealmConfig("file-realm", settings.build(), globalSettings);
+        FileRealm realm = new FileRealm(config, userPasswdStore, userRolesStore);
+
+        Map<String, Object> usage = realm.usageStats();
+        assertThat(usage, is(notNullValue()));
+        assertThat(usage, hasEntry("type", "file"));
+        assertThat(usage, hasEntry("name", "file-realm"));
+        assertThat(usage, hasEntry("order", order));
+        assertThat(usage, hasEntry("size", UsernamePasswordRealm.UserbaseSize.resolve(userCount)));
+
+    }
+
     static class UserPasswdStore extends FileUserPasswdStore {
         public UserPasswdStore(RealmConfig config) {
             super(config, mock(ResourceWatcherService.class));

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/file/tool/UsersToolTests.java

@@ -5,16 +5,6 @@
  */
 package org.elasticsearch.shield.authc.file.tool;
 
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.FileSystem;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-
 import com.google.common.jimfs.Configuration;
 import com.google.common.jimfs.Jimfs;
 import org.apache.lucene.util.IOUtils;
@@ -35,17 +25,28 @@ import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
 public class UsersToolTests extends CommandTestCase {
 
     // the mock filesystem we use so permissions/users/groups can be modified
     static FileSystem jimfs;
+    String pathHomeParameter;
+    String fileTypeParameter;
 
     // the config dir for each test to use
     Path confDir;
 
     // settings used to create an Environment for tools
-    Settings.Builder settingsBuilder;
+    Settings settings;
-
 
     @BeforeClass
     public static void setupJimfs() throws IOException {
@@ -78,9 +79,13 @@ public class UsersToolTests extends CommandTestCase {
             "test_r2:",
             "  cluster: all"
         ), StandardCharsets.UTF_8);
-        settingsBuilder = Settings.builder()
+        settings =
-            .put("path.home", homeDir)
+                Settings.builder()
-            .put("xpack.security.authc.realms.file.type", "file");
+                        .put("path.home", homeDir)
+                        .put("xpack.security.authc.realms.file.type", "file")
+                        .build();
+        pathHomeParameter = "-Epath.home=" + homeDir;
+        fileTypeParameter = "-Expack.security.authc.realms.file.type=file";
     }
 
     @AfterClass
@@ -93,7 +98,7 @@ public class UsersToolTests extends CommandTestCase {
 
     @Override
     protected Command newCommand() {
-        return new UsersTool(new Environment(settingsBuilder.build()));
+        return new UsersTool();
     }
 
     /** checks the user exists with the given password */
@@ -220,7 +225,7 @@ public class UsersToolTests extends CommandTestCase {
     }
 
     public void testParseUnknownRole() throws Exception {
-        UsersTool.parseRoles(terminal, new Environment(settingsBuilder.build()), "test_r1,r2,r3");
+        UsersTool.parseRoles(terminal, new Environment(settings), "test_r1,r2,r3");
         String output = terminal.getOutput();
         assertTrue(output, output.contains("The following roles [r2,r3] are not in the ["));
     }
@@ -228,21 +233,21 @@ public class UsersToolTests extends CommandTestCase {
     public void testParseReservedRole() throws Exception {
         final String reservedRoleName = randomFrom(ReservedRolesStore.names().toArray(Strings.EMPTY_ARRAY));
         String rolesArg = randomBoolean() ? "test_r1," + reservedRoleName : reservedRoleName;
-        UsersTool.parseRoles(terminal, new Environment(settingsBuilder.build()), rolesArg);
+        UsersTool.parseRoles(terminal, new Environment(settings), rolesArg);
         String output = terminal.getOutput();
         assertTrue(output, output.isEmpty());
     }
 
     public void testParseInvalidRole() throws Exception {
         UserError e = expectThrows(UserError.class, () -> {
-            UsersTool.parseRoles(terminal, new Environment(settingsBuilder.build()), "$345");
+            UsersTool.parseRoles(terminal, new Environment(settings), "$345");
         });
         assertEquals(ExitCodes.DATA_ERROR, e.exitCode);
         assertTrue(e.getMessage(), e.getMessage().contains("Invalid role [$345]"));
     }
 
     public void testParseMultipleRoles() throws Exception {
-        String[] roles = UsersTool.parseRoles(terminal, new Environment(settingsBuilder.build()), "test_r1,test_r2");
+        String[] roles = UsersTool.parseRoles(terminal, new Environment(settings), "test_r1,test_r2");
         assertEquals(Objects.toString(roles), 2, roles.length);
         assertEquals("test_r1", roles[0]);
         assertEquals("test_r2", roles[1]);
@@ -251,18 +256,18 @@ public class UsersToolTests extends CommandTestCase {
     public void testUseraddNoPassword() throws Exception {
         terminal.addSecretInput("changeme");
         terminal.addSecretInput("changeme");
-        execute("useradd", "username");
+        execute("useradd", pathHomeParameter, fileTypeParameter, "username");
         assertUser("username", "changeme");
     }
 
     public void testUseraddPasswordOption() throws Exception {
-        execute("useradd", "username", "-p", "changeme");
+        execute("useradd", pathHomeParameter, fileTypeParameter, "username", "-p", "changeme");
         assertUser("username", "changeme");
     }
 
     public void testUseraddUserExists() throws Exception {
         UserError e = expectThrows(UserError.class, () -> {
-            execute("useradd", "existing_user", "-p", "changeme");
+            execute("useradd", pathHomeParameter, fileTypeParameter, "existing_user", "-p", "changeme");
         });
         assertEquals(ExitCodes.CODE_ERROR, e.exitCode);
         assertEquals("User [existing_user] already exists", e.getMessage());
@@ -271,27 +276,27 @@ public class UsersToolTests extends CommandTestCase {
     public void testUseraddNoRoles() throws Exception {
         Files.delete(confDir.resolve("users_roles"));
         Files.createFile(confDir.resolve("users_roles"));
-        execute("useradd", "username", "-p", "changeme");
+        execute("useradd", pathHomeParameter, fileTypeParameter, "username", "-p", "changeme");
         List<String> lines = Files.readAllLines(confDir.resolve("users_roles"), StandardCharsets.UTF_8);
         assertTrue(lines.toString(), lines.isEmpty());
     }
 
     public void testUserdelUnknownUser() throws Exception {
         UserError e = expectThrows(UserError.class, () -> {
-            execute("userdel", "unknown");
+            execute("userdel", pathHomeParameter, fileTypeParameter, "unknown");
         });
         assertEquals(ExitCodes.NO_USER, e.exitCode);
         assertTrue(e.getMessage(), e.getMessage().contains("User [unknown] doesn't exist"));
     }
 
     public void testUserdel() throws Exception {
-        execute("userdel", "existing_user");
+        execute("userdel", pathHomeParameter, fileTypeParameter, "existing_user");
         assertNoUser("existing_user");
     }
 
     public void testPasswdUnknownUser() throws Exception {
         UserError e = expectThrows(UserError.class, () -> {
-            execute("passwd", "unknown", "-p", "changeme");
+            execute("passwd", pathHomeParameter, fileTypeParameter, "unknown", "-p", "changeme");
         });
         assertEquals(ExitCodes.NO_USER, e.exitCode);
         assertTrue(e.getMessage(), e.getMessage().contains("User [unknown] doesn't exist"));
@@ -300,64 +305,64 @@ public class UsersToolTests extends CommandTestCase {
     public void testPasswdNoPasswordOption() throws Exception {
         terminal.addSecretInput("newpassword");
         terminal.addSecretInput("newpassword");
-        execute("passwd", "existing_user");
+        execute("passwd", pathHomeParameter, fileTypeParameter, "existing_user");
         assertUser("existing_user", "newpassword");
         assertRole("test_admin", "existing_user", "existing_user2"); // roles unchanged
     }
 
     public void testPasswd() throws Exception {
-        execute("passwd", "existing_user", "-p", "newpassword");
+        execute("passwd", pathHomeParameter, fileTypeParameter, "existing_user", "-p", "newpassword");
         assertUser("existing_user", "newpassword");
         assertRole("test_admin", "existing_user"); // roles unchanged
     }
 
     public void testRolesUnknownUser() throws Exception {
         UserError e = expectThrows(UserError.class, () -> {
-            execute("roles", "unknown");
+            execute("roles", pathHomeParameter, fileTypeParameter, "unknown");
         });
         assertEquals(ExitCodes.NO_USER, e.exitCode);
         assertTrue(e.getMessage(), e.getMessage().contains("User [unknown] doesn't exist"));
     }
 
     public void testRolesAdd() throws Exception {
-        execute("roles", "existing_user", "-a", "test_r1");
+        execute("roles", pathHomeParameter, fileTypeParameter, "existing_user", "-a", "test_r1");
         assertRole("test_admin", "existing_user");
         assertRole("test_r1", "existing_user");
     }
 
     public void testRolesRemove() throws Exception {
-        execute("roles", "existing_user", "-r", "test_admin");
+        execute("roles", pathHomeParameter, fileTypeParameter, "existing_user", "-r", "test_admin");
         assertRole("test_admin", "existing_user2");
     }
 
     public void testRolesAddAndRemove() throws Exception {
-        execute("roles", "existing_user", "-a", "test_r1", "-r", "test_admin");
+        execute("roles", pathHomeParameter, fileTypeParameter, "existing_user", "-a", "test_r1", "-r", "test_admin");
         assertRole("test_admin", "existing_user2");
         assertRole("test_r1", "existing_user");
     }
 
     public void testRolesRemoveLeavesExisting() throws Exception {
-        execute("useradd", "username", "-p", "changeme", "-r", "test_admin");
+        execute("useradd", pathHomeParameter, fileTypeParameter, "username", "-p", "changeme", "-r", "test_admin");
-        execute("roles", "existing_user", "-r", "test_admin");
+        execute("roles", pathHomeParameter, fileTypeParameter, "existing_user", "-r", "test_admin");
         assertRole("test_admin", "username");
     }
 
     public void testRolesNoAddOrRemove() throws Exception {
-        String output = execute("roles", "existing_user");
+        String output = execute("roles", pathHomeParameter, fileTypeParameter, "existing_user");
         assertTrue(output, output.contains("existing_user"));
         assertTrue(output, output.contains("test_admin"));
     }
 
     public void testListUnknownUser() throws Exception {
         UserError e = expectThrows(UserError.class, () -> {
-            execute("list", "unknown");
+            execute("list", pathHomeParameter, fileTypeParameter, "unknown");
         });
         assertEquals(ExitCodes.NO_USER, e.exitCode);
         assertTrue(e.getMessage(), e.getMessage().contains("User [unknown] doesn't exist"));
     }
 
     public void testListAllUsers() throws Exception {
-        String output = execute("list");
+        String output = execute("list", pathHomeParameter, fileTypeParameter);
         assertTrue(output, output.contains("existing_user"));
         assertTrue(output, output.contains("test_admin"));
         assertTrue(output, output.contains("existing_user2"));
@@ -368,7 +373,7 @@ public class UsersToolTests extends CommandTestCase {
     }
 
     public void testListSingleUser() throws Exception {
-        String output = execute("list", "existing_user");
+        String output = execute("list", pathHomeParameter, fileTypeParameter, "existing_user");
         assertTrue(output, output.contains("existing_user"));
         assertTrue(output, output.contains("test_admin"));
         assertFalse(output, output.contains("existing_user2"));
@@ -379,8 +384,8 @@ public class UsersToolTests extends CommandTestCase {
     }
 
     public void testListUnknownRoles() throws Exception {
-        execute("useradd", "username", "-p", "changeme", "-r", "test_r1,r2,r3");
+        execute("useradd", pathHomeParameter, fileTypeParameter, "username", "-p", "changeme", "-r", "test_r1,r2,r3");
-        String output = execute("list", "username");
+        String output = execute("list", pathHomeParameter, fileTypeParameter, "username");
         assertTrue(output, output.contains("username"));
         assertTrue(output, output.contains("r2*,r3*,test_r1"));
     }
@@ -390,14 +395,14 @@ public class UsersToolTests extends CommandTestCase {
         Files.createFile(confDir.resolve("users"));
         Files.delete(confDir.resolve("users_roles"));
         Files.createFile(confDir.resolve("users_roles"));
-        String output = execute("list");
+        String output = execute("list", pathHomeParameter, fileTypeParameter);
         assertTrue(output, output.contains("No users found"));
     }
 
     public void testListUserWithoutRoles() throws Exception {
-        String output = execute("list", "existing_user3");
+        String output = execute("list", pathHomeParameter, fileTypeParameter, "existing_user3");
         assertTrue(output, output.contains("existing_user3"));
-        output = execute("list");
+        output = execute("list", pathHomeParameter, fileTypeParameter);
         assertTrue(output, output.contains("existing_user3"));
 
         // output should not contain '*' which indicates unknown role

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/ldap/LdapRealmTests.java

@@ -6,7 +6,6 @@
 package org.elasticsearch.shield.authc.ldap;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.shield.user.User;
 import org.elasticsearch.shield.authc.RealmConfig;
 import org.elasticsearch.shield.authc.ldap.support.LdapSearchScope;
 import org.elasticsearch.shield.authc.ldap.support.LdapTestCase;
@@ -15,16 +14,20 @@ import org.elasticsearch.shield.authc.support.DnRoleMapper;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.shield.authc.support.SecuredStringTests;
 import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
+import org.elasticsearch.shield.user.User;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 import org.junit.After;
 import org.junit.Before;
 
+import java.util.Map;
+
 import static org.elasticsearch.shield.authc.ldap.LdapSessionFactory.USER_DN_TEMPLATES_SETTING;
 import static org.elasticsearch.shield.authc.ldap.support.SessionFactory.HOSTNAME_VERIFICATION_SETTING;
 import static org.elasticsearch.shield.authc.ldap.support.SessionFactory.URLS_SETTING;
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
@@ -35,6 +38,7 @@ import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 
 public class LdapRealmTests extends LdapTestCase {
+
     public static final String VALID_USER_TEMPLATE = "cn={0},ou=people,o=sevenSeas";
     public static final String VALID_USERNAME = "Thomas Masterman Hardy";
     public static final String PASSWORD = "pass";
@@ -217,4 +221,37 @@ public class LdapRealmTests extends LdapTestCase {
         assertThat(user, notNullValue());
         assertThat(user.roles(), arrayContaining("avenger"));
     }
+
+    public void testUsageStats() throws Exception {
+        String groupSearchBase = "o=sevenSeas";
+        Settings.Builder settings = Settings.builder()
+                .putArray(URLS_SETTING, ldapUrls())
+                .put("bind_dn", "cn=Thomas Masterman Hardy,ou=people,o=sevenSeas")
+                .put("bind_password", PASSWORD)
+                .put("group_search.base_dn", groupSearchBase)
+                .put("group_search.scope", LdapSearchScope.SUB_TREE)
+                .put(HOSTNAME_VERIFICATION_SETTING, false);
+
+        int order = randomIntBetween(0, 10);
+        settings.put("order", order);
+
+        boolean userSearch = randomBoolean();
+        if (userSearch) {
+            settings.put("user_search.base_dn", "");
+        }
+
+        RealmConfig config = new RealmConfig("ldap-realm", settings.build(), globalSettings);
+
+        LdapSessionFactory ldapFactory = new LdapSessionFactory(config, null).init();
+        LdapRealm realm = new LdapRealm(config, ldapFactory, new DnRoleMapper(LdapRealm.TYPE, config, resourceWatcherService, null));
+
+        Map<String, Object> stats = realm.usageStats();
+        assertThat(stats, is(notNullValue()));
+        assertThat(stats, hasEntry("type", "ldap"));
+        assertThat(stats, hasEntry("name", "ldap-realm"));
+        assertThat(stats, hasEntry("order", realm.order()));
+        assertThat(stats, hasEntry("size", "tiny"));
+        assertThat(stats, hasEntry("ssl", false));
+        assertThat(stats, hasEntry("user_search", userSearch));
+    }
 }

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/ldap/LdapUserSearchSessionFactoryTests.java

@@ -59,6 +59,7 @@ import static org.hamcrest.Matchers.nullValue;
         LdapUserSearchSessionFactoryTests.BackgroundConnectThreadLeakFilter.class
 })
 public class LdapUserSearchSessionFactoryTests extends LdapTestCase {
+
     private ClientSSLService clientSSLService;
     private Settings globalSettings;
 

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/ldap/OpenLdapTests.java

@@ -13,6 +13,7 @@ import org.elasticsearch.shield.authc.ldap.support.LdapSearchScope;
 import org.elasticsearch.shield.authc.ldap.support.LdapSession;
 import org.elasticsearch.shield.authc.ldap.support.LdapTestCase;
 import org.elasticsearch.shield.authc.ldap.support.SessionFactory;
+import org.elasticsearch.shield.authc.support.DnRoleMapper;
 import org.elasticsearch.shield.authc.support.SecuredStringTests;
 import org.elasticsearch.shield.ssl.ClientSSLService;
 import org.elasticsearch.shield.ssl.SSLConfiguration.Global;
@@ -22,9 +23,14 @@ import org.junit.Before;
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.core.Is.is;
+import static org.mockito.Mockito.mock;
 
 @Network
 public class OpenLdapTests extends ESTestCase {
@@ -91,6 +97,36 @@ public class OpenLdapTests extends ESTestCase {
         }
     }
 
+    public void testUsageStats() throws Exception {
+        String groupSearchBase = "ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com";
+        String userTemplate = "uid={0},ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com";
+        Settings.Builder settings = Settings.builder()
+                .put(buildLdapSettings(OPEN_LDAP_URL, userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL))
+                .put("group_search.filter", "(&(objectclass=posixGroup)(memberUID={0}))")
+                .put("group_search.user_attribute", "uid");
+
+        boolean userSearch = randomBoolean();
+        if (userSearch) {
+            settings.put("user_search.base_dn", "");
+        }
+
+        String loadBalanceType = randomFrom("failover", "round_robin");
+        settings.put("load_balance.type", loadBalanceType);
+
+        RealmConfig config = new RealmConfig("oldap-test", settings.build(), globalSettings);
+        LdapSessionFactory sessionFactory = new LdapSessionFactory(config, clientSSLService).init();
+        LdapRealm realm = new LdapRealm(config, sessionFactory, mock(DnRoleMapper.class));
+
+        Map<String, Object> stats = realm.usageStats();
+        assertThat(stats, is(notNullValue()));
+        assertThat(stats, hasEntry("size", "small"));
+        assertThat(stats, hasEntry("ssl", true));
+        assertThat(stats, hasEntry("user_search", userSearch));
+        assertThat(stats, hasEntry("load_balance_type", loadBalanceType));
+    }
+
+
+
     public void testCustomFilter() throws Exception {
         String groupSearchBase = "ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com";
         String userTemplate = "uid={0},ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com";

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/ldap/support/LDAPServersTests.java

@@ -12,6 +12,7 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
 public class LDAPServersTests extends ESTestCase {
+
     public void testConfigure1ldaps() {
         String[] urls = new String[] { "ldaps://example.com:636" };
 

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/ldap/support/LdapLoadBalancingTests.java

@@ -26,7 +26,7 @@ public class LdapLoadBalancingTests extends ESTestCase {
             LdapLoadBalancing.serverSet(null, null, settings, null, null);
             fail("using type [" + badType + "] should have thrown an exception");
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("unknown server set type"));
+            assertThat(e.getMessage(), containsString("unknown load balance type"));
         }
     }
 

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealmTests.java

@@ -7,9 +7,9 @@ package org.elasticsearch.shield.authc.support;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.shield.user.User;
 import org.elasticsearch.shield.authc.Realm;
 import org.elasticsearch.shield.authc.RealmConfig;
+import org.elasticsearch.shield.user.User;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
@@ -25,6 +25,7 @@ import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
 public class CachingUsernamePasswordRealmTests extends ESTestCase {
+
     private Settings globalSettings;
 
     @Before

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/support/UsernamePasswordRealmTests.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.shield.authc.support;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Locale;
+
+import static org.hamcrest.core.Is.is;
+
+/**
+ *
+ */
+public class UsernamePasswordRealmTests extends ESTestCase {
+
+    public void testUserbaseScaelResolve() throws Exception {
+        int count = randomIntBetween(0, 1000);
+        UsernamePasswordRealm.UserbaseSize size = UsernamePasswordRealm.UserbaseSize.resolve(count);
+        if (count < 10) {
+            assertThat(size, is(UsernamePasswordRealm.UserbaseSize.SMALL));
+        } else if (count < 100) {
+            assertThat(size, is(UsernamePasswordRealm.UserbaseSize.SMALL));
+        } else if (count < 500) {
+            assertThat(size, is(UsernamePasswordRealm.UserbaseSize.MEDIUM));
+        } else if (count < 1000) {
+            assertThat(size, is(UsernamePasswordRealm.UserbaseSize.LARGE));
+        } else {
+            assertThat(size, is(UsernamePasswordRealm.UserbaseSize.XLARGE));
+        }
+    }
+
+    public void testUserbaseScaleToString() throws Exception {
+        UsernamePasswordRealm.UserbaseSize size = randomFrom(UsernamePasswordRealm.UserbaseSize.values());
+        String value = size.toString();
+        if (size == UsernamePasswordRealm.UserbaseSize.XLARGE) {
+            assertThat(value , is("x-large"));
+        } else {
+            assertThat(value , is(size.name().toLowerCase(Locale.ROOT)));
+        }
+    }
+}

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/crypto/tool/SystemKeyToolTests.java

@@ -7,12 +7,12 @@ package org.elasticsearch.shield.crypto.tool;
 
 import com.google.common.jimfs.Configuration;
 import com.google.common.jimfs.Jimfs;
+import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.cli.Command;
 import org.elasticsearch.cli.CommandTestCase;
 import org.elasticsearch.common.io.PathUtilsForTesting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
 import org.elasticsearch.shield.crypto.InternalCryptoService;
+import org.junit.After;
 
 import java.nio.file.FileSystem;
 import java.nio.file.Files;
@@ -23,32 +23,33 @@ import java.util.Set;
 public class SystemKeyToolTests extends CommandTestCase {
 
     private FileSystem jimfs;
-    private Settings.Builder settingsBuilder;
-    private Path homeDir;
 
-    private void initFileSystem(boolean needsPosix) throws Exception {
+    private Path initFileSystem(boolean needsPosix) throws Exception {
         String view = needsPosix ? "posix" : randomFrom("basic", "posix");
         Configuration conf = Configuration.unix().toBuilder().setAttributeViews(view).build();
         jimfs = Jimfs.newFileSystem(conf);
         PathUtilsForTesting.installMock(jimfs);
-        homeDir = jimfs.getPath("eshome");
+        return jimfs.getPath("eshome");
+    }
 
-        settingsBuilder = Settings.builder()
+    @After
-            .put(Environment.PATH_HOME_SETTING.getKey(), homeDir);
+    public void tearDown() throws Exception {
+        IOUtils.close(jimfs);
+        super.tearDown();
     }
 
     @Override
     protected Command newCommand() {
-        return new SystemKeyTool(new Environment(settingsBuilder.build()));
+        return new SystemKeyTool();
     }
 
     public void testGenerate() throws Exception {
-        initFileSystem(true);
+        final Path homeDir = initFileSystem(true);
 
         Path path = jimfs.getPath(randomAsciiOfLength(10)).resolve("key");
         Files.createDirectory(path.getParent());
 
-        execute(path.toString());
+        execute("-Epath.home=" + homeDir, path.toString());
         byte[] bytes = Files.readAllBytes(path);
         // TODO: maybe we should actually check the key is...i dunno...valid?
         assertEquals(InternalCryptoService.KEY_SIZE / 8, bytes.length);
@@ -60,35 +61,35 @@ public class SystemKeyToolTests extends CommandTestCase {
     }
 
     public void testGeneratePathInSettings() throws Exception {
-        initFileSystem(false);
+        final Path homeDir = initFileSystem(false);
 
         Path path = jimfs.getPath(randomAsciiOfLength(10)).resolve("key");
         Files.createDirectories(path.getParent());
-        settingsBuilder.put(InternalCryptoService.FILE_SETTING.getKey(), path.toAbsolutePath().toString());
+        execute("-Epath.home=" + homeDir.toString(), "-Expack.security.system_key.file=" + path.toAbsolutePath().toString());
-        execute();
         byte[] bytes = Files.readAllBytes(path);
         assertEquals(InternalCryptoService.KEY_SIZE / 8, bytes.length);
     }
 
     public void testGenerateDefaultPath() throws Exception {
-        initFileSystem(false);
+        final Path homeDir = initFileSystem(false);
         Path keyPath = homeDir.resolve("config/x-pack/system_key");
         Files.createDirectories(keyPath.getParent());
-        execute();
+        execute("-Epath.home=" + homeDir.toString());
         byte[] bytes = Files.readAllBytes(keyPath);
         assertEquals(InternalCryptoService.KEY_SIZE / 8, bytes.length);
     }
 
     public void testThatSystemKeyMayOnlyBeReadByOwner() throws Exception {
-        initFileSystem(true);
+        final Path homeDir = initFileSystem(true);
 
         Path path = jimfs.getPath(randomAsciiOfLength(10)).resolve("key");
         Files.createDirectories(path.getParent());
 
-        execute(path.toString());
+        execute("-Epath.home=" + homeDir, path.toString());
         Set<PosixFilePermission> perms = Files.getPosixFilePermissions(path);
         assertTrue(perms.toString(), perms.contains(PosixFilePermission.OWNER_READ));
         assertTrue(perms.toString(), perms.contains(PosixFilePermission.OWNER_WRITE));
         assertEquals(perms.toString(), 2, perms.size());
     }
+
 }

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/ssl/CertUtilsTests.java

@@ -7,7 +7,9 @@ package org.elasticsearch.shield.ssl;
 
 import org.bouncycastle.asn1.x509.GeneralName;
 import org.bouncycastle.asn1.x509.GeneralNames;
+import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.InputStream;
@@ -113,7 +115,8 @@ public class CertUtilsTests extends ESTestCase {
         GeneralName[] generalNameArray = generalNames.getNames();
         assertThat(generalNameArray, notNullValue());
 
-        if (resolveName) {
+        logger.info("resolve name [{}], address [{}], subject alt names [{}]", resolveName, NetworkAddress.format(address), generalNames);
+        if (resolveName && isResolvable(address)) {
             assertThat(generalNameArray.length, is(2));
             int firstType = generalNameArray[0].getTagNo();
             if (firstType == GeneralName.iPAddress) {
@@ -129,6 +132,12 @@ public class CertUtilsTests extends ESTestCase {
         }
     }
 
+    @SuppressForbidden(reason = "need to use getHostName to resolve DNS name and getHostAddress to ensure we resolved the name")
+    private boolean isResolvable(InetAddress inetAddress) {
+        String hostname = inetAddress.getHostName();
+        return hostname.equals(inetAddress.getHostAddress()) == false;
+    }
+
     public void testIsAnyLocalAddress() throws Exception {
         InetAddress address = mock(InetAddress.class);
         when(address.isAnyLocalAddress()).thenReturn(true);

elasticsearch/x-pack/shield/src/test/resources/org/elasticsearch/transport/actions

@@ -75,6 +75,7 @@ cluster:admin/script/delete
 cluster:admin/script/put
 indices:data/write/update
 cluster:monitor/xpack/info
+cluster:monitor/xpack/usage
 cluster:monitor/xpack/license/get
 cluster:admin/xpack/license/delete
 cluster:admin/xpack/license/put

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackFeatureSet.java

@@ -5,6 +5,14 @@
  */
 package org.elasticsearch.xpack;
 
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+
 /**
  *
  */
@@ -18,4 +26,65 @@ public interface XPackFeatureSet {
 
     boolean enabled();
 
+    Usage usage();
+
+    abstract class Usage implements ToXContent, NamedWriteable {
+
+        private static final String AVAILABLE_XFIELD = "available";
+        private static final String ENABLED_XFIELD = "enabled";
+
+        protected final String name;
+        protected final boolean available;
+        protected final boolean enabled;
+
+        public Usage(StreamInput input) throws IOException {
+            this(input.readString(), input.readBoolean(), input.readBoolean());
+        }
+
+        public Usage(String name, boolean available, boolean enabled) {
+            this.name = name;
+            this.available = available;
+            this.enabled = enabled;
+        }
+
+        public String name() {
+            return name;
+        }
+
+        public boolean available() {
+            return available;
+        }
+
+        public boolean enabled() {
+            return enabled;
+        }
+
+        @Override
+        public String getWriteableName() {
+            return writeableName(name);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeBoolean(available);
+            out.writeBoolean(enabled);
+        }
+
+        @Override
+        public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            innerXContent(builder, params);
+            return builder.endObject();
+        }
+
+        protected void innerXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.field(AVAILABLE_XFIELD, available);
+            builder.field(ENABLED_XFIELD, enabled);
+        }
+
+        public static String writeableName(String featureName) {
+            return "xpack.usage." + featureName;
+        }
+    }
+
 }

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackPlugin.java

@@ -27,7 +27,9 @@ import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.shield.Security;
 import org.elasticsearch.shield.authc.AuthenticationModule;
 import org.elasticsearch.xpack.action.TransportXPackInfoAction;
+import org.elasticsearch.xpack.action.TransportXPackUsageAction;
 import org.elasticsearch.xpack.action.XPackInfoAction;
+import org.elasticsearch.xpack.action.XPackUsageAction;
 import org.elasticsearch.xpack.common.http.HttpClientModule;
 import org.elasticsearch.xpack.common.init.LazyInitializationModule;
 import org.elasticsearch.xpack.common.init.LazyInitializationService;
@@ -39,6 +41,7 @@ import org.elasticsearch.xpack.notification.email.Account;
 import org.elasticsearch.xpack.notification.email.support.BodyPartSource;
 import org.elasticsearch.xpack.rest.action.RestXPackInfoAction;
 import org.elasticsearch.xpack.common.text.TextTemplateModule;
+import org.elasticsearch.xpack.rest.action.RestXPackUsageAction;
 import org.elasticsearch.xpack.watcher.Watcher;
 
 import java.nio.file.Path;
@@ -199,6 +202,7 @@ public class XPackPlugin extends Plugin {
     public void onModule(NetworkModule module) {
         if (!transportClientMode) {
             module.registerRestHandler(RestXPackInfoAction.class);
+            module.registerRestHandler(RestXPackUsageAction.class);
         }
         licensing.onModule(module);
         monitoring.onModule(module);
@@ -210,6 +214,7 @@ public class XPackPlugin extends Plugin {
     public void onModule(ActionModule module) {
         if (!transportClientMode) {
             module.registerAction(XPackInfoAction.INSTANCE, TransportXPackInfoAction.class);
+            module.registerAction(XPackUsageAction.INSTANCE, TransportXPackUsageAction.class);
         }
         licensing.onModule(module);
         monitoring.onModule(module);

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/action/TransportXPackUsageAction.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.action;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.XPackFeatureSet;
+
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ */
+public class TransportXPackUsageAction extends HandledTransportAction<XPackUsageRequest, XPackUsageResponse> {
+
+    private final Set<XPackFeatureSet> featureSets;
+
+    @Inject
+    public TransportXPackUsageAction(Settings settings, ThreadPool threadPool, TransportService transportService,
+                                     ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                     Set<XPackFeatureSet> featureSets) {
+        super(settings, XPackInfoAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver,
+                XPackUsageRequest::new);
+        this.featureSets = featureSets;
+    }
+
+    @Override
+    protected void doExecute(XPackUsageRequest request, ActionListener<XPackUsageResponse> listener) {
+        List<XPackFeatureSet.Usage> usages = featureSets.stream().map(XPackFeatureSet::usage).collect(Collectors.toList());
+        listener.onResponse(new XPackUsageResponse(usages));
+    }
+}

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/action/XPackInfoResponse.java

@@ -14,7 +14,6 @@ import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.license.core.License;
 import org.elasticsearch.xpack.XPackBuild;
-import org.elasticsearch.xpack.XPackFeatureSet;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -219,7 +218,7 @@ public class XPackInfoResponse extends ActionResponse {
             }
         }
 
-        public static class FeatureSet implements XPackFeatureSet, ToXContent, Writeable {
+        public static class FeatureSet implements ToXContent, Writeable {
 
             private final String name;
             private final @Nullable String description;
@@ -237,23 +236,19 @@ public class XPackInfoResponse extends ActionResponse {
                 this.enabled = enabled;
             }
 
-            @Override
             public String name() {
                 return name;
             }
 
-            @Override
             @Nullable
             public String description() {
                 return description;
             }
 
-            @Override
             public boolean available() {
                 return available;
             }
 
-            @Override
             public boolean enabled() {
                 return enabled;
             }

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/action/XPackUsageAction.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.action;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.client.ElasticsearchClient;
+
+/**
+ *
+ */
+public class XPackUsageAction extends Action<XPackUsageRequest, XPackUsageResponse, XPackUsageRequestBuilder> {
+
+    public static final String NAME = "cluster:monitor/xpack/usage";
+    public static final XPackUsageAction INSTANCE = new XPackUsageAction();
+
+    public XPackUsageAction() {
+        super(NAME);
+    }
+
+    @Override
+    public XPackUsageRequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new XPackUsageRequestBuilder(client);
+    }
+
+    @Override
+    public XPackUsageResponse newResponse() {
+        return new XPackUsageResponse();
+    }
+}

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/action/XPackUsageRequest.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.action;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+
+/**
+ *
+ */
+public class XPackUsageRequest extends ActionRequest<XPackUsageRequest> {
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+
+}

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/action/XPackUsageRequestBuilder.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.action;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+
+/**
+ */
+public class XPackUsageRequestBuilder extends ActionRequestBuilder<XPackUsageRequest, XPackUsageResponse, XPackUsageRequestBuilder> {
+
+    public XPackUsageRequestBuilder(ElasticsearchClient client) {
+        this(client, XPackUsageAction.INSTANCE);
+    }
+
+    public XPackUsageRequestBuilder(ElasticsearchClient client, XPackUsageAction action) {
+        super(client, action, new XPackUsageRequest());
+    }
+}

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/action/XPackUsageResponse.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.action;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.license.core.License;
+import org.elasticsearch.xpack.XPackBuild;
+import org.elasticsearch.xpack.XPackFeatureSet;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ */
+public class XPackUsageResponse extends ActionResponse {
+
+    private List<XPackFeatureSet.Usage> usages;
+
+    public XPackUsageResponse() {}
+
+    public XPackUsageResponse(List<XPackFeatureSet.Usage> usages) {
+        this.usages = usages;
+    }
+
+    public List<XPackFeatureSet.Usage> getUsages() {
+        return usages;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeVInt(usages.size());
+        for (XPackFeatureSet.Usage usage : usages) {
+            out.writeNamedWriteable(usage);
+        }
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        int size = in.readVInt();
+        usages = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            usages.add(in.readNamedWriteable(XPackFeatureSet.Usage.class));
+        }
+    }
+}

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/InstallXPackExtensionCommand.java

@@ -8,17 +8,18 @@ package org.elasticsearch.xpack.extensions;
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
 import org.apache.lucene.util.IOUtils;
-
 import org.elasticsearch.bootstrap.JarHell;
-import org.elasticsearch.cli.Command;
 import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.SettingCommand;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserError;
 import org.elasticsearch.common.io.FileSystemUtils;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.internal.InternalSettingsPreparer;
 
-import java.io.InputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URL;
 import java.net.URLDecoder;
@@ -26,13 +27,14 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
-import java.util.List;
 import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
-import static org.elasticsearch.xpack.XPackPlugin.resolveXPackExtensionsFile;
 import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
+import static org.elasticsearch.xpack.XPackPlugin.resolveXPackExtensionsFile;
 
 /**
  * A command for the extension cli to install an extension into x-pack.
@@ -49,22 +51,20 @@ import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
  *     <li>Jar hell does not exist, either between the extension's own jars or with the parent classloader (elasticsearch + x-pack)</li>
  * </ul>
  */
-class InstallXPackExtensionCommand extends Command {
+class InstallXPackExtensionCommand extends SettingCommand {
 
-    private final Environment env;
     private final OptionSpec<Void> batchOption;
     private final OptionSpec<String> arguments;
 
-    InstallXPackExtensionCommand(Environment env) {
+    InstallXPackExtensionCommand() {
         super("Install a plugin");
-        this.env = env;
         this.batchOption = parser.acceptsAll(Arrays.asList("b", "batch"),
                 "Enable batch mode explicitly, automatic confirmation of security permission");
         this.arguments = parser.nonOptions("plugin id");
     }
 
     @Override
-    protected void execute(Terminal terminal, OptionSet options) throws Exception {
+    protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
         // TODO: in jopt-simple 5.0 we can enforce a min/max number of positional args
         List<String> args = arguments.values(options);
         if (args.size() != 1) {
@@ -72,12 +72,13 @@ class InstallXPackExtensionCommand extends Command {
         }
         String extensionURL = args.get(0);
         boolean isBatch = options.has(batchOption) || System.console() == null;
-        execute(terminal, extensionURL, isBatch);
+        execute(terminal, extensionURL, isBatch, settings);
     }
 
 
     // pkg private for testing
-    void execute(Terminal terminal, String extensionId, boolean isBatch) throws Exception {
+    void execute(Terminal terminal, String extensionId, boolean isBatch, Map<String, String> properties) throws Exception {
+        Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, terminal, properties);
         if (Files.exists(resolveXPackExtensionsFile(env)) == false) {
             terminal.println("xpack extensions directory [" + resolveXPackExtensionsFile(env) + "] does not exist. Creating...");
             Files.createDirectories(resolveXPackExtensionsFile(env));

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/ListXPackExtensionCommand.java

@@ -6,32 +6,33 @@
 package org.elasticsearch.xpack.extensions;
 
 import joptsimple.OptionSet;
-
+import org.elasticsearch.cli.SettingCommand;
-import org.elasticsearch.cli.Command;
 import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.internal.InternalSettingsPreparer;
 
 import java.io.IOException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Map;
 
-import static org.elasticsearch.xpack.XPackPlugin.resolveXPackExtensionsFile;
 import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
+import static org.elasticsearch.xpack.XPackPlugin.resolveXPackExtensionsFile;
 
 /**
  * A command for the extension cli to list extensions installed in x-pack.
  */
-class ListXPackExtensionCommand extends Command {
+class ListXPackExtensionCommand extends SettingCommand {
-    private final Environment env;
 
-    ListXPackExtensionCommand(Environment env) {
+    ListXPackExtensionCommand() {
         super("Lists installed x-pack extensions");
-        this.env = env;
     }
 
     @Override
-    protected void execute(Terminal terminal, OptionSet options) throws Exception {
+    protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
+        Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, terminal, settings);
         if (Files.exists(resolveXPackExtensionsFile(env)) == false) {
             throw new IOException("Extensions directory missing: " + resolveXPackExtensionsFile(env));
         }
@@ -43,4 +44,5 @@ class ListXPackExtensionCommand extends Command {
             }
         }
     }
+
 }

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/RemoveXPackExtensionCommand.java

@@ -7,51 +7,53 @@ package org.elasticsearch.xpack.extensions;
 
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
-
 import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.cli.Command;
 import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.SettingCommand;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserError;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.internal.InternalSettingsPreparer;
 
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
-import static org.elasticsearch.xpack.XPackPlugin.resolveXPackExtensionsFile;
 import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
+import static org.elasticsearch.xpack.XPackPlugin.resolveXPackExtensionsFile;
 
 /**
  * A command for the extension cli to remove an extension from x-pack.
  */
-class RemoveXPackExtensionCommand  extends Command {
+class RemoveXPackExtensionCommand  extends SettingCommand {
-    private final Environment env;
     private final OptionSpec<String> arguments;
 
-    RemoveXPackExtensionCommand(Environment env) {
+    RemoveXPackExtensionCommand() {
         super("Removes an extension from x-pack");
-        this.env = env;
         this.arguments = parser.nonOptions("extension name");
     }
 
     @Override
-    protected void execute(Terminal terminal, OptionSet options) throws Exception {
+    protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
+
         // TODO: in jopt-simple 5.0 we can enforce a min/max number of positional args
         List<String> args = arguments.values(options);
         if (args.size() != 1) {
             throw new UserError(ExitCodes.USAGE, "Must supply a single extension id argument");
         }
-        execute(terminal, args.get(0));
+        execute(terminal, args.get(0), settings);
     }
 
     // pkg private for testing
-    void execute(Terminal terminal, String extensionName) throws Exception {
+    void execute(Terminal terminal, String extensionName, Map<String, String> settings) throws Exception {
         terminal.println("-> Removing " + Strings.coalesceToEmpty(extensionName) + "...");
 
+        Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, terminal, settings);
         Path extensionDir = resolveXPackExtensionsFile(env).resolve(extensionName);
         if (Files.exists(extensionDir) == false) {
             throw new UserError(ExitCodes.USAGE,

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/extensions/XPackExtensionCli.java

@@ -9,25 +9,22 @@ import org.apache.log4j.BasicConfigurator;
 import org.apache.log4j.varia.NullAppender;
 import org.elasticsearch.cli.MultiCommand;
 import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
 
 /**
  * A cli tool for adding, removing and listing extensions for x-pack.
  */
 public class XPackExtensionCli extends MultiCommand {
 
-    public XPackExtensionCli(Environment env) {
+    public XPackExtensionCli() {
         super("A tool for managing installed x-pack extensions");
-        subcommands.put("list", new ListXPackExtensionCommand(env));
+        subcommands.put("list", new ListXPackExtensionCommand());
-        subcommands.put("install", new InstallXPackExtensionCommand(env));
+        subcommands.put("install", new InstallXPackExtensionCommand());
-        subcommands.put("remove", new RemoveXPackExtensionCommand(env));
+        subcommands.put("remove", new RemoveXPackExtensionCommand());
     }
 
     public static void main(String[] args) throws Exception {
         BasicConfigurator.configure(new NullAppender());
-        Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, Terminal.DEFAULT);
+        exit(new XPackExtensionCli().main(args, Terminal.DEFAULT));
-        exit(new XPackExtensionCli(env).main(args, Terminal.DEFAULT));
     }
+
 }

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/rest/action/RestXPackUsageAction.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.rest.action;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.action.support.RestBuilderListener;
+import org.elasticsearch.xpack.XPackClient;
+import org.elasticsearch.xpack.XPackFeatureSet;
+import org.elasticsearch.xpack.action.XPackUsageRequestBuilder;
+import org.elasticsearch.xpack.action.XPackUsageResponse;
+import org.elasticsearch.xpack.rest.XPackRestHandler;
+
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.rest.RestStatus.OK;
+
+public class RestXPackUsageAction extends XPackRestHandler {
+
+    @Inject
+    public RestXPackUsageAction(Settings settings, RestController controller, Client client) {
+        super(settings, client);
+        controller.registerHandler(GET, URI_BASE + "/usage", this);
+    }
+
+    @Override
+    protected void handleRequest(RestRequest request, RestChannel restChannel, XPackClient client) throws Exception {
+        new XPackUsageRequestBuilder(client.es()).execute(new RestBuilderListener<XPackUsageResponse>(restChannel) {
+            @Override
+            public RestResponse buildResponse(XPackUsageResponse response, XContentBuilder builder) throws Exception {
+                builder.startObject();
+                for (XPackFeatureSet.Usage usage : response.getUsages()) {
+                    builder.field(usage.name(), usage);
+                }
+                builder.endObject();
+                return new BytesRestResponse(OK, builder);
+            }
+        });
+    }
+}

elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/InstallXPackExtensionCommandTests.java

@@ -12,34 +12,37 @@ import org.elasticsearch.cli.UserError;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.MalformedURLException;
 import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.Path;
 import java.nio.file.DirectoryStream;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.StandardCopyOption;
-import java.nio.file.FileVisitResult;
-import java.nio.file.NoSuchFileException;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
 @LuceneTestCase.SuppressFileSystems("*")
 public class InstallXPackExtensionCommandTests extends ESTestCase {
-    /**
+
-     * Creates a test environment with plugins and xpack extensions directories.
+    Path home;
-     */
+    Environment env;
-    static Environment createEnv() throws IOException {
+
-        Path home = createTempDir();
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        home = createTempDir();
         Files.createDirectories(home.resolve("org/elasticsearch/xpack/extensions").resolve("xpack").resolve("extensions"));
-        Settings settings = Settings.builder()
+        env = new Environment(Settings.builder().put("path.home", home.toString()).build());
-                .put("path.home", home)
-                .build();
-        return new Environment(settings);
     }
 
     /**
@@ -84,9 +87,11 @@ public class InstallXPackExtensionCommandTests extends ESTestCase {
         return writeZip(structure);
     }
 
-    static MockTerminal installExtension(String extensionUrl, Environment env) throws Exception {
+    static MockTerminal installExtension(String extensionUrl, Path home) throws Exception {
+        Map<String, String> settings = new HashMap<>();
+        settings.put("path.home", home.toString());
         MockTerminal terminal = new MockTerminal();
-        new InstallXPackExtensionCommand(env).execute(terminal, extensionUrl, true);
+        new InstallXPackExtensionCommand().execute(terminal, extensionUrl, true, settings);
         return terminal;
     }
 
@@ -108,77 +113,65 @@ public class InstallXPackExtensionCommandTests extends ESTestCase {
     }
 
     public void testSomethingWorks() throws Exception {
-        Environment env = createEnv();
         Path extDir = createTempDir();
         String extZip = createExtension("fake", extDir);
-        installExtension(extZip, env);
+        installExtension(extZip, home);
         assertExtension("fake", extDir, env);
     }
 
     public void testSpaceInUrl() throws Exception {
-        Environment env = createEnv();
         Path extDir = createTempDir();
         String extZip = createExtension("fake", extDir);
         Path extZipWithSpaces = createTempFile("foo bar", ".zip");
         try (InputStream in = new URL(extZip).openStream()) {
             Files.copy(in, extZipWithSpaces, StandardCopyOption.REPLACE_EXISTING);
         }
-        installExtension(extZipWithSpaces.toUri().toURL().toString(), env);
+        installExtension(extZipWithSpaces.toUri().toURL().toString(), home);
         assertExtension("fake", extDir, env);
     }
 
     public void testMalformedUrlNotMaven() throws Exception {
         // has two colons, so it appears similar to maven coordinates
         MalformedURLException e = expectThrows(MalformedURLException.class, () -> {
-            installExtension("://host:1234", createEnv());
+            installExtension("://host:1234", home);
         });
         assertTrue(e.getMessage(), e.getMessage().contains("no protocol"));
     }
 
     public void testJarHell() throws Exception {
-        Environment env = createEnv();
         Path extDir = createTempDir();
         writeJar(extDir.resolve("other.jar"), "FakeExtension");
         String extZip = createExtension("fake", extDir); // adds extension.jar with FakeExtension
-        IllegalStateException e = expectThrows(IllegalStateException.class, () -> {
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> installExtension(extZip, home));
-            installExtension(extZip, env);
-        });
         assertTrue(e.getMessage(), e.getMessage().contains("jar hell"));
         assertInstallCleaned(env);
     }
 
     public void testIsolatedExtension() throws Exception {
-        Environment env = createEnv();
         // these both share the same FakeExtension class
         Path extDir1 = createTempDir();
         String extZip1 = createExtension("fake1", extDir1);
-        installExtension(extZip1, env);
+        installExtension(extZip1, home);
         Path extDir2 = createTempDir();
         String extZip2 = createExtension("fake2", extDir2);
-        installExtension(extZip2, env);
+        installExtension(extZip2, home);
         assertExtension("fake1", extDir1, env);
         assertExtension("fake2", extDir2, env);
     }
 
     public void testExistingExtension() throws Exception {
-        Environment env = createEnv();
         String extZip = createExtension("fake", createTempDir());
-        installExtension(extZip, env);
+        installExtension(extZip, home);
-        UserError e = expectThrows(UserError.class, () -> {
+        UserError e = expectThrows(UserError.class, () -> installExtension(extZip, home));
-            installExtension(extZip, env);
-        });
         assertTrue(e.getMessage(), e.getMessage().contains("already exists"));
         assertInstallCleaned(env);
     }
 
     public void testMissingDescriptor() throws Exception {
-        Environment env = createEnv();
         Path extDir = createTempDir();
         Files.createFile(extDir.resolve("fake.yml"));
         String extZip = writeZip(extDir);
-        NoSuchFileException e = expectThrows(NoSuchFileException.class, () -> {
+        NoSuchFileException e = expectThrows(NoSuchFileException.class, () -> installExtension(extZip, home));
-            installExtension(extZip, env);
-        });
         assertTrue(e.getMessage(), e.getMessage().contains("x-pack-extension-descriptor.properties"));
         assertInstallCleaned(env);
     }

elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/ListXPackExtensionCommandTests.java

@@ -11,6 +11,7 @@ import org.elasticsearch.cli.MockTerminal;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -19,60 +20,55 @@ import java.nio.file.Path;
 @LuceneTestCase.SuppressFileSystems("*")
 public class ListXPackExtensionCommandTests extends ESTestCase {
 
-    Environment createEnv() throws IOException {
+    private Path home;
-        Path home = createTempDir();
+
-        Settings settings = Settings.builder()
+    @Before
-            .put("path.home", home)
+    public void setUp() throws Exception {
-            .build();
+        super.setUp();
-        return new Environment(settings);
+        home = createTempDir();
     }
 
-    Path createExtensionDir(Environment env) throws IOException {
+    private static Path createExtensionDir(final Path home) throws IOException {
-        Path path = env.pluginsFile().resolve("x-pack").resolve("extensions");
+        final Environment env = new Environment(Settings.builder().put("path.home", home.toString()).build());
+        final Path path = env.pluginsFile().resolve("x-pack").resolve("extensions");
         return Files.createDirectories(path);
     }
 
-    static MockTerminal listExtensions(Environment env) throws Exception {
+    static MockTerminal listExtensions(Path home) throws Exception {
         MockTerminal terminal = new MockTerminal();
-        String[] args = {};
+        int status = new ListXPackExtensionCommand().main(new String[] { "-Epath.home=" + home }, terminal);
-        int status = new ListXPackExtensionCommand(env).main(args, terminal);
         assertEquals(ExitCodes.OK, status);
         return terminal;
     }
 
     public void testExtensionsDirMissing() throws Exception {
-        Environment env = createEnv();
+        Path extDir = createExtensionDir(home);
-        Path extDir = createExtensionDir(env);
         Files.delete(extDir);
-        IOException e = expectThrows(IOException.class, () -> {
+        IOException e = expectThrows(IOException.class, () -> listExtensions(home));
-           listExtensions(env);
-        });
         assertTrue(e.getMessage(), e.getMessage().contains("Extensions directory missing"));
     }
 
     public void testNoExtensions() throws Exception {
-        Environment env = createEnv();
+        createExtensionDir(home);
-        createExtensionDir(env);
+        MockTerminal terminal = listExtensions(home);
-        MockTerminal terminal = listExtensions(env);
         assertTrue(terminal.getOutput(), terminal.getOutput().isEmpty());
     }
 
     public void testOneExtension() throws Exception {
-        Environment env = createEnv();
+        Path extDir = createExtensionDir(home);
-        Path extDir = createExtensionDir(env);
         Files.createDirectory(extDir.resolve("fake"));
-        MockTerminal terminal = listExtensions(env);
+        MockTerminal terminal = listExtensions(home);
         assertTrue(terminal.getOutput(), terminal.getOutput().contains("fake"));
     }
 
     public void testTwoExtensions() throws Exception {
-        Environment env = createEnv();
+        Path extDir = createExtensionDir(home);
-        Path extDir = createExtensionDir(env);
         Files.createDirectory(extDir.resolve("fake1"));
         Files.createDirectory(extDir.resolve("fake2"));
-        MockTerminal terminal = listExtensions(env);
+        MockTerminal terminal = listExtensions(home);
         String output = terminal.getOutput();
         assertTrue(output, output.contains("fake1"));
         assertTrue(output, output.contains("fake2"));
     }
+
 }

elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/extensions/RemoveXPackExtensionCommandTests.java

@@ -11,22 +11,26 @@ import org.elasticsearch.cli.UserError;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
 
 @LuceneTestCase.SuppressFileSystems("*")
 public class RemoveXPackExtensionCommandTests extends ESTestCase {
 
-    /** Creates a test environment with bin, config and plugins directories. */
+    private Path home;
-    static Environment createEnv() throws IOException {
+    private Environment env;
-        Path home = createTempDir();
+
-        Settings settings = Settings.builder()
+    @Before
-            .put("path.home", home)
+    public void setUp() throws Exception {
-            .build();
+        super.setUp();
-        return new Environment(settings);
+        home = createTempDir();
+        env = new Environment(Settings.builder().put("path.home", home.toString()).build());
     }
 
     Path createExtensionDir(Environment env) throws IOException {
@@ -34,9 +38,11 @@ public class RemoveXPackExtensionCommandTests extends ESTestCase {
         return Files.createDirectories(path);
     }
 
-    static MockTerminal removeExtension(String name, Environment env) throws Exception {
+    static MockTerminal removeExtension(String name, Path home) throws Exception {
+        Map<String, String> settings = new HashMap<>();
+        settings.put("path.home", home.toString());
         MockTerminal terminal = new MockTerminal();
-        new RemoveXPackExtensionCommand(env).execute(terminal, name);
+        new RemoveXPackExtensionCommand().execute(terminal, name, settings);
         return terminal;
     }
 
@@ -51,25 +57,22 @@ public class RemoveXPackExtensionCommandTests extends ESTestCase {
     }
 
     public void testMissing() throws Exception {
-        Environment env = createEnv();
         Path extDir = createExtensionDir(env);
-        UserError e = expectThrows(UserError.class, () -> {
+        UserError e = expectThrows(UserError.class, () -> removeExtension("dne", home));
-           removeExtension("dne", env);
-        });
         assertTrue(e.getMessage(), e.getMessage().contains("Extension dne not found"));
         assertRemoveCleaned(extDir);
     }
 
     public void testBasic() throws Exception {
-        Environment env = createEnv();
         Path extDir = createExtensionDir(env);
         Files.createDirectory(extDir.resolve("fake"));
         Files.createFile(extDir.resolve("fake").resolve("extension.jar"));
         Files.createDirectory(extDir.resolve("fake").resolve("subdir"));
         Files.createDirectory(extDir.resolve("other"));
-        removeExtension("fake", env);
+        removeExtension("fake", home);
         assertFalse(Files.exists(extDir.resolve("fake")));
         assertTrue(Files.exists(extDir.resolve("other")));
         assertRemoveCleaned(extDir);
     }
+
 }

elasticsearch/x-pack/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestCase.java

@@ -10,6 +10,8 @@ import java.io.InputStreamReader;
 import java.net.URI;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
@@ -21,18 +23,20 @@ import org.apache.http.client.methods.HttpPut;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
-import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.license.plugin.TestUtils;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.RestTestCandidate;
 import org.elasticsearch.test.rest.parser.RestTestParseException;
-import org.elasticsearch.xpack.common.xcontent.XContentUtils;
 import org.junit.After;
 import org.junit.Before;
 
@@ -82,6 +86,19 @@ public abstract class XPackRestTestCase extends ESRestTestCase {
         }
     }
 
+    @Before
+    public void installLicense() throws Exception {
+        final XContentBuilder builder = XContentFactory.jsonBuilder();
+        TestUtils.generateSignedLicense("trial", TimeValue.timeValueHours(2)).toXContent(builder, ToXContent.EMPTY_PARAMS);
+        final BytesReference bytes = builder.bytes();
+        try (XContentParser parser = XContentFactory.xContent(bytes).createParser(bytes)) {
+            final List<Map<String, Object>> bodies = Collections.singletonList(Collections.singletonMap("license",
+                    parser.map()));
+            getAdminExecutionContext().callApi("license.post", Collections.singletonMap("acknowledge", "true"),
+                    bodies, Collections.singletonMap("Authorization", BASIC_AUTH_VALUE));
+        }
+    }
+
     @After
     public void clearShieldUsersAndRoles() throws Exception {
         // we cannot delete the .security index from a rest test since we aren't the internal user, lets wipe the data

elasticsearch/x-pack/src/test/resources/rest-api-spec/api/xpack.usage.json

@@ -0,0 +1,14 @@
+{
+  "xpack.info": {
+    "documentation": "Retrieve information about xpack features usage",
+    "methods": [ "GET" ],
+    "url": {
+      "path": "/_xpack/usage",
+      "paths": [ "/_xpack/usage" ],
+      "parts": {},
+      "params": {
+      }
+    },
+    "body": null
+  }
+}

elasticsearch/x-pack/watcher/bin/x-pack/croneval

@@ -70,29 +70,6 @@ elif [ -f "/etc/default/elasticsearch" ]; then
    . "/etc/default/elasticsearch"
 fi
 
-# Parse any long getopt options and put them into properties
-ARGCOUNT=$#
-COUNT=0
-while [ $COUNT -lt $ARGCOUNT ]
-do
-    case $1 in
-      --*) properties="$properties $1 $2"
-           shift ; shift; COUNT=$(($COUNT+2))
-           ;;
-      *) set -- "$@" "$1"; shift; COUNT=$(($COUNT+1))
-    esac
-done
-
-# check if properties already has a config file or config dir
-if [ -e "$CONF_DIR" ]; then
-    case "$properties" in
-        *-Des.default.path.conf=*) ;;
-        *)
-            properties="$properties -Des.default.path.conf=$CONF_DIR"
-            ;;
-    esac
-fi
-
 export HOSTNAME=`hostname -s`
 
 # include watcher jars in classpath
@@ -112,8 +89,14 @@ if [ ! -z "$CONF_FILE" ]; then
     exit 1
 fi
 
+declare -a args=("$@")
+
+if [ -e "$CONF_DIR" ]; then
+  args=("${args[@]}" -Edefault.path.conf="$CONF_DIR")
+fi
+
 cd "$ES_HOME" > /dev/null
-"$JAVA" $ES_JAVA_OPTS -cp "$ES_CLASSPATH" -Des.path.home="$ES_HOME" org.elasticsearch.xpack.watcher.trigger.schedule.tool.CronEvalTool "$@" $properties
+"$JAVA" $ES_JAVA_OPTS -cp "$ES_CLASSPATH" -Des.path.home="$ES_HOME" org.elasticsearch.xpack.watcher.trigger.schedule.tool.CronEvalTool "${args[@]}"
 status=$?
 cd - > /dev/null
 exit $status

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherFeatureSet.java

@@ -7,9 +7,13 @@ package org.elasticsearch.xpack.watcher;
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.xpack.XPackFeatureSet;
 
+import java.io.IOException;
+
 /**
  *
  */
@@ -19,9 +23,10 @@ public class WatcherFeatureSet implements XPackFeatureSet {
     private final WatcherLicensee licensee;
 
     @Inject
-    public WatcherFeatureSet(Settings settings, @Nullable WatcherLicensee licensee) {
+    public WatcherFeatureSet(Settings settings, @Nullable WatcherLicensee licensee, NamedWriteableRegistry namedWriteableRegistry) {
         this.enabled = Watcher.enabled(settings);
         this.licensee = licensee;
+        namedWriteableRegistry.register(Usage.class, Usage.writeableName(Watcher.NAME), Usage::new);
     }
 
     @Override
@@ -36,11 +41,28 @@ public class WatcherFeatureSet implements XPackFeatureSet {
 
     @Override
     public boolean available() {
-        return licensee != null && licensee.available();
+        return licensee != null && licensee.isAvailable();
     }
 
     @Override
     public boolean enabled() {
         return enabled;
     }
+
+    @Override
+    public XPackFeatureSet.Usage usage() {
+        return new Usage(available(), enabled());
+    }
+
+    static class Usage extends XPackFeatureSet.Usage {
+
+        public Usage(StreamInput input) throws IOException {
+            super(input);
+        }
+
+        public Usage(boolean available, boolean enabled) {
+            super(Watcher.NAME, available, enabled);
+        }
+
+    }
 }

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherLicensee.java

@@ -62,12 +62,22 @@ public class WatcherLicensee extends AbstractLicenseeComponent<WatcherLicensee>
      *
      * @return {@code true} as long as the license is valid. Otherwise {@code false}.
      */
-    public boolean available() {
+    public boolean isAvailable() {
         // status is volatile, so a local variable is used for a consistent view
         Status localStatus = status;
 
-        return localStatus.getLicenseState() != LicenseState.DISABLED && (localStatus.getMode() == OperationMode.TRIAL ||
+        if (localStatus.getLicenseState() == LicenseState.DISABLED) {
-                localStatus.getMode() == OperationMode.GOLD || localStatus.getMode() == OperationMode.PLATINUM);
+            return false;
+        }
+
+        switch (localStatus.getMode()) {
+            case TRIAL:
+            case GOLD:
+            case PLATINUM:
+                return true;
+            default:
+                return false;
+        }
     }
 
     public boolean isExecutingActionsAllowed() {
@@ -84,7 +94,7 @@ public class WatcherLicensee extends AbstractLicenseeComponent<WatcherLicensee>
 
 
     public boolean isWatcherTransportActionAllowed() {
-        return available();
+        return isAvailable();
     }
 
 

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/WatcherService.java

@@ -64,7 +64,7 @@ public class WatcherService extends AbstractComponent {
     public void start(ClusterState clusterState) throws Exception {
         if (state.compareAndSet(WatcherState.STOPPED, WatcherState.STARTING)) {
             try {
-                logger.info("starting watch service...");
+                logger.debug("starting watch service...");
                 watcherIndexTemplateRegistry.addTemplatesIfMissing();
                 watchLockService.start();
 
@@ -74,7 +74,7 @@ public class WatcherService extends AbstractComponent {
 
                 triggerService.start(watchStore.activeWatches());
                 state.set(WatcherState.STARTED);
-                logger.info("watch service has started");
+                logger.debug("watch service has started");
             } catch (Exception e) {
                 state.set(WatcherState.STOPPED);
                 throw e;
@@ -90,7 +90,7 @@ public class WatcherService extends AbstractComponent {
 
     public void stop() {
         if (state.compareAndSet(WatcherState.STARTED, WatcherState.STOPPING)) {
-            logger.info("stopping watch service...");
+            logger.debug("stopping watch service...");
             triggerService.stop();
             executionService.stop();
             try {
@@ -100,7 +100,7 @@ public class WatcherService extends AbstractComponent {
             }
             watchStore.stop();
             state.set(WatcherState.STOPPED);
-            logger.info("watch service has stopped");
+            logger.debug("watch service has stopped");
         } else {
             logger.debug("not stopping watcher, because its state is [{}] while [{}] is expected", state, WatcherState.STARTED);
         }

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/xcontent/XContentSource.java

@@ -21,6 +21,8 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
 /**
  * Encapsulates the xcontent source
  */
@@ -48,6 +50,10 @@ public class XContentSource implements ToXContent {
         this(builder.bytes(), builder.contentType());
     }
 
+    public XContentSource(ToXContent content) throws IOException {
+        this(content.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS));
+    }
+
     /**
      * @return The bytes reference of the source
      */

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/xpack/watcher/trigger/schedule/ScheduleModule.java

@@ -36,7 +36,7 @@ public class ScheduleModule extends AbstractModule {
 
     public static Class<? extends TriggerEngine> triggerEngineType(Settings nodeSettings) {
         Engine engine = Engine.resolve(nodeSettings);
-        Loggers.getLogger(ScheduleModule.class, nodeSettings).info("using [{}] schedule trigger engine",
+        Loggers.getLogger(ScheduleModule.class, nodeSettings).debug("using [{}] schedule trigger engine",
                 engine.name().toLowerCase(Locale.ROOT));
         return engine.engineType();
     }

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/script/MockMustacheScriptEngine.java

@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.script;
 
-import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.xpack.common.text.DefaultTextTemplateEngine;
 
 import java.util.Collections;
@@ -29,8 +28,7 @@ public class MockMustacheScriptEngine extends MockScriptEngine {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MockMustacheScriptEngine.class,
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MockMustacheScriptEngine.class, NAME, true));
-                            NAME, ScriptMode.ON));
         }
 
     }

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/script/SleepScriptEngine.java

@@ -7,7 +7,6 @@ package org.elasticsearch.script;
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
@@ -39,7 +38,7 @@ public class SleepScriptEngine implements ScriptEngineService {
 
         public void onModule(ScriptModule module) {
             module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(SleepScriptEngine.class,
-                            SleepScriptEngine.NAME, ScriptMode.ON));
+                            SleepScriptEngine.NAME, true));
         }
 
     }

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/xpack/watcher/WatcherFeatureSetTests.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.watcher;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import static org.hamcrest.core.Is.is;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ *
+ */
+public class WatcherFeatureSetTests extends ESTestCase {
+
+    private WatcherLicensee licensee;
+    private NamedWriteableRegistry namedWriteableRegistry;
+
+    @Before
+    public void init() throws Exception {
+        licensee = mock(WatcherLicensee.class);
+        namedWriteableRegistry = mock(NamedWriteableRegistry.class);
+    }
+
+    public void testWritableRegistration() throws Exception {
+        new WatcherFeatureSet(Settings.EMPTY, licensee, namedWriteableRegistry);
+        verify(namedWriteableRegistry).register(eq(WatcherFeatureSet.Usage.class), eq("xpack.usage.watcher"), anyObject());
+    }
+
+    public void testAvailable() throws Exception {
+        WatcherFeatureSet featureSet = new WatcherFeatureSet(Settings.EMPTY, licensee, namedWriteableRegistry);
+        boolean available = randomBoolean();
+        when(licensee.isAvailable()).thenReturn(available);
+        assertThat(featureSet.available(), is(available));
+    }
+
+    public void testEnabled() throws Exception {
+        boolean enabled = randomBoolean();
+        Settings.Builder settings = Settings.builder();
+        if (enabled) {
+            if (randomBoolean()) {
+                settings.put("xpack.watcher.enabled", enabled);
+            }
+        } else {
+            settings.put("xpack.watcher.enabled", enabled);
+        }
+        WatcherFeatureSet featureSet = new WatcherFeatureSet(settings.build(), licensee, namedWriteableRegistry);
+        assertThat(featureSet.enabled(), is(enabled));
+    }
+
+}