Merge branch 'master' into kibana-shield-user-role-ui

Original commit: elastic/x-pack-elasticsearch@631196d5b1

elasticsearch/qa/messy-test-watcher-with-groovy/src/test/java/org/elasticsearch/messy/tests/HistoryTemplateTransformMappingsTests.java

@@ -98,7 +98,7 @@ public class HistoryTemplateTransformMappingsTests extends AbstractWatcherIntegr
                 assertThat(mappingsResponse, notNullValue());
                 assertThat(mappingsResponse.getMappings().isEmpty(), is(false));
                 for (ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetaData>> metadatas : mappingsResponse.getMappings()) {
-                    if (!metadatas.key.startsWith(".watch_history")) {
+                    if (!metadatas.key.startsWith(".watcher-history")) {
                         continue;
                     }
                     MappingMetaData metadata = metadatas.value.get("watch_record");

elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/NoMasterNodeTests.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.watcher.test.integration;
+package org.elasticsearch.messy.tests;
 
 import org.apache.lucene.util.LuceneTestCase.BadApple;
 import org.elasticsearch.ExceptionsHelper;
@@ -18,6 +18,9 @@ import org.elasticsearch.discovery.zen.elect.ElectMasterService;
 import org.elasticsearch.discovery.zen.ping.ZenPing;
 import org.elasticsearch.discovery.zen.ping.ZenPingService;
 import org.elasticsearch.discovery.zen.ping.unicast.UnicastZenPing;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.MockMustacheScriptEngine;
+import org.elasticsearch.script.mustache.MustachePlugin;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.SuppressLocalMode;
 import org.elasticsearch.test.discovery.ClusterDiscoveryConfiguration;
@@ -33,6 +36,10 @@ import org.elasticsearch.watcher.test.WatcherTestUtils;
 import org.elasticsearch.watcher.transport.actions.delete.DeleteWatchResponse;
 import org.elasticsearch.watcher.transport.actions.stats.WatcherStatsResponse;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
@@ -81,6 +88,23 @@ public class NoMasterNodeTests extends AbstractWatcherIntegrationTestCase {
                 .build();
     }
 
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        Collection<Class<? extends Plugin>> types = new ArrayList<>();
+        types.addAll(super.nodePlugins());
+        // TODO remove dependency on mustache
+        types.add(MustachePlugin.class);
+        return types;
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getMockPlugins() {
+        Set<Class<? extends Plugin>> plugins = new HashSet<>(super.getMockPlugins());
+        // remove the mock because we use mustache here...
+        plugins.remove(MockMustacheScriptEngine.TestPlugin.class);
+        return plugins;
+    }
+
     public void testSimpleFailure() throws Exception {
         // we need 3 hosts here because we stop the master and start another - it doesn't restart the pre-existing node...
         config = new ClusterDiscoveryConfiguration.UnicastZen(3, Settings.EMPTY);

elasticsearch/qa/messy-test-xpack-with-mustache/src/test/java/org/elasticsearch/messy/tests/WatchAckTests.java

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-package org.elasticsearch.watcher.transport.action.ack;
+package org.elasticsearch.messy.tests;
 
 
 import org.apache.lucene.util.LuceneTestCase.BadApple;
@@ -13,6 +13,9 @@ import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.MockMustacheScriptEngine;
+import org.elasticsearch.script.mustache.MustachePlugin;
 import org.elasticsearch.watcher.actions.ActionStatus;
 import org.elasticsearch.watcher.client.WatcherClient;
 import org.elasticsearch.watcher.condition.compare.CompareCondition;
@@ -29,6 +32,10 @@ import org.elasticsearch.watcher.watch.Watch;
 import org.elasticsearch.watcher.watch.WatchStore;
 import org.hamcrest.Matchers;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@@ -52,6 +59,24 @@ import static org.hamcrest.core.IsEqual.equalTo;
 //test is just too slow, please fix it to not be sleep-based
 @BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007")
 public class WatchAckTests extends AbstractWatcherIntegrationTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        Collection<Class<? extends Plugin>> types = new ArrayList<>();
+        types.addAll(super.nodePlugins());
+        // TODO remove dependency on mustache
+        types.add(MustachePlugin.class);
+        return types;
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getMockPlugins() {
+        Set<Class<? extends Plugin>> plugins = new HashSet<>(super.getMockPlugins());
+        // remove the mock because we use mustache here...
+        plugins.remove(MockMustacheScriptEngine.TestPlugin.class);
+        return plugins;
+    }
+
     private IndexResponse indexTestDoc() {
         createIndex("actions", "events");
         ensureGreen("actions", "events");

elasticsearch/qa/shield-audit-tests/build.gradle

@@ -10,7 +10,7 @@ integTest {
     systemProperty 'es.shield.audit.enabled', 'true'
     systemProperty 'es.shield.audit.outputs', 'index'
     setupCommand 'setupDummyUser',
-                 'bin/x-pack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
+                 'bin/xpack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
     waitCondition = { node, ant ->
       File tmpFile = new File(node.cwd, 'wait.success')
       ant.get(src: "http://${node.httpUri()}",

elasticsearch/qa/shield-audit-tests/src/test/java/org/elasticsearch/shield/audit/IndexAuditIT.java

@@ -5,10 +5,14 @@
  */
 package org.elasticsearch.shield.audit;
 
+import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse;
+import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.shield.audit.index.IndexAuditTrail;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -49,6 +53,35 @@ public class IndexAuditIT extends ESIntegTestCase {
         assertThat((String) searchResponse.getHits().getAt(0).sourceAsMap().get("principal"), is(USER));
     }
 
+    public void testAuditTrailTemplateIsRecreatedAfterDelete() throws Exception {
+        // this is already "tested" by the test framework since we wipe the templates before and after,
+        // but lets be explicit about the behavior
+        awaitIndexTemplateCreation();
+
+        // delete the template
+        DeleteIndexTemplateResponse deleteResponse = client().admin().indices()
+                .prepareDeleteTemplate(IndexAuditTrail.INDEX_TEMPLATE_NAME).execute().actionGet();
+        assertThat(deleteResponse.isAcknowledged(), is(true));
+        awaitIndexTemplateCreation();
+    }
+
+    private void awaitIndexTemplateCreation() throws InterruptedException {
+        boolean found = awaitBusy(() -> {
+            GetIndexTemplatesResponse response = client().admin().indices()
+                    .prepareGetTemplates(IndexAuditTrail.INDEX_TEMPLATE_NAME).execute().actionGet();
+            if (response.getIndexTemplates().size() > 0) {
+                for (IndexTemplateMetaData indexTemplateMetaData : response.getIndexTemplates()) {
+                    if (IndexAuditTrail.INDEX_TEMPLATE_NAME.equals(indexTemplateMetaData.name())) {
+                        return true;
+                    }
+                }
+            }
+            return false;
+        });
+
+        assertThat("index template [" + IndexAuditTrail.INDEX_TEMPLATE_NAME + "] was not created", found, is(true));
+    }
+
     @Override
     protected Settings externalClusterClientSettings() {
         return Settings.builder()

elasticsearch/qa/shield-client-tests/build.gradle

@@ -8,9 +8,9 @@ integTest {
   cluster {
     plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack')
     setupCommand 'setupDummyUser',
-                 'bin/x-pack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
+                 'bin/xpack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
     setupCommand 'setupTransportClientUser',
-                 'bin/x-pack/esusers', 'useradd', 'transport', '-p', 'changeme', '-r', 'transport_client'
+                 'bin/xpack/esusers', 'useradd', 'transport', '-p', 'changeme', '-r', 'transport_client'
     waitCondition = { node, ant ->
       File tmpFile = new File(node.cwd, 'wait.success')
       ant.get(src: "http://${node.httpUri()}",

elasticsearch/qa/shield-core-rest-tests/build.gradle

@@ -35,7 +35,7 @@ integTest {
     systemProperty 'es.watcher.enabled', 'false'
     systemProperty 'es.marvel.enabled', 'false'
     setupCommand 'setupDummyUser',
-                 'bin/x-pack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
+                 'bin/xpack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
     waitCondition = { node, ant ->
       File tmpFile = new File(node.cwd, 'wait.success')
       ant.get(src: "http://${node.httpUri()}",

elasticsearch/qa/shield-example-realm/build.gradle

@@ -23,7 +23,7 @@ integTest {
     systemProperty 'es.shield.authc.realms.esusers.type', 'esusers'
 
     setupCommand 'setupDummyUser',
-                 'bin/x-pack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
+                 'bin/xpack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
     waitCondition = { node, ant ->
       File tmpFile = new File(node.cwd, 'wait.success')
       ant.get(src: "http://${node.httpUri()}",

elasticsearch/qa/shield-tribe-node-tests/integration-tests.xml

@@ -75,10 +75,10 @@
         <property name="home" location="${integ.scratch}/elasticsearch-${elasticsearch.version}"/>
 
         <echo>Adding roles.yml</echo>
-        <copy file="shield-roles.yml" tofile="${home}/config/shield/roles.yml" overwrite="true"/>
+        <copy file="shield-roles.yml" tofile="${home}/config/xpack/roles.yml" overwrite="true"/>
 
         <echo>Adding shield users...</echo>
-        <run-script script="${home}/bin/shield/esusers">
+        <run-script script="${home}/bin/xpack/esusers">
             <nested>
                 <arg value="useradd"/>
                 <arg value="test_admin"/>

elasticsearch/qa/smoke-test-found-license-with-shield-and-watcher/integration-tests.xml

@@ -63,10 +63,10 @@
     <property name="home" location="${integ.scratch}/elasticsearch-${elasticsearch.version}"/>
 
     <echo>Adding roles.yml with watcher roles</echo>
-    <copy file="watcher-with-shield-roles.yml" tofile="${home}/config/shield/roles.yml" overwrite="true"/>
+    <copy file="watcher-with-shield-roles.yml" tofile="${home}/config/xpack/roles.yml" overwrite="true"/>
 
     <echo>Adding shield users...</echo>
-    <run-script script="${home}/bin/shield/esusers">
+    <run-script script="${home}/bin/xpack/esusers">
       <nested>
         <arg value="useradd"/>
         <arg value="test_admin"/>
@@ -76,7 +76,7 @@
         <arg value="admin"/>
       </nested>
     </run-script>
-    <run-script script="${home}/bin/shield/esusers">
+    <run-script script="${home}/bin/xpack/esusers">
       <nested>
         <arg value="useradd"/>
         <arg value="watcher_manager"/>
@@ -86,7 +86,7 @@
         <arg value="watcher_manager"/>
       </nested>
     </run-script>
-    <run-script script="${home}/bin/shield/esusers">
+    <run-script script="${home}/bin/xpack/esusers">
       <nested>
         <arg value="useradd"/>
         <arg value="powerless_user"/>

elasticsearch/qa/smoke-test-found-license-with-shield-and-watcher/watcher-with-shield-roles.yml

@@ -6,12 +6,12 @@ admin:
 watcher_manager:
   cluster: manage_watcher, cluster:monitor/nodes/info, cluster:monitor/health
   indices:
-    '.watch_history-*': all
+    '.watcher-history-*': all
 
 watcher_monitor:
   cluster: monitor_watcher
   indices:
-    '.watch_history-*': read
+    '.watcher-history-*': read
 
 crapy_role:
   cluster:

elasticsearch/qa/smoke-test-plugins-ssl/build.gradle

@@ -57,9 +57,9 @@ integTest {
     // copy keystore into config/
     extraConfigFile keystore.name, keystore
     setupCommand 'setupTestUser',
-                 'bin/x-pack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
+                 'bin/xpack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
     setupCommand 'setupMarvelUser',
-                 'bin/x-pack/esusers', 'useradd', 'marvel_export', '-p', 'changeme', '-r', 'marvel_agent'
+                 'bin/xpack/esusers', 'useradd', 'marvel_export', '-p', 'changeme', '-r', 'marvel_agent'
     waitCondition = { node, ant ->
       // we just return true, doing an https check is tricky here
       return true

elasticsearch/qa/smoke-test-plugins/build.gradle

@@ -18,7 +18,7 @@ integTest {
     plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack')
 
     setupCommand 'setupDummyUser',
-                 'bin/x-pack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
+                 'bin/xpack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
     waitCondition = { node, ant ->
       File tmpFile = new File(node.cwd, 'wait.success')
       ant.get(src: "http://${node.httpUri()}",

elasticsearch/qa/smoke-test-watcher-with-mustache/src/test/resources/rest-api-spec/test/watcher_mustache/20_array_access.yaml

@@ -58,5 +58,5 @@
 
   - do:
       search:
-        index: .watch_history-*
+        index: .watcher-history-*
   - match: { hits.hits.0._source.result.actions.0.logging.logged_text: "value1 value2" }

elasticsearch/qa/smoke-test-watcher-with-shield/build.gradle

@@ -20,13 +20,13 @@ integTest {
 
   cluster {
     plugin 'x-pack', project(':x-plugins:elasticsearch:x-pack')
-    extraConfigFile 'x-pack/roles.yml', 'roles.yml'
+    extraConfigFile 'xpack/roles.yml', 'roles.yml'
     setupCommand 'setupTestAdminUser',
-                 'bin/x-pack/esusers', 'useradd', 'test_admin', '-p', 'changeme', '-r', 'admin'
+                 'bin/xpack/esusers', 'useradd', 'test_admin', '-p', 'changeme', '-r', 'admin'
     setupCommand 'setupWatcherManagerUser',
-                 'bin/x-pack/esusers', 'useradd', 'watcher_manager', '-p', 'changeme', '-r', 'watcher_manager'
+                 'bin/xpack/esusers', 'useradd', 'watcher_manager', '-p', 'changeme', '-r', 'watcher_manager'
     setupCommand 'setupPowerlessUser',
-                 'bin/x-pack/esusers', 'useradd', 'powerless_user', '-p', 'changeme', '-r', 'crappy_role'
+                 'bin/xpack/esusers', 'useradd', 'powerless_user', '-p', 'changeme', '-r', 'crappy_role'
     waitCondition = { node, ant ->
       File tmpFile = new File(node.cwd, 'wait.success')
       ant.get(src: "http://${node.httpUri()}",

elasticsearch/qa/smoke-test-watcher-with-shield/roles.yml

@@ -6,13 +6,13 @@ admin:
 watcher_manager:
   cluster: manage_watcher, cluster:monitor/nodes/info, cluster:monitor/health
   indices:
-    '.watch_history-*': all
+    '.watcher-history-*': all
   run_as: powerless_user, watcher_manager
 
 watcher_monitor:
   cluster: monitor_watcher
   indices:
-    '.watch_history-*': read
+    '.watcher-history-*': read
 
 crappy_role:
   cluster:

elasticsearch/x-dev-tools/src/main/resources/ant/shield-overrides.xml

@@ -1,97 +0,0 @@
-<?xml version="1.0"?>
-<project name="shield-overrides"
-         xmlns:ac="antlib:net.sf.antcontrib">
-
-  <!-- redefined to work with auth -->
-  <macrodef name="waitfor-elasticsearch">
-      <attribute name="port"/>
-      <attribute name="timeoutproperty"/>
-    <sequential>
-      <echo>Waiting for elasticsearch to become available on port @{port}...</echo>
-      <waitfor maxwait="30" maxwaitunit="second"
-               checkevery="500" checkeveryunit="millisecond"
-               timeoutproperty="@{timeoutproperty}">
-        <socket server="127.0.0.1" port="@{port}"/>
-      </waitfor>
-    </sequential>
-  </macrodef>
-
-  <target name="start-external-cluster-with-plugins" depends="setup-workspace">
-    <ac:for list="${xplugins.list}" param="xplugin.name">
-      <sequential>
-        <fail message="Expected @{xplugin.name}-${version}.zip as a dependency, but could not be found in ${integ.deps}/plugins}">
-          <condition>
-            <not>
-              <available file="${integ.deps}/plugins/@{xplugin.name}-${elasticsearch.version}.zip" />
-            </not>
-          </condition>
-        </fail>
-        <ac:if>
-          <equals arg1="@{xplugin.name}" arg2="elasticsearch-marvel"/>
-          <ac:then>
-            <property name="marvel.enabled">true</property>
-          </ac:then>
-        </ac:if>
-      </sequential>
-    </ac:for>
-
-    <ac:for param="file">
-      <path>
-        <fileset dir="${integ.deps}/plugins"/>
-      </path>
-      <sequential>
-        <local name="plugin.name"/>
-        <convert-plugin-name file="@{file}" outputproperty="plugin.name"/>
-        <install-plugin name="${plugin.name}" file="@{file}"/>
-      </sequential>
-    </ac:for>
-
-    <local name="home"/>
-    <property name="home" location="${integ.scratch}/elasticsearch-${elasticsearch.version}"/>
-
-    <echo>Setting up Shield auth</echo>
-    <run-script script="${home}/bin/shield/esusers">
-      <nested>
-        <arg value="useradd"/>
-        <arg value="test_user"/>
-        <arg value="-p"/>
-        <arg value="changeme"/>
-        <arg value="-r"/>
-        <arg value="admin"/>
-      </nested>
-    </run-script>
-
-    <ac:if>
-      <equals arg1="${marvel.enabled}" arg2="true"/>
-      <ac:then>
-        <run-script script="${home}/bin/shield/esusers">
-          <nested>
-            <arg value="useradd"/>
-            <arg value="marvel_export"/>
-            <arg value="-p"/>
-            <arg value="changeme"/>
-            <arg value="-r"/>
-            <arg value="marvel_agent"/>
-          </nested>
-        </run-script>
-
-        <startup-elasticsearch>
-          <additional-args>
-            <arg value="-Des.marvel.agent.exporter.es.hosts=http://marvel_export:changeme@localhost:${integ.http.port}"/>
-          </additional-args>
-        </startup-elasticsearch>
-      </ac:then>
-      <ac:else>
-        <startup-elasticsearch/>
-      </ac:else>
-    </ac:if>
-    
-
-    <echo>Checking we can connect with basic auth on port ${integ.http.port}...</echo>
-    <local name="temp.file"/>
-    <tempfile property="temp.file" destdir="${java.io.tmpdir}"/>
-    <get src="http://127.0.0.1:${integ.http.port}" dest="${temp.file}" 
-         username="test_user" password="changeme" verbose="true" retries="10"/>
-  </target>
-
-</project>

elasticsearch/x-dev-tools/src/main/resources/commercial-license-check/elasticsearch_license_header.txt

@@ -1,14 +0,0 @@
-ELASTICSEARCH CONFIDENTIAL
-__________________
-
- [2014] Elasticsearch Incorporated. All Rights Reserved.
-
-NOTICE:  All information contained herein is, and remains
-the property of Elasticsearch Incorporated and its suppliers,
-if any.  The intellectual and technical concepts contained
-herein are proprietary to Elasticsearch Incorporated
-and its suppliers and may be covered by U.S. and Foreign Patents,
-patents in process, and are protected by trade secret or copyright law.
-Dissemination of this information or reproduction of this material
-is strictly forbidden unless prior written permission is obtained
-from Elasticsearch Incorporated.

elasticsearch/x-dev-tools/src/main/resources/commercial-license-check/license_header_definition.xml

@@ -1,13 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<additionalHeaders>
-    <javadoc_style>
-        <firstLine>/*</firstLine>
-        <beforeEachLine> * </beforeEachLine>
-        <endLine> */</endLine>
-        <!--skipLine></skipLine-->
-        <firstLineDetectionPattern>(\s|\t)*/\*.*$</firstLineDetectionPattern>
-        <lastLineDetectionPattern>.*\*/(\s|\t)*$</lastLineDetectionPattern>
-        <allowBlankLines>false</allowBlankLines>
-        <isMultiline>true</isMultiline>
-    </javadoc_style>
-</additionalHeaders>

elasticsearch/x-pack/build.gradle

@@ -3,7 +3,7 @@ import org.elasticsearch.gradle.test.NodeInfo
 
 apply plugin: 'elasticsearch.esplugin'
 esplugin {
-  name 'x-pack'
+  name 'xpack'
   description 'Elasticsearch Expanded Pack Plugin'
   classname 'org.elasticsearch.xpack.XPackPlugin'
   // FIXME we still can't be isolated due to shield custom realms
@@ -74,6 +74,9 @@ ext.expansions = [
   // Used in marvel index templates
   'marvel.plugin.version': version,
   'marvel.template.version': '1',
+  // Used in watcher index template
+  'watcher.plugin.version': version,
+  'watcher.template.version': '1',
 ]
 
 processResources {
@@ -111,13 +114,13 @@ bundlePlugin {
     include 'LICENSE.txt'
     include 'NOTICE.txt'
   }
-  from('shield/bin/shield') {
+  from('shield/bin/xpack') {
     into 'bin'
   }
-  from('shield/config/shield') {
+  from('shield/config/xpack') {
     into 'config'
   }
-  from('watcher/bin/watcher') {
+  from('watcher/bin/xpack') {
     into 'bin'
   }
 }
@@ -126,7 +129,7 @@ integTest {
   // TODO: fix this rest test to not depend on a hardcoded port!
   systemProperty 'tests.rest.blacklist', 'getting_started/10_monitor_cluster_health/*'
   cluster {
-    setupCommand 'setupDummyUser', 'bin/x-pack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
+    setupCommand 'setupDummyUser', 'bin/xpack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
     waitCondition = { NodeInfo node, AntBuilder ant ->
         File tmpFile = new File(node.cwd, 'wait.success')
         ant.get(src: "http://${node.httpUri()}",
@@ -154,7 +157,7 @@ artifacts {
 }
 
 run {
-  setupCommand 'setupDummyUser', 'bin/x-pack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
+  setupCommand 'setupDummyUser', 'bin/xpack/esusers', 'useradd', 'test_user', '-p', 'changeme', '-r', 'admin'
 }
 
 // classes are missing, e.g. com.ibm.icu.lang.UCharacter

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexRecoveryCollector.java

@@ -19,6 +19,7 @@ import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.MarvelLicensee;
 import org.elasticsearch.shield.InternalClient;
+import org.elasticsearch.shield.ShieldPlugin;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -26,8 +27,6 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 
-import static org.elasticsearch.shield.ShieldPlugin.shieldEnabled;
-
 /**
  * Collector for the Recovery API.
  * <p>
@@ -73,7 +72,7 @@ public class IndexRecoveryCollector extends AbstractCollector<IndexRecoveryColle
                 results.add(indexRecoveryDoc);
             }
         } catch (IndexNotFoundException e) {
-            if (shieldEnabled(settings) && IndexNameExpressionResolver.isAllIndices(Arrays.asList(marvelSettings.indices()))) {
+            if (ShieldPlugin.enabled(settings) && IndexNameExpressionResolver.isAllIndices(Arrays.asList(marvelSettings.indices()))) {
                 logger.debug("collector [{}] - unable to collect data for missing index [{}]", name(), e.getIndex());
             } else {
                 throw e;

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollector.java

@@ -21,6 +21,7 @@ import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.MarvelLicensee;
 import org.elasticsearch.shield.InternalClient;
+import org.elasticsearch.shield.ShieldPlugin;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -28,8 +29,6 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 
-import static org.elasticsearch.shield.ShieldPlugin.shieldEnabled;
-
 /**
  * Collector for indices statistics.
  * <p>
@@ -87,7 +86,7 @@ public class IndexStatsCollector extends AbstractCollector<IndexStatsCollector>
                 results.add(indexStatsDoc);
             }
         } catch (IndexNotFoundException e) {
-            if (shieldEnabled(settings) && IndexNameExpressionResolver.isAllIndices(Arrays.asList(marvelSettings.indices()))) {
+            if (ShieldPlugin.enabled(settings) && IndexNameExpressionResolver.isAllIndices(Arrays.asList(marvelSettings.indices()))) {
                 logger.debug("collector [{}] - unable to collect data for missing index [{}]", name(), e.getIndex());
             } else {
                 throw e;

elasticsearch/x-pack/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsCollector.java

@@ -19,13 +19,12 @@ import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
 import org.elasticsearch.marvel.agent.settings.MarvelSettings;
 import org.elasticsearch.marvel.license.MarvelLicensee;
 import org.elasticsearch.shield.InternalClient;
+import org.elasticsearch.shield.ShieldPlugin;
 
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 
-import static org.elasticsearch.shield.ShieldPlugin.shieldEnabled;
-
 /**
  * Collector for indices statistics.
  * <p>
@@ -72,7 +71,7 @@ public class IndicesStatsCollector extends AbstractCollector<IndicesStatsCollect
 
             return Collections.singletonList(indicesStatsDoc);
         } catch (IndexNotFoundException e) {
-            if (shieldEnabled(settings) && IndexNameExpressionResolver.isAllIndices(Arrays.asList(marvelSettings.indices()))) {
+            if (ShieldPlugin.enabled(settings) && IndexNameExpressionResolver.isAllIndices(Arrays.asList(marvelSettings.indices()))) {
                 logger.debug("collector [{}] - unable to collect data for missing index [{}]", name(), e.getIndex());
                 return Collections.emptyList();
             }

elasticsearch/x-pack/marvel/src/test/java/org/elasticsearch/marvel/MarvelPluginTests.java

@@ -66,7 +66,7 @@ public class MarvelPluginTests extends MarvelIntegTestCase {
                     break;
                 }
             }
-            assertThat("x-pack plugin not found", found, equalTo(true));
+            assertThat("xpack plugin not found", found, equalTo(true));
         }
     }
 

elasticsearch/x-pack/shield/bin/xpack/.in.bat

@@ -76,7 +76,7 @@ REM JAVA_OPTS=%JAVA_OPTS% -XX:HeapDumpPath=$ES_HOME/logs/heapdump.hprof
 
 REM Disables explicit GC
 set JAVA_OPTS=%JAVA_OPTS% -XX:+DisableExplicitGC
-set ES_CLASSPATH=%ES_CLASSPATH%;%ES_HOME%/lib/elasticsearch-1.4.0-SNAPSHOT.jar;%ES_HOME%/lib/*;%ES_HOME%/lib/sigar/*;%ES_HOME%/plugins/x-pack/*
+set ES_CLASSPATH=%ES_CLASSPATH%;%ES_HOME%/lib/*;%ES_HOME%/lib/sigar/*;%ES_HOME%/plugins/xpack/*
 set ES_PARAMS=-Des.path.home="%ES_HOME%"
 
 SET HOSTNAME=%COMPUTERNAME%

elasticsearch/x-pack/shield/bin/xpack/esusers

@@ -102,9 +102,9 @@ if [ -e "$CONF_DIR" ]; then
     case "$properties" in
         *-Des.default.path.conf=*) ;;
         *)
-            if [ ! -d "$CONF_DIR/shield" ]; then
+            if [ ! -d "$CONF_DIR/xpack" ]; then
-                echo "ERROR: The configuration directory [$CONF_DIR/shield] does not exist. The esusers tool expects Shield configuration files in that location."
+                echo "ERROR: The configuration directory [$CONF_DIR/xpack] does not exist. The esusers tool expects security configuration files in that location."
-                echo "The plugin may not have been installed with the correct configuration path. If [$ES_HOME/config/shield] exists, please copy the shield directory to [$CONF_DIR]"
+                echo "The plugin may not have been installed with the correct configuration path. If [$ES_HOME/config/xpack] exists, please copy the 'xpack' directory to [$CONF_DIR]"
                 exit 1
             fi
             properties="$properties -Des.default.path.conf=$CONF_DIR"
@@ -123,7 +123,7 @@ fi
 export HOSTNAME=`hostname -s`
 
 # include shield jars in classpath
-ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/plugins/x-pack/*"
+ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/plugins/xpack/*"
 
 cd "$ES_HOME" > /dev/null
 "$JAVA" $ES_JAVA_OPTS -cp "$ES_CLASSPATH" -Des.path.home="$ES_HOME" $properties org.elasticsearch.shield.authc.esusers.tool.ESUsersTool "$@"

elasticsearch/x-pack/shield/bin/xpack/syskeygen

@@ -102,9 +102,9 @@ if [ -e "$CONF_DIR" ]; then
     case "$properties" in
         *-Des.default.path.conf=*) ;;
         *)
-            if [ ! -d "$CONF_DIR/shield" ]; then
+            if [ ! -d "$CONF_DIR/xpack" ]; then
-                echo "ERROR: The configuration directory [$CONF_DIR/shield] does not exist. The syskeygen tool expects Shield configuration files in that location."
+                echo "ERROR: The configuration directory [$CONF_DIR/xpack] does not exist. The syskeygen tool expects security configuration files in that location."
-                echo "The plugin may not have been installed with the correct configuration path. If [$ES_HOME/config/shield] exists, please copy the shield directory to [$CONF_DIR]"
+                echo "The plugin may not have been installed with the correct configuration path. If [$ES_HOME/config/xpack] exists, please copy the 'xpack' directory to [$CONF_DIR]"
                 exit 1
             fi
             properties="$properties -Des.default.path.conf=$CONF_DIR"
@@ -123,7 +123,7 @@ fi
 export HOSTNAME=`hostname -s`
 
 # include shield jars in classpath
-ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/plugins/x-pack/*"
+ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/plugins/xpack/*"
 
 cd "$ES_HOME" > /dev/null
 $JAVA $ES_JAVA_OPTS -cp "$ES_CLASSPATH" -Des.path.home="$ES_HOME" $properties org.elasticsearch.shield.crypto.tool.SystemKeyTool "$@"

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/ShieldPlugin.java

@@ -6,7 +6,6 @@
 package org.elasticsearch.shield;
 
 import org.elasticsearch.action.ActionModule;
-import org.elasticsearch.client.Client;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.network.NetworkModule;
@@ -14,7 +13,6 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.shield.action.ShieldActionFilter;
@@ -71,7 +69,6 @@ import org.elasticsearch.shield.transport.netty.ShieldNettyHttpServerTransport;
 import org.elasticsearch.shield.transport.netty.ShieldNettyTransport;
 import org.elasticsearch.xpack.XPackPlugin;
 
-import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -85,22 +82,19 @@ import java.util.Map;
 public class ShieldPlugin extends Plugin {
 
     public static final String NAME = "shield";
-    public static final String ENABLED_SETTING_NAME = NAME + ".enabled";
+    public static final String DLS_FLS_FEATURE = "shield.dls_fls";
     public static final String OPT_OUT_QUERY_CACHE = "opt_out_cache";
-    public static final String DLS_FLS_ENABLED_SETTING = "shield.dls_fls.enabled";
-
-    private static final boolean DEFAULT_ENABLED_SETTING = true;
 
     private final Settings settings;
     private final boolean enabled;
-    private final boolean clientMode;
+    private final boolean transportClientMode;
     private ShieldLicenseState shieldLicenseState;
 
     public ShieldPlugin(Settings settings) {
         this.settings = settings;
-        this.enabled = shieldEnabled(settings);
+        this.transportClientMode = XPackPlugin.transportClientMode(settings);
-        this.clientMode = clientMode(settings);
+        this.enabled = XPackPlugin.featureEnabled(settings, NAME, true);
-        if (enabled && clientMode == false) {
+        if (enabled && !transportClientMode) {
             failIfShieldQueryCacheIsNotActive(settings, true);
         }
     }
@@ -118,11 +112,11 @@ public class ShieldPlugin extends Plugin {
     @Override
     public Collection<Module> nodeModules() {
 
-        if (!enabled) {
+        if (enabled == false) {
             return Collections.singletonList(new ShieldDisabledModule(settings));
         }
 
-        if (clientMode) {
+        if (transportClientMode == true) {
             return Arrays.<Module>asList(
                     new ShieldTransportModule(settings),
                     new SSLModule(settings));
@@ -147,9 +141,13 @@ public class ShieldPlugin extends Plugin {
 
     @Override
     public Collection<Class<? extends LifecycleComponent>> nodeServices() {
-        if (enabled && clientMode == false) {
+        if (enabled == false || transportClientMode == true) {
+            return Collections.emptyList();
+        }
         List<Class<? extends LifecycleComponent>> list = new ArrayList<>();
-            if (AuditTrailModule.fileAuditLoggingEnabled(settings)) {
+
+        //TODO why only focus on file audit logs? shouldn't we just check if audit trail is enabled in general?
+        if (AuditTrailModule.fileAuditLoggingEnabled(settings) == true) {
             list.add(LoggingAuditTrail.class);
         }
         list.add(ShieldLicensee.class);
@@ -157,8 +155,7 @@ public class ShieldPlugin extends Plugin {
         list.add(FileRolesStore.class);
         list.add(Realms.class);
         return list;
-        }
+
-        return Collections.emptyList();
     }
 
     @Override
@@ -178,6 +175,7 @@ public class ShieldPlugin extends Plugin {
     }
 
     public void onModule(SettingsModule settingsModule) {
+        //TODO shouldn't we register these settings only if shield is enabled and we're not in a client mode?
         settingsModule.registerSetting(IPFilter.IP_FILTER_ENABLED_SETTING);
         settingsModule.registerSetting(IPFilter.IP_FILTER_ENABLED_HTTP_SETTING);
         settingsModule.registerSetting(IPFilter.HTTP_FILTER_ALLOW_SETTING);
@@ -187,6 +185,9 @@ public class ShieldPlugin extends Plugin {
         settingsModule.registerSetting(Setting.boolSetting("plugins.load_classpath_plugins", true, false, Setting.Scope.CLUSTER));
         // TODO add real settings for this wildcard here
         settingsModule.registerSetting(Setting.groupSetting("shield.", false, Setting.Scope.CLUSTER));
+        // TODO please let's just drop the old settings before releasing
+        settingsModule.registerSetting(Setting.groupSetting("xpack.shield.", false, Setting.Scope.CLUSTER));
+
         String[] asArray = settings.getAsArray("shield.hide_settings");
         for (String pattern : asArray) {
             settingsModule.registerSettingsFilter(pattern);
@@ -207,14 +208,15 @@ public class ShieldPlugin extends Plugin {
         if (enabled == false) {
             return;
         }
+
         assert shieldLicenseState != null;
         if (flsDlsEnabled(settings)) {
             module.setSearcherWrapper((indexService) -> new ShieldIndexSearcherWrapper(indexService.getIndexSettings(),
-                    indexService.getQueryShardContext(), indexService.mapperService(),
+                    indexService.newQueryShardContext(), indexService.mapperService(),
                     indexService.cache().bitsetFilterCache(), indexService.getIndexServices().getThreadPool().getThreadContext(),
                     shieldLicenseState));
         }
-        if (clientMode == false) {
+        if (transportClientMode == false) {
             module.registerQueryCache(ShieldPlugin.OPT_OUT_QUERY_CACHE, OptOutQueryCache::new);
             failIfShieldQueryCacheIsNotActive(module.getSettings(), false);
         }
@@ -225,7 +227,7 @@ public class ShieldPlugin extends Plugin {
             return;
         }
         // registering the security filter only for nodes
-        if (clientMode == false) {
+        if (transportClientMode == false) {
             module.registerFilter(ShieldActionFilter.class);
         }
 
@@ -241,23 +243,21 @@ public class ShieldPlugin extends Plugin {
     }
 
     public void onModule(NetworkModule module) {
-        if (clientMode == false) {
-            // we want to expose the shield rest action even when the plugin is disabled
-            module.registerRestHandler(RestShieldInfoAction.class);
-        }
 
-        if (enabled == false) {
+        if (transportClientMode) {
+            if (enabled) {
+                module.registerTransport(ShieldPlugin.NAME, ShieldNettyTransport.class);
+                module.registerTransportService(ShieldPlugin.NAME, ShieldClientTransportService.class);
+            }
             return;
         }
 
-        module.registerTransport(ShieldPlugin.NAME, ShieldNettyTransport.class);
+        // we want to expose the shield rest action even when the plugin is disabled
-        if (clientMode) {
+        module.registerRestHandler(RestShieldInfoAction.class);
-            module.registerTransportService(ShieldPlugin.NAME, ShieldClientTransportService.class);
-        } else {
-            module.registerTransportService(ShieldPlugin.NAME, ShieldServerTransportService.class);
-        }
 
-        if (clientMode == false) {
+        if (enabled) {
+            module.registerTransport(ShieldPlugin.NAME, ShieldNettyTransport.class);
+            module.registerTransportService(ShieldPlugin.NAME, ShieldServerTransportService.class);
             module.registerRestHandler(RestAuthenticateAction.class);
             module.registerRestHandler(RestClearRealmCacheAction.class);
             module.registerRestHandler(RestClearRolesCacheAction.class);
@@ -290,39 +290,41 @@ public class ShieldPlugin extends Plugin {
                 .toCharArray())));
     }
 
-    /*
+    /**
-     We inject additional settings on each tribe client if the current node is a tribe node, to make sure that every tribe has shield
+     * If the current node is a tribe node, we inject additional settings on each tribe client. We do this to make sure
-     installed and enabled too:
+     * that every tribe cluster has shield installed and is enabled. We do that by:
-     - if shield is loaded on the tribe node we make sure it is also loaded on every tribe, by making it mandatory there
+     *
-     (this means that the tribe node will fail at startup if shield is not loaded on any tribe due to missing mandatory plugin)
+     *    - making it mandatory on the tribe client (this means that the tribe node will fail at startup if shield is
-     - if shield is loaded and enabled on the tribe node, we make sure it is also enabled on every tribe, by forcibly enabling it
+     *      not loaded on any tribe due to missing mandatory plugin)
-       (that means it's not possible to disable shield on the tribe clients)
+     *
+     *    - forcibly enabling it (that means it's not possible to disable shield on the tribe clients)
      */
     private void addTribeSettings(Settings.Builder settingsBuilder) {
         Map<String, Settings> tribesSettings = settings.getGroups("tribe", true);
         if (tribesSettings.isEmpty()) {
+            // it's not a tribe node
             return;
         }
 
         for (Map.Entry<String, Settings> tribeSettings : tribesSettings.entrySet()) {
             String tribePrefix = "tribe." + tribeSettings.getKey() + ".";
 
-            // we copy over existing mandatory plugins under additional settings, as they would get overridden otherwise (arrays don't get
+            // we copy over existing mandatory plugins under additional settings, as they would get overridden
-            // merged)
+            // otherwise (arrays don't get merged)
             String[] existingMandatoryPlugins = tribeSettings.getValue().getAsArray("plugin.mandatory", null);
             if (existingMandatoryPlugins == null) {
                 //shield is mandatory on every tribe if installed and enabled on the tribe node
-                settingsBuilder.putArray(tribePrefix + "plugin.mandatory", NAME);
+                settingsBuilder.putArray(tribePrefix + "plugin.mandatory", XPackPlugin.NAME);
             } else {
-                if (!isShieldMandatory(existingMandatoryPlugins)) {
+                if (Arrays.binarySearch(existingMandatoryPlugins, XPackPlugin.NAME) < 0) {
-                    throw new IllegalStateException("when [plugin.mandatory] is explicitly configured, [" + NAME + "] must be included in" +
+                    throw new IllegalStateException("when [plugin.mandatory] is explicitly configured, [" +
-                            " this list");
+                            XPackPlugin.NAME + "] must be included in this list");
                 }
             }
 
-            final String tribeEnabledSetting = tribePrefix + ENABLED_SETTING_NAME;
+            final String tribeEnabledSetting = tribePrefix + XPackPlugin.featureEnabledSetting(NAME);
             if (settings.get(tribeEnabledSetting) != null) {
-                boolean enabled = shieldEnabled(tribeSettings.getValue());
+                boolean enabled = enabled(tribeSettings.getValue());
                 if (!enabled) {
                     throw new IllegalStateException("tribe setting [" + tribeEnabledSetting + "] must be set to true but the value is ["
                             + settings.get(tribeEnabledSetting) + "]");
@@ -334,43 +336,22 @@ public class ShieldPlugin extends Plugin {
         }
     }
 
-    /*
+    /**
-        We need to forcefully overwrite the query cache implementation to use Shield's opt out query cache implementation.
+     *  We need to forcefully overwrite the query cache implementation to use Shield's opt out query cache implementation.
-        This impl. disabled the query cache if field level security is used for a particular request. If we wouldn't do
+     *  This impl. disabled the query cache if field level security is used for a particular request. If we wouldn't do
-        forcefully overwrite the query cache implementation then we leave the system vulnerable to leakages of data to
+     *  forcefully overwrite the query cache implementation then we leave the system vulnerable to leakages of data to
-        unauthorized users.
+     *  unauthorized users.
      */
     private void addQueryCacheSettings(Settings.Builder settingsBuilder) {
         settingsBuilder.put(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey(), OPT_OUT_QUERY_CACHE);
     }
 
-    private static boolean isShieldMandatory(String[] existingMandatoryPlugins) {
+    public static boolean enabled(Settings settings) {
-        for (String existingMandatoryPlugin : existingMandatoryPlugins) {
+        return XPackPlugin.featureEnabled(settings, NAME, true);
-            if (NAME.equals(existingMandatoryPlugin)) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    public static Path configDir(Environment env) {
-        return env.configFile().resolve(XPackPlugin.NAME);
-    }
-
-    public static Path resolveConfigFile(Environment env, String name) {
-        return configDir(env).resolve(name);
-    }
-
-    public static boolean clientMode(Settings settings) {
-        return !"node".equals(settings.get(Client.CLIENT_TYPE_SETTING_S.getKey()));
-    }
-
-    public static boolean shieldEnabled(Settings settings) {
-        return settings.getAsBoolean(ENABLED_SETTING_NAME, DEFAULT_ENABLED_SETTING);
     }
 
     public static boolean flsDlsEnabled(Settings settings) {
-        return settings.getAsBoolean(DLS_FLS_ENABLED_SETTING, true);
+        return XPackPlugin.featureEnabled(settings, DLS_FLS_FEATURE, true);
     }
 
     private void failIfShieldQueryCacheIsNotActive(Settings settings, boolean nodeSettings) {

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/esnative/ESNativeUsersStore.java

@@ -54,7 +54,7 @@ import org.elasticsearch.shield.action.user.DeleteUserRequest;
 import org.elasticsearch.shield.authc.AuthenticationService;
 import org.elasticsearch.shield.authc.support.Hasher;
 import org.elasticsearch.shield.authc.support.SecuredString;
-import org.elasticsearch.shield.client.ShieldClient;
+import org.elasticsearch.shield.client.SecurityClient;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
@@ -458,10 +458,10 @@ public class ESNativeUsersStore extends AbstractComponent implements ClusterStat
     }
 
     private <Response> void clearRealmCache(String username, ActionListener<Response> listener, Response response) {
-        ShieldClient shieldClient = new ShieldClient(client);
+        SecurityClient securityClient = new SecurityClient(client);
-        ClearRealmCacheRequest request = shieldClient.prepareClearRealmCache()
+        ClearRealmCacheRequest request = securityClient.prepareClearRealmCache()
                 .usernames(username).request();
-        shieldClient.clearRealmCache(request, new ActionListener<ClearRealmCacheResponse>() {
+        securityClient.clearRealmCache(request, new ActionListener<ClearRealmCacheResponse>() {
             @Override
             public void onResponse(ClearRealmCacheResponse nodes) {
                 listener.onResponse(response);

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/esusers/FileUserPasswdStore.java

@@ -20,6 +20,7 @@ import org.elasticsearch.shield.support.Validation;
 import org.elasticsearch.watcher.FileChangesListener;
 import org.elasticsearch.watcher.FileWatcher;
 import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xpack.XPackPlugin;
 
 import java.io.IOException;
 import java.io.PrintWriter;
@@ -98,7 +99,7 @@ public class FileUserPasswdStore {
     public static Path resolveFile(Settings settings, Environment env) {
         String location = settings.get("files.users");
         if (location == null) {
-            return ShieldPlugin.resolveConfigFile(env, "users");
+            return XPackPlugin.resolveConfigFile(env, "users");
         }
         return env.binFile().getParent().resolve(location);
     }
@@ -166,7 +167,7 @@ public class FileUserPasswdStore {
         }
 
         if (users.isEmpty()) {
-            logger.warn("no users found in users file [{}]. use bin/shield/esusers to add users and role mappings", path.toAbsolutePath());
+            logger.warn("no users found in users file [{}]. use bin/xpack/esusers to add users and role mappings", path.toAbsolutePath());
         }
         return unmodifiableMap(users);
     }

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/esusers/FileUserRolesStore.java

@@ -19,6 +19,7 @@ import org.elasticsearch.shield.support.Validation;
 import org.elasticsearch.watcher.FileChangesListener;
 import org.elasticsearch.watcher.FileWatcher;
 import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xpack.XPackPlugin;
 
 import java.io.IOException;
 import java.io.PrintWriter;
@@ -91,7 +92,7 @@ public class FileUserRolesStore {
     public static Path resolveFile(Settings settings, Environment env) {
         String location = settings.get("files.users_roles");
         if (location == null) {
-            return ShieldPlugin.resolveConfigFile(env, "users_roles");
+            return XPackPlugin.resolveConfigFile(env, "users_roles");
         }
         return env.binFile().getParent().resolve(location);
     }
@@ -181,7 +182,7 @@ public class FileUserRolesStore {
         }
 
         if (usersRoles.isEmpty()) {
-            logger.warn("no entries found in users_roles file [{}]. use bin/shield/esusers to add users and role mappings", path
+            logger.warn("no entries found in users_roles file [{}]. use bin/xpack/esusers to add users and role mappings", path
                     .toAbsolutePath());
         }
 

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/support/DnRoleMapper.java

@@ -17,6 +17,7 @@ import org.elasticsearch.shield.authc.RealmConfig;
 import org.elasticsearch.watcher.FileChangesListener;
 import org.elasticsearch.watcher.FileWatcher;
 import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xpack.XPackPlugin;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -82,7 +83,7 @@ public class DnRoleMapper {
     public static Path resolveFile(Settings settings, Environment env) {
         String location = settings.get(ROLE_MAPPING_FILE_SETTING);
         if (location == null) {
-            return ShieldPlugin.resolveConfigFile(env, DEFAULT_FILE_NAME);
+            return XPackPlugin.resolveConfigFile(env, DEFAULT_FILE_NAME);
         }
         return env.binFile().getParent().resolve(location);
     }

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/support/Hasher.java

@@ -7,6 +7,7 @@ package org.elasticsearch.shield.authc.support;
 
 import org.elasticsearch.common.Base64;
 import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.hash.MessageDigests;
 
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
@@ -142,7 +143,7 @@ public enum Hasher {
         @Override
         public char[] hash(SecuredString text) {
             byte[] textBytes = CharArrays.toUtf8Bytes(text.internalChars());
-            MessageDigest md = SHA1Provider.sha1();
+            MessageDigest md = MessageDigests.sha1();
             md.update(textBytes);
             String hash = Base64.encodeBytes(md.digest());
             return (SHA1_PREFIX + hash).toCharArray();
@@ -155,7 +156,7 @@ public enum Hasher {
                 return false;
             }
             byte[] textBytes = CharArrays.toUtf8Bytes(text.internalChars());
-            MessageDigest md = SHA1Provider.sha1();
+            MessageDigest md = MessageDigests.sha1();
             md.update(textBytes);
             String passwd64 = Base64.encodeBytes(md.digest());
             String hashNoPrefix = hashStr.substring(SHA1_PREFIX.length());
@@ -166,7 +167,7 @@ public enum Hasher {
     MD5() {
         @Override
         public char[] hash(SecuredString text) {
-            MessageDigest md = MD5Provider.md5();
+            MessageDigest md = MessageDigests.md5();
             md.update(CharArrays.toUtf8Bytes(text.internalChars()));
             String hash = Base64.encodeBytes(md.digest());
             return (MD5_PREFIX + hash).toCharArray();
@@ -179,7 +180,7 @@ public enum Hasher {
                 return false;
             }
             hashStr = hashStr.substring(MD5_PREFIX.length());
-            MessageDigest md = MD5Provider.md5();
+            MessageDigest md = MessageDigests.md5();
             md.update(CharArrays.toUtf8Bytes(text.internalChars()));
             String computedHashStr = Base64.encodeBytes(md.digest());
             return SecuredString.constantTimeEquals(hashStr, computedHashStr);
@@ -189,7 +190,7 @@ public enum Hasher {
     SSHA256() {
         @Override
         public char[] hash(SecuredString text) {
-            MessageDigest md = SHA256Provider.sha256();
+            MessageDigest md = MessageDigests.sha256();
             md.update(CharArrays.toUtf8Bytes(text.internalChars()));
             char[] salt = SaltProvider.salt(8);
             md.update(CharArrays.toUtf8Bytes(salt));
@@ -209,7 +210,7 @@ public enum Hasher {
             }
             hashStr = hashStr.substring(SSHA256_PREFIX.length());
             char[] saltAndHash = hashStr.toCharArray();
-            MessageDigest md = SHA256Provider.sha256();
+            MessageDigest md = MessageDigests.sha256();
             md.update(CharArrays.toUtf8Bytes(text.internalChars()));
             md.update(new String(saltAndHash, 0, 8).getBytes(StandardCharsets.UTF_8));
             String computedHash = Base64.encodeBytes(md.digest());
@@ -279,76 +280,6 @@ public enum Hasher {
 
     public abstract boolean verify(SecuredString data, char[] hash);
 
-    static final class MD5Provider {
-
-        private static final MessageDigest digest;
-
-        static {
-            try {
-                digest = MessageDigest.getInstance("MD5");
-            } catch (NoSuchAlgorithmException e) {
-                throw new IllegalStateException("unsupported digest algorithm [MD5]. Please verify you are running on Java 7 or above", e);
-            }
-        }
-
-        private static MessageDigest md5() {
-            try {
-                MessageDigest md5 = (MessageDigest) digest.clone();
-                md5.reset();
-                return md5;
-            } catch (CloneNotSupportedException e) {
-                throw new IllegalStateException("could not create MD5 digest", e);
-            }
-        }
-    }
-
-    static final class SHA1Provider {
-
-        private static final MessageDigest digest;
-
-        static {
-            try {
-                digest = MessageDigest.getInstance("SHA-1");
-            } catch (NoSuchAlgorithmException e) {
-                throw new IllegalStateException("unsupported digest algorithm [SHA-1]", e);
-            }
-        }
-
-        private static MessageDigest sha1() {
-            try {
-                MessageDigest sha1 = (MessageDigest) digest.clone();
-                sha1.reset();
-                return sha1;
-            } catch (CloneNotSupportedException e) {
-                throw new IllegalStateException("could not create SHA-1 digest", e);
-            }
-        }
-    }
-
-    static final class SHA256Provider {
-
-        private static final MessageDigest digest;
-
-        static {
-            try {
-                digest = MessageDigest.getInstance("SHA-256");
-            } catch (NoSuchAlgorithmException e) {
-                String msg = "unsupported digest algorithm [SHA-256]. Please verify you are running on Java 7 or above";
-                throw new IllegalStateException(msg, e);
-            }
-        }
-
-        private static MessageDigest sha256() {
-            try {
-                MessageDigest sha = (MessageDigest) digest.clone();
-                sha.reset();
-                return sha;
-            } catch (CloneNotSupportedException e) {
-                throw new IllegalStateException("could not create [SHA-256] digest", e);
-            }
-        }
-    }
-
     static final class SaltProvider {
 
         static final char[] ALPHABET = new char[]{

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/esnative/ESNativeRolesStore.java

@@ -48,7 +48,7 @@ import org.elasticsearch.shield.authc.AuthenticationService;
 import org.elasticsearch.shield.authz.RoleDescriptor;
 import org.elasticsearch.shield.authz.permission.Role;
 import org.elasticsearch.shield.authz.store.RolesStore;
-import org.elasticsearch.shield.client.ShieldClient;
+import org.elasticsearch.shield.client.SecurityClient;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
@@ -84,7 +84,7 @@ public class ESNativeRolesStore extends AbstractComponent implements RolesStore,
     private final ConcurrentHashMap<String, RoleAndVersion> roleCache = new ConcurrentHashMap<>();
 
     private Client client;
-    private ShieldClient shieldClient;
+    private SecurityClient securityClient;
     private int scrollSize;
     private TimeValue scrollKeepAlive;
     private ScheduledFuture<?> versionChecker;
@@ -353,7 +353,7 @@ public class ESNativeRolesStore extends AbstractComponent implements RolesStore,
         try {
             if (state.compareAndSet(State.INITIALIZED, State.STARTING)) {
                 this.client = clientProvider.get();
-                this.shieldClient = new ShieldClient(client);
+                this.securityClient = new SecurityClient(client);
                 this.scrollSize = settings.getAsInt("shield.authc.native.scroll.size", 1000);
                 this.scrollKeepAlive = settings.getAsTime("shield.authc.native.scroll.keep_alive", TimeValue.timeValueSeconds(10L));
                 TimeValue pollInterval = settings.getAsTime("shield.authc.native.reload.interval", TimeValue.timeValueSeconds(30L));
@@ -407,7 +407,7 @@ public class ESNativeRolesStore extends AbstractComponent implements RolesStore,
 
     private <Response> void clearRoleCache(final String role, ActionListener<Response> listener, Response response) {
         ClearRolesCacheRequest request = new ClearRolesCacheRequest().roles(role);
-        shieldClient.clearRolesCache(request, new ActionListener<ClearRolesCacheResponse>() {
+        securityClient.clearRolesCache(request, new ActionListener<ClearRolesCacheResponse>() {
             @Override
             public void onResponse(ClearRolesCacheResponse nodes) {
                 listener.onResponse(response);

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/IndexPrivilege.java

@@ -32,16 +32,18 @@ public class IndexPrivilege extends AbstractAutomatonPrivilege<IndexPrivilege> {
     public static final IndexPrivilege CREATE_INDEX = new IndexPrivilege("create_index", CreateIndexAction.NAME);
     public static final IndexPrivilege MANAGE_ALIASES = new IndexPrivilege("manage_aliases", "indices:admin/aliases*");
     public static final IndexPrivilege MONITOR = new IndexPrivilege("monitor", "indices:monitor/*");
-    public static final IndexPrivilege DATA_ACCESS = new IndexPrivilege("data_access", "indices:data/*");
+    public static final IndexPrivilege DATA_ACCESS = new IndexPrivilege("data_access", "indices:data/*", "indices:admin/mapping/put");
-    public static final IndexPrivilege CRUD = new IndexPrivilege("crud", "indices:data/write/*", "indices:data/read/*");
+    public static final IndexPrivilege CRUD =
+            new IndexPrivilege("crud", "indices:data/write/*", "indices:data/read/*", "indices:admin/mapping/put");
     public static final IndexPrivilege READ = new IndexPrivilege("read", "indices:data/read/*");
-    public static final IndexPrivilege SEARCH = new IndexPrivilege("search", SearchAction.NAME + "*", MultiSearchAction.NAME + "*",
+    public static final IndexPrivilege SEARCH =
-            SuggestAction.NAME + "*");
+            new IndexPrivilege("search", SearchAction.NAME + "*", MultiSearchAction.NAME + "*", SuggestAction.NAME + "*");
     public static final IndexPrivilege GET = new IndexPrivilege("get", GetAction.NAME + "*", MultiGetAction.NAME + "*");
     public static final IndexPrivilege SUGGEST = new IndexPrivilege("suggest", SuggestAction.NAME + "*");
-    public static final IndexPrivilege INDEX = new IndexPrivilege("index", "indices:data/write/index*", "indices:data/write/update*");
+    public static final IndexPrivilege INDEX =
+            new IndexPrivilege("index", "indices:data/write/index*", "indices:data/write/update*", "indices:admin/mapping/put");
     public static final IndexPrivilege DELETE = new IndexPrivilege("delete", "indices:data/write/delete*");
-    public static final IndexPrivilege WRITE = new IndexPrivilege("write", "indices:data/write/*");
+    public static final IndexPrivilege WRITE = new IndexPrivilege("write", "indices:data/write/*", "indices:admin/mapping/put");
 
     private static final Set<IndexPrivilege> values = new CopyOnWriteArraySet<>();
 

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/privilege/SystemPrivilege.java

@@ -22,8 +22,7 @@ public class SystemPrivilege extends Privilege<SystemPrivilege> {
             "internal:*",
             "indices:monitor/*", // added for marvel
             "cluster:monitor/*",  // added for marvel
-            "cluster:admin/reroute", // added for DiskThresholdDecider.DiskListener
+            "cluster:admin/reroute" // added for DiskThresholdDecider.DiskListener
-            "indices:admin/mapping/put" // ES 2.0 MappingUpdatedAction - updateMappingOnMasterSynchronously
     ));
 
     SystemPrivilege() {

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authz/store/FileRolesStore.java

@@ -34,6 +34,7 @@ import org.elasticsearch.shield.support.Validation;
 import org.elasticsearch.watcher.FileChangesListener;
 import org.elasticsearch.watcher.FileWatcher;
 import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xpack.XPackPlugin;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -109,7 +110,7 @@ public class FileRolesStore extends AbstractLifecycleComponent<RolesStore> imple
     public static Path resolveFile(Settings settings, Environment env) {
         String location = settings.get("shield.authz.store.files.roles");
         if (location == null) {
-            return ShieldPlugin.resolveConfigFile(env, "roles.yml");
+            return XPackPlugin.resolveConfigFile(env, "roles.yml");
         }
 
         return env.binFile().getParent().resolve(location);
@@ -285,10 +286,10 @@ public class FileRolesStore extends AbstractLifecycleComponent<RolesStore> imple
                                                                 if (token == XContentParser.Token.VALUE_STRING) {
                                                                     names.add(parser.text());
                                                                 } else {
-                                                                    logger.error("invalid role definition [{}] in roles file [{}]. could " +
+                                                                    logger.error("invalid role definition [{}] in roles file [{}]. " +
-                                                                            "not parse " +
+                                                                            "could not parse [{}] as index privilege. privilege " +
-                                                                            "[{}] as index privilege. privilege names must be strings. " +
+                                                                            "names must be strings. skipping role...", roleName,
-                                                                            "skipping role...", roleName, path.toAbsolutePath(), token);
+                                                                            path.toAbsolutePath(), token);
                                                                     return null;
                                                                 }
                                                             }
@@ -301,26 +302,27 @@ public class FileRolesStore extends AbstractLifecycleComponent<RolesStore> imple
                                                 if (name != null) {
                                                     if ((query != null || (fields != null && fields.isEmpty() == false)) &&
                                                             ShieldPlugin.flsDlsEnabled(settings) == false) {
-                                                        logger.error("invalid role definition [{}] in roles file [{}]. document and field" +
+                                                        logger.error("invalid role definition [{}] in roles file [{}]. " +
-                                                                " level security is not enabled. set [{}] to [true] in the configuration " +
+                                                                "document and field level security is not enabled. " +
-                                                                "file. skipping role...", roleName, path.toAbsolutePath(), ShieldPlugin
+                                                                "set [{}] to [true] in the configuration file. skipping role...",
-                                                                .DLS_FLS_ENABLED_SETTING);
+                                                                roleName, path.toAbsolutePath(),
+                                                                XPackPlugin.featureEnabledSetting(ShieldPlugin.DLS_FLS_FEATURE));
                                                         return null;
                                                     }
 
                                                     try {
                                                         role.add(fields, query, IndexPrivilege.get(name), indices);
                                                     } catch (IllegalArgumentException e) {
-                                                        logger.error("invalid role definition [{}] in roles file [{}]. could not resolve " +
+                                                        logger.error("invalid role definition [{}] in roles file [{}]. could not " +
-                                                                "indices privileges [{}]. skipping role...", roleName,
+                                                                "resolve indices privileges [{}]. skipping role...", roleName,
                                                                 path.toAbsolutePath(), name);
                                                         return null;
                                                     }
                                                 }
                                                 continue;
                                             } else {
-                                                logger.error("invalid role definition [{}] in roles file [{}]. could not parse [{}] as " +
+                                                logger.error("invalid role definition [{}] in roles file [{}]. " +
-                                                        "index privileges. privilege lists must either " +
+                                                        "could not parse [{}] as index privileges. privilege lists must either " +
                                                         "be a comma delimited string or an array of strings. skipping role...", roleName,
                                                         path.toAbsolutePath(), token);
                                                 return null;

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/client/SecurityClient.java

@@ -44,19 +44,12 @@ import org.elasticsearch.shield.action.role.ClearRolesCacheResponse;
 /**
  * A wrapper to elasticsearch clients that exposes all Shield related APIs
  */
-public class ShieldClient {
+public class SecurityClient {
 
     private final ElasticsearchClient client;
-    private final ShieldAuthcClient authcClient;
 
-    public ShieldClient(ElasticsearchClient client) {
+    public SecurityClient(ElasticsearchClient client) {
         this.client = client;
-        this.authcClient = new ShieldAuthcClient(client);
-    }
-
-    @Deprecated
-    public ShieldAuthcClient authc() {
-        return authcClient;
     }
 
     /****************

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/client/ShieldAuthcClient.java

@@ -1,54 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.shield.client;
-
-import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.shield.action.realm.ClearRealmCacheAction;
-import org.elasticsearch.shield.action.realm.ClearRealmCacheRequest;
-import org.elasticsearch.shield.action.realm.ClearRealmCacheRequestBuilder;
-import org.elasticsearch.shield.action.realm.ClearRealmCacheResponse;
-
-/**
- * A client to manage Shield's authentication
- *
- * @deprecated Use {@link ShieldClient} directly instead
- */
-@Deprecated
-public class ShieldAuthcClient {
-
-    private final ElasticsearchClient client;
-
-    ShieldAuthcClient(ElasticsearchClient client) {
-        this.client = client;
-    }
-
-    /**
-     * Clears the realm caches. It's possible to clear all user entries from all realms in the cluster or alternatively
-     * select the realms (by their unique names) and/or users (by their usernames) that should be evicted.
-     */
-    public ClearRealmCacheRequestBuilder prepareClearRealmCache() {
-        return new ClearRealmCacheRequestBuilder(client);
-    }
-
-    /**
-     * Clears the realm caches. It's possible to clear all user entries from all realms in the cluster or alternatively
-     * select the realms (by their unique names) and/or users (by their usernames) that should be evicted.
-     */
-    public void clearRealmCache(ClearRealmCacheRequest request, ActionListener<ClearRealmCacheResponse> listener) {
-        client.execute(ClearRealmCacheAction.INSTANCE, request, listener);
-    }
-
-    /**
-     * Clears the realm caches. It's possible to clear all user entries from all realms in the cluster or alternatively
-     * select the realms (by their unique names) and/or users (by their usernames) that should be evicted.
-     */
-    public ActionFuture<ClearRealmCacheResponse> clearRealmCache(ClearRealmCacheRequest request) {
-        return client.execute(ClearRealmCacheAction.INSTANCE, request);
-    }
-
-}

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/crypto/InternalCryptoService.java

@@ -17,6 +17,7 @@ import org.elasticsearch.shield.authc.support.CharArrays;
 import org.elasticsearch.watcher.FileChangesListener;
 import org.elasticsearch.watcher.FileWatcher;
 import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xpack.XPackPlugin;
 
 import javax.crypto.BadPaddingException;
 import javax.crypto.Cipher;
@@ -159,7 +160,7 @@ public class InternalCryptoService extends AbstractLifecycleComponent<InternalCr
     public static Path resolveSystemKey(Settings settings, Environment env) {
         String location = settings.get(FILE_SETTING);
         if (location == null) {
-            return ShieldPlugin.resolveConfigFile(env, FILE_NAME);
+            return XPackPlugin.resolveConfigFile(env, FILE_NAME);
         }
         return env.binFile().getParent().resolve(location);
     }
@@ -564,30 +565,25 @@ public class InternalCryptoService extends AbstractLifecycleComponent<InternalCr
     }
 
     /**
-     * Provider class for the HmacSHA1 {@link Mac} that provides an optimization by using clone instead of calling
+     * Provider class for the HmacSHA1 {@link Mac} that provides an optimization by using a thread local instead of calling
-     * Mac#getInstance and obtaining a lock
+     * Mac#getInstance and obtaining a lock (in the internals)
      */
     private static class HmacSHA1Provider {
 
-        private static final Mac mac;
+        private static final ThreadLocal<Mac> MAC = ThreadLocal.withInitial(() -> {
-
-        static {
             try {
-                mac = Mac.getInstance(HMAC_ALGO);
+                return Mac.getInstance(HMAC_ALGO);
             } catch (NoSuchAlgorithmException e) {
-                throw new IllegalStateException("could not create message authentication code instance with algorithm [HmacSHA1]", e);
+                throw new IllegalStateException("could not create Mac instance with algorithm [" + HMAC_ALGO + "]", e);
-            }
             }
+        });
 
         private static Mac hmacSHA1() {
-            try {
+            Mac instance = MAC.get();
-                Mac hmac = (Mac) mac.clone();
+            instance.reset();
-                hmac.reset();
+            return instance;
-                return hmac;
-            } catch (CloneNotSupportedException e) {
-                throw new IllegalStateException("could not create [HmacSHA1] MAC", e);
-            }
         }
+
     }
 
     /**

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/rest/action/RestShieldInfoAction.java

@@ -37,7 +37,7 @@ public class RestShieldInfoAction extends BaseRestHandler {
         super(settings, client);
         this.clusterName = clusterName;
         this.shieldLicenseState = licenseState;
-        this.shieldEnabled = ShieldPlugin.shieldEnabled(settings);
+        this.shieldEnabled = ShieldPlugin.enabled(settings);
         controller.registerHandler(GET, "/_shield", this);
         controller.registerHandler(HEAD, "/_shield", this);
     }

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/rest/action/realm/RestClearRealmCacheAction.java

@@ -20,7 +20,7 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.support.RestBuilderListener;
 import org.elasticsearch.shield.action.realm.ClearRealmCacheRequest;
 import org.elasticsearch.shield.action.realm.ClearRealmCacheResponse;
-import org.elasticsearch.shield.client.ShieldClient;
+import org.elasticsearch.shield.client.SecurityClient;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
@@ -41,7 +41,7 @@ public class RestClearRealmCacheAction extends BaseRestHandler {
 
         ClearRealmCacheRequest req = new ClearRealmCacheRequest().realms(realms).usernames(usernames);
 
-        new ShieldClient(client).clearRealmCache(req, new RestBuilderListener<ClearRealmCacheResponse>(channel) {
+        new SecurityClient(client).clearRealmCache(req, new RestBuilderListener<ClearRealmCacheResponse>(channel) {
             @Override
             public RestResponse buildResponse(ClearRealmCacheResponse response, XContentBuilder builder) throws Exception {
                 response.toXContent(builder, ToXContent.EMPTY_PARAMS);

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/rest/action/role/RestAddRoleAction.java

@@ -19,7 +19,7 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.support.RestBuilderListener;
 import org.elasticsearch.shield.action.role.AddRoleRequest;
 import org.elasticsearch.shield.action.role.AddRoleResponse;
-import org.elasticsearch.shield.client.ShieldClient;
+import org.elasticsearch.shield.client.SecurityClient;
 
 /**
  * Rest endpoint to add a Role to the shield index
@@ -38,7 +38,7 @@ public class RestAddRoleAction extends BaseRestHandler {
         AddRoleRequest addRoleReq = new AddRoleRequest(request.content());
         addRoleReq.name(request.param("id"));
 
-        new ShieldClient(client).addRole(addRoleReq, new RestBuilderListener<AddRoleResponse>(channel) {
+        new SecurityClient(client).addRole(addRoleReq, new RestBuilderListener<AddRoleResponse>(channel) {
             @Override
             public RestResponse buildResponse(AddRoleResponse addRoleResponse, XContentBuilder builder) throws Exception {
                 return new BytesRestResponse(RestStatus.OK,

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/rest/action/role/RestClearRolesCacheAction.java

@@ -20,7 +20,7 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.support.RestBuilderListener;
 import org.elasticsearch.shield.action.role.ClearRolesCacheRequest;
 import org.elasticsearch.shield.action.role.ClearRolesCacheResponse;
-import org.elasticsearch.shield.client.ShieldClient;
+import org.elasticsearch.shield.client.SecurityClient;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
@@ -42,7 +42,7 @@ public class RestClearRolesCacheAction extends BaseRestHandler {
 
         ClearRolesCacheRequest req = new ClearRolesCacheRequest().roles(roles);
 
-        new ShieldClient(client).clearRolesCache(req, new RestBuilderListener<ClearRolesCacheResponse>(channel) {
+        new SecurityClient(client).clearRolesCache(req, new RestBuilderListener<ClearRolesCacheResponse>(channel) {
             @Override
             public RestResponse buildResponse(ClearRolesCacheResponse response, XContentBuilder builder) throws Exception {
                 response.toXContent(builder, ToXContent.EMPTY_PARAMS);

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/rest/action/role/RestDeleteRoleAction.java

@@ -19,7 +19,7 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.support.RestBuilderListener;
 import org.elasticsearch.shield.action.role.DeleteRoleRequest;
 import org.elasticsearch.shield.action.role.DeleteRoleResponse;
-import org.elasticsearch.shield.client.ShieldClient;
+import org.elasticsearch.shield.client.SecurityClient;
 
 /**
  * Rest endpoint to delete a Role from the shield index
@@ -37,7 +37,7 @@ public class RestDeleteRoleAction extends BaseRestHandler {
         String role = request.param("id");
         DeleteRoleRequest delRoleRequest = new DeleteRoleRequest(role);
 
-        new ShieldClient(client).deleteRole(delRoleRequest, new RestBuilderListener<DeleteRoleResponse>(channel) {
+        new SecurityClient(client).deleteRole(delRoleRequest, new RestBuilderListener<DeleteRoleResponse>(channel) {
             @Override
             public RestResponse buildResponse(DeleteRoleResponse response, XContentBuilder builder) throws Exception {
                 return new BytesRestResponse(response.found() ? RestStatus.OK : RestStatus.NOT_FOUND,

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/rest/action/role/RestGetRolesAction.java

@@ -20,7 +20,7 @@ import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.support.RestBuilderListener;
 import org.elasticsearch.shield.action.role.GetRolesResponse;
-import org.elasticsearch.shield.client.ShieldClient;
+import org.elasticsearch.shield.client.SecurityClient;
 
 /**
  * Rest endpoint to retrieve a Role from the shield index
@@ -38,7 +38,7 @@ public class RestGetRolesAction extends BaseRestHandler {
     protected void handleRequest(RestRequest request, final RestChannel channel, Client client) throws Exception {
         String[] roles = Strings.splitStringByCommaToArray(request.param("id"));
 
-        new ShieldClient(client).prepareGetRoles().roles(roles).execute(new RestBuilderListener<GetRolesResponse>(channel) {
+        new SecurityClient(client).prepareGetRoles().roles(roles).execute(new RestBuilderListener<GetRolesResponse>(channel) {
             @Override
             public RestResponse buildResponse(GetRolesResponse getRolesResponse, XContentBuilder builder) throws Exception {
                 builder.startObject();

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/rest/action/user/RestAddUserAction.java

@@ -20,7 +20,7 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.support.RestBuilderListener;
 import org.elasticsearch.shield.action.user.AddUserRequest;
 import org.elasticsearch.shield.action.user.AddUserResponse;
-import org.elasticsearch.shield.client.ShieldClient;
+import org.elasticsearch.shield.client.SecurityClient;
 
 /**
  * Rest endpoint to add a User to the shield index
@@ -40,7 +40,7 @@ public class RestAddUserAction extends BaseRestHandler {
         addUserReq.username(request.param("username"));
         addUserReq.source(request.content());
 
-        new ShieldClient(client).addUser(addUserReq, new RestBuilderListener<AddUserResponse>(channel) {
+        new SecurityClient(client).addUser(addUserReq, new RestBuilderListener<AddUserResponse>(channel) {
             @Override
             public RestResponse buildResponse(AddUserResponse addUserResponse, XContentBuilder builder) throws Exception {
                 return new BytesRestResponse(RestStatus.OK,

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/rest/action/user/RestDeleteUserAction.java

@@ -19,7 +19,7 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.support.RestBuilderListener;
 import org.elasticsearch.shield.action.user.DeleteUserRequest;
 import org.elasticsearch.shield.action.user.DeleteUserResponse;
-import org.elasticsearch.shield.client.ShieldClient;
+import org.elasticsearch.shield.client.SecurityClient;
 
 /**
  * Rest action to delete a user from the shield index
@@ -37,7 +37,7 @@ public class RestDeleteUserAction extends BaseRestHandler {
         String user = request.param("username");
         DeleteUserRequest delUserRequest = new DeleteUserRequest(user);
 
-        new ShieldClient(client).deleteUser(delUserRequest, new RestBuilderListener<DeleteUserResponse>(channel) {
+        new SecurityClient(client).deleteUser(delUserRequest, new RestBuilderListener<DeleteUserResponse>(channel) {
             @Override
             public RestResponse buildResponse(DeleteUserResponse response, XContentBuilder builder) throws Exception {
                 return new BytesRestResponse(response.found() ? RestStatus.OK : RestStatus.NOT_FOUND,

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/rest/action/user/RestGetUsersAction.java

@@ -21,7 +21,7 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.support.RestBuilderListener;
 import org.elasticsearch.shield.User;
 import org.elasticsearch.shield.action.user.GetUsersResponse;
-import org.elasticsearch.shield.client.ShieldClient;
+import org.elasticsearch.shield.client.SecurityClient;
 
 /**
  * Rest action to retrieve a user from the shield index
@@ -39,7 +39,7 @@ public class RestGetUsersAction extends BaseRestHandler {
     protected void handleRequest(RestRequest request, final RestChannel channel, Client client) throws Exception {
         String[] users = Strings.splitStringByCommaToArray(request.param("username"));
 
-        new ShieldClient(client).prepareGetUsers().users(users).execute(new RestBuilderListener<GetUsersResponse>(channel) {
+        new SecurityClient(client).prepareGetUsers().users(users).execute(new RestBuilderListener<GetUsersResponse>(channel) {
             @Override
             public RestResponse buildResponse(GetUsersResponse getUsersResponse, XContentBuilder builder) throws Exception {
                 builder.startObject();

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/support/AbstractShieldModule.java

@@ -22,7 +22,7 @@ public abstract class AbstractShieldModule extends AbstractModule {
     public AbstractShieldModule(Settings settings) {
         this.settings = settings;
         this.clientMode = !"node".equals(settings.get(Client.CLIENT_TYPE_SETTING_S.getKey()));
-        this.shieldEnabled = ShieldPlugin.shieldEnabled(settings);
+        this.shieldEnabled = ShieldPlugin.enabled(settings);
     }
 
     @Override

elasticsearch/x-pack/shield/src/main/resources/org/elasticsearch/shield/crypto/tool/syskey-generate.help

@@ -9,7 +9,7 @@ SYNOPSIS
 DESCRIPTION
 
     Generates the system key and stores in the system_key file. By default
-    it will be stored in 'config/shield/.system_key' file. If the file location
+    it will be stored in 'config/xpack/system_key' file. If the file location
     is customized in the elasticsearch.yml (under the 'shield.system_key.file'
     setting), the generated key will be stored in that custom location.
 

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/BulkUpdateTests.java

@@ -16,6 +16,7 @@ import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
 import org.elasticsearch.test.ShieldIntegTestCase;
 import org.elasticsearch.test.ShieldSettingsSource;
 import org.elasticsearch.test.rest.client.http.HttpResponse;
+import org.elasticsearch.xpack.XPackPlugin;
 
 import java.io.IOException;
 
@@ -30,7 +31,7 @@ public class BulkUpdateTests extends ShieldIntegTestCase {
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
                 .put(NetworkModule.HTTP_ENABLED.getKey(), true)
-                .put(ShieldPlugin.DLS_FLS_ENABLED_SETTING, randomBoolean())
+                .put(XPackPlugin.featureEnabledSetting(ShieldPlugin.DLS_FLS_FEATURE), randomBoolean())
                 .build();
     }
 

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ClearRealmsCacheTests.java

@@ -21,7 +21,7 @@ import org.elasticsearch.shield.authc.support.Hasher;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.shield.authc.support.SecuredStringTests;
 import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
-import org.elasticsearch.shield.client.ShieldClient;
+import org.elasticsearch.shield.client.SecurityClient;
 import org.elasticsearch.test.ShieldIntegTestCase;
 import org.elasticsearch.test.ShieldSettingsSource;
 import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
@@ -140,11 +140,11 @@ public class ClearRealmsCacheTests extends ShieldIntegTestCase {
         public abstract void executeRequest() throws Exception;
 
         static void executeTransportRequest(ClearRealmCacheRequest request) throws Exception {
-            ShieldClient shieldClient = new ShieldClient(client());
+            SecurityClient securityClient = securityClient(client());
 
             final CountDownLatch latch = new CountDownLatch(1);
             final AtomicReference<Throwable> error = new AtomicReference<>();
-            shieldClient.clearRealmCache(request, new ActionListener<ClearRealmCacheResponse>() {
+            securityClient.clearRealmCache(request, new ActionListener<ClearRealmCacheResponse>() {
                 @Override
                 public void onResponse(ClearRealmCacheResponse response) {
                     assertThat(response.getNodes().length, equalTo(internalCluster().getNodeNames().length));

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/ClearRolesCacheTests.java

@@ -21,7 +21,7 @@ import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
 import org.elasticsearch.shield.authz.RoleDescriptor;
 import org.elasticsearch.shield.authz.esnative.ESNativeRolesStore;
-import org.elasticsearch.shield.client.ShieldClient;
+import org.elasticsearch.shield.client.SecurityClient;
 import org.elasticsearch.test.ShieldIntegTestCase;
 import org.elasticsearch.test.ShieldSettingsSource;
 import org.elasticsearch.test.junit.annotations.TestLogging;
@@ -57,7 +57,7 @@ public class ClearRolesCacheTests extends ShieldIntegTestCase {
     @Before
     public void setupForTest() throws Exception {
         // Clear the realm cache for all realms since we use a SUITE scoped cluster
-        ShieldClient client = new ShieldClient(internalCluster().transportClient());
+        SecurityClient client = securityClient(internalCluster().transportClient());
         client.prepareClearRealmCache().get();
 
         for (ESNativeUsersStore store : internalCluster().getInstances(ESNativeUsersStore.class)) {
@@ -78,7 +78,7 @@ public class ClearRolesCacheTests extends ShieldIntegTestCase {
             });
         }
 
-        ShieldClient c = new ShieldClient(client());
+        SecurityClient c = securityClient();
         // create roles
         for (String role : roles) {
             c.prepareAddRole().name(role)
@@ -109,13 +109,13 @@ public class ClearRolesCacheTests extends ShieldIntegTestCase {
 
     public void testModifyingViaApiClearsCache() throws Exception {
         Client client = internalCluster().transportClient();
-        ShieldClient shieldClient = new ShieldClient(client);
+        SecurityClient securityClient = securityClient(client);
 
         int modifiedRolesCount = randomIntBetween(1, roles.length);
         List<String> toModify = randomSubsetOf(modifiedRolesCount, roles);
         logger.debug("--> modifying roles {} to have run_as", toModify);
         for (String role : toModify) {
-            AddRoleResponse response = shieldClient.prepareAddRole().name(role)
+            AddRoleResponse response = securityClient.prepareAddRole().name(role)
                     .cluster("none")
                     .addIndices(new String[] { "*" }, new String[] { "ALL" }, null, null)
                     .runAs(role)
@@ -124,7 +124,7 @@ public class ClearRolesCacheTests extends ShieldIntegTestCase {
             logger.debug("--> updated role [{}] with run_as", role);
         }
 
-        assertRolesAreCorrect(shieldClient, toModify);
+        assertRolesAreCorrect(securityClient, toModify);
     }
 
     public void testModifyingDocumentsDirectly() throws Exception {
@@ -144,7 +144,7 @@ public class ClearRolesCacheTests extends ShieldIntegTestCase {
 
         // in this test, the poller runs too frequently to check the cache still has roles without run as
         // clear the cache and we should definitely see the latest values!
-        ShieldClient shieldClient = new ShieldClient(client);
+        SecurityClient securityClient = securityClient(client);
         final boolean useHttp = randomBoolean();
         final boolean clearAll = randomBoolean();
         logger.debug("--> starting to clear roles. using http [{}] clearing all [{}]", useHttp, clearAll);
@@ -163,18 +163,18 @@ public class ClearRolesCacheTests extends ShieldIntegTestCase {
                     .execute();
             assertThat(response.getStatusCode(), is(RestStatus.OK.getStatus()));
         } else {
-            shieldClient.prepareClearRolesCache().roles(rolesToClear).get();
+            securityClient.prepareClearRolesCache().roles(rolesToClear).get();
         }
 
-        assertRolesAreCorrect(shieldClient, toModify);
+        assertRolesAreCorrect(securityClient, toModify);
     }
 
     public void testDeletingRoleDocumentDirectly() throws Exception {
         Client client = internalCluster().transportClient();
-        ShieldClient shieldClient = new ShieldClient(client);
+        SecurityClient securityClient = securityClient(client);
 
         final String role = randomFrom(roles);
-        List<RoleDescriptor> foundRoles = shieldClient.prepareGetRoles().roles(role).get().roles();
+        List<RoleDescriptor> foundRoles = securityClient.prepareGetRoles().roles(role).get().roles();
         assertThat(foundRoles.size(), is(1));
         logger.debug("--> deleting role [{}]", role);
         DeleteResponse response = client.prepareDelete(ShieldTemplateService.SHIELD_ADMIN_INDEX_NAME,
@@ -184,15 +184,15 @@ public class ClearRolesCacheTests extends ShieldIntegTestCase {
         assertBusy(new Runnable() {
             @Override
             public void run() {
-                assertThat(shieldClient.prepareGetRoles().roles(role).get().roles().isEmpty(), is(true));
+                assertThat(securityClient.prepareGetRoles().roles(role).get().roles().isEmpty(), is(true));
             }
         });
     }
 
-    private void assertRolesAreCorrect(ShieldClient shieldClient, List<String> toModify) {
+    private void assertRolesAreCorrect(SecurityClient securityClient, List<String> toModify) {
         for (String role : roles) {
             logger.debug("--> getting role [{}]", role);
-            GetRolesResponse roleResponse = shieldClient.prepareGetRoles().roles(role).get();
+            GetRolesResponse roleResponse = securityClient.prepareGetRoles().roles(role).get();
             assertThat(roleResponse.isExists(), is(true));
             final String[] runAs = roleResponse.roles().get(0).getRunAs();
             if (toModify.contains(role)) {

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java

@@ -12,6 +12,7 @@ import org.elasticsearch.shield.ShieldPlugin;
 import org.elasticsearch.shield.authc.support.Hasher;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.test.ShieldIntegTestCase;
+import org.elasticsearch.xpack.XPackPlugin;
 
 import java.util.Collections;
 
@@ -75,7 +76,7 @@ public class DocumentAndFieldLevelSecurityTests extends ShieldIntegTestCase {
     public Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put(ShieldPlugin.DLS_FLS_ENABLED_SETTING, true)
+                .put(XPackPlugin.featureEnabledSetting(ShieldPlugin.DLS_FLS_FEATURE), true)
                 .build();
     }
 

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityRandomTests.java

@@ -14,6 +14,7 @@ import org.elasticsearch.shield.ShieldPlugin;
 import org.elasticsearch.shield.authc.support.Hasher;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.test.ShieldIntegTestCase;
+import org.elasticsearch.xpack.XPackPlugin;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -74,7 +75,7 @@ public class DocumentLevelSecurityRandomTests extends ShieldIntegTestCase {
     public Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put(ShieldPlugin.DLS_FLS_ENABLED_SETTING, true)
+                .put(XPackPlugin.featureEnabledSetting(ShieldPlugin.DLS_FLS_FEATURE), true)
                 .build();
     }
 

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java

@@ -28,6 +28,7 @@ import org.elasticsearch.shield.ShieldPlugin;
 import org.elasticsearch.shield.authc.support.Hasher;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.test.ShieldIntegTestCase;
+import org.elasticsearch.xpack.XPackPlugin;
 
 import java.util.Collections;
 
@@ -87,7 +88,7 @@ public class DocumentLevelSecurityTests extends ShieldIntegTestCase {
     public Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put(ShieldPlugin.DLS_FLS_ENABLED_SETTING, true)
+                .put(XPackPlugin.featureEnabledSetting(ShieldPlugin.DLS_FLS_FEATURE), true)
                 .build();
     }
 

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java

@@ -14,6 +14,7 @@ import org.elasticsearch.shield.ShieldPlugin;
 import org.elasticsearch.shield.authc.support.Hasher;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.test.ShieldIntegTestCase;
+import org.elasticsearch.xpack.XPackPlugin;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -111,7 +112,7 @@ public class FieldLevelSecurityRandomTests extends ShieldIntegTestCase {
     public Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put(ShieldPlugin.DLS_FLS_ENABLED_SETTING, true)
+                .put(XPackPlugin.featureEnabledSetting(ShieldPlugin.DLS_FLS_FEATURE), true)
                 .build();
     }
 

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/FieldLevelSecurityTests.java

@@ -28,6 +28,7 @@ import org.elasticsearch.shield.authc.support.Hasher;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ShieldIntegTestCase;
+import org.elasticsearch.xpack.XPackPlugin;
 
 import java.util.Collections;
 
@@ -116,7 +117,7 @@ public class FieldLevelSecurityTests extends ShieldIntegTestCase {
     public Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put(ShieldPlugin.DLS_FLS_ENABLED_SETTING, true)
+                .put(XPackPlugin.featureEnabledSetting(ShieldPlugin.DLS_FLS_FEATURE), true)
                 .build();
     }
 

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/IndexPrivilegeTests.java

@@ -8,10 +8,11 @@ package org.elasticsearch.integration;
 import org.apache.lucene.util.LuceneTestCase.BadApple;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.node.Node;
+import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.junit.Before;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
@@ -21,6 +22,7 @@ import static org.hamcrest.Matchers.is;
 
 //test is just too slow, please fix it to not be sleep-based
 @BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007")
+@ESIntegTestCase.ClusterScope(randomDynamicTemplates = false)
 public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
 
     private String jsonDoc = "{ \"name\" : \"elasticsearch\"}";
@@ -94,8 +96,7 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
             "u11:" + USERS_PASSWD_HASHED + "\n" +
             "u12:" + USERS_PASSWD_HASHED + "\n" +
             "u13:" + USERS_PASSWD_HASHED + "\n" +
-            "u14:" + USERS_PASSWD_HASHED + "\n" +
+            "u14:" + USERS_PASSWD_HASHED + "\n";
-            "u15:" + USERS_PASSWD_HASHED + "\n";
 
     public static final String USERS_ROLES =
             "all_indices_role:admin,u8\n" +
@@ -120,7 +121,6 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder().put(super.nodeSettings(nodeOrdinal))
                 .put(NetworkModule.HTTP_ENABLED.getKey(), true)
-                .put("action.disable_shutdown", true)
                 .build();
     }
 
@@ -139,6 +139,17 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
         return super.configUsersRoles() + USERS_ROLES;
     }
 
+    // we reduce the number of shards and replicas to help speed up this test since that is not the focus of this test
+    @Override
+    public int maximumNumberOfReplicas() {
+        return 1;
+    }
+
+    @Override
+    public int maximumNumberOfShards() {
+        return 2;
+    }
+
     @Before
     public void insertBaseDocumentsAsAdmin() throws Exception {
         // indices: a,b,c,abc
@@ -296,24 +307,13 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
         assertUserIsDenied("u14", "all", "c");
     }
 
-    public void testUserU15() throws Exception {
-        //u15 has access to manage and search a, so that adding warmer templates work
-        assertUserIsAllowed("u15", "manage", "a");
-        assertUserIsAllowed("u15", "search", "a");
-        assertAccessIsAllowed("u15", "PUT", "/a/_warmer/w1", "{ \"query\" : { \"match_all\" : {} } }");
-        assertAccessIsAllowed("u15", "DELETE", "/a/_warmer/w1");
-
-        assertUserIsDenied("u15", "all", "b");
-        assertUserIsDenied("u15", "all", "c");
-    }
-
     public void testThatUnknownUserIsRejectedProperly() throws Exception {
         HttpResponse response = executeRequest("idonotexist", "GET", "/", null, new HashMap<>());
         assertThat(response.getStatusCode(), is(401));
     }
 
     private void assertUserExecutes(String user, String action, String index, boolean userIsAllowed) throws Exception {
-        Map<String, String> refreshParams = singletonMap("refresh", "true");
+        Map<String, String> refreshParams = Collections.emptyMap();//singletonMap("refresh", "true");
 
         switch (action) {
             case "all" :
@@ -353,11 +353,6 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
                     assertAccessIsAllowed("admin", "PUT", "/" + index + "/foo/1", jsonDoc, refreshParams);
                     client().admin().cluster().prepareHealth(index).setWaitForGreenStatus().get();
                     assertAccessIsAllowed(user, "GET", "/" + index + "/_mapping/foo/field/name");
-                    // putting warmers only works if the user is allowed to search as well, as the query gets validated, added an own
-                    // test for this
-                    assertAccessIsAllowed("admin", "PUT", "/" + index + "/_warmer/w1", "{ \"query\" : { \"match_all\" : {} } }");
-                    assertAccessIsAllowed(user, "GET", "/" + index + "/_warmer/w1");
-                    assertAccessIsAllowed(user, "DELETE", "/" + index + "/_warmer/w1");
                     assertAccessIsAllowed(user, "GET", "/" + index + "/_settings");
                 } else {
                     assertAccessIsDenied(user, "DELETE", "/" + index);
@@ -372,9 +367,6 @@ public class IndexPrivilegeTests extends AbstractPrivilegeTestCase {
                     assertAccessIsDenied(user, "POST", "/" + index + "/_open");
                     assertAccessIsDenied(user, "POST", "/" + index + "/_cache/clear");
                     assertAccessIsDenied(user, "GET", "/" + index + "/_mapping/foo/field/name");
-                    assertAccessIsDenied(user, "PUT", "/" + index + "/_warmer/w1", "{ \"query\" : { \"match_all\" : {} } }");
-                    assertAccessIsDenied(user, "GET", "/" + index + "/_warmer/w1");
-                    assertAccessIsDenied(user, "DELETE", "/" + index + "/_warmer/w1");
                     assertAccessIsDenied(user, "GET", "/" + index + "/_settings");
                 }
                 break;

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/IndicesPermissionsWithAliasesWildcardsAndRegexsTests.java

@@ -12,6 +12,7 @@ import org.elasticsearch.shield.ShieldPlugin;
 import org.elasticsearch.shield.authc.support.Hasher;
 import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.test.ShieldIntegTestCase;
+import org.elasticsearch.xpack.XPackPlugin;
 
 import java.util.Collections;
 
@@ -60,7 +61,7 @@ public class IndicesPermissionsWithAliasesWildcardsAndRegexsTests extends Shield
     public Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put(ShieldPlugin.DLS_FLS_ENABLED_SETTING, true)
+                .put(XPackPlugin.featureEnabledSetting(ShieldPlugin.DLS_FLS_FEATURE), true)
                 .build();
     }
 

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/integration/PermissionPrecedenceTests.java

@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.integration;
 
-import org.apache.lucene.util.LuceneTestCase.BadApple;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
@@ -33,8 +32,6 @@ import static org.hamcrest.Matchers.hasSize;
  * actions that are normally categorized as index actions as cluster actions - for example,
  * index template actions.
  */
-//test is just too slow, please fix it to not be sleep-based
-@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007")
 public class PermissionPrecedenceTests extends ShieldIntegTestCase {
     protected static final String USERS_PASSWD_HASHED = new String(Hasher.BCRYPT.hash(new SecuredString("test123".toCharArray())));
 

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/ShieldPluginEnabledDisabledTests.java

@@ -66,7 +66,7 @@ public class ShieldPluginEnabledDisabledTests extends ShieldIntegTestCase {
         logger.info("******* shield is " + (enabled ? "enabled" : "disabled"));
         return Settings.settingsBuilder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put(ShieldPlugin.ENABLED_SETTING_NAME, enabled)
+                .put(XPackPlugin.featureEnabledSetting(ShieldPlugin.NAME), enabled)
                 .put(NetworkModule.HTTP_ENABLED.getKey(), true)
                 .build();
     }
@@ -75,7 +75,7 @@ public class ShieldPluginEnabledDisabledTests extends ShieldIntegTestCase {
     protected Settings transportClientSettings() {
         return Settings.settingsBuilder()
                 .put(super.transportClientSettings())
-                .put(ShieldPlugin.ENABLED_SETTING_NAME, enabled)
+                .put(XPackPlugin.featureEnabledSetting(ShieldPlugin.NAME), enabled)
                 .build();
     }
 

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/ShieldPluginSettingsTests.java

@@ -7,6 +7,7 @@ package org.elasticsearch.shield;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.XPackPlugin;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
@@ -14,8 +15,9 @@ import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.Matchers.arrayContaining;
 
 public class ShieldPluginSettingsTests extends ESTestCase {
-    private static final String TRIBE_T1_SHIELD_ENABLED = "tribe.t1." + ShieldPlugin.ENABLED_SETTING_NAME;
+
-    private static final String TRIBE_T2_SHIELD_ENABLED = "tribe.t2." + ShieldPlugin.ENABLED_SETTING_NAME;
+    private static final String TRIBE_T1_SHIELD_ENABLED = "tribe.t1." + XPackPlugin.featureEnabledSetting(ShieldPlugin.NAME);
+    private static final String TRIBE_T2_SHIELD_ENABLED = "tribe.t2." + XPackPlugin.featureEnabledSetting(ShieldPlugin.NAME);
 
     public void testShieldIsMandatoryOnTribes() {
         Settings settings = Settings.builder().put("tribe.t1.cluster.name", "non_existing")
@@ -26,8 +28,8 @@ public class ShieldPluginSettingsTests extends ESTestCase {
         Settings additionalSettings = shieldPlugin.additionalSettings();
 
 
-        assertThat(additionalSettings.getAsArray("tribe.t1.plugin.mandatory", null), arrayContaining(ShieldPlugin.NAME));
+        assertThat(additionalSettings.getAsArray("tribe.t1.plugin.mandatory", null), arrayContaining(XPackPlugin.NAME));
-        assertThat(additionalSettings.getAsArray("tribe.t2.plugin.mandatory", null), arrayContaining(ShieldPlugin.NAME));
+        assertThat(additionalSettings.getAsArray("tribe.t2.plugin.mandatory", null), arrayContaining(XPackPlugin.NAME));
     }
 
     public void testAdditionalMandatoryPluginsOnTribes() {
@@ -41,14 +43,14 @@ public class ShieldPluginSettingsTests extends ESTestCase {
             Settings.builder().put(settings).put(shieldPlugin.additionalSettings()).build();
             fail("shield cannot change the value of a setting that is already defined, so a exception should be thrown");
         } catch (IllegalStateException e) {
-            assertThat(e.getMessage(), containsString("shield"));
+            assertThat(e.getMessage(), containsString(XPackPlugin.NAME));
             assertThat(e.getMessage(), containsString("plugin.mandatory"));
         }
     }
 
     public void testMandatoryPluginsOnTribesShieldAlreadyMandatory() {
         Settings settings = Settings.builder().put("tribe.t1.cluster.name", "non_existing")
-                .putArray("tribe.t1.plugin.mandatory", "test_plugin", ShieldPlugin.NAME).build();
+                .putArray("tribe.t1.plugin.mandatory", "test_plugin", XPackPlugin.NAME).build();
 
         ShieldPlugin shieldPlugin = new ShieldPlugin(settings);
 
@@ -59,7 +61,7 @@ public class ShieldPluginSettingsTests extends ESTestCase {
         assertThat(finalMandatoryPlugins, notNullValue());
         assertThat(finalMandatoryPlugins.length, equalTo(2));
         assertThat(finalMandatoryPlugins[0], equalTo("test_plugin"));
-        assertThat(finalMandatoryPlugins[1], equalTo(ShieldPlugin.NAME));
+        assertThat(finalMandatoryPlugins[1], equalTo(XPackPlugin.NAME));
     }
 
     public void testShieldIsEnabledByDefaultOnTribes() {
@@ -93,7 +95,7 @@ public class ShieldPluginSettingsTests extends ESTestCase {
         Settings settings = Settings.builder().put("tribe.t1.cluster.name", "non_existing")
                 .put(TRIBE_T1_SHIELD_ENABLED, false)
                 .put("tribe.t2.cluster.name", "non_existing")
-                .putArray("tribe.t1.plugin.mandatory", "test_plugin", ShieldPlugin.NAME).build();
+                .putArray("tribe.t1.plugin.mandatory", "test_plugin", XPackPlugin.NAME).build();
 
         ShieldPlugin shieldPlugin = new ShieldPlugin(settings);
 

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailEnabledTests.java

@@ -1,108 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.shield.audit.index;
-
-import org.apache.lucene.util.LuceneTestCase.BadApple;
-import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse;
-import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.shield.audit.logfile.LoggingAuditTrail;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.elasticsearch.test.ShieldIntegTestCase;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-
-import java.util.Collections;
-import java.util.Set;
-
-import static org.hamcrest.Matchers.is;
-
-//test is just too slow, please fix it to not be sleep-based
-@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007")
-@ClusterScope(scope = Scope.TEST, randomDynamicTemplates = false)
-public class IndexAuditTrailEnabledTests extends ShieldIntegTestCase {
-    IndexNameResolver.Rollover rollover = randomFrom(IndexNameResolver.Rollover.values());
-
-    @Override
-    protected Set<String> excludeTemplates() {
-        return Collections.singleton(IndexAuditTrail.INDEX_TEMPLATE_NAME);
-    }
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        Settings.Builder builder = Settings.builder()
-                .put(super.nodeSettings(nodeOrdinal));
-        builder.put("shield.audit.enabled", true);
-        if (randomBoolean()) {
-            builder.putArray("shield.audit.outputs", LoggingAuditTrail.NAME, IndexAuditTrail.NAME);
-        } else {
-            builder.putArray("shield.audit.outputs", IndexAuditTrail.NAME);
-        }
-        builder.put(IndexAuditTrail.ROLLOVER_SETTING, rollover);
-
-        return builder.build();
-    }
-
-    @Override
-    public void beforeIndexDeletion() {
-        // For this test, this is a NO-OP because the index audit trail will continue to capture events and index after
-        // the tests have completed. The default implementation of this method expects that nothing is performing operations
-        // after the test has completed
-    }
-
-    public void testAuditTrailIndexAndTemplateExists() throws Exception {
-        awaitIndexTemplateCreation();
-
-        // Wait for the index to be created since we have our own startup
-        awaitAuditDocumentCreation();
-    }
-
-    public void testAuditTrailTemplateIsRecreatedAfterDelete() throws Exception {
-        // this is already "tested" by the test framework since we wipe the templates before and after, but lets be explicit about the
-        // behavior
-        awaitIndexTemplateCreation();
-
-        // delete the template
-        DeleteIndexTemplateResponse deleteResponse = client().admin().indices()
-                .prepareDeleteTemplate(IndexAuditTrail.INDEX_TEMPLATE_NAME).execute().actionGet();
-        assertThat(deleteResponse.isAcknowledged(), is(true));
-        awaitIndexTemplateCreation();
-    }
-
-    void awaitAuditDocumentCreation() throws Exception {
-        final String indexName = IndexNameResolver.resolve(IndexAuditTrail.INDEX_NAME_PREFIX, DateTime.now(DateTimeZone.UTC), rollover);
-        boolean success = awaitBusy(() -> {
-            try {
-                SearchResponse searchResponse = client().prepareSearch(indexName).setSize(0).setTerminateAfter(1).execute().actionGet();
-                return searchResponse.getHits().totalHits() > 0;
-            } catch (Exception e) {
-                return false;
-            }
-        });
-
-        assertThat("no audit document exists!", success, is(true));
-    }
-
-    void awaitIndexTemplateCreation() throws InterruptedException {
-        boolean found = awaitBusy(() -> {
-            GetIndexTemplatesResponse response = client().admin().indices()
-                    .prepareGetTemplates(IndexAuditTrail.INDEX_TEMPLATE_NAME).execute().actionGet();
-            if (response.getIndexTemplates().size() > 0) {
-                for (IndexTemplateMetaData indexTemplateMetaData : response.getIndexTemplates()) {
-                    if (IndexAuditTrail.INDEX_TEMPLATE_NAME.equals(indexTemplateMetaData.name())) {
-                        return true;
-                    }
-                }
-            }
-            return false;
-        });
-
-        assertThat("index template [" + IndexAuditTrail.INDEX_TEMPLATE_NAME + "] was not created", found, is(true));
-    }
-}

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/audit/index/IndexAuditTrailTests.java

@@ -42,6 +42,7 @@ import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportInfo;
 import org.elasticsearch.transport.TransportMessage;
 import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.xpack.XPackPlugin;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.ISODateTimeFormat;
@@ -166,18 +167,21 @@ public class IndexAuditTrailTests extends ShieldIntegTestCase {
                     public Settings nodeSettings(int nodeOrdinal) {
                         Settings.Builder builder = Settings.builder()
                                 .put(super.nodeSettings(nodeOrdinal))
-                                .put(ShieldPlugin.ENABLED_SETTING_NAME, useShield);
+                                .put(XPackPlugin.featureEnabledSetting(ShieldPlugin.NAME), useShield);
-                        // For tests we forcefully configure Shield's custom query cache because the test framework randomizes the query
+
-                        // cache impl,
+                        // For tests we forcefully configure Shield's custom query cache because the test framework
-                        // but if shield is disabled then we don't need to forcefully set the query cache
+                        // randomizes the query cache impl but if shield is disabled then we don't need to forcefully
+                        // set the query cache
                         if (useShield == false) {
                             builder.remove(IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING.getKey());
                         }
+
                         return builder.build();
                     }
             };
             cluster2 = new InternalTestCluster("network", randomLong(), createTempDir(), numNodes, numNodes, cluster2Name,
-                    cluster2SettingsSource, 0, false, SECOND_CLUSTER_NODE_PREFIX, getMockPlugins(), Function.identity());
+                    cluster2SettingsSource, 0, false, SECOND_CLUSTER_NODE_PREFIX, getMockPlugins(),
+                    useShield ? getClientWrapper() : Function.identity());
             cluster2.beforeTest(getRandom(), 0.5);
             remoteClient = cluster2.client();
 
@@ -187,7 +191,7 @@ public class IndexAuditTrailTests extends ShieldIntegTestCase {
 
             Settings.Builder builder = Settings.builder()
                     .put(settings)
-                    .put(ShieldPlugin.ENABLED_SETTING_NAME, useShield)
+                    .put(XPackPlugin.featureEnabledSetting(ShieldPlugin.NAME), useShield)
                     .put(remoteSettings(NetworkAddress.formatAddress(inet.address().getAddress()), inet.address().getPort(), cluster2Name))
                     .put("shield.audit.index.client.shield.user", DEFAULT_USER_NAME + ":" + DEFAULT_PASSWORD);
 

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/esnative/ESNativeTests.java

@@ -22,7 +22,7 @@ import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.shield.authz.RoleDescriptor;
 import org.elasticsearch.shield.authz.esnative.ESNativeRolesStore;
 import org.elasticsearch.shield.authz.permission.Role;
-import org.elasticsearch.shield.client.ShieldClient;
+import org.elasticsearch.shield.client.SecurityClient;
 import org.elasticsearch.test.ShieldIntegTestCase;
 import org.elasticsearch.test.ShieldSettingsSource;
 import org.junit.After;
@@ -43,7 +43,7 @@ import static org.hamcrest.Matchers.isOneOf;
 public class ESNativeTests extends ShieldIntegTestCase {
 
     public void testDeletingNonexistingUserAndRole() throws Exception {
-        ShieldClient c = new ShieldClient(client());
+        SecurityClient c = securityClient();
         DeleteUserResponse resp = c.prepareDeleteUser().user("joe").get();
         assertFalse("user shouldn't be found", resp.found());
         DeleteRoleResponse resp2 = c.prepareDeleteRole().role("role").get();
@@ -51,7 +51,7 @@ public class ESNativeTests extends ShieldIntegTestCase {
     }
 
     public void testGettingUserThatDoesntExist() throws Exception {
-        ShieldClient c = new ShieldClient(client());
+        SecurityClient c = securityClient();
         GetUsersResponse resp = c.prepareGetUsers().users("joe").get();
         assertFalse("user should not exist", resp.isExists());
         GetRolesResponse resp2 = c.prepareGetRoles().roles("role").get();
@@ -59,7 +59,7 @@ public class ESNativeTests extends ShieldIntegTestCase {
     }
 
     public void testAddAndGetUser() throws Exception {
-        ShieldClient c = new ShieldClient(client());
+        SecurityClient c = securityClient();
         logger.error("--> creating user");
         c.prepareAddUser()
                 .username("joe")
@@ -117,7 +117,7 @@ public class ESNativeTests extends ShieldIntegTestCase {
     }
 
     public void testAddAndGetRole() throws Exception {
-        ShieldClient c = new ShieldClient(client());
+        SecurityClient c = securityClient();
         logger.error("--> creating role");
         c.prepareAddRole()
                 .name("test_role")
@@ -171,7 +171,7 @@ public class ESNativeTests extends ShieldIntegTestCase {
     }
 
     public void testAddUserAndRoleThenAuth() throws Exception {
-        ShieldClient c = new ShieldClient(client());
+        SecurityClient c = securityClient();
         logger.error("--> creating role");
         c.prepareAddRole()
                 .name("test_role")
@@ -204,7 +204,7 @@ public class ESNativeTests extends ShieldIntegTestCase {
     }
 
     public void testUpdatingUserAndAuthentication() throws Exception {
-        ShieldClient c = new ShieldClient(client());
+        SecurityClient c = securityClient();
         logger.error("--> creating user");
         c.prepareAddUser()
                 .username("joe")
@@ -248,7 +248,7 @@ public class ESNativeTests extends ShieldIntegTestCase {
     }
 
     public void testCreateDeleteAuthenticate() {
-        ShieldClient c = new ShieldClient(client());
+        SecurityClient c = securityClient();
         logger.error("--> creating user");
         c.prepareAddUser()
                 .username("joe")
@@ -285,7 +285,7 @@ public class ESNativeTests extends ShieldIntegTestCase {
 
     public void testCreateAndUpdateRole() {
         final boolean authenticate = randomBoolean();
-        ShieldClient c = new ShieldClient(client());
+        SecurityClient c = securityClient();
         logger.error("--> creating role");
         c.prepareAddRole()
                 .name("test_role")
@@ -340,7 +340,7 @@ public class ESNativeTests extends ShieldIntegTestCase {
     }
 
     public void testAuthenticateWithDeletedRole() {
-        ShieldClient c = new ShieldClient(client());
+        SecurityClient c = securityClient();
         logger.error("--> creating role");
         c.prepareAddRole()
                 .name("test_role")
@@ -373,7 +373,7 @@ public class ESNativeTests extends ShieldIntegTestCase {
     @Before
     public void ensureStoresStarted() throws Exception {
         // Clear the realm cache for all realms since we use a SUITE scoped cluster
-        ShieldClient client = new ShieldClient(client());
+        SecurityClient client = securityClient();
         client.prepareClearRealmCache().get();
 
         for (ESNativeUsersStore store : internalCluster().getInstances(ESNativeUsersStore.class)) {

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authc/support/CachingUsernamePasswordRealmTests.java

@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.shield.authc.support;
 
-import org.apache.lucene.util.LuceneTestCase.BadApple;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.shield.User;
@@ -25,8 +24,6 @@ import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
-//test is just too slow, please fix it to not be sleep-based
-@BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007")
 public class CachingUsernamePasswordRealmTests extends ESTestCase {
     private Settings globalSettings;
 
@@ -206,7 +203,9 @@ public class CachingUsernamePasswordRealmTests extends ESTestCase {
         };
 
         final CountDownLatch latch = new CountDownLatch(1);
-        final int numberOfThreads = randomIntBetween(8, 24);
+        final int numberOfProcessors = Runtime.getRuntime().availableProcessors();
+        final int numberOfThreads = scaledRandomIntBetween((numberOfProcessors + 1) / 2, numberOfProcessors * 3);
+        final int numberOfIterations = scaledRandomIntBetween(20, 100);
         List<Thread> threads = new ArrayList<>();
         for (int i = 0; i < numberOfThreads; i++) {
             final boolean invalidPassword = randomBoolean();
@@ -215,9 +214,9 @@ public class CachingUsernamePasswordRealmTests extends ESTestCase {
                 public void run() {
                     try {
                         latch.await();
-                        for (int i = 0; i < 100; i++) {
+                        for (int i = 0; i < numberOfIterations; i++) {
-                            User user = realm.authenticate(
+                            UsernamePasswordToken token = new UsernamePasswordToken(username, invalidPassword ? randomPassword : password);
-                                    new UsernamePasswordToken(username, invalidPassword ? randomPassword : password));
+                            User user = realm.authenticate(token);
                             if (invalidPassword && user != null) {
                                 throw new RuntimeException("invalid password led to an authenticated user: " + user.toString());
                             } else if (invalidPassword == false && user == null) {

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/privilege/PrivilegeTests.java

@@ -245,7 +245,7 @@ public class PrivilegeTests extends ESTestCase {
         assertThat(predicate.test("whatever"), is(false));
         assertThat(predicate.test("cluster:admin/reroute"), is(true));
         assertThat(predicate.test("cluster:admin/whatever"), is(false));
-        assertThat(predicate.test("indices:admin/mapping/put"), is(true));
+        assertThat(predicate.test("indices:admin/mapping/put"), is(false));
         assertThat(predicate.test("indices:admin/mapping/whatever"), is(false));
     }
 

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/authz/store/FileRolesStoreTests.java

@@ -19,6 +19,7 @@ import org.elasticsearch.shield.authz.privilege.IndexPrivilege;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xpack.XPackPlugin;
 
 import java.io.BufferedWriter;
 import java.io.OutputStream;
@@ -52,8 +53,9 @@ public class FileRolesStoreTests extends ESTestCase {
 
     public void testParseFile() throws Exception {
         Path path = getDataPath("roles.yml");
-        Map<String, Role> roles = FileRolesStore.parseFile(path, logger,
+        Map<String, Role> roles = FileRolesStore.parseFile(path, logger, Settings.builder()
-                Settings.builder().put(ShieldPlugin.DLS_FLS_ENABLED_SETTING, true).build());
+                .put(XPackPlugin.featureEnabledSetting(ShieldPlugin.DLS_FLS_FEATURE), true)
+                .build());
         assertThat(roles, notNullValue());
         assertThat(roles.size(), is(10));
 
@@ -208,8 +210,9 @@ public class FileRolesStoreTests extends ESTestCase {
     public void testParseFileWithFLSAndDLSDisabled() throws Exception {
         Path path = getDataPath("roles.yml");
         CapturingLogger logger = new CapturingLogger(CapturingLogger.Level.ERROR);
-        Map<String, Role> roles = FileRolesStore.parseFile(path,
+        Map<String, Role> roles = FileRolesStore.parseFile(path, logger, Settings.builder()
-                logger, Settings.builder().put(ShieldPlugin.DLS_FLS_ENABLED_SETTING, false).build());
+                .put(XPackPlugin.featureEnabledSetting(ShieldPlugin.DLS_FLS_FEATURE), false)
+                .build());
         assertThat(roles, notNullValue());
         assertThat(roles.size(), is(7));
         assertThat(roles.get("role_fields"), nullValue());

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/ssl/ClientSSLServiceTests.java

@@ -251,17 +251,17 @@ public class ClientSSLServiceTests extends ESTestCase {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/2")
     public void testThatSSLSocketFactoryHasProperCiphersAndProtocols() throws Exception {
         ClientSSLService sslService = createClientSSLService(settingsBuilder()
                 .put("shield.ssl.keystore.path", testclientStore)
                 .put("shield.ssl.keystore.password", "testclient")
                 .build());
         SSLSocketFactory factory = sslService.sslSocketFactory();
-        assertThat(factory.getDefaultCipherSuites(), is(sslService.ciphers()));
+        final String[] ciphers = sslService.supportedCiphers(factory.getSupportedCipherSuites(), sslService.ciphers());
+        assertThat(factory.getDefaultCipherSuites(), is(ciphers));
 
         try (SSLSocket socket = (SSLSocket) factory.createSocket()) {
-            assertThat(socket.getEnabledCipherSuites(), is(sslService.ciphers()));
+            assertThat(socket.getEnabledCipherSuites(), is(ciphers));
             assertThat(socket.getEnabledProtocols(), is(sslService.supportedProtocols()));
         }
     }

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/shield/ssl/ServerSSLServiceTests.java

@@ -32,7 +32,6 @@ import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
-import static org.mockito.Mockito.mock;
 
 public class ServerSSLServiceTests extends ESTestCase {
     Path testnodeStore;
@@ -223,17 +222,17 @@ public class ServerSSLServiceTests extends ESTestCase {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/x-plugins/issues/2")
     public void testThatSSLSocketFactoryHasProperCiphersAndProtocols() throws Exception {
         ServerSSLService sslService = new ServerSSLService(settingsBuilder()
                 .put("shield.ssl.keystore.path", testnodeStore)
                 .put("shield.ssl.keystore.password", "testnode")
                 .build(), env);
         SSLSocketFactory factory = sslService.sslSocketFactory();
-        assertThat(factory.getDefaultCipherSuites(), is(sslService.ciphers()));
+        final String[] ciphers = sslService.supportedCiphers(factory.getSupportedCipherSuites(), sslService.ciphers());
+        assertThat(factory.getDefaultCipherSuites(), is(ciphers));
 
         try (SSLSocket socket = (SSLSocket) factory.createSocket()) {
-            assertThat(socket.getEnabledCipherSuites(), is(sslService.ciphers()));
+            assertThat(socket.getEnabledCipherSuites(), is(ciphers));
             assertThat(socket.getEnabledProtocols(), is(sslService.supportedProtocols()));
         }
     }

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/test/ShieldIntegTestCase.java

@@ -15,9 +15,11 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.shield.InternalClient;
 import org.elasticsearch.shield.authc.support.SecuredString;
+import org.elasticsearch.shield.client.SecurityClient;
 import org.elasticsearch.test.ESIntegTestCase.SuppressLocalMode;
 import org.elasticsearch.test.transport.AssertingLocalTransport;
 import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.xpack.XPackClient;
 import org.elasticsearch.xpack.XPackPlugin;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -267,6 +269,7 @@ public abstract class ShieldIntegTestCase extends ESIntegTestCase {
     }
 
     private class CustomShieldSettingsSource extends ShieldSettingsSource {
+
         private CustomShieldSettingsSource(boolean sslTransportEnabled, Path configDir, Scope scope) {
             super(maxNumberOfNodes(), sslTransportEnabled, configDir, scope);
         }
@@ -339,4 +342,12 @@ public abstract class ShieldIntegTestCase extends ESIntegTestCase {
     protected InternalClient internalClient(String node) {
         return internalCluster().getInstance(InternalClient.class, node);
     }
+
+    protected SecurityClient securityClient() {
+        return securityClient(client());
+    }
+
+    public static SecurityClient securityClient(Client client) {
+        return randomBoolean() ? new XPackClient(client).security() : new SecurityClient(client);
+    }
 }

elasticsearch/x-pack/shield/src/test/java/org/elasticsearch/transport/ShieldServerTransportServiceTests.java

@@ -9,6 +9,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.shield.ShieldPlugin;
 import org.elasticsearch.shield.transport.ShieldServerTransportService;
 import org.elasticsearch.test.ShieldIntegTestCase;
+import org.elasticsearch.xpack.XPackPlugin;
 
 import java.util.Map;
 
@@ -21,7 +22,7 @@ public class ShieldServerTransportServiceTests extends ShieldIntegTestCase {
     protected Settings transportClientSettings() {
         return Settings.settingsBuilder()
                 .put(super.transportClientSettings())
-                .put(ShieldPlugin.ENABLED_SETTING_NAME, true)
+                .put(XPackPlugin.featureEnabledSetting(ShieldPlugin.NAME), true)
                 .build();
     }
 

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackClient.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack;
+
+import org.elasticsearch.client.Client;
+import org.elasticsearch.shield.authc.support.SecuredString;
+import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
+import org.elasticsearch.shield.client.SecurityClient;
+import org.elasticsearch.watcher.client.WatcherClient;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.BASIC_AUTH_HEADER;
+import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
+
+/**
+ *
+ */
+public class XPackClient {
+
+    private final Client client;
+    private final SecurityClient securityClient;
+    private final WatcherClient watcherClient;
+
+    public XPackClient(Client client) {
+        this.client = client;
+        this.securityClient = new SecurityClient(client);
+        this.watcherClient = new WatcherClient(client);
+    }
+
+    public SecurityClient security() {
+        return securityClient;
+    }
+
+    public WatcherClient watcher() {
+        return watcherClient;
+    }
+
+    public XPackClient withHeaders(Map<String, String> headers) {
+        return new XPackClient(client.filterWithHeader(headers));
+    }
+
+    /**
+     * Returns a client that will call xpack APIs on behalf of the given user.
+     *
+     * @param username The username of the user
+     * @param passwd    The password of the user. This char array can be cleared after calling this method.
+     */
+    public XPackClient withAuth(String username, char[] passwd) {
+       return withHeaders(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue(username, new SecuredString(passwd))));
+    }
+}

elasticsearch/x-pack/src/main/java/org/elasticsearch/xpack/XPackPlugin.java

@@ -7,13 +7,13 @@ package org.elasticsearch.xpack;
 
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.action.ActionModule;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.license.plugin.LicensePlugin;
 import org.elasticsearch.marvel.MarvelPlugin;
@@ -22,6 +22,7 @@ import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.shield.ShieldPlugin;
 import org.elasticsearch.watcher.WatcherPlugin;
 
+import java.nio.file.Path;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.ArrayList;
@@ -29,9 +30,7 @@ import java.util.Collection;
 
 public class XPackPlugin extends Plugin {
 
-    public static final String NAME = "x-pack";
+    public static final String NAME = "xpack";
-
-    private final static ESLogger logger = Loggers.getLogger(XPackPlugin.class);
 
     // TODO: clean up this library to not ask for write access to all system properties!
     static {
@@ -143,4 +142,30 @@ public class XPackPlugin extends Plugin {
         watcherPlugin.onIndexModule(module);
         marvelPlugin.onIndexModule(module);
     }
+
+    public static boolean transportClientMode(Settings settings) {
+        return !"node".equals(settings.get(Client.CLIENT_TYPE_SETTING_S.getKey()));
+    }
+
+    public static Path resolveConfigFile(Environment env, String name) {
+        return env.configFile().resolve(NAME).resolve(name);
+    }
+
+    /**
+     * A consistent way to enable disable features using the following setting:
+     *
+     *          {@code "xpack.<feature>.enabled": true | false}
+     *
+     *  Also supports the following setting as a fallback (for BWC with 1.x/2.x):
+     *
+     *          {@code "<feature>.enabled": true | false}
+     */
+    public static boolean featureEnabled(Settings settings, String featureName, boolean defaultValue) {
+        return settings.getAsBoolean(featureEnabledSetting(featureName),
+                settings.getAsBoolean(featureName + ".enabled", defaultValue)); // for bwc
+    }
+
+    public static String featureEnabledSetting(String featureName) {
+        return NAME + "." + featureName + ".enabled";
+    }
 }

elasticsearch/x-pack/src/test/resources/rest-api-spec/test/xpack/10_basic.yaml

@@ -10,4 +10,4 @@
     - do:
         nodes.info: {}
 
-    - match:  { nodes.$master.plugins.0.name: x-pack  }
+    - match:  { nodes.$master.plugins.0.name: xpack  }

elasticsearch/x-pack/watcher/bin/xpack/.in.bat

@@ -76,7 +76,7 @@ REM JAVA_OPTS=%JAVA_OPTS% -XX:HeapDumpPath=$ES_HOME/logs/heapdump.hprof
 
 REM Disables explicit GC
 set JAVA_OPTS=%JAVA_OPTS% -XX:+DisableExplicitGC
-set ES_CLASSPATH=%ES_CLASSPATH%;%ES_HOME%/lib/elasticsearch-1.4.0-SNAPSHOT.jar;%ES_HOME%/lib/*;%ES_HOME%/lib/sigar/*;%ES_HOME%/plugins/x-pack/*
+set ES_CLASSPATH=%ES_CLASSPATH%;%ES_HOME%/lib/*;%ES_HOME%/lib/sigar/*;%ES_HOME%/plugins/xpack/*
 set ES_PARAMS=-Des.path.home="%ES_HOME%"
 
 SET HOSTNAME=%COMPUTERNAME%

elasticsearch/x-pack/watcher/bin/xpack/croneval

@@ -114,7 +114,7 @@ fi
 export HOSTNAME=`hostname -s`
 
 # include watcher jars in classpath
-ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/plugins/x-pack/*"
+ES_CLASSPATH="$ES_CLASSPATH:$ES_HOME/plugins/xpack/*"
 
 cd "$ES_HOME" > /dev/null
 $JAVA $ES_JAVA_OPTS -cp "$ES_CLASSPATH" org.elasticsearch.watcher.trigger.schedule.tool.CronEvalTool "$@" $properties

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/WatcherModule.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.watcher;
 
 
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.multibindings.Multibinder;
 import org.elasticsearch.common.settings.Setting;
@@ -14,10 +15,17 @@ import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry;
 import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry.TemplateConfig;
 import org.elasticsearch.watcher.support.validation.WatcherSettingsValidation;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
 
 public class WatcherModule extends AbstractModule {
 
-    public static final String HISTORY_TEMPLATE_NAME = "watch_history";
+    static final String PROPERTIES_FILE = "/watcher.properties";
+    static final String VERSION_FIELD = "watcher.template.version";
+
+    public static final String HISTORY_TEMPLATE_NAME = "watch_history_" + getHistoryIndexTemplateVersion();
     public static final String TRIGGERED_TEMPLATE_NAME = "triggered_watches";
     public static final String WATCHES_TEMPLATE_NAME = "watches";
     public static final Setting<Settings> HISTORY_TEMPLATE_SETTING = Setting.groupSetting("watcher.history.index.", true,
@@ -30,7 +38,7 @@ public class WatcherModule extends AbstractModule {
 
     public final static TemplateConfig[] TEMPLATE_CONFIGS = new TemplateConfig[]{
             new TemplateConfig(TRIGGERED_TEMPLATE_NAME, TRIGGERED_TEMPLATE_SETTING),
-            new TemplateConfig(HISTORY_TEMPLATE_NAME, HISTORY_TEMPLATE_SETTING),
+            new TemplateConfig(HISTORY_TEMPLATE_NAME, "watch_history", HISTORY_TEMPLATE_SETTING),
             new TemplateConfig(WATCHES_TEMPLATE_NAME, WATCHES_TEMPLATE_SETTING)
     };
 
@@ -53,4 +61,19 @@ public class WatcherModule extends AbstractModule {
         }
     }
 
+    public static final Integer getHistoryIndexTemplateVersion() {
+        try (InputStream is = WatcherModule.class.getResourceAsStream(PROPERTIES_FILE)) {
+            Properties properties = new Properties();
+            properties.load(is);
+            String version = properties.getProperty(VERSION_FIELD);
+            if (Strings.hasLength(version)) {
+                return Integer.parseInt(version);
+            }
+            return null;
+        } catch (NumberFormatException e) {
+            throw new IllegalArgumentException("failed to parse watcher template version");
+        } catch (IOException e) {
+            throw new IllegalArgumentException("failed to load watcher template version");
+        }
+    }
 }

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/WatcherPlugin.java

@@ -22,6 +22,8 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptModule;
+import org.elasticsearch.shield.ShieldPlugin;
+import org.elasticsearch.shield.authz.privilege.ClusterPrivilege;
 import org.elasticsearch.watcher.actions.WatcherActionModule;
 import org.elasticsearch.watcher.actions.email.service.EmailService;
 import org.elasticsearch.watcher.actions.email.service.InternalEmailService;
@@ -50,8 +52,7 @@ import org.elasticsearch.watcher.rest.action.RestPutWatchAction;
 import org.elasticsearch.watcher.rest.action.RestWatchServiceAction;
 import org.elasticsearch.watcher.rest.action.RestWatcherInfoAction;
 import org.elasticsearch.watcher.rest.action.RestWatcherStatsAction;
-import org.elasticsearch.watcher.shield.ShieldSecretService;
+import org.elasticsearch.watcher.support.secret.SecretService;
-import org.elasticsearch.watcher.shield.WatcherShieldModule;
 import org.elasticsearch.watcher.support.WatcherIndexTemplateRegistry.TemplateConfig;
 import org.elasticsearch.watcher.support.clock.ClockModule;
 import org.elasticsearch.watcher.support.http.HttpClient;
@@ -91,13 +92,19 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.function.Function;
 
+import static org.elasticsearch.common.settings.Setting.Scope.CLUSTER;
 import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 
 public class WatcherPlugin extends Plugin {
 
     public static final String NAME = "watcher";
     public static final String ENABLED_SETTING = NAME + ".enabled";
+    public static final Setting<String> INDEX_WATCHER_VERSION_SETTING =
+            new Setting<>("index.watcher.plugin.version", "", Function.identity(), false, Setting.Scope.INDEX);
+    public static final Setting<String> INDEX_WATCHER_TEMPLATE_VERSION_SETTING =
+            new Setting<>("index.watcher.template.version", "", Function.identity(), false, Setting.Scope.INDEX);
 
     private final static ESLogger logger = Loggers.getLogger(XPackPlugin.class);
 
@@ -114,7 +121,14 @@ public class WatcherPlugin extends Plugin {
         transportClient = "transport".equals(settings.get(Client.CLIENT_TYPE_SETTING_S.getKey()));
         enabled = watcherEnabled(settings);
         validAutoCreateIndex(settings);
+
+        // adding the watcher privileges to shield
+        if (ShieldPlugin.enabled(settings)) {
+            registerClusterPrivilege("manage_watcher", "cluster:admin/watcher/*", "cluster:monitor/watcher/*");
+            registerClusterPrivilege("monitor_watcher", "cluster:monitor/watcher/*");
         }
+    }
+
 
     @Override public String name() {
         return NAME;
@@ -146,7 +160,6 @@ public class WatcherPlugin extends Plugin {
                 new WatcherActionModule(),
                 new HistoryModule(),
                 new ExecutionModule(),
-                new WatcherShieldModule(settings),
                 new SecretModule(settings));
     }
 
@@ -193,37 +206,41 @@ public class WatcherPlugin extends Plugin {
         module.registerSetting(InternalEmailService.EMAIL_ACCOUNT_SETTING);
         module.registerSetting(InternalHipChatService.HIPCHAT_ACCOUNT_SETTING);
         module.registerSetting(InternalPagerDutyService.PAGERDUTY_ACCOUNT_SETTING);
-        module.registerSetting(Setting.intSetting("watcher.execution.scroll.size", 0, false, Setting.Scope.CLUSTER));
+        module.registerSetting(INDEX_WATCHER_VERSION_SETTING);
-        module.registerSetting(Setting.intSetting("watcher.watch.scroll.size", 0, false, Setting.Scope.CLUSTER));
+        module.registerSetting(INDEX_WATCHER_TEMPLATE_VERSION_SETTING);
-        module.registerSetting(Setting.boolSetting("watcher.enabled", false, false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.intSetting("watcher.execution.scroll.size", 0, false, CLUSTER));
-        module.registerSetting(ShieldSecretService.ENCRYPT_SENSITIVE_DATA_SETTING);
+        module.registerSetting(Setting.intSetting("watcher.watch.scroll.size", 0, false, CLUSTER));
+        module.registerSetting(Setting.boolSetting("watcher.enabled", false, false, CLUSTER));
+        module.registerSetting(SecretService.Secure.ENCRYPT_SENSITIVE_DATA_SETTING);
+
         // TODO add real settings for these
-        module.registerSetting(Setting.simpleString("resource.reload.interval", false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.simpleString("resource.reload.interval", false, CLUSTER));
-        module.registerSetting(Setting.simpleString("resource.reload.enabled", false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.simpleString("resource.reload.enabled", false, CLUSTER));
-        module.registerSetting(Setting.simpleString("resource.reload.interval.low", false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.simpleString("resource.reload.interval.low", false, CLUSTER));
-        module.registerSetting(Setting.simpleString("resource.reload.interval.medium", false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.simpleString("resource.reload.interval.medium", false, CLUSTER));
-        module.registerSetting(Setting.simpleString("watcher.internal.ops.search.default_timeout", false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.simpleString("watcher.internal.ops.search.default_timeout", false, CLUSTER));
-        module.registerSetting(Setting.simpleString("watcher.internal.ops.bulk.default_timeout", false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.simpleString("watcher.internal.ops.bulk.default_timeout", false, CLUSTER));
-        module.registerSetting(Setting.simpleString("watcher.internal.ops.index.default_timeout", false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.simpleString("watcher.internal.ops.index.default_timeout", false, CLUSTER));
-        module.registerSetting(Setting.simpleString("watcher.execution.default_throttle_period", false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.simpleString("watcher.execution.default_throttle_period", false, CLUSTER));
-        module.registerSetting(Setting.simpleString("watcher.http.default_read_timeout", false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.simpleString("watcher.http.default_read_timeout", false, CLUSTER));
-        module.registerSetting(Setting.groupSetting("watcher.http.ssl.", false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.groupSetting("watcher.http.ssl.", false, CLUSTER));
-        module.registerSetting(Setting.groupSetting("watcher.http.proxy.", false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.groupSetting("watcher.http.proxy.", false, CLUSTER));
-        module.registerSetting(Setting.simpleString("watcher.actions.index.default_timeout", false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.simpleString("watcher.actions.index.default_timeout", false, CLUSTER));
-        module.registerSetting(Setting.simpleString("watcher.index.rest.direct_access", false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.simpleString("watcher.index.rest.direct_access", false, CLUSTER));
-        module.registerSetting(Setting.simpleString("watcher.trigger.schedule.engine", false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.simpleString("watcher.trigger.schedule.engine", false, CLUSTER));
-        module.registerSetting(Setting.simpleString("watcher.input.search.default_timeout", false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.simpleString("watcher.input.search.default_timeout", false, CLUSTER));
-        module.registerSetting(Setting.simpleString("watcher.transform.search.default_timeout", false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.simpleString("watcher.transform.search.default_timeout", false, CLUSTER));
-        module.registerSetting(Setting.simpleString("watcher.trigger.schedule.ticker.tick_interval", false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.simpleString("watcher.trigger.schedule.ticker.tick_interval", false, CLUSTER));
-        module.registerSetting(Setting.simpleString("watcher.execution.scroll.timeout", false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.simpleString("watcher.execution.scroll.timeout", false, CLUSTER));
-        module.registerSetting(Setting.simpleString("watcher.start_immediately", false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.simpleString("watcher.start_immediately", false, CLUSTER));
-        module.registerSetting(Setting.simpleString("watcher.http.default_connection_timeout", false, Setting.Scope.CLUSTER));
+        module.registerSetting(Setting.simpleString("watcher.http.default_connection_timeout", false, CLUSTER));
 
         module.registerSettingsFilter("watcher.actions.email.service.account.*.smtp.password");
         module.registerSettingsFilter("watcher.actions.slack.service.account.*.url");
         module.registerSettingsFilter("watcher.actions.pagerduty.service.account.*.url");
         module.registerSettingsFilter("watcher.actions.pagerduty.service." + PagerDutyAccount.SERVICE_KEY_SETTING);
-        module.registerSettingsFilter("watcher.actions.pagerduty.service.account.*." + PagerDutyAccount.SERVICE_KEY_SETTING);
+        module.registerSettingsFilter("watcher.actions.pagerduty.service.account.*." +
+                PagerDutyAccount.SERVICE_KEY_SETTING);
         module.registerSettingsFilter("watcher.actions.hipchat.service.account.*.auth_token");
     }
 
@@ -265,8 +282,9 @@ public class WatcherPlugin extends Plugin {
             return;
         }
 
-        String errorMessage = LoggerMessageFormat.format("the [action.auto_create_index] setting value [{}] is too restrictive. disable " +
+        String errorMessage = LoggerMessageFormat.format("the [action.auto_create_index] setting value [{}] is too" +
-                "[action.auto_create_index] or set it to [.watches,.triggered_watches,.watch_history*]", (Object) settings);
+                " restrictive. disable [action.auto_create_index] or set it to " +
+                "[.watches,.triggered_watches,.watcher-history*]", (Object) settings);
         if (Booleans.isExplicitFalse(value)) {
             throw new IllegalArgumentException(errorMessage);
         }
@@ -312,9 +330,22 @@ public class WatcherPlugin extends Plugin {
                 throw new IllegalArgumentException(errorMessage);
             }
         }
-        logger.warn("the [action.auto_create_index] setting is configured to be restrictive [{}]. for the next 6 months daily history " +
+        logger.warn("the [action.auto_create_index] setting is configured to be restrictive [{}]. " +
-                "indices are allowed to be created, but please make sure that any future history indices after 6 months with the pattern " +
+                " for the next 6 months daily history indices are allowed to be created, but please make sure" +
-                "[.watch_history-YYYY.MM.dd] are allowed to be created", value);
+                " that any future history indices after 6 months with the pattern " +
+                "[.watcher-history-YYYY.MM.dd] are allowed to be created", value);
+    }
+
+    void registerClusterPrivilege(String name, String... patterns) {
+        try {
+            ClusterPrivilege.addCustom(name, patterns);
+        } catch (Exception se) {
+            logger.warn("could not register cluster privilege [{}]", name);
+
+            // we need to prevent bubbling the shield exception here for the tests. In the tests
+            // we create multiple nodes in the same jvm and since the custom cluster is a static binding
+            // multiple nodes will try to add the same privileges multiple times.
+        }
     }
 
 }

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/execution/WatchExecutionContext.java

@@ -70,7 +70,7 @@ public abstract class WatchExecutionContext {
     public abstract boolean skipThrottling(String actionId);
 
     /**
-     * @return true if this execution should be recorded in the .watch_history index
+     * @return true if this execution should be recorded in the .watcher-history index
      */
     public abstract boolean recordExecution();
 

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/history/HistoryStore.java

@@ -12,6 +12,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
+import org.elasticsearch.watcher.WatcherModule;
 import org.elasticsearch.watcher.execution.ExecutionState;
 import org.elasticsearch.watcher.support.init.proxy.ClientProxy;
 import org.joda.time.DateTime;
@@ -30,7 +31,7 @@ import static org.elasticsearch.watcher.support.Exceptions.ioException;
  */
 public class HistoryStore extends AbstractComponent {
 
-    public static final String INDEX_PREFIX = ".watch_history-";
+    public static final String INDEX_PREFIX = ".watcher-history-" + WatcherModule.getHistoryIndexTemplateVersion() + "-";
     public static final String DOC_TYPE = "watch_record";
 
     static final DateTimeFormatter indexTimeFormat = DateTimeFormat.forPattern("YYYY.MM.dd");

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/shield/ShieldSecretService.java

@@ -1,41 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.watcher.shield;
-
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.shield.crypto.CryptoService;
-import org.elasticsearch.watcher.support.secret.SecretService;
-
-/**
- *
- */
-public class ShieldSecretService extends AbstractComponent implements SecretService {
-
-    private final CryptoService cryptoService;
-    private final boolean encryptSensitiveData;
-    public static final Setting<Boolean> ENCRYPT_SENSITIVE_DATA_SETTING =
-            Setting.boolSetting("watcher.shield.encrypt_sensitive_data", false, false, Setting.Scope.CLUSTER);
-
-    @Inject
-    public ShieldSecretService(Settings settings, CryptoService cryptoService) {
-        super(settings);
-        this.encryptSensitiveData = ENCRYPT_SENSITIVE_DATA_SETTING.get(settings);
-        this.cryptoService = cryptoService;
-    }
-
-    @Override
-    public char[] encrypt(char[] text) {
-        return encryptSensitiveData ? cryptoService.encrypt(text) : text;
-    }
-
-    @Override
-    public char[] decrypt(char[] text) {
-        return encryptSensitiveData ? cryptoService.decrypt(text) : text;
-    }
-}

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/shield/WatcherShieldModule.java

@@ -1,45 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.watcher.shield;
-
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.shield.ShieldPlugin;
-import org.elasticsearch.shield.authz.privilege.ClusterPrivilege;
-
-/**
- *
- */
-public class WatcherShieldModule extends AbstractModule {
-
-    private final ESLogger logger;
-
-    public WatcherShieldModule(Settings settings) {
-        this.logger = Loggers.getLogger(WatcherShieldModule.class, settings);
-        if (ShieldPlugin.shieldEnabled(settings)) {
-            registerClusterPrivilege("manage_watcher", "cluster:admin/watcher/*", "cluster:monitor/watcher/*");
-            registerClusterPrivilege("monitor_watcher", "cluster:monitor/watcher/*");
-        }
-    }
-
-    void registerClusterPrivilege(String name, String... patterns) {
-        try {
-            ClusterPrivilege.addCustom(name, patterns);
-        } catch (Exception se) {
-            logger.warn("could not register cluster privilege [{}]", name);
-
-            // we need to prevent bubbling the shield exception here for the tests. In the tests
-            // we create multiple nodes in the same jvm and since the custom cluster is a static binding
-            // multiple nodes will try to add the same privileges multiple times.
-        }
-    }
-
-    @Override
-    protected void configure() {
-    }
-}

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/support/WatcherIndexTemplateRegistry.java

@@ -153,9 +153,9 @@ public class WatcherIndexTemplateRegistry extends AbstractComponent implements C
         executor.execute(new Runnable() {
             @Override
             public void run() {
-                try (InputStream is = WatchStore.class.getResourceAsStream("/" + config.getTemplateName() + ".json")) {
+                try (InputStream is = WatchStore.class.getResourceAsStream("/" + config.getFileName()+ ".json")) {
                     if (is == null) {
-                        logger.error("Resource [/" + config.getTemplateName() + ".json] not found in classpath");
+                        logger.error("Resource [/" + config.getFileName() + ".json] not found in classpath");
                         return;
                     }
                     final byte[] template;
@@ -184,13 +184,23 @@ public class WatcherIndexTemplateRegistry extends AbstractComponent implements C
     public static class TemplateConfig {
 
         private final String templateName;
+        private String fileName;
         private final Setting<Settings> setting;
 
         public TemplateConfig(String templateName, Setting<Settings> setting) {
+            this(templateName, templateName, setting);
+        }
+
+        public TemplateConfig(String templateName, String fileName, Setting<Settings> setting) {
             this.templateName = templateName;
+            this.fileName = fileName;
             this.setting = setting;
         }
 
+        public String getFileName() {
+            return fileName;
+        }
+
         public String getTemplateName() {
             return templateName;
         }

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/support/secret/SecretModule.java

@@ -8,7 +8,6 @@ package org.elasticsearch.watcher.support.secret;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.shield.ShieldPlugin;
-import org.elasticsearch.watcher.shield.ShieldSecretService;
 
 /**
  *
@@ -18,17 +17,16 @@ public class SecretModule extends AbstractModule {
     private final boolean shieldEnabled;
 
     public SecretModule(Settings settings) {
-        shieldEnabled = ShieldPlugin.shieldEnabled(settings);
+        shieldEnabled = ShieldPlugin.enabled(settings);
     }
 
     @Override
     protected void configure() {
         if (shieldEnabled) {
-            bind(ShieldSecretService.class).asEagerSingleton();
+            bind(SecretService.Secure.class).asEagerSingleton();
-            bind(SecretService.class).to(ShieldSecretService.class);
+            bind(SecretService.class).to(SecretService.Secure.class);
         } else {
-            bind(SecretService.PlainText.class).asEagerSingleton();
+            bind(SecretService.class).toInstance(SecretService.Insecure.INSTANCE);
-            bind(SecretService.class).to(SecretService.PlainText.class);
         }
     }
 }

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/support/secret/SecretService.java

@@ -5,6 +5,12 @@
  */
 package org.elasticsearch.watcher.support.secret;
 
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.shield.crypto.CryptoService;
+
 /**
  *
  */
@@ -14,7 +20,12 @@ public interface SecretService {
 
     char[] decrypt(char[] text);
 
-    class PlainText implements SecretService {
+    class Insecure implements SecretService {
+
+        public static final Insecure INSTANCE = new Insecure();
+
+        Insecure() {
+        }
 
         @Override
         public char[] encrypt(char[] text) {
@@ -26,4 +37,31 @@ public interface SecretService {
             return text;
         }
     }
+
+    /**
+     *
+     */
+    class Secure extends AbstractComponent implements SecretService {
+
+        private final CryptoService cryptoService;
+        private final boolean encryptSensitiveData;
+        public static final Setting<Boolean> ENCRYPT_SENSITIVE_DATA_SETTING =
+                Setting.boolSetting("watcher.shield.encrypt_sensitive_data", false, false, Setting.Scope.CLUSTER);
+        @Inject
+        public Secure(Settings settings, CryptoService cryptoService) {
+            super(settings);
+            this.encryptSensitiveData = ENCRYPT_SENSITIVE_DATA_SETTING.get(settings);
+            this.cryptoService = cryptoService;
+        }
+
+        @Override
+        public char[] encrypt(char[] text) {
+            return encryptSensitiveData ? cryptoService.encrypt(text) : text;
+        }
+
+        @Override
+        public char[] decrypt(char[] text) {
+            return encryptSensitiveData ? cryptoService.decrypt(text) : text;
+        }
+    }
 }

elasticsearch/x-pack/watcher/src/main/java/org/elasticsearch/watcher/watch/Watch.java

@@ -257,7 +257,7 @@ public class Watch implements TriggerEngine.Job, ToXContent {
          * Such that the returned watch will potentially hide this sensitive data behind a "secret". A secret
          * is an abstraction around sensitive data (text). There can be different implementations of how the
          * secret holds the data, depending on the wired up {@link SecretService}. When shield is installed, a
-         * {@link org.elasticsearch.watcher.shield.ShieldSecretService} is used, that potentially encrypts the data
+         * {@link SecretService.Secure} is used, that potentially encrypts the data
          * using Shield's configured system key.
          *
          * This method is only called once - when the user adds a new watch. From that moment on, all representations

elasticsearch/x-pack/watcher/src/main/resources/watch_history.json

@@ -1,7 +1,9 @@
 {
-  "template": ".watch_history*",
+  "template": ".watcher-history-${watcher.template.version}*",
   "order": 2147483647,
   "settings": {
+    "watcher.plugin.version": "${watcher.plugin.version}",
+    "watcher.template.version": "${watcher.template.version}",
     "index.number_of_shards": 1,
     "index.mapper.dynamic": false
   },
@@ -29,7 +31,7 @@
           }
         }
       ],
-      "dynamic": "strict",
+      "dynamic": false,
       "_all": {
         "enabled": false
       },

elasticsearch/x-pack/watcher/src/main/resources/watcher.properties

@@ -0,0 +1 @@
+watcher.template.version=${watcher.template.version}

elasticsearch/x-pack/watcher/src/test/java/org/elasticsearch/watcher/WatcherPluginDisableTests.java

@@ -41,8 +41,10 @@ public class WatcherPluginDisableTests extends ESIntegTestCase {
         return Settings.settingsBuilder()
                 .put(super.nodeSettings(nodeOrdinal))
                 .put(WatcherPlugin.ENABLED_SETTING, false)
+
                 // disable shield because of query cache check and authentication/authorization
-                .put(ShieldPlugin.ENABLED_SETTING_NAME, false)
+                .put(XPackPlugin.featureEnabledSetting(ShieldPlugin.NAME), false)
+
                 .put(NetworkModule.HTTP_ENABLED.getKey(), true)
                 .build();
     }