Shield refactoring for 5.0 - phase 3

- Consolidated the `bin` and `config` directories of watcher, shield and marvel under a single `config/xpack` and `bin/xpack` directories.

 - updated docs accordingly

Original commit: elastic/x-pack-elasticsearch@c2aa6132fa

elasticsearch/qa/shield-tribe-node-tests/integration-tests.xml

@@ -75,10 +75,10 @@
         <property name="home" location="${integ.scratch}/elasticsearch-${elasticsearch.version}"/>
 
         <echo>Adding roles.yml</echo>
-        <copy file="shield-roles.yml" tofile="${home}/config/shield/roles.yml" overwrite="true"/>
+        <copy file="shield-roles.yml" tofile="${home}/config/xpack/roles.yml" overwrite="true"/>
 
         <echo>Adding shield users...</echo>
-        <run-script script="${home}/bin/shield/esusers">
+        <run-script script="${home}/bin/xpack/esusers">
             <nested>
                 <arg value="useradd"/>
                 <arg value="test_admin"/>

elasticsearch/qa/smoke-test-found-license-with-shield-and-watcher/integration-tests.xml

@@ -63,10 +63,10 @@
     <property name="home" location="${integ.scratch}/elasticsearch-${elasticsearch.version}"/>
 
     <echo>Adding roles.yml with watcher roles</echo>
-    <copy file="watcher-with-shield-roles.yml" tofile="${home}/config/shield/roles.yml" overwrite="true"/>
+    <copy file="watcher-with-shield-roles.yml" tofile="${home}/config/xpack/roles.yml" overwrite="true"/>
 
     <echo>Adding shield users...</echo>
-    <run-script script="${home}/bin/shield/esusers">
+    <run-script script="${home}/bin/xpack/esusers">
       <nested>
         <arg value="useradd"/>
         <arg value="test_admin"/>
@@ -76,7 +76,7 @@
         <arg value="admin"/>
       </nested>
     </run-script>
-    <run-script script="${home}/bin/shield/esusers">
+    <run-script script="${home}/bin/xpack/esusers">
       <nested>
         <arg value="useradd"/>
         <arg value="watcher_manager"/>
@@ -86,7 +86,7 @@
         <arg value="watcher_manager"/>
       </nested>
     </run-script>
-    <run-script script="${home}/bin/shield/esusers">
+    <run-script script="${home}/bin/xpack/esusers">
       <nested>
         <arg value="useradd"/>
         <arg value="powerless_user"/>

elasticsearch/x-dev-tools/src/main/resources/ant/shield-overrides.xml

@@ -1,97 +0,0 @@
-<?xml version="1.0"?>
-<project name="shield-overrides"
-         xmlns:ac="antlib:net.sf.antcontrib">
-
-  <!-- redefined to work with auth -->
-  <macrodef name="waitfor-elasticsearch">
-      <attribute name="port"/>
-      <attribute name="timeoutproperty"/>
-    <sequential>
-      <echo>Waiting for elasticsearch to become available on port @{port}...</echo>
-      <waitfor maxwait="30" maxwaitunit="second"
-               checkevery="500" checkeveryunit="millisecond"
-               timeoutproperty="@{timeoutproperty}">
-        <socket server="127.0.0.1" port="@{port}"/>
-      </waitfor>
-    </sequential>
-  </macrodef>
-
-  <target name="start-external-cluster-with-plugins" depends="setup-workspace">
-    <ac:for list="${xplugins.list}" param="xplugin.name">
-      <sequential>
-        <fail message="Expected @{xplugin.name}-${version}.zip as a dependency, but could not be found in ${integ.deps}/plugins}">
-          <condition>
-            <not>
-              <available file="${integ.deps}/plugins/@{xplugin.name}-${elasticsearch.version}.zip" />
-            </not>
-          </condition>
-        </fail>
-        <ac:if>
-          <equals arg1="@{xplugin.name}" arg2="elasticsearch-marvel"/>
-          <ac:then>
-            <property name="marvel.enabled">true</property>
-          </ac:then>
-        </ac:if>
-      </sequential>
-    </ac:for>
-
-    <ac:for param="file">
-      <path>
-        <fileset dir="${integ.deps}/plugins"/>
-      </path>
-      <sequential>
-        <local name="plugin.name"/>
-        <convert-plugin-name file="@{file}" outputproperty="plugin.name"/>
-        <install-plugin name="${plugin.name}" file="@{file}"/>
-      </sequential>
-    </ac:for>
-
-    <local name="home"/>
-    <property name="home" location="${integ.scratch}/elasticsearch-${elasticsearch.version}"/>
-
-    <echo>Setting up Shield auth</echo>
-    <run-script script="${home}/bin/shield/esusers">
-      <nested>
-        <arg value="useradd"/>
-        <arg value="test_user"/>
-        <arg value="-p"/>
-        <arg value="changeme"/>
-        <arg value="-r"/>
-        <arg value="admin"/>
-      </nested>
-    </run-script>
-
-    <ac:if>
-      <equals arg1="${marvel.enabled}" arg2="true"/>
-      <ac:then>
-        <run-script script="${home}/bin/shield/esusers">
-          <nested>
-            <arg value="useradd"/>
-            <arg value="marvel_export"/>
-            <arg value="-p"/>
-            <arg value="changeme"/>
-            <arg value="-r"/>
-            <arg value="marvel_agent"/>
-          </nested>
-        </run-script>
-
-        <startup-elasticsearch>
-          <additional-args>
-            <arg value="-Des.marvel.agent.exporter.es.hosts=http://marvel_export:changeme@localhost:${integ.http.port}"/>
-          </additional-args>
-        </startup-elasticsearch>
-      </ac:then>
-      <ac:else>
-        <startup-elasticsearch/>
-      </ac:else>
-    </ac:if>
-    
-
-    <echo>Checking we can connect with basic auth on port ${integ.http.port}...</echo>
-    <local name="temp.file"/>
-    <tempfile property="temp.file" destdir="${java.io.tmpdir}"/>
-    <get src="http://127.0.0.1:${integ.http.port}" dest="${temp.file}" 
-         username="test_user" password="changeme" verbose="true" retries="10"/>
-  </target>
-
-</project>

elasticsearch/x-dev-tools/src/main/resources/commercial-license-check/elasticsearch_license_header.txt

@@ -1,14 +0,0 @@
-ELASTICSEARCH CONFIDENTIAL
-__________________
-
- [2014] Elasticsearch Incorporated. All Rights Reserved.
-
-NOTICE:  All information contained herein is, and remains
-the property of Elasticsearch Incorporated and its suppliers,
-if any.  The intellectual and technical concepts contained
-herein are proprietary to Elasticsearch Incorporated
-and its suppliers and may be covered by U.S. and Foreign Patents,
-patents in process, and are protected by trade secret or copyright law.
-Dissemination of this information or reproduction of this material
-is strictly forbidden unless prior written permission is obtained
-from Elasticsearch Incorporated.

elasticsearch/x-dev-tools/src/main/resources/commercial-license-check/license_header_definition.xml

@@ -1,13 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<additionalHeaders>
-    <javadoc_style>
-        <firstLine>/*</firstLine>
-        <beforeEachLine> * </beforeEachLine>
-        <endLine> */</endLine>
-        <!--skipLine></skipLine-->
-        <firstLineDetectionPattern>(\s|\t)*/\*.*$</firstLineDetectionPattern>
-        <lastLineDetectionPattern>.*\*/(\s|\t)*$</lastLineDetectionPattern>
-        <allowBlankLines>false</allowBlankLines>
-        <isMultiline>true</isMultiline>
-    </javadoc_style>
-</additionalHeaders>

elasticsearch/x-pack/build.gradle

@@ -114,13 +114,13 @@ bundlePlugin {
     include 'LICENSE.txt'
     include 'NOTICE.txt'
   }
-  from('shield/bin/shield') {
+  from('shield/bin/xpack') {
     into 'bin'
   }
-  from('shield/config/shield') {
+  from('shield/config/xpack') {
     into 'config'
   }
-  from('watcher/bin/watcher') {
+  from('watcher/bin/xpack') {
     into 'bin'
   }
 }

elasticsearch/x-pack/shield/bin/xpack/esusers

@@ -102,9 +102,9 @@ if [ -e "$CONF_DIR" ]; then
     case "$properties" in
         *-Des.default.path.conf=*) ;;
         *)
-            if [ ! -d "$CONF_DIR/shield" ]; then
+            if [ ! -d "$CONF_DIR/xpack" ]; then
-                echo "ERROR: The configuration directory [$CONF_DIR/shield] does not exist. The esusers tool expects Shield configuration files in that location."
+                echo "ERROR: The configuration directory [$CONF_DIR/xpack] does not exist. The esusers tool expects security configuration files in that location."
-                echo "The plugin may not have been installed with the correct configuration path. If [$ES_HOME/config/shield] exists, please copy the shield directory to [$CONF_DIR]"
+                echo "The plugin may not have been installed with the correct configuration path. If [$ES_HOME/config/xpack] exists, please copy the 'xpack' directory to [$CONF_DIR]"
                 exit 1
             fi
             properties="$properties -Des.default.path.conf=$CONF_DIR"

elasticsearch/x-pack/shield/bin/xpack/syskeygen

@@ -102,9 +102,9 @@ if [ -e "$CONF_DIR" ]; then
     case "$properties" in
         *-Des.default.path.conf=*) ;;
         *)
-            if [ ! -d "$CONF_DIR/shield" ]; then
+            if [ ! -d "$CONF_DIR/xpack" ]; then
-                echo "ERROR: The configuration directory [$CONF_DIR/shield] does not exist. The syskeygen tool expects Shield configuration files in that location."
+                echo "ERROR: The configuration directory [$CONF_DIR/xpack] does not exist. The syskeygen tool expects security configuration files in that location."
-                echo "The plugin may not have been installed with the correct configuration path. If [$ES_HOME/config/shield] exists, please copy the shield directory to [$CONF_DIR]"
+                echo "The plugin may not have been installed with the correct configuration path. If [$ES_HOME/config/xpack] exists, please copy the 'xpack' directory to [$CONF_DIR]"
                 exit 1
             fi
             properties="$properties -Des.default.path.conf=$CONF_DIR"

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/esusers/FileUserPasswdStore.java

@@ -167,7 +167,7 @@ public class FileUserPasswdStore {
         }
 
         if (users.isEmpty()) {
-            logger.warn("no users found in users file [{}]. use bin/shield/esusers to add users and role mappings", path.toAbsolutePath());
+            logger.warn("no users found in users file [{}]. use bin/xpack/esusers to add users and role mappings", path.toAbsolutePath());
         }
         return unmodifiableMap(users);
     }

elasticsearch/x-pack/shield/src/main/java/org/elasticsearch/shield/authc/esusers/FileUserRolesStore.java

@@ -182,7 +182,7 @@ public class FileUserRolesStore {
         }
 
         if (usersRoles.isEmpty()) {
-            logger.warn("no entries found in users_roles file [{}]. use bin/shield/esusers to add users and role mappings", path
+            logger.warn("no entries found in users_roles file [{}]. use bin/xpack/esusers to add users and role mappings", path
                     .toAbsolutePath());
         }
 

elasticsearch/x-pack/shield/src/main/resources/org/elasticsearch/shield/crypto/tool/syskey-generate.help

@@ -9,7 +9,7 @@ SYNOPSIS
 DESCRIPTION
 
     Generates the system key and stores in the system_key file. By default
-    it will be stored in 'config/shield/.system_key' file. If the file location
+    it will be stored in 'config/xpack/system_key' file. If the file location
     is customized in the elasticsearch.yml (under the 'shield.system_key.file'
     setting), the generated key will be stored in that custom location.