fix transport client documentation for 2.0 and add notes about file locations

This also change ES_HOME/config -> CONFIG_DIR in the documentation to more accurately reflect the
location of configuration files as they are not always in ES_HOME/config.

Closes elastic/elasticsearch#455
Closes elastic/elasticsearch#432

Original commit: elastic/x-pack-elasticsearch@63ce35450c

shield/docs/public/configuring-auditing.asciidoc

@@ -207,7 +207,7 @@ The following tables describe the possible attributes each entry type can carry
 [float]
 === Audit Logs Settings
 
-As mentioned above, the audit logs are configured in the `logging.yml` file located in `ES_HOME/config/shield`. The following snippet shows the default logging configuration:
+As mentioned above, the audit logs are configured in the `logging.yml` file located in `CONFIG_DIR/shield`. The following snippet shows the default logging configuration:
 
 [[logging-file]]
 

shield/docs/public/configuring-clients-integrations/java.asciidoc

@@ -82,9 +82,14 @@ dependencies {
 import org.elasticsearch.client.transport.TransportClient;
 ...
 
-TransportClient client = new TransportClient(ImmutableSettings.builder()
+TransportClient client = TransportClient.builder()
-    .put("cluster.name", "myClusterName")
+    .settings(Settings.builder()
-    .put("shield.user", "transport_client_user:changeme")
+        .put("cluster.name", "myClusterName")
+        .put("shield.user", "transport_client_user:changeme")
+        .put("plugin.types", "org.elasticsearch.shield.ShieldPlugin")
+        .put("path.home", "/path/to/client/home")
+        ...
+        .build())
     .addTransportAddress(new InetSocketTransportAddress("localhost", 9300))
     .addTransportAddress(new InetSocketTransportAddress("localhost", 9301));
 -------------------------------------------------------------------------------------------------
@@ -97,15 +102,23 @@ For example, the following snippet adds the `Authorization` header to a search r
 +
 [source,java]
 --------------------------------------------------------------------------------------------------
-import org.elasticsearch.shield.authc.support.SecuredString;
 import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.shield.authc.support.SecuredString;
 
 import static org.elasticsearch.shield.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 ...
 
-TransportClient client = new TransportClient(ImmutableSettings.builder()
+TransportClient client = TransportClient.builder()
-    .put("shield.user", "transport_client_user:changeme")
+    .settings(Settings.builder()
-    ...
+        .put("cluster.name", "myClusterName")
+        .put("shield.user", "transport_client_user:changeme")
+        .put("plugin.types", "org.elasticsearch.shield.ShieldPlugin")
+        .put("path.home", "/path/to/client/home")
+        ...
+        .build())
+    .build()
     .addTransportAddress(new InetSocketTransportAddress("localhost", 9300))
     .addTransportAddress(new InetSocketTransportAddress("localhost", 9301));
 
@@ -126,15 +139,19 @@ NOTE: Client authentication is enabled by default. For information about disabli
 import org.elasticsearch.client.transport.TransportClient;
 ...
 
-TransportClient client = new TransportClient(ImmutableSettings.builder()
+TransportClient client = TransportClient.builder()
-    .put("cluster.name", "myClusterName")
+    .settings(Settings.builder()
-    .put("shield.user", "transport_client_user:changeme")
+        .put("cluster.name", "myClusterName")
-    .put("shield.ssl.keystore.path", "/path/to/client.jks") (1)
+        .put("plugin.types", "org.elasticsearch.shield.ShieldPlugin")
-    .put("shield.ssl.keystore.password", "password")
+        .put("shield.user", "transport_client_user:changeme")
-    ...
+        .put("shield.ssl.keystore.path", "/path/to/client.jks") (1)
+        .put("shield.ssl.keystore.password", "password")
+        .put("path.home", "/path/to/client/home")
+        ...
+        .build());
 --------------------------------------------------------------------------------------------------
 +
-(1) The `client.jks` keystore must contain the client's signed CA certificate and the CA certificate.
+(1) The `client.jks` keystore must contain the client's signed certificate and the CA certificate.
 +
 .. Enable the SSL transport by setting `shield.transport.ssl` to `true` in the client configuration.
 +
@@ -143,12 +160,17 @@ TransportClient client = new TransportClient(ImmutableSettings.builder()
 import org.elasticsearch.client.transport.TransportClient;
 ...
 
-TransportClient client = new TransportClient(ImmutableSettings.builder()
+TransportClient client = TransportClient.builder()
-    .put("cluster.name", "myClusterName")
+    .settings(Settings.builder()
-    .put("shield.user", "transport_client_user:changeme")
+        .put("cluster.name", "myClusterName")
-    .put("shield.ssl.keystore.path", "/path/to/client.jks") (1)
+        .put("shield.user", "transport_client_user:changeme")
-    .put("shield.ssl.keystore.password", "password")
+        .put("shield.ssl.keystore.path", "/path/to/client.jks") (1)
-    .put("shield.transport.ssl", "true"))
+        .put("shield.ssl.keystore.password", "password")
+        .put("shield.transport.ssl", "true")
+        .put("plugin.types", "org.elasticsearch.shield.ShieldPlugin")
+        .put("path.home", "/path/to/client/home")
+        ...
+        .build())
     .addTransportAddress(new InetSocketTransportAddress("localhost", 9300))
     .addTransportAddress(new InetSocketTransportAddress("localhost", 9301));
 --------------------------------------------------------------------------------------------------
@@ -166,12 +188,17 @@ If you are not using client authentication and sign the Elasticsearch node certi
 import org.elasticsearch.client.transport.TransportClient;
 ...
 
-TransportClient client = new TransportClient(ImmutableSettings.builder()
+TransportClient client = TransportClient.builder()
-    .put("cluster.name", "myClusterName")
+    .settings(Settings.builder()
-    .put("shield.user", "test_user:changeme")
+        .put("cluster.name", "myClusterName")
-    .put("shield.ssl.truststore.path", "/path/to/truststore.jks") (1)
+        .put("shield.user", "test_user:changeme")
-    .put("shield.ssl.truststore.password", "password")
+        .put("shield.ssl.truststore.path", "/path/to/truststore.jks") (1)
-    .put("shield.transport.ssl", "true"))
+        .put("shield.ssl.truststore.password", "password")
+        .put("shield.transport.ssl", "true")
+        .put("plugin.types", "org.elasticsearch.shield.ShieldPlugin")
+        .put("path.home", "/path/to/client/home")
+        ...
+        .build())
     .addTransportAddress(new InetSocketTransportAddress("localhost", 9300))
     .addTransportAddress(new InetSocketTransportAddress("localhost", 9301));
 ------------------------------------------------------------------------------------------------------
@@ -198,7 +225,7 @@ The following example shows how you can clear Shield's realm caches using `Shiel
 import static org.elasticsearch.node.NodeBuilder.*;
 ...
 
-Client client = ... // create the client (either transport or node)
+Client client = ... // create the transport client
 
 ShieldClient shieldClient = new ShieldClient(client);
 ClearRealmCacheResponse response = shieldClient.authc().prepareClearRealmCache()

shield/docs/public/configuring-rbac.asciidoc

@@ -25,7 +25,7 @@ As an administrator, you will need to define the roles that you want to use, the
 
 [[defining-roles]]
 === Defining Roles
-Roles are defined in the role definition file `roles.yml` located in `ES_HOME/config/shield`.
+Roles are defined in the role definition file `roles.yml` located in `CONFIG_DIR/shield`.
 This is a YAML file where each entry defines the unique role name and the cluster and indices permissions associated
 with it.
 

shield/docs/public/getting-started/enable-message-authentication.asciidoc

@@ -12,9 +12,7 @@ To enable message authentication:
 bin/shield/syskeygen
 ----------------
 +
-This creates a system key file in `ES_HOME/config/shield/system_key`. You 
+This creates a system key file in `CONFIG_DIR/shield/system_key`.
-can customize this file's location by changing the value of the `shield.system_key.file` setting in  
-`elasticsearch.yml`.
 
 . Copy the genererated system key to the rest of the nodes in the cluster.
 

shield/docs/public/managing-users.asciidoc

@@ -140,7 +140,7 @@ userdel <username>
 [float]
 === About `esusers`
 
-The `esusers` tool manipulates two files, `users` and `users_roles`, in `ES_HOME/config/shield/`. These two files store all user data for the _esusers_ realm and are read by Shield 
+The `esusers` tool manipulates two files, `users` and `users_roles`, in `CONFIG_DIR/shield/`. These two files store all user data for the _esusers_ realm and are read by Shield
 on startup.
 
 By default, Shield checks these files for changes every 5 seconds. You can change this default behavior by changing the

shield/docs/public/reference.asciidoc

@@ -163,7 +163,7 @@ The parameters listed in this section are configured in the `config/elasticsearc
 [options="header"]
 |======
 | Name                     | Default                                               | Description
-| `shield.system_key.file` |`ES_HOME/config/shield/system_key`  | Sets the location of the `system_key` file. For more information, see <<enable-message-authentication,Enabling Message Authentication>>.
+| `shield.system_key.file` |`CONFIG_DIR/shield/system_key`  | Sets the <<ref-shield-files-location,location>> of the `system_key` file. For more information, see <<enable-message-authentication,Enabling Message Authentication>>.
 |======
 
 [[ref-anonymous-access]]
@@ -221,8 +221,8 @@ shield.authc.realms:
 [options="header"]
 |======
 | Name                 | Required    | Default                            | Description
-| `files.users`        | no          | `ES_HOME/config/shield/users`      | The location of the <<users-file, users>> file.
+| `files.users`        | no          | `CONFIG_DIR/shield/users`      | The <<ref-shield-files-location,location>> of the <<users-file, users>> file.
-| `files.users_roles`  | no          | `ES_HOME/config/shield/users_roles`| The location of the <<users_defining-roles, users_roles>> file.
+| `files.users_roles`  | no          | `CONFIG_DIR/shield/users_roles`| The <<ref-shield-files-location,location>> of the <<users_defining-roles, users_roles>> file.
 | `cache.ttl`          | no          | `20m`                              | The time-to-live for cached user entries--user credentials are cached for this configured period of time. Defaults to `20m`. Specify values using the standard Elasticsearch {ref}/common-options.html#time-units[time units].
 | `cache.max_users`    | no          | 100000                             | The maximum number of user entries that can live in the cache at a given time. Defaults to 100,000.
 | `cache.hash_algo`    | no          | `ssha256`                          | (Expert Setting) The hashing algorithm that is used for the in-memory cached user credentials. See the <<ref-cache-hash-algo,Cache hash algorithms>> table for all possible values.
@@ -251,7 +251,7 @@ shield.authc.realms:
 | `group_search.filter`                    | no         | See description                                            | When not set, the realm will search for `group`, `groupOfNames`, or `groupOfUniqueNames`, with the attributes `member` or `memberOf`.  Any instance of `{0}` in the filter will be replaced by the user attribute defined in `group_search.user_attribute`
 | `group_search.user_attribute`            | no         | Empty                                                      | Specifies the user attribute that will be fetched and provided as a parameter to the filter.  If not set, the user DN is passed into the filter.
 | `unmapped_groups_as_roles`               | no         | false                                                      | Takes a boolean variable. When this element is set to `true`, the names of any unmapped LDAP groups are used as role names and assigned to the user. THe default value is `false`.
-| `files.role_mapping`                     | no         | `ES_HOME/config/shield/users/role_mapping.yml` | The path and file name for the <<ldap-role-mapping, YAML role mapping configuration file>>. 
+| `files.role_mapping`                     | no         | `CONFIG_DIR/shield/users/role_mapping.yml` | The <<ref-shield-files-location,location>> for the <<ldap-role-mapping, YAML role mapping configuration file>>.
 | `follow_referrals`                       | no         | `true`                                                     | Boolean value that specifies whether Shield should follow referrals returned by the LDAP server. Referrals are URLs returned by the server that are to be used to continue the LDAP operation (e.g. search).
 | `connect_timeout`                        | no         | "5s" - for 5 seconds                                       | The timeout period for establishing an LDAP connection.  An `s` at the end indicates seconds, or `ms` indicates milliseconds.
 | `read_timeout`                           | no         | "5s" - for 5 seconds                                       | The timeout period for an LDAP operation.  An `s` at the end indicates seconds, or `ms` indicates milliseconds.
@@ -271,7 +271,7 @@ NOTE: `user_dn_templates` is required to operate in user template mode and `user
 | `url`                         | no          | `ldap://<domain_name>:389`                                  | A URL in the format `ldap[s]://<server>:<port>`  If not specified the URL will be derived from the domain_name, assuming clear-text `ldap` and port `389` (e.g. `ldap://<domain_name>:389`).
 | `domain_name`                 | yes         | -                                                           | The domain name of Active Directory. The cluster can derive the URL and `user_search_dn` fields from values in this element if those fields are not otherwise specified.
 | `unmapped_groups_as_roles`    | no          | false                                                       | Takes a boolean variable. When this element is set to `true`, the names of any unmapped groups and the user's relative distinguished name are used as role names and assigned to the user. THe default value is `false`.
-| `files.role_mapping`          | no          | `ES_HOME/config/shield/users/role_mapping.yml`  | The path and file name for the <<ad-role-mapping, YAML role mapping configuration file>>. 
+| `files.role_mapping`          | no          | `CONFIG_DIR/shield/users/role_mapping.yml`  | The <<ref-shield-files-location,location>> for the <<ad-role-mapping, YAML role mapping configuration file>>.
 | `user_search.base_dn`         | no          | Root of Active Directory                                    | The context to search for a user. The default value for this element is the root of the Active Directory domain.
 | `user_search.scope`           | no          | `sub_tree`                                                  | Specifies whether the user search should be `sub_tree`, `one_level` or `base`.  `one_level` only searches users directly contained within the `base_dn`. `sub_tree` searches all objects contained under `base_dn`. `base` specifies that the `base_dn` is a user object, and that it is the only user considered.
 | `user_search.filter`          | no          | See description                                             | Specifies a filter to use to lookup a user given a username.  The default filter looks up `user` objects with either `sAMAccountName` or `userPrincipalName`
@@ -296,7 +296,7 @@ NOTE: `user_dn_templates` is required to operate in user template mode and `user
 | `truststore.path`          | no        | `shield.ssl.keystore`                                      | The path of a truststore to use. The default truststore is the one defined by <<ref-ssl-tls-settings,SSL/TLS settings>>
 | `truststore.password`      | no        | -                                                          | The password to the truststore. Must be provided if `truststore.path` is set.
 | `truststore.algorithm`     | no        | SunX509                                                    | Algorithm for the trustsore. Default is `SunX509`
-| `files.role_mapping`       | no        | `ES_HOME/config/shield/users/role_mapping.yml` | Specifies the path and file name for the <<pki-role-mapping, YAML role  mapping configuration file>>. 
+| `files.role_mapping`       | no        | `CONFIG_DIR/shield/users/role_mapping.yml` | Specifies the <<ref-shield-files-location,location>> for the <<pki-role-mapping, YAML role  mapping configuration file>>.
 |======
 
 [[ref-cache-hash-algo]]
@@ -324,7 +324,7 @@ NOTE: `user_dn_templates` is required to operate in user template mode and `user
 [options="header"]
 |======
 | Name                                        | Default                                 | Description
-| `shield.authz.store.file.roles`             | `ES_HOME/config/shield/users/roles.yml` | The location of the roles definition file.
+| `shield.authz.store.file.roles`             | `CONFIG_DIR/shield/users/roles.yml` | The <<ref-shield-files-location,location>> of the roles definition file.
 |======
 
 [[ref-ssl-tls-settings]]
@@ -392,13 +392,18 @@ NOTE: `user_dn_templates` is required to operate in user template mode and `user
 
 The Shield security plugin uses the following files:
 
-* `config/shield/roles.yml` defines the roles in use on the cluster (read more <<defining-roles,here>>).
+* `CONFIG_DIR/shield/roles.yml` defines the roles in use on the cluster (read more <<defining-roles,here>>).
-* `config/shield/users` defines the hashed passwords for users on the cluster (read more <<users-file,here>>).
+* `CONFIG_DIR/shield/users` defines the hashed passwords for users on the cluster (read more <<users-file,here>>).
-* `config/shield/users_roles` defines the role assignments for users on the cluster (read more <<users_defining-roles,here>>).
+* `CONFIG_DIR/shield/users_roles` defines the role assignments for users on the cluster (read more <<users_defining-roles,here>>).
-* `config/shield/role_mapping.yml` defines the role assignments for a Distinguished Name (DN) to a role. This allows for
+* `CONFIG_DIR/shield/role_mapping.yml` defines the role assignments for a Distinguished Name (DN) to a role. This allows for
 LDAP and Active Directory groups and users and PKI users to be mapped to roles (read more <<ldap-role-mapping,here>>).
-* `config/shield/logging.yml` contains audit information (read more <<logging-file,here>>).
+* `CONFIG_DIR/shield/logging.yml` contains audit information (read more <<logging-file,here>>).
-* `config/shield/system_key` holds a cluster secret key used for message authentication. For more information, see <<enable-message-authentication,Enabling Message Authentication>>.
+* `CONFIG_DIR/shield/system_key` holds a cluster secret key used for message authentication. For more information, see <<enable-message-authentication,Enabling Message Authentication>>.
+
+[[ref-shield-files-location]]
+IMPORTANT: Any files that Shield uses must be stored in the Elasticsearch {ref}/setup-dir-layout.html#setup-dir-layout[configuration directory].
+Elasticsearch runs with restricted permissions and is only permitted to read from the locations configured in the directory
+layout for enhanced security.
 
 Several of these files are in the YAML format. When you edit these files, be aware that YAML is indentation-level
 sensitive and indentation errors can lead to configuration errors. Avoid the tab character to set indentation levels,

shield/docs/public/setting-up-authentication/configuring-active-directory-realm.asciidoc

@@ -64,7 +64,7 @@ shield:
 | `group_search.base_dn`     | no        | Specifies the context to search for groups in which the user has membership. The default value for this element is the root of the Active Directory domain.
 | `group_search.scope`       | no        | Specifies whether the group search should be `sub_tree` (default), `one_level` or `base`.  `one_level` searches for groups directly contained within the `base_dn`. `sub_tree` searches all objects contained under `base_dn`. `base` specifies that the `base_dn` is a group object, and that it is the only group considered.
 | `unmapped_groups_as_roles` | no        | When set to `true`, the names of any unmapped LDAP groups are used as role names and assigned to the user. The default value is `false`.
-| `files.role_mapping`       | no        | Specifies the path and file name for the <<ad-role-mapping, YAML role  mapping configuration file>>. By default it is `ES_HOME/config/shield/users/role_mapping.yml`.
+| `files.role_mapping`       | no        | Specifies the <<ref-shield-files-location,location>> for the <<ad-role-mapping, YAML role  mapping configuration file>>. By default it is `CONFIG_DIR/shield/users/role_mapping.yml`.
 | `follow_referrals`         | no        | Boolean value that specifies whether Shield should follow referrals returned by the LDAP server. Referrals are URLs returned by the server that are to be used to continue the LDAP operation (e.g. search). Default is `true`.
 | `hostname_verification`    | no        | When set to `true`, hostname verification will be performed when connecting to a LDAP server. The hostname or IP address used in the `url` must match one of the names in the certificate or the connection will not be allowed. Defaults to `true`.
 | `cache.ttl`                | no        | Specified the time-to-live for cached user entries (a user and its credentials will be cached for this configured period of time). Defaults to `20m` (use the standard Elasticsearch {ref}/common-options.html#time-units[time units])

shield/docs/public/setting-up-authentication/configuring-esusers-realm.asciidoc

@@ -29,8 +29,8 @@ shield:
 | `type`                         | yes       | Indicates the realm type and must be set to `esusers`.
 | `order`                        | no        | Indicates the priority of this realm within the realm chain. Realms with lower order will be consulted first. Although not required, it is highly recommended to explicitly set this value when multiple realms are configured. Defaults to `Integer.MAX_VALUE`.
 | `enabled`                      | no        | Indicates whether this realm is enabled/disabled. Provides an easy way to disable realms in the chain without removing their configuration. Defaults to `true`.
-| `files.users`                  | no        | Points to the location of the `users` file where the users and their passwords are stored. By default, it is `ES_HOME/config/shield/users`.
+| `files.users`                  | no        | Points to the <<ref-shield-files-location,location>> of the `users` file where the users and their passwords are stored. By default, it is `CONFIG_DIR/shield/users`.
-| `files.users_roles`            | no        | Points to the location of the `users_roles` file where the users and their roles are stored. By default, it is `ES_HOME/config/shield/users_roles`.
+| `files.users_roles`            | no        | Points to the <<ref-shield-files-location,location>> of the `users_roles` file where the users and their roles are stored. By default, it is `CONFIG_DIR/shield/users_roles`.
 | `cache.ttl`                    | no        | Specified the time-to-live for cached user entries (a user and its credentials will be cached for this configured period of time). Defaults to `20m` (use the standard Elasticsearch {ref}/common-options.html#time-units[time units]).
 | `cache.max_users`              | no        | Specified the maximum number of user entries that can live in the cache at a given time. Defaults to 100,000.
 | `cache.hash_algo`              | no        | (Expert Setting) Specifies the hashing algorithm that will be used for the in-memory cached user credentials (see <<esusers-cache-hash-algo,here>> for possible values).

shield/docs/public/setting-up-authentication/configuring-ldap-realm.asciidoc

@@ -101,7 +101,7 @@ shield:
 | `unmapped_groups_as_roles`     | no       | When set to `true`, the names of any unmapped LDAP groups are used as role names and assigned to the user. The default value is `false`.
 | `connect_timeout`              | no       | The timeout period for establishing an LDAP connection.  An `s` at the end indicates seconds, or `ms` indicates milliseconds. Defaults to "5s" - for 5 seconds
 | `read_timeout`                 | no       | The timeout period for an LDAP operation.  An `s` at the end indicates seconds, or `ms` indicates milliseconds. Defaults to "5s" - for 5 seconds
-| `files.role_mapping`           | no       | Specifies the path and file name for the <<ldap-role-mapping, YAML role mapping configuration file>>. By default, it is `ES_HOME/config/shield/role_mapping.yml`.
+| `files.role_mapping`           | no       | Specifies the <<ref-shield-files-location,location>> for the <<ldap-role-mapping, YAML role mapping configuration file>>. By default, it is `CONFIG_DIR/shield/role_mapping.yml`.
 | `follow_referrals`             | no       | Boolean value that specifies whether Shield should follow referrals returned by the LDAP server. Referrals are URLs returned by the server that are to be used to continue the LDAP operation (e.g. search). Default is `true`.
 | `hostname_verification`        | no       | When set to `true`, hostname verification will be performed when connecting to a LDAP server. The hostname or IP address used in the `url` must match one of the names in the certificate or the connection will not be allowed. Defaults to `true`.
 | `cache.ttl`                    | no       | Specified the time-to-live for cached user entries (a user and its credentials will be cached for this configured period of time). Defaults to `20m` (use the standard Elasticsearch {ref}/common-options.html#time-units[time units]).

shield/docs/public/setting-up-authentication/configuring-pki-realm.asciidoc

@@ -73,6 +73,6 @@ shield:
 | `truststore.path`          | no        | The path of a truststore to use. The default truststore is the one defined by <<ref-ssl-tls-settings,SSL/TLS settings>>
 | `truststore.password`      | no        | The password to the truststore. Must be provided if `truststore.path` is set.
 | `truststore.algorithm`     | no        | Algorithm for the trustsore. Default is `SunX509`
-| `files.role_mapping`       | no        | Specifies the path and file name for the <<pki-role-mapping, YAML role  mapping configuration file>>. By default, it is `ES_HOME/config/shield/role_mapping.yml`.
+| `files.role_mapping`       | no        | Specifies the <<ref-shield-files-location,location>> for the <<pki-role-mapping, YAML role  mapping configuration file>>. By default, it is `CONFIG_DIR/shield/role_mapping.yml`.
 |=======================
 

shield/docs/public/troubleshooting.asciidoc

@@ -99,8 +99,8 @@ misconfigured. See <<ref-shield-files, Shield Files>> for more.
 
 |======================
 
-To help track down these possibilities, add `shield.authc: DEBUG` to the `logging.yml` configuration file in `ES_HOME/config/shield/`.  A successful
+To help track down these possibilities, add `shield.authc: DEBUG` to the `logging.yml` configuration file in `CONFIG_DIR`.
-authentication should produce debug statements that list groups and role mappings.
+A successful authentication should produce debug statements that list groups and role mappings.
 --