Add `path.shared_data`

This allows `path.shared_data` to be added to the security manager while
still allowing a custom `data_path` for indices using shadow replicas.

For example, configuring `path.shared_data: /tmp/foo`, then created an
index with:

```
POST /myindex
{
  "index": {
    "number_of_shards": 1,
    "number_of_replicas": 1,
    "data_path": "/tmp/foo/bar/baz",
    "shadow_replicas": true
  }
}
```

The index will then reside in `/tmp/foo/bar/baz`.

`path.shared_data` defaults to `null` if not specified.

Resolves #12714
Relates to #11065

core/src/main/java/org/elasticsearch/bootstrap/Security.java

@@ -126,6 +126,9 @@ final class Security {
         // read-write dirs
         addPath(policy, environment.tmpFile(), "read,readlink,write,delete");
         addPath(policy, environment.logsFile(), "read,readlink,write,delete");
+        if (environment.sharedDataFile() != null) {
+            addPath(policy, environment.sharedDataFile(), "read,readlink,write,delete");
+        }
         for (Path path : environment.dataFiles()) {
             addPath(policy, path, "read,readlink,write,delete");
         }

core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java

@@ -50,12 +50,14 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.FileSystemUtils;
+import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.mapper.DocumentMapper;
@@ -99,12 +101,14 @@ public class MetaDataCreateIndexService extends AbstractComponent {
     private final AliasValidator aliasValidator;
     private final IndexTemplateFilter indexTemplateFilter;
     private final NodeEnvironment nodeEnv;
+    private final Environment env;
 
     @Inject
     public MetaDataCreateIndexService(Settings settings, ThreadPool threadPool, ClusterService clusterService,
                                       IndicesService indicesService, AllocationService allocationService, MetaDataService metaDataService,
                                       Version version, AliasValidator aliasValidator,
-                                      Set<IndexTemplateFilter> indexTemplateFilters, NodeEnvironment nodeEnv) {
+                                      Set<IndexTemplateFilter> indexTemplateFilters, Environment env,
+                                      NodeEnvironment nodeEnv) {
         super(settings);
         this.threadPool = threadPool;
         this.clusterService = clusterService;
@@ -114,6 +118,7 @@ public class MetaDataCreateIndexService extends AbstractComponent {
         this.version = version;
         this.aliasValidator = aliasValidator;
         this.nodeEnv = nodeEnv;
+        this.env = env;
 
         if (indexTemplateFilters.isEmpty()) {
             this.indexTemplateFilter = DEFAULT_INDEX_TEMPLATE_FILTER;
@@ -511,8 +516,13 @@ public class MetaDataCreateIndexService extends AbstractComponent {
     public void validateIndexSettings(String indexName, Settings settings) throws IndexCreationException {
         String customPath = settings.get(IndexMetaData.SETTING_DATA_PATH, null);
         List<String> validationErrors = Lists.newArrayList();
-        if (customPath != null && nodeEnv.isCustomPathsEnabled() == false) {
-            validationErrors.add("custom data_paths for indices is disabled");
+        if (customPath != null && env.sharedDataFile() == null) {
+            validationErrors.add("path.shared_data must be set in order to use custom data paths");
+        } else if (customPath != null) {
+            Path resolvedPath = PathUtils.get(new Path[]{env.sharedDataFile()}, customPath);
+            if (resolvedPath == null) {
+                validationErrors.add("custom path [" + customPath + "] is not a sub-path of path.shared_data [" + env.sharedDataFile() + "]");
+            }
         }
         Integer number_of_primaries = settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, null);
         Integer number_of_replicas = settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, null);

core/src/main/java/org/elasticsearch/env/Environment.java

@@ -57,6 +57,8 @@ public class Environment {
 
     private final Path pluginsFile;
 
+    private final Path sharedDataFile;
+
     /** location of bin/, used by plugin manager */
     private final Path binFile;
 
@@ -126,6 +128,11 @@ public class Environment {
             dataFiles = new Path[]{homeFile.resolve("data")};
             dataWithClusterFiles = new Path[]{homeFile.resolve("data").resolve(ClusterName.clusterNameFromSettings(settings).value())};
         }
+        if (settings.get("path.shared_data") != null) {
+            sharedDataFile = PathUtils.get(cleanPath(settings.get("path.shared_data")));
+        } else {
+            sharedDataFile = null;
+        }
         String[] repoPaths = settings.getAsArray("path.repo");
         if (repoPaths.length > 0) {
             repoFiles = new Path[repoPaths.length];
@@ -165,6 +172,13 @@ public class Environment {
         return dataFiles;
     }
 
+    /**
+     * The shared data location
+     */
+    public Path sharedDataFile() {
+        return sharedDataFile;
+    }
+
     /**
      * The data location with the cluster name as a sub directory.
      */

core/src/main/java/org/elasticsearch/env/NodeEnvironment.java

@@ -105,6 +105,7 @@ public class NodeEnvironment extends AbstractComponent implements Closeable {
     }
 
     private final NodePath[] nodePaths;
+    private final Path sharedDataPath;
     private final Lock[] locks;
 
     private final boolean addNodeId;
@@ -137,6 +138,7 @@ public class NodeEnvironment extends AbstractComponent implements Closeable {
 
         if (!DiscoveryNode.nodeRequiresLocalStorage(settings)) {
             nodePaths = null;
+            sharedDataPath = null;
             locks = null;
             localNodeId = -1;
             return;
@@ -144,6 +146,7 @@ public class NodeEnvironment extends AbstractComponent implements Closeable {
 
         final NodePath[] nodePaths = new NodePath[environment.dataWithClusterFiles().length];
         final Lock[] locks = new Lock[nodePaths.length];
+        sharedDataPath = environment.sharedDataFile();
 
         int localNodeId = -1;
         IOException lastException = null;
@@ -792,7 +795,6 @@ public class NodeEnvironment extends AbstractComponent implements Closeable {
      *
      * @param indexSettings settings for the index
      */
-    @SuppressForbidden(reason = "Lee is working on it: https://github.com/elastic/elasticsearch/pull/11065")
     private Path resolveCustomLocation(@IndexSettings Settings indexSettings) {
         assert indexSettings != Settings.EMPTY;
         String customDataDir = indexSettings.get(IndexMetaData.SETTING_DATA_PATH);
@@ -800,9 +802,9 @@ public class NodeEnvironment extends AbstractComponent implements Closeable {
             // This assert is because this should be caught by MetaDataCreateIndexService
             assert customPathsEnabled;
             if (addNodeId) {
-                return PathUtils.get(customDataDir).resolve(Integer.toString(this.localNodeId));
+                return sharedDataPath.resolve(customDataDir).resolve(Integer.toString(this.localNodeId));
             } else {
-                return PathUtils.get(customDataDir);
+                return sharedDataPath.resolve(customDataDir);
             }
         } else {
             throw new IllegalArgumentException("no custom " + IndexMetaData.SETTING_DATA_PATH + " setting available");

core/src/test/java/org/elasticsearch/bootstrap/SecurityTests.java

@@ -77,6 +77,7 @@ public class SecurityTests extends ESTestCase {
         settingsBuilder.put("path.scripts", esHome.resolve("scripts").toString());
         settingsBuilder.put("path.plugins", esHome.resolve("plugins").toString());
         settingsBuilder.putArray("path.data", esHome.resolve("data1").toString(), esHome.resolve("data2").toString());
+        settingsBuilder.put("path.shared_data", esHome.resolve("custom").toString());
         settingsBuilder.put("path.logs", esHome.resolve("logs").toString());
         settingsBuilder.put("pidfile", esHome.resolve("test.pid").toString());
         Settings settings = settingsBuilder.build();
@@ -122,6 +123,7 @@ public class SecurityTests extends ESTestCase {
         for (Path dataPath : environment.dataWithClusterFiles()) {
             assertExactPermissions(new FilePermission(dataPath.toString(), "read,readlink,write,delete"), permissions);
         }
+        assertExactPermissions(new FilePermission(environment.sharedDataFile().toString(), "read,readlink,write,delete"), permissions);
         // logs: r/w
         assertExactPermissions(new FilePermission(environment.logsFile().toString(), "read,readlink,write,delete"), permissions);
         // temp dir: r/w

core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java

@@ -300,7 +300,7 @@ public class NodeEnvironmentTests extends ESTestCase {
     @Test
     public void testCustomDataPaths() throws Exception {
         String[] dataPaths = tmpPaths();
-        NodeEnvironment env = newNodeEnvironment(dataPaths, Settings.EMPTY);
+        NodeEnvironment env = newNodeEnvironment(dataPaths, "/tmp", Settings.EMPTY);
 
         Settings s1 = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).build();
         Settings s2 = Settings.builder().put(IndexMetaData.SETTING_DATA_PATH, "/tmp/foo").build();
@@ -323,7 +323,7 @@ public class NodeEnvironmentTests extends ESTestCase {
                 env.indexPaths(i), equalTo(stringsToPaths(dataPaths, "elasticsearch/nodes/0/indices/myindex")));
 
         env.close();
-        NodeEnvironment env2 = newNodeEnvironment(dataPaths,
+        NodeEnvironment env2 = newNodeEnvironment(dataPaths, "/tmp",
                 Settings.builder().put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH, false).build());
 
         assertThat(env2.availableShardPaths(sid), equalTo(env2.availableShardPaths(sid)));
@@ -381,4 +381,14 @@ public class NodeEnvironmentTests extends ESTestCase {
                 .putArray("path.data", dataPaths).build();
         return new NodeEnvironment(build, new Environment(build));
     }
+
+    public NodeEnvironment newNodeEnvironment(String[] dataPaths, String sharedDataPath, Settings settings) throws IOException {
+        Settings build = Settings.builder()
+                .put(settings)
+                .put("path.home", createTempDir().toAbsolutePath().toString())
+                .put("path.shared_data", sharedDataPath)
+                .put(NodeEnvironment.SETTING_CUSTOM_DATA_PATH_ENABLED, true)
+                .putArray("path.data", dataPaths).build();
+        return new NodeEnvironment(build, new Environment(build));
+    }
 }

core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java

@@ -68,23 +68,44 @@ import static org.hamcrest.Matchers.*;
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class IndexWithShadowReplicasIT extends ESIntegTestCase {
 
-    private Settings nodeSettings() {
+    private Settings nodeSettings(Path dataPath) {
+        return nodeSettings(dataPath.toString());
+    }
+
+    private Settings nodeSettings(String dataPath) {
         return Settings.builder()
                 .put("node.add_id_to_custom_path", false)
                 .put("node.enable_custom_paths", true)
+                .put("path.shared_data", dataPath)
                 .put("index.store.fs.fs_lock", randomFrom("native", "simple"))
                 .build();
     }
 
+    public void testCannotCreateWithBadPath() throws Exception {
+        Settings nodeSettings = nodeSettings("/badpath");
+        internalCluster().startNodesAsync(1, nodeSettings).get();
+        Settings idxSettings = Settings.builder()
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_DATA_PATH, "/etc/foo")
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).build();
+        try {
+            assertAcked(prepareCreate("foo").setSettings(idxSettings));
+            fail("should have failed");
+        } catch (IllegalArgumentException e) {
+            assertTrue(e.getMessage(),
+                    e.getMessage().contains("custom path [/etc/foo] is not a sub-path of path.shared_data"));
+        }
+    }
+
     /**
      * Tests the case where we create an index without shadow replicas, snapshot it and then restore into
      * an index with shadow replicas enabled.
      */
     public void testRestoreToShadow() throws ExecutionException, InterruptedException {
-        Settings nodeSettings = nodeSettings();
+        final Path dataPath = createTempDir();
+        Settings nodeSettings = nodeSettings(dataPath);
 
         internalCluster().startNodesAsync(3, nodeSettings).get();
-        final Path dataPath = createTempDir();
         Settings idxSettings = Settings.builder()
                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).build();
@@ -137,11 +158,11 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
 
     @Test
     public void testIndexWithFewDocuments() throws Exception {
-        Settings nodeSettings = nodeSettings();
+        final Path dataPath = createTempDir();
+        Settings nodeSettings = nodeSettings(dataPath);
 
         internalCluster().startNodesAsync(3, nodeSettings).get();
         final String IDX = "test";
-        final Path dataPath = createTempDir();
 
         Settings idxSettings = Settings.builder()
                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
@@ -200,10 +221,10 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
 
     @Test
     public void testReplicaToPrimaryPromotion() throws Exception {
-        Settings nodeSettings = nodeSettings();
+        Path dataPath = createTempDir();
+        Settings nodeSettings = nodeSettings(dataPath);
 
         String node1 = internalCluster().startNode(nodeSettings);
-        Path dataPath = createTempDir();
         String IDX = "test";
 
         Settings idxSettings = Settings.builder()
@@ -259,10 +280,10 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
 
     @Test
     public void testPrimaryRelocation() throws Exception {
-        Settings nodeSettings = nodeSettings();
+        Path dataPath = createTempDir();
+        Settings nodeSettings = nodeSettings(dataPath);
 
         String node1 = internalCluster().startNode(nodeSettings);
-        Path dataPath = createTempDir();
         String IDX = "test";
 
         Settings idxSettings = Settings.builder()
@@ -320,10 +341,10 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
 
     @Test
     public void testPrimaryRelocationWithConcurrentIndexing() throws Throwable {
-        Settings nodeSettings = nodeSettings();
+        Path dataPath = createTempDir();
+        Settings nodeSettings = nodeSettings(dataPath);
 
         String node1 = internalCluster().startNode(nodeSettings);
-        Path dataPath = createTempDir();
         final String IDX = "test";
 
         Settings idxSettings = Settings.builder()
@@ -393,14 +414,15 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
 
     @Test
     public void testPrimaryRelocationWhereRecoveryFails() throws Exception {
+        Path dataPath = createTempDir();
         Settings nodeSettings = Settings.builder()
                 .put("node.add_id_to_custom_path", false)
                 .put("node.enable_custom_paths", true)
                 .put("plugin.types", MockTransportService.Plugin.class.getName())
+                .put("path.shared_data", dataPath)
                 .build();
 
         String node1 = internalCluster().startNode(nodeSettings);
-        Path dataPath = createTempDir();
         final String IDX = "test";
 
         Settings idxSettings = Settings.builder()
@@ -490,11 +512,11 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
 
     @Test
     public void testIndexWithShadowReplicasCleansUp() throws Exception {
-        Settings nodeSettings = nodeSettings();
+        Path dataPath = createTempDir();
+        Settings nodeSettings = nodeSettings(dataPath);
 
         int nodeCount = randomIntBetween(2, 5);
         internalCluster().startNodesAsync(nodeCount, nodeSettings).get();
-        Path dataPath = createTempDir();
         String IDX = "test";
 
         Settings idxSettings = Settings.builder()
@@ -532,10 +554,10 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
      */
     @Test
     public void testShadowReplicaNaturalRelocation() throws Exception {
-        Settings nodeSettings = nodeSettings();
+        Path dataPath = createTempDir();
+        Settings nodeSettings = nodeSettings(dataPath);
 
         internalCluster().startNodesAsync(2, nodeSettings).get();
-        Path dataPath = createTempDir();
         String IDX = "test";
 
         Settings idxSettings = Settings.builder()
@@ -586,10 +608,10 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
 
     @Test
     public void testShadowReplicasUsingFieldData() throws Exception {
-        Settings nodeSettings = nodeSettings();
+        Path dataPath = createTempDir();
+        Settings nodeSettings = nodeSettings(dataPath);
 
         internalCluster().startNodesAsync(3, nodeSettings).get();
-        Path dataPath = createTempDir();
         String IDX = "test";
 
         Settings idxSettings = Settings.builder()
@@ -655,7 +677,8 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
 
     @Test
     public void testIndexOnSharedFSRecoversToAnyNode() throws Exception {
-        Settings nodeSettings = nodeSettings();
+        Path dataPath = createTempDir();
+        Settings nodeSettings = nodeSettings(dataPath);
         Settings fooSettings = Settings.builder().put(nodeSettings).put("node.affinity", "foo").build();
         Settings barSettings = Settings.builder().put(nodeSettings).put("node.affinity", "bar").build();
 
@@ -663,7 +686,6 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
         final Future<List<String>> barNodes = internalCluster().startNodesAsync(2, barSettings);
         fooNodes.get();
         barNodes.get();
-        Path dataPath = createTempDir();
         String IDX = "test";
 
         Settings includeFoo = Settings.builder()

core/src/test/java/org/elasticsearch/indices/IndicesCustomDataPathIT.java

@@ -42,10 +42,24 @@ import static org.hamcrest.Matchers.equalTo;
 /**
  * Tests for custom data path locations and templates
  */
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class IndicesCustomDataPathIT extends ESIntegTestCase {
 
     private String path;
 
+    private Settings nodeSettings(Path dataPath) {
+        return nodeSettings(dataPath.toString());
+    }
+
+    private Settings nodeSettings(String dataPath) {
+        return Settings.builder()
+                .put("node.add_id_to_custom_path", false)
+                .put("node.enable_custom_paths", true)
+                .put("path.shared_data", dataPath)
+                .put("index.store.fs.fs_lock", randomFrom("native", "simple"))
+                .build();
+    }
+
     @Before
     public void setup() {
         path = createTempDir().toAbsolutePath().toString();
@@ -61,6 +75,7 @@ public class IndicesCustomDataPathIT extends ESIntegTestCase {
     public void testDataPathCanBeChanged() throws Exception {
         final String INDEX = "idx";
         Path root = createTempDir();
+        internalCluster().startNodesAsync(1, nodeSettings(root));
         Path startDir = root.resolve("start");
         Path endDir = root.resolve("end");
         logger.info("--> start dir: [{}]", startDir.toAbsolutePath().toString());
@@ -128,9 +143,12 @@ public class IndicesCustomDataPathIT extends ESIntegTestCase {
     @Test
     public void testIndexCreatedWithCustomPathAndTemplate() throws Exception {
         final String INDEX = "myindex2";
+        internalCluster().startNodesAsync(1, nodeSettings(path));
 
         logger.info("--> creating an index with data_path [{}]", path);
-        Settings.Builder sb = Settings.builder().put(IndexMetaData.SETTING_DATA_PATH, path);
+        Settings.Builder sb = Settings.builder()
+                .put(IndexMetaData.SETTING_DATA_PATH, path)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0);
 
         client().admin().indices().prepareCreate(INDEX).setSettings(sb).get();
         ensureGreen(INDEX);

core/src/test/java/org/elasticsearch/test/ESIntegTestCase.java

@@ -30,8 +30,10 @@ import com.google.common.collect.Lists;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.http.impl.client.HttpClients;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
+import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.shard.MergeSchedulerConfig;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
@@ -691,15 +693,12 @@ public abstract class ESIntegTestCase extends ESTestCase {
         if (numberOfReplicas >= 0) {
             builder.put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas).build();
         }
-        // norelease: disabled because custom data paths don't play well against
-        // an external test cluster: the security manager is not happy that random
-        // files are touched. See http://build-us-00.elastic.co/job/es_core_master_strong/4357/console
         // 30% of the time
-        // if (randomInt(9) < 3) {
-        //     final Path dataPath = createTempDir();
-        //     logger.info("using custom data_path for index: [{}]", dataPath);
-        //    builder.put(IndexMetaData.SETTING_DATA_PATH, dataPath);
-        // }
+        if (randomInt(9) < 3) {
+            final String dataPath = randomAsciiOfLength(10);
+            logger.info("using custom data_path for index: [{}]", dataPath);
+            builder.put(IndexMetaData.SETTING_DATA_PATH, dataPath);
+        }
         return builder.build();
     }
 
@@ -1616,6 +1615,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
                 // from failing on nodes without enough disk space
                 .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK, "1b")
                 .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK, "1b")
+                .put(NodeEnvironment.SETTING_CUSTOM_DATA_PATH_ENABLED, true)
                 .put("script.indexed", "on")
                 .put("script.inline", "on")
                         // wait short time for other active shards before actually deleting, default 30s not needed in tests

core/src/test/java/org/elasticsearch/test/ESSingleNodeTestCase.java

@@ -35,6 +35,7 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.node.Node;
@@ -118,16 +119,20 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
 
     private static Node newNode() {
         Node build = NodeBuilder.nodeBuilder().local(true).data(true).settings(Settings.builder()
-            .put(ClusterName.SETTING, InternalTestCluster.clusterName("single-node-cluster", randomLong()))
-            .put("path.home", createTempDir())
-            .put("node.name", nodeName())
-            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-            .put("script.inline", "on")
-            .put("script.indexed", "on")
-            .put(EsExecutors.PROCESSORS, 1) // limit the number of threads created
-            .put("http.enabled", false)
-            .put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING, true) // make sure we get what we set :)
+                .put(ClusterName.SETTING, InternalTestCluster.clusterName("single-node-cluster", randomLong()))
+                .put("path.home", createTempDir())
+                // TODO: use a consistent data path for custom paths
+                // This needs to tie into the ESIntegTestCase#indexSettings() method
+                .put("path.shared_data", createTempDir().getParent())
+                .put("node.name", nodeName())
+                .put(NodeEnvironment.SETTING_CUSTOM_DATA_PATH_ENABLED, true)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put("script.inline", "on")
+                .put("script.indexed", "on")
+                .put(EsExecutors.PROCESSORS, 1) // limit the number of threads created
+                .put("http.enabled", false)
+                .put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING, true) // make sure we get what we set :)
         ).build();
         build.start();
         assertThat(DiscoveryNode.localNode(build.settings()), is(true));

core/src/test/java/org/elasticsearch/test/InternalTestCluster.java

@@ -301,6 +301,7 @@ public final class InternalTestCluster extends TestCluster {
                 builder.put("path.data", dataPath.toString());
             }
         }
+        builder.put("path.shared_data", baseDir.resolve("custom"));
         builder.put("path.home", baseDir);
         builder.put("path.repo", baseDir.resolve("repos"));
         builder.put("transport.tcp.port", BASE_PORT + "-" + (BASE_PORT + 100));

docs/reference/indices/shadow-replicas.asciidoc

@@ -14,26 +14,20 @@ settings, you need to enable using it in elasticsearch.yml:
 [source,yaml]
 --------------------------------------------------
 node.enable_custom_paths: true
+node.add_id_to_custom_path: false
 --------------------------------------------------
 
-You will also need to disable the default security manager that Elasticsearch
-runs with. You can do this by either passing
-`-Des.security.manager.enabled=false` with the parameters while starting
-Elasticsearch, or you can disable it in elasticsearch.yml:
+You will also need to indicate to the security manager where the custom indices
+will be, so that the correct permissions can be applied. You can do this by
+setting the `path.shared_data` setting in elasticsearch.yml:
 
 [source,yaml]
 --------------------------------------------------
-security.manager.enabled: false
+path.shared_data: /opt/data
 --------------------------------------------------
 
-[WARNING]
-========================
-Disabling the security manager means that the Elasticsearch process is not
-limited to the directories and files that it can read and write. However,
-because the `index.data_path` setting is set when creating the index, the
-security manager would prevent writing or reading from the index's location, so
-it must be disabled.
-========================
+This means that Elasticsearch can read and write to files in any subdirectory of
+the `path.shared_data` setting.
 
 You can then create an index with a custom data path, where each node will use
 this path for the data:
@@ -54,7 +48,7 @@ curl -XPUT 'localhost:9200/my_index' -d '
     "index" : {
         "number_of_shards" : 1,
         "number_of_replicas" : 4,
-        "data_path": "/var/data/my_index",
+        "data_path": "/opt/data/my_index",
         "shadow_replicas": true
     } 
 }'
@@ -62,7 +56,7 @@ curl -XPUT 'localhost:9200/my_index' -d '
 
 [WARNING]
 ========================
-In the above example, the "/var/data/my_index" path is a shared filesystem that
+In the above example, the "/opt/data/my_index" path is a shared filesystem that
 must be available on every node in the Elasticsearch cluster. You must also
 ensure that the Elasticsearch process has the correct permissions to read from
 and write to the directory used in the `index.data_path` setting.