Expose shards data and state path via ShardStats

Since we now don't stripe shards across data paths we need a way
to access the information on which path a shard is allocated to
eventually do better allocation decisions based on disk usage etc.
This commit exposes the shard paths as part of the shard stats.

Relates to #13106

core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java

@@ -106,7 +106,7 @@ public class TransportClusterStatsAction extends TransportNodesAction<ClusterSta
             for (IndexShard indexShard : indexService) {
                 if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) {
                     // only report on fully started shards
-                    shardsStats.add(new ShardStats(indexShard, indexShard.routingEntry(), SHARD_STATS_FLAGS));
+                    shardsStats.add(new ShardStats(indexShard, SHARD_STATS_FLAGS));
                 }
             }
         }

core/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java

@@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentBuilderString;
 import org.elasticsearch.index.engine.CommitStats;
 import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.ShardPath;
 
 import java.io.IOException;
 
@@ -37,20 +38,23 @@ import static org.elasticsearch.cluster.routing.ShardRouting.readShardRoutingEnt
 /**
  */
 public class ShardStats extends BroadcastShardResponse implements ToXContent {
-
     private ShardRouting shardRouting;
-
+    private CommonStats commonStats;
-    CommonStats commonStats;
-
     @Nullable
-    CommitStats commitStats;
+    private CommitStats commitStats;
+    private String dataPath;
+    private String statePath;
+    private boolean isCustomDataPath;
 
     ShardStats() {
     }
 
-    public ShardStats(IndexShard indexShard, ShardRouting shardRouting, CommonStatsFlags flags) {
+    public ShardStats(IndexShard indexShard, CommonStatsFlags flags) {
         super(indexShard.shardId());
-        this.shardRouting = shardRouting;
+        this.shardRouting = indexShard.routingEntry();
+        this.dataPath = indexShard.shardPath().getRootDataPath().toString();
+        this.statePath = indexShard.shardPath().getRootStatePath().toString();
+        this.isCustomDataPath = indexShard.shardPath().isCustomDataPath();
         this.commonStats = new CommonStats(indexShard, flags);
         this.commitStats = indexShard.commitStats();
     }
@@ -70,6 +74,18 @@ public class ShardStats extends BroadcastShardResponse implements ToXContent {
         return this.commitStats;
     }
 
+    public String getDataPath() {
+        return dataPath;
+    }
+
+    public String getStatePath() {
+        return statePath;
+    }
+
+    public boolean isCustomDataPath() {
+        return isCustomDataPath;
+    }
+
     public static ShardStats readShardStats(StreamInput in) throws IOException {
         ShardStats stats = new ShardStats();
         stats.readFrom(in);
@@ -82,6 +98,9 @@ public class ShardStats extends BroadcastShardResponse implements ToXContent {
         shardRouting = readShardRoutingEntry(in);
         commonStats = CommonStats.readCommonStats(in);
         commitStats = CommitStats.readOptionalCommitStatsFrom(in);
+        statePath = in.readString();
+        dataPath = in.readString();
+        isCustomDataPath = in.readBoolean();
     }
 
     @Override
@@ -90,6 +109,9 @@ public class ShardStats extends BroadcastShardResponse implements ToXContent {
         shardRouting.writeTo(out);
         commonStats.writeTo(out);
         out.writeOptionalStreamable(commitStats);
+        out.writeString(statePath);
+        out.writeString(dataPath);
+        out.writeBoolean(isCustomDataPath);
     }
 
     @Override
@@ -105,12 +127,21 @@ public class ShardStats extends BroadcastShardResponse implements ToXContent {
         if (commitStats != null) {
             commitStats.toXContent(builder, params);
         }
+        builder.startObject(Fields.SHARD_PATH);
+        builder.field(Fields.STATE_PATH, statePath);
+        builder.field(Fields.DATA_PATH, dataPath);
+        builder.field(Fields.IS_CUSTOM_DATA_PATH, isCustomDataPath);
+        builder.endObject();
         return builder;
     }
 
     static final class Fields {
         static final XContentBuilderString ROUTING = new XContentBuilderString("routing");
         static final XContentBuilderString STATE = new XContentBuilderString("state");
+        static final XContentBuilderString STATE_PATH = new XContentBuilderString("state_path");
+        static final XContentBuilderString DATA_PATH = new XContentBuilderString("data_path");
+        static final XContentBuilderString IS_CUSTOM_DATA_PATH = new XContentBuilderString("is_custom_data_path");
+        static final XContentBuilderString SHARD_PATH = new XContentBuilderString("shard_path");
         static final XContentBuilderString PRIMARY = new XContentBuilderString("primary");
         static final XContentBuilderString NODE = new XContentBuilderString("node");
         static final XContentBuilderString RELOCATING_NODE = new XContentBuilderString("relocating_node");

core/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java

@@ -189,7 +189,7 @@ public class TransportIndicesStatsAction extends TransportBroadcastAction<Indice
             flags.set(CommonStatsFlags.Flag.Recovery);
         }
 
-        return new ShardStats(indexShard, indexShard.routingEntry(), flags);
+        return new ShardStats(indexShard, flags);
     }
 
     static class IndexShardStatsRequest extends BroadcastShardRequest {

core/src/main/java/org/elasticsearch/common/util/MultiDataPathUpgrader.java

@@ -198,7 +198,7 @@ public class MultiDataPathUpgrader {
                 }
             }
         }
-        return new ShardPath(target.path, target.path, IndexMetaData.INDEX_UUID_NA_VALUE /* we don't know */, shard);
+        return new ShardPath(false, target.path, target.path, IndexMetaData.INDEX_UUID_NA_VALUE /* we don't know */, shard);
     }
 
     private ShardFileInfo[] getShardFileInfo(ShardId shard, NodeEnvironment.NodePath[] paths) throws IOException {

core/src/main/java/org/elasticsearch/index/shard/ShardPath.java

@@ -20,6 +20,7 @@ package org.elasticsearch.index.shard;
 
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.NodeEnvironment;
@@ -41,10 +42,18 @@ public final class ShardPath {
     private final String indexUUID;
     private final ShardId shardId;
     private final Path shardStatePath;
+    private final boolean isCustomDataPath;
 
-
+    public ShardPath(boolean isCustomDataPath, Path dataPath, Path shardStatePath, String indexUUID, ShardId shardId) {
-    public ShardPath(Path path, Path shardStatePath, String indexUUID, ShardId shardId) {
+        assert dataPath.getFileName().toString().equals(Integer.toString(shardId.id())) : "dataPath must end with the shard ID but didn't: " + dataPath.toString();
-        this.path = path;
+        assert shardStatePath.getFileName().toString().equals(Integer.toString(shardId.id())) : "shardStatePath must end with the shard ID but didn't: " + dataPath.toString();
+        assert dataPath.getParent().getFileName().toString().equals(shardId.getIndex()) : "dataPath must end with index/shardID but didn't: " + dataPath.toString();
+        assert shardStatePath.getParent().getFileName().toString().equals(shardId.getIndex()) : "shardStatePath must end with index/shardID but didn't: " + dataPath.toString();
+        if (isCustomDataPath && dataPath.equals(shardStatePath)) {
+            throw new IllegalArgumentException("shard state path must be different to the data path when using custom data paths");
+        }
+        this.isCustomDataPath = isCustomDataPath;
+        this.path = dataPath;
         this.indexUUID = indexUUID;
         this.shardId = shardId;
         this.shardStatePath = shardStatePath;
@@ -78,6 +87,30 @@ public final class ShardPath {
         return shardStatePath;
     }
 
+    /**
+     * Returns the data-path root for this shard. The root is a parent of {@link #getDataPath()} without the index name
+     * and the shard ID.
+     */
+    public Path getRootDataPath() {
+        Path noIndexShardId = getDataPath().getParent().getParent();
+        return isCustomDataPath ? noIndexShardId : noIndexShardId.getParent(); // also strip the indices folder
+    }
+
+    /**
+     * Returns the state-path root for this shard. The root is a parent of {@link #getRootStatePath()} ()} without the index name
+     * and the shard ID.
+     */
+    public Path getRootStatePath() {
+        return getShardStatePath().getParent().getParent().getParent(); // also strip the indices folder
+    }
+
+    /**
+     * Returns <code>true</code> iff the data location is a custom data location and therefore outside of the nodes configured data paths.
+     */
+    public boolean isCustomDataPath() {
+        return isCustomDataPath;
+    }
+
     /**
      * This method walks through the nodes shard paths to find the data and state path for the given shard. If multiple
      * directories with a valid shard state exist the one with the highest version will be used.
@@ -113,7 +146,7 @@ public final class ShardPath {
                 dataPath = statePath;
             }
             logger.debug("{} loaded data path [{}], state path [{}]", shardId, dataPath, statePath);
-            return new ShardPath(dataPath, statePath, indexUUID, shardId);
+            return new ShardPath(NodeEnvironment.hasCustomDataPath(indexSettings), dataPath, statePath, indexUUID, shardId);
         }
     }
 
@@ -202,7 +235,7 @@ public final class ShardPath {
             dataPath = statePath;
         }
 
-        return new ShardPath(dataPath, statePath, indexUUID, shardId);
+        return new ShardPath(NodeEnvironment.hasCustomDataPath(indexSettings), dataPath, statePath, indexUUID, shardId);
     }
 
     @Override

core/src/main/java/org/elasticsearch/indices/IndicesService.java

@@ -236,7 +236,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
                     if (indexShard.routingEntry() == null) {
                         continue;
                     }
-                    IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard, indexShard.routingEntry(), flags) });
+                    IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard, flags) });
                     if (!statsByShard.containsKey(indexService.index())) {
                         statsByShard.put(indexService.index(), Lists.<IndexShardStats>newArrayList(indexShardStats));
                     } else {

core/src/test/java/org/elasticsearch/common/util/MultiDataPathUpgraderTests.java

@@ -83,7 +83,7 @@ public class MultiDataPathUpgraderTests extends ESTestCase {
                 ShardStateMetaData.FORMAT.write(new ShardStateMetaData(metaStateVersion, true, uuid), metaStateVersion, shardDataPaths);
             }
             final Path path = randomFrom(shardDataPaths);
-            ShardPath targetPath = new ShardPath(path, path, uuid, new ShardId("foo", 0));
+            ShardPath targetPath = new ShardPath(false, path, path, uuid, new ShardId("foo", 0));
             MultiDataPathUpgrader helper = new MultiDataPathUpgrader(nodeEnvironment);
             helper.upgrade(shardId, targetPath);
             assertFalse(helper.needsUpgrading(shardId));
@@ -177,7 +177,7 @@ public class MultiDataPathUpgraderTests extends ESTestCase {
             }
             logger.info("--> injecting index [{}] into multiple data paths", indexName);
             OldIndexBackwardsCompatibilityIT.copyIndex(logger, src, indexName, multiDataPath);
-            final ShardPath shardPath = new ShardPath(nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))[0], nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))[0], IndexMetaData.INDEX_UUID_NA_VALUE, new ShardId(indexName, 0));
+            final ShardPath shardPath = new ShardPath(false, nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))[0], nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))[0], IndexMetaData.INDEX_UUID_NA_VALUE, new ShardId(indexName, 0));
 
             logger.info("{}", FileSystemUtils.files(shardPath.resolveIndex()));
 

core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java

@@ -22,7 +22,9 @@ import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
+import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.*;
@@ -31,11 +33,16 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.ShardLock;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.query.QueryParsingException;
@@ -539,4 +546,39 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         assertPathHasBeenCleared(startDir.toAbsolutePath().toString());
         assertPathHasBeenCleared(endDir.toAbsolutePath().toString());
     }
+
+    public void testShardStats() throws IOException {
+        createIndex("test");
+        ensureGreen();
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        IndexService test = indicesService.indexService("test");
+        IndexShard shard = test.shard(0);
+        ShardStats stats = new ShardStats(shard, new CommonStatsFlags());
+        assertEquals(shard.shardPath().getRootDataPath().toString(), stats.getDataPath());
+        assertEquals(shard.shardPath().getRootStatePath().toString(), stats.getStatePath());
+        assertEquals(shard.shardPath().isCustomDataPath(), stats.isCustomDataPath());
+
+        if (randomBoolean() || true) { // try to serialize it to ensure values survive the serialization
+            BytesStreamOutput out = new BytesStreamOutput();
+            stats.writeTo(out);
+            StreamInput in = StreamInput.wrap(out.bytes());
+            stats = ShardStats.readShardStats(in);
+        }
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.startObject();
+        stats.toXContent(builder, EMPTY_PARAMS);
+        builder.endObject();
+        String xContent = builder.string();
+        StringBuilder expectedSubSequence = new StringBuilder("\"shard_path\":{\"state_path\":\"");
+        expectedSubSequence.append(shard.shardPath().getRootStatePath().toString());
+        expectedSubSequence.append("\",\"data_path\":\"");
+        expectedSubSequence.append(shard.shardPath().getRootDataPath().toString());
+        expectedSubSequence.append("\",\"is_custom_data_path\":").append(shard.shardPath().isCustomDataPath()).append("}");
+        System.out.println(expectedSubSequence);
+        System.out.println(xContent);
+        assertTrue(xContent.contains(expectedSubSequence));
+    }
+
+    // "shard_path":{"state_path":"/private/var/folders/qj/rsr2js6n275f3r88r1z5bbgw0000gn/T/org.elasticsearch.index.shard.IndexShardTests_EE3FC3D62D02988A-001/tempDir-001/data/single-node-cluster-CHILD_VM=[0]-CLUSTER_SEED=[9012992977055748205]-HASH=[13FDF896A8B7DEC0]/nodes/0","data_path":"/private/var/folders/qj/rsr2js6n275f3r88r1z5bbgw0000gn/T/org.elasticsearch.index.shard.IndexShardTests_EE3FC3D62D02988A-001/tempDir-001/data/single-node-cluster-CHILD_VM=[0]-CLUSTER_SEED=[9012992977055748205]-HASH=[13FDF896A8B7DEC0]/nodes/0","is_custom_data_path":"false}
+    // "shard_path":{"state_path":"/private/var/folders/qj/rsr2js6n275f3r88r1z5bbgw0000gn/T/org.elasticsearch.index.shard.IndexShardTests_EE3FC3D62D02988A-001/tempDir-001/data/single-node-cluster-CHILD_VM=[0]-CLUSTER_SEED=[9012992977055748205]-HASH=[13FDF896A8B7DEC0]/nodes/0","data_path":"/private/var/folders/qj/rsr2js6n275f3r88r1z5bbgw0000gn/T/org.elasticsearch.index.shard.IndexShardTests_EE3FC3D62D02988A-001/tempDir-001/data/single-node-cluster-CHILD_VM=[0]-CLUSTER_SEED=[9012992977055748205]-HASH=[13FDF896A8B7DEC0]/nodes/0","is_custom_data_path":false}
 }

core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.index.shard;
 
+import com.carrotsearch.randomizedtesting.annotations.Repeat;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.NodeEnvironment;
@@ -26,6 +27,7 @@ import org.junit.Test;
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.Set;
 
 import static org.elasticsearch.common.settings.Settings.settingsBuilder;
 
@@ -78,4 +80,71 @@ public class ShardPathTests extends ESTestCase {
         }
     }
 
+    @Test(expected = IllegalArgumentException.class)
+    public void testIllegalCustomDataPath() {
+        final Path path = createTempDir().resolve("foo").resolve("0");
+        new ShardPath(true, path, path, "foo", new ShardId("foo", 0));
+    }
+
+    public void testValidCtor() {
+        final Path path = createTempDir().resolve("foo").resolve("0");
+        ShardPath shardPath = new ShardPath(false, path, path, "foo", new ShardId("foo", 0));
+        assertFalse(shardPath.isCustomDataPath());
+        assertEquals(shardPath.getDataPath(), path);
+        assertEquals(shardPath.getShardStatePath(), path);
+    }
+
+    public void testGetRootPaths() throws IOException {
+        boolean useCustomDataPath = randomBoolean();
+        final Settings indexSetttings;
+        final Settings nodeSettings;
+        Settings.Builder indexSettingsBuilder = settingsBuilder().put(IndexMetaData.SETTING_INDEX_UUID, "0xDEADBEEF");
+        final Path customPath;
+        if (useCustomDataPath) {
+            final Path path = createTempDir();
+            final boolean includeNodeId = randomBoolean();
+            indexSetttings = indexSettingsBuilder.put(IndexMetaData.SETTING_DATA_PATH, "custom").build();
+            nodeSettings = settingsBuilder().put("path.shared_data", path.toAbsolutePath().toAbsolutePath())
+                    .put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH, includeNodeId).build();
+            if (includeNodeId) {
+                customPath = path.resolve("custom").resolve("0");
+            } else {
+                customPath = path.resolve("custom");
+            }
+        } else {
+            customPath = null;
+            indexSetttings = indexSettingsBuilder.build();
+            nodeSettings = Settings.EMPTY;
+        }
+        try (final NodeEnvironment env = newNodeEnvironment(nodeSettings)) {
+            ShardId shardId = new ShardId("foo", 0);
+            Path[] paths = env.availableShardPaths(shardId);
+            Path path = randomFrom(paths);
+            ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF"), 2, path);
+            ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, indexSetttings);
+            boolean found = false;
+            for (Path p : env.nodeDataPaths()) {
+                if (p.equals(shardPath.getRootStatePath())) {
+                    found = true;
+                    break;
+                }
+            }
+            assertTrue("root state paths must be a node path but wasn't: " + shardPath.getRootStatePath(), found);
+            found = false;
+            if (useCustomDataPath) {
+                assertNotEquals(shardPath.getRootDataPath(), shardPath.getRootStatePath());
+                assertEquals(customPath, shardPath.getRootDataPath());
+            } else {
+                assertNull(customPath);
+                for (Path p : env.nodeDataPaths()) {
+                    if (p.equals(shardPath.getRootDataPath())) {
+                        found = true;
+                        break;
+                    }
+                }
+                assertTrue("root state paths must be a node path but wasn't: " + shardPath.getRootDataPath(), found);
+            }
+        }
+    }
+
 }

core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java

@@ -35,11 +35,11 @@ import java.util.Locale;
 public class IndexStoreTests extends ESTestCase {
 
     public void testStoreDirectory() throws IOException {
-        final Path tempDir = createTempDir();
+        final Path tempDir = createTempDir().resolve("foo").resolve("0");
         final IndexStoreModule.Type[] values = IndexStoreModule.Type.values();
         final IndexStoreModule.Type type = RandomPicks.randomFrom(random(), values);
         Settings settings = Settings.settingsBuilder().put(IndexStoreModule.STORE_TYPE, type.name().toLowerCase(Locale.ROOT)).build();
-        FsDirectoryService service = new FsDirectoryService(settings, null, new ShardPath(tempDir, tempDir, "foo", new ShardId("foo", 0)));
+        FsDirectoryService service = new FsDirectoryService(settings, null, new ShardPath(false, tempDir, tempDir, "foo", new ShardId("foo", 0)));
         try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
             switch (type) {
                 case NIOFS:
@@ -70,9 +70,9 @@ public class IndexStoreTests extends ESTestCase {
     }
 
     public void testStoreDirectoryDefault() throws IOException {
-        final Path tempDir = createTempDir();
+        final Path tempDir = createTempDir().resolve("foo").resolve("0");
         Settings settings = Settings.EMPTY;
-        FsDirectoryService service = new FsDirectoryService(settings, null, new ShardPath(tempDir, tempDir, "foo", new ShardId("foo", 0)));
+        FsDirectoryService service = new FsDirectoryService(settings, null, new ShardPath(false, tempDir, tempDir, "foo", new ShardId("foo", 0)));
         try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
             if (Constants.WINDOWS) {
                 assertTrue(directory.toString(), directory instanceof MMapDirectory || directory instanceof SimpleFSDirectory);