diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 82a29890b0d..26e78264534 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -106,7 +106,7 @@ public class TransportClusterStatsAction extends TransportNodesActiontrue iff the data location is a custom data location and therefore outside of the nodes configured data paths. + */ + public boolean isCustomDataPath() { + return isCustomDataPath; + } + /** * This method walks through the nodes shard paths to find the data and state path for the given shard. If multiple * directories with a valid shard state exist the one with the highest version will be used. @@ -113,7 +146,7 @@ public final class ShardPath { dataPath = statePath; } logger.debug("{} loaded data path [{}], state path [{}]", shardId, dataPath, statePath); - return new ShardPath(dataPath, statePath, indexUUID, shardId); + return new ShardPath(NodeEnvironment.hasCustomDataPath(indexSettings), dataPath, statePath, indexUUID, shardId); } } @@ -202,7 +235,7 @@ public final class ShardPath { dataPath = statePath; } - return new ShardPath(dataPath, statePath, indexUUID, shardId); + return new ShardPath(NodeEnvironment.hasCustomDataPath(indexSettings), dataPath, statePath, indexUUID, shardId); } @Override diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java index 43fdb3df675..289b4373ce8 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -236,7 +236,7 @@ public class IndicesService extends AbstractLifecycleComponent i if (indexShard.routingEntry() == null) { continue; } - IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard, indexShard.routingEntry(), flags) }); + IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard, flags) }); if (!statsByShard.containsKey(indexService.index())) { statsByShard.put(indexService.index(), Lists.newArrayList(indexShardStats)); } else { diff --git a/core/src/test/java/org/elasticsearch/common/util/MultiDataPathUpgraderTests.java b/core/src/test/java/org/elasticsearch/common/util/MultiDataPathUpgraderTests.java index fc05a79423d..f91697884c0 100644 --- a/core/src/test/java/org/elasticsearch/common/util/MultiDataPathUpgraderTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/MultiDataPathUpgraderTests.java @@ -83,7 +83,7 @@ public class MultiDataPathUpgraderTests extends ESTestCase { ShardStateMetaData.FORMAT.write(new ShardStateMetaData(metaStateVersion, true, uuid), metaStateVersion, shardDataPaths); } final Path path = randomFrom(shardDataPaths); - ShardPath targetPath = new ShardPath(path, path, uuid, new ShardId("foo", 0)); + ShardPath targetPath = new ShardPath(false, path, path, uuid, new ShardId("foo", 0)); MultiDataPathUpgrader helper = new MultiDataPathUpgrader(nodeEnvironment); helper.upgrade(shardId, targetPath); assertFalse(helper.needsUpgrading(shardId)); @@ -177,7 +177,7 @@ public class MultiDataPathUpgraderTests extends ESTestCase { } logger.info("--> injecting index [{}] into multiple data paths", indexName); OldIndexBackwardsCompatibilityIT.copyIndex(logger, src, indexName, multiDataPath); - final ShardPath shardPath = new ShardPath(nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))[0], nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))[0], IndexMetaData.INDEX_UUID_NA_VALUE, new ShardId(indexName, 0)); + final ShardPath shardPath = new ShardPath(false, nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))[0], nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))[0], IndexMetaData.INDEX_UUID_NA_VALUE, new ShardId(indexName, 0)); logger.info("{}", FileSystemUtils.files(shardPath.resolveIndex())); diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 1efe8d7f776..0f7d75d9edf 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -22,7 +22,9 @@ import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.IOUtils; import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.IndexStats; +import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.*; @@ -31,11 +33,16 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.ShardLock; +import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.query.QueryParsingException; @@ -539,4 +546,39 @@ public class IndexShardTests extends ESSingleNodeTestCase { assertPathHasBeenCleared(startDir.toAbsolutePath().toString()); assertPathHasBeenCleared(endDir.toAbsolutePath().toString()); } + + public void testShardStats() throws IOException { + createIndex("test"); + ensureGreen(); + IndicesService indicesService = getInstanceFromNode(IndicesService.class); + IndexService test = indicesService.indexService("test"); + IndexShard shard = test.shard(0); + ShardStats stats = new ShardStats(shard, new CommonStatsFlags()); + assertEquals(shard.shardPath().getRootDataPath().toString(), stats.getDataPath()); + assertEquals(shard.shardPath().getRootStatePath().toString(), stats.getStatePath()); + assertEquals(shard.shardPath().isCustomDataPath(), stats.isCustomDataPath()); + + if (randomBoolean() || true) { // try to serialize it to ensure values survive the serialization + BytesStreamOutput out = new BytesStreamOutput(); + stats.writeTo(out); + StreamInput in = StreamInput.wrap(out.bytes()); + stats = ShardStats.readShardStats(in); + } + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + stats.toXContent(builder, EMPTY_PARAMS); + builder.endObject(); + String xContent = builder.string(); + StringBuilder expectedSubSequence = new StringBuilder("\"shard_path\":{\"state_path\":\""); + expectedSubSequence.append(shard.shardPath().getRootStatePath().toString()); + expectedSubSequence.append("\",\"data_path\":\""); + expectedSubSequence.append(shard.shardPath().getRootDataPath().toString()); + expectedSubSequence.append("\",\"is_custom_data_path\":").append(shard.shardPath().isCustomDataPath()).append("}"); + System.out.println(expectedSubSequence); + System.out.println(xContent); + assertTrue(xContent.contains(expectedSubSequence)); + } + + // "shard_path":{"state_path":"/private/var/folders/qj/rsr2js6n275f3r88r1z5bbgw0000gn/T/org.elasticsearch.index.shard.IndexShardTests_EE3FC3D62D02988A-001/tempDir-001/data/single-node-cluster-CHILD_VM=[0]-CLUSTER_SEED=[9012992977055748205]-HASH=[13FDF896A8B7DEC0]/nodes/0","data_path":"/private/var/folders/qj/rsr2js6n275f3r88r1z5bbgw0000gn/T/org.elasticsearch.index.shard.IndexShardTests_EE3FC3D62D02988A-001/tempDir-001/data/single-node-cluster-CHILD_VM=[0]-CLUSTER_SEED=[9012992977055748205]-HASH=[13FDF896A8B7DEC0]/nodes/0","is_custom_data_path":"false} + // "shard_path":{"state_path":"/private/var/folders/qj/rsr2js6n275f3r88r1z5bbgw0000gn/T/org.elasticsearch.index.shard.IndexShardTests_EE3FC3D62D02988A-001/tempDir-001/data/single-node-cluster-CHILD_VM=[0]-CLUSTER_SEED=[9012992977055748205]-HASH=[13FDF896A8B7DEC0]/nodes/0","data_path":"/private/var/folders/qj/rsr2js6n275f3r88r1z5bbgw0000gn/T/org.elasticsearch.index.shard.IndexShardTests_EE3FC3D62D02988A-001/tempDir-001/data/single-node-cluster-CHILD_VM=[0]-CLUSTER_SEED=[9012992977055748205]-HASH=[13FDF896A8B7DEC0]/nodes/0","is_custom_data_path":false} } diff --git a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java index e03698f62ee..32d20190890 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.index.shard; +import com.carrotsearch.randomizedtesting.annotations.Repeat; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.NodeEnvironment; @@ -26,6 +27,7 @@ import org.junit.Test; import java.io.IOException; import java.nio.file.Path; +import java.util.Set; import static org.elasticsearch.common.settings.Settings.settingsBuilder; @@ -78,4 +80,71 @@ public class ShardPathTests extends ESTestCase { } } + @Test(expected = IllegalArgumentException.class) + public void testIllegalCustomDataPath() { + final Path path = createTempDir().resolve("foo").resolve("0"); + new ShardPath(true, path, path, "foo", new ShardId("foo", 0)); + } + + public void testValidCtor() { + final Path path = createTempDir().resolve("foo").resolve("0"); + ShardPath shardPath = new ShardPath(false, path, path, "foo", new ShardId("foo", 0)); + assertFalse(shardPath.isCustomDataPath()); + assertEquals(shardPath.getDataPath(), path); + assertEquals(shardPath.getShardStatePath(), path); + } + + public void testGetRootPaths() throws IOException { + boolean useCustomDataPath = randomBoolean(); + final Settings indexSetttings; + final Settings nodeSettings; + Settings.Builder indexSettingsBuilder = settingsBuilder().put(IndexMetaData.SETTING_INDEX_UUID, "0xDEADBEEF"); + final Path customPath; + if (useCustomDataPath) { + final Path path = createTempDir(); + final boolean includeNodeId = randomBoolean(); + indexSetttings = indexSettingsBuilder.put(IndexMetaData.SETTING_DATA_PATH, "custom").build(); + nodeSettings = settingsBuilder().put("path.shared_data", path.toAbsolutePath().toAbsolutePath()) + .put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH, includeNodeId).build(); + if (includeNodeId) { + customPath = path.resolve("custom").resolve("0"); + } else { + customPath = path.resolve("custom"); + } + } else { + customPath = null; + indexSetttings = indexSettingsBuilder.build(); + nodeSettings = Settings.EMPTY; + } + try (final NodeEnvironment env = newNodeEnvironment(nodeSettings)) { + ShardId shardId = new ShardId("foo", 0); + Path[] paths = env.availableShardPaths(shardId); + Path path = randomFrom(paths); + ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF"), 2, path); + ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, indexSetttings); + boolean found = false; + for (Path p : env.nodeDataPaths()) { + if (p.equals(shardPath.getRootStatePath())) { + found = true; + break; + } + } + assertTrue("root state paths must be a node path but wasn't: " + shardPath.getRootStatePath(), found); + found = false; + if (useCustomDataPath) { + assertNotEquals(shardPath.getRootDataPath(), shardPath.getRootStatePath()); + assertEquals(customPath, shardPath.getRootDataPath()); + } else { + assertNull(customPath); + for (Path p : env.nodeDataPaths()) { + if (p.equals(shardPath.getRootDataPath())) { + found = true; + break; + } + } + assertTrue("root state paths must be a node path but wasn't: " + shardPath.getRootDataPath(), found); + } + } + } + } diff --git a/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java b/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java index 75237921fdc..e921f95394c 100644 --- a/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java +++ b/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java @@ -35,11 +35,11 @@ import java.util.Locale; public class IndexStoreTests extends ESTestCase { public void testStoreDirectory() throws IOException { - final Path tempDir = createTempDir(); + final Path tempDir = createTempDir().resolve("foo").resolve("0"); final IndexStoreModule.Type[] values = IndexStoreModule.Type.values(); final IndexStoreModule.Type type = RandomPicks.randomFrom(random(), values); Settings settings = Settings.settingsBuilder().put(IndexStoreModule.STORE_TYPE, type.name().toLowerCase(Locale.ROOT)).build(); - FsDirectoryService service = new FsDirectoryService(settings, null, new ShardPath(tempDir, tempDir, "foo", new ShardId("foo", 0))); + FsDirectoryService service = new FsDirectoryService(settings, null, new ShardPath(false, tempDir, tempDir, "foo", new ShardId("foo", 0))); try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) { switch (type) { case NIOFS: @@ -70,9 +70,9 @@ public class IndexStoreTests extends ESTestCase { } public void testStoreDirectoryDefault() throws IOException { - final Path tempDir = createTempDir(); + final Path tempDir = createTempDir().resolve("foo").resolve("0"); Settings settings = Settings.EMPTY; - FsDirectoryService service = new FsDirectoryService(settings, null, new ShardPath(tempDir, tempDir, "foo", new ShardId("foo", 0))); + FsDirectoryService service = new FsDirectoryService(settings, null, new ShardPath(false, tempDir, tempDir, "foo", new ShardId("foo", 0))); try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) { if (Constants.WINDOWS) { assertTrue(directory.toString(), directory instanceof MMapDirectory || directory instanceof SimpleFSDirectory);