Expose shards data and state path via ShardStats

Since we now don't stripe shards across data paths we need a way
to access the information on which path a shard is allocated to
eventually do better allocation decisions based on disk usage etc.
This commit exposes the shard paths as part of the shard stats.

Relates to #13106
This commit is contained in:
Simon Willnauer 2015-08-26 11:24:23 +02:00
parent df4345f3ab
commit 03ccb99cd5
10 changed files with 197 additions and 22 deletions

View File

@ -106,7 +106,7 @@ public class TransportClusterStatsAction extends TransportNodesAction<ClusterSta
for (IndexShard indexShard : indexService) {
if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) {
// only report on fully started shards
shardsStats.add(new ShardStats(indexShard, indexShard.routingEntry(), SHARD_STATS_FLAGS));
shardsStats.add(new ShardStats(indexShard, SHARD_STATS_FLAGS));
}
}
}

View File

@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentBuilderString;
import org.elasticsearch.index.engine.CommitStats;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.ShardPath;
import java.io.IOException;
@ -37,20 +38,23 @@ import static org.elasticsearch.cluster.routing.ShardRouting.readShardRoutingEnt
/**
*/
public class ShardStats extends BroadcastShardResponse implements ToXContent {
private ShardRouting shardRouting;
CommonStats commonStats;
private CommonStats commonStats;
@Nullable
CommitStats commitStats;
private CommitStats commitStats;
private String dataPath;
private String statePath;
private boolean isCustomDataPath;
ShardStats() {
}
public ShardStats(IndexShard indexShard, ShardRouting shardRouting, CommonStatsFlags flags) {
public ShardStats(IndexShard indexShard, CommonStatsFlags flags) {
super(indexShard.shardId());
this.shardRouting = shardRouting;
this.shardRouting = indexShard.routingEntry();
this.dataPath = indexShard.shardPath().getRootDataPath().toString();
this.statePath = indexShard.shardPath().getRootStatePath().toString();
this.isCustomDataPath = indexShard.shardPath().isCustomDataPath();
this.commonStats = new CommonStats(indexShard, flags);
this.commitStats = indexShard.commitStats();
}
@ -70,6 +74,18 @@ public class ShardStats extends BroadcastShardResponse implements ToXContent {
return this.commitStats;
}
public String getDataPath() {
return dataPath;
}
public String getStatePath() {
return statePath;
}
public boolean isCustomDataPath() {
return isCustomDataPath;
}
public static ShardStats readShardStats(StreamInput in) throws IOException {
ShardStats stats = new ShardStats();
stats.readFrom(in);
@ -82,6 +98,9 @@ public class ShardStats extends BroadcastShardResponse implements ToXContent {
shardRouting = readShardRoutingEntry(in);
commonStats = CommonStats.readCommonStats(in);
commitStats = CommitStats.readOptionalCommitStatsFrom(in);
statePath = in.readString();
dataPath = in.readString();
isCustomDataPath = in.readBoolean();
}
@Override
@ -90,6 +109,9 @@ public class ShardStats extends BroadcastShardResponse implements ToXContent {
shardRouting.writeTo(out);
commonStats.writeTo(out);
out.writeOptionalStreamable(commitStats);
out.writeString(statePath);
out.writeString(dataPath);
out.writeBoolean(isCustomDataPath);
}
@Override
@ -105,12 +127,21 @@ public class ShardStats extends BroadcastShardResponse implements ToXContent {
if (commitStats != null) {
commitStats.toXContent(builder, params);
}
builder.startObject(Fields.SHARD_PATH);
builder.field(Fields.STATE_PATH, statePath);
builder.field(Fields.DATA_PATH, dataPath);
builder.field(Fields.IS_CUSTOM_DATA_PATH, isCustomDataPath);
builder.endObject();
return builder;
}
static final class Fields {
static final XContentBuilderString ROUTING = new XContentBuilderString("routing");
static final XContentBuilderString STATE = new XContentBuilderString("state");
static final XContentBuilderString STATE_PATH = new XContentBuilderString("state_path");
static final XContentBuilderString DATA_PATH = new XContentBuilderString("data_path");
static final XContentBuilderString IS_CUSTOM_DATA_PATH = new XContentBuilderString("is_custom_data_path");
static final XContentBuilderString SHARD_PATH = new XContentBuilderString("shard_path");
static final XContentBuilderString PRIMARY = new XContentBuilderString("primary");
static final XContentBuilderString NODE = new XContentBuilderString("node");
static final XContentBuilderString RELOCATING_NODE = new XContentBuilderString("relocating_node");

View File

@ -189,7 +189,7 @@ public class TransportIndicesStatsAction extends TransportBroadcastAction<Indice
flags.set(CommonStatsFlags.Flag.Recovery);
}
return new ShardStats(indexShard, indexShard.routingEntry(), flags);
return new ShardStats(indexShard, flags);
}
static class IndexShardStatsRequest extends BroadcastShardRequest {

View File

@ -198,7 +198,7 @@ public class MultiDataPathUpgrader {
}
}
}
return new ShardPath(target.path, target.path, IndexMetaData.INDEX_UUID_NA_VALUE /* we don't know */, shard);
return new ShardPath(false, target.path, target.path, IndexMetaData.INDEX_UUID_NA_VALUE /* we don't know */, shard);
}
private ShardFileInfo[] getShardFileInfo(ShardId shard, NodeEnvironment.NodePath[] paths) throws IOException {

View File

@ -20,6 +20,7 @@ package org.elasticsearch.index.shard;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.NodeEnvironment;
@ -41,10 +42,18 @@ public final class ShardPath {
private final String indexUUID;
private final ShardId shardId;
private final Path shardStatePath;
private final boolean isCustomDataPath;
public ShardPath(Path path, Path shardStatePath, String indexUUID, ShardId shardId) {
this.path = path;
public ShardPath(boolean isCustomDataPath, Path dataPath, Path shardStatePath, String indexUUID, ShardId shardId) {
assert dataPath.getFileName().toString().equals(Integer.toString(shardId.id())) : "dataPath must end with the shard ID but didn't: " + dataPath.toString();
assert shardStatePath.getFileName().toString().equals(Integer.toString(shardId.id())) : "shardStatePath must end with the shard ID but didn't: " + dataPath.toString();
assert dataPath.getParent().getFileName().toString().equals(shardId.getIndex()) : "dataPath must end with index/shardID but didn't: " + dataPath.toString();
assert shardStatePath.getParent().getFileName().toString().equals(shardId.getIndex()) : "shardStatePath must end with index/shardID but didn't: " + dataPath.toString();
if (isCustomDataPath && dataPath.equals(shardStatePath)) {
throw new IllegalArgumentException("shard state path must be different to the data path when using custom data paths");
}
this.isCustomDataPath = isCustomDataPath;
this.path = dataPath;
this.indexUUID = indexUUID;
this.shardId = shardId;
this.shardStatePath = shardStatePath;
@ -78,6 +87,30 @@ public final class ShardPath {
return shardStatePath;
}
/**
* Returns the data-path root for this shard. The root is a parent of {@link #getDataPath()} without the index name
* and the shard ID.
*/
public Path getRootDataPath() {
Path noIndexShardId = getDataPath().getParent().getParent();
return isCustomDataPath ? noIndexShardId : noIndexShardId.getParent(); // also strip the indices folder
}
/**
* Returns the state-path root for this shard. The root is a parent of {@link #getRootStatePath()} ()} without the index name
* and the shard ID.
*/
public Path getRootStatePath() {
return getShardStatePath().getParent().getParent().getParent(); // also strip the indices folder
}
/**
* Returns <code>true</code> iff the data location is a custom data location and therefore outside of the nodes configured data paths.
*/
public boolean isCustomDataPath() {
return isCustomDataPath;
}
/**
* This method walks through the nodes shard paths to find the data and state path for the given shard. If multiple
* directories with a valid shard state exist the one with the highest version will be used.
@ -113,7 +146,7 @@ public final class ShardPath {
dataPath = statePath;
}
logger.debug("{} loaded data path [{}], state path [{}]", shardId, dataPath, statePath);
return new ShardPath(dataPath, statePath, indexUUID, shardId);
return new ShardPath(NodeEnvironment.hasCustomDataPath(indexSettings), dataPath, statePath, indexUUID, shardId);
}
}
@ -202,7 +235,7 @@ public final class ShardPath {
dataPath = statePath;
}
return new ShardPath(dataPath, statePath, indexUUID, shardId);
return new ShardPath(NodeEnvironment.hasCustomDataPath(indexSettings), dataPath, statePath, indexUUID, shardId);
}
@Override

View File

@ -236,7 +236,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
if (indexShard.routingEntry() == null) {
continue;
}
IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard, indexShard.routingEntry(), flags) });
IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard, flags) });
if (!statsByShard.containsKey(indexService.index())) {
statsByShard.put(indexService.index(), Lists.<IndexShardStats>newArrayList(indexShardStats));
} else {

View File

@ -83,7 +83,7 @@ public class MultiDataPathUpgraderTests extends ESTestCase {
ShardStateMetaData.FORMAT.write(new ShardStateMetaData(metaStateVersion, true, uuid), metaStateVersion, shardDataPaths);
}
final Path path = randomFrom(shardDataPaths);
ShardPath targetPath = new ShardPath(path, path, uuid, new ShardId("foo", 0));
ShardPath targetPath = new ShardPath(false, path, path, uuid, new ShardId("foo", 0));
MultiDataPathUpgrader helper = new MultiDataPathUpgrader(nodeEnvironment);
helper.upgrade(shardId, targetPath);
assertFalse(helper.needsUpgrading(shardId));
@ -177,7 +177,7 @@ public class MultiDataPathUpgraderTests extends ESTestCase {
}
logger.info("--> injecting index [{}] into multiple data paths", indexName);
OldIndexBackwardsCompatibilityIT.copyIndex(logger, src, indexName, multiDataPath);
final ShardPath shardPath = new ShardPath(nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))[0], nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))[0], IndexMetaData.INDEX_UUID_NA_VALUE, new ShardId(indexName, 0));
final ShardPath shardPath = new ShardPath(false, nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))[0], nodeEnvironment.availableShardPaths(new ShardId(indexName, 0))[0], IndexMetaData.INDEX_UUID_NA_VALUE, new ShardId(indexName, 0));
logger.info("{}", FileSystemUtils.files(shardPath.resolveIndex()));

View File

@ -22,7 +22,9 @@ import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.action.admin.indices.stats.ShardStats;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.cluster.*;
@ -31,11 +33,16 @@ import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardRoutingState;
import org.elasticsearch.cluster.routing.TestShardRouting;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.env.ShardLock;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.query.QueryParsingException;
@ -539,4 +546,39 @@ public class IndexShardTests extends ESSingleNodeTestCase {
assertPathHasBeenCleared(startDir.toAbsolutePath().toString());
assertPathHasBeenCleared(endDir.toAbsolutePath().toString());
}
public void testShardStats() throws IOException {
createIndex("test");
ensureGreen();
IndicesService indicesService = getInstanceFromNode(IndicesService.class);
IndexService test = indicesService.indexService("test");
IndexShard shard = test.shard(0);
ShardStats stats = new ShardStats(shard, new CommonStatsFlags());
assertEquals(shard.shardPath().getRootDataPath().toString(), stats.getDataPath());
assertEquals(shard.shardPath().getRootStatePath().toString(), stats.getStatePath());
assertEquals(shard.shardPath().isCustomDataPath(), stats.isCustomDataPath());
if (randomBoolean() || true) { // try to serialize it to ensure values survive the serialization
BytesStreamOutput out = new BytesStreamOutput();
stats.writeTo(out);
StreamInput in = StreamInput.wrap(out.bytes());
stats = ShardStats.readShardStats(in);
}
XContentBuilder builder = XContentFactory.jsonBuilder();
builder.startObject();
stats.toXContent(builder, EMPTY_PARAMS);
builder.endObject();
String xContent = builder.string();
StringBuilder expectedSubSequence = new StringBuilder("\"shard_path\":{\"state_path\":\"");
expectedSubSequence.append(shard.shardPath().getRootStatePath().toString());
expectedSubSequence.append("\",\"data_path\":\"");
expectedSubSequence.append(shard.shardPath().getRootDataPath().toString());
expectedSubSequence.append("\",\"is_custom_data_path\":").append(shard.shardPath().isCustomDataPath()).append("}");
System.out.println(expectedSubSequence);
System.out.println(xContent);
assertTrue(xContent.contains(expectedSubSequence));
}
// "shard_path":{"state_path":"/private/var/folders/qj/rsr2js6n275f3r88r1z5bbgw0000gn/T/org.elasticsearch.index.shard.IndexShardTests_EE3FC3D62D02988A-001/tempDir-001/data/single-node-cluster-CHILD_VM=[0]-CLUSTER_SEED=[9012992977055748205]-HASH=[13FDF896A8B7DEC0]/nodes/0","data_path":"/private/var/folders/qj/rsr2js6n275f3r88r1z5bbgw0000gn/T/org.elasticsearch.index.shard.IndexShardTests_EE3FC3D62D02988A-001/tempDir-001/data/single-node-cluster-CHILD_VM=[0]-CLUSTER_SEED=[9012992977055748205]-HASH=[13FDF896A8B7DEC0]/nodes/0","is_custom_data_path":"false}
// "shard_path":{"state_path":"/private/var/folders/qj/rsr2js6n275f3r88r1z5bbgw0000gn/T/org.elasticsearch.index.shard.IndexShardTests_EE3FC3D62D02988A-001/tempDir-001/data/single-node-cluster-CHILD_VM=[0]-CLUSTER_SEED=[9012992977055748205]-HASH=[13FDF896A8B7DEC0]/nodes/0","data_path":"/private/var/folders/qj/rsr2js6n275f3r88r1z5bbgw0000gn/T/org.elasticsearch.index.shard.IndexShardTests_EE3FC3D62D02988A-001/tempDir-001/data/single-node-cluster-CHILD_VM=[0]-CLUSTER_SEED=[9012992977055748205]-HASH=[13FDF896A8B7DEC0]/nodes/0","is_custom_data_path":false}
}

View File

@ -18,6 +18,7 @@
*/
package org.elasticsearch.index.shard;
import com.carrotsearch.randomizedtesting.annotations.Repeat;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.NodeEnvironment;
@ -26,6 +27,7 @@ import org.junit.Test;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Set;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
@ -78,4 +80,71 @@ public class ShardPathTests extends ESTestCase {
}
}
@Test(expected = IllegalArgumentException.class)
public void testIllegalCustomDataPath() {
final Path path = createTempDir().resolve("foo").resolve("0");
new ShardPath(true, path, path, "foo", new ShardId("foo", 0));
}
public void testValidCtor() {
final Path path = createTempDir().resolve("foo").resolve("0");
ShardPath shardPath = new ShardPath(false, path, path, "foo", new ShardId("foo", 0));
assertFalse(shardPath.isCustomDataPath());
assertEquals(shardPath.getDataPath(), path);
assertEquals(shardPath.getShardStatePath(), path);
}
public void testGetRootPaths() throws IOException {
boolean useCustomDataPath = randomBoolean();
final Settings indexSetttings;
final Settings nodeSettings;
Settings.Builder indexSettingsBuilder = settingsBuilder().put(IndexMetaData.SETTING_INDEX_UUID, "0xDEADBEEF");
final Path customPath;
if (useCustomDataPath) {
final Path path = createTempDir();
final boolean includeNodeId = randomBoolean();
indexSetttings = indexSettingsBuilder.put(IndexMetaData.SETTING_DATA_PATH, "custom").build();
nodeSettings = settingsBuilder().put("path.shared_data", path.toAbsolutePath().toAbsolutePath())
.put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH, includeNodeId).build();
if (includeNodeId) {
customPath = path.resolve("custom").resolve("0");
} else {
customPath = path.resolve("custom");
}
} else {
customPath = null;
indexSetttings = indexSettingsBuilder.build();
nodeSettings = Settings.EMPTY;
}
try (final NodeEnvironment env = newNodeEnvironment(nodeSettings)) {
ShardId shardId = new ShardId("foo", 0);
Path[] paths = env.availableShardPaths(shardId);
Path path = randomFrom(paths);
ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF"), 2, path);
ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, indexSetttings);
boolean found = false;
for (Path p : env.nodeDataPaths()) {
if (p.equals(shardPath.getRootStatePath())) {
found = true;
break;
}
}
assertTrue("root state paths must be a node path but wasn't: " + shardPath.getRootStatePath(), found);
found = false;
if (useCustomDataPath) {
assertNotEquals(shardPath.getRootDataPath(), shardPath.getRootStatePath());
assertEquals(customPath, shardPath.getRootDataPath());
} else {
assertNull(customPath);
for (Path p : env.nodeDataPaths()) {
if (p.equals(shardPath.getRootDataPath())) {
found = true;
break;
}
}
assertTrue("root state paths must be a node path but wasn't: " + shardPath.getRootDataPath(), found);
}
}
}
}

View File

@ -35,11 +35,11 @@ import java.util.Locale;
public class IndexStoreTests extends ESTestCase {
public void testStoreDirectory() throws IOException {
final Path tempDir = createTempDir();
final Path tempDir = createTempDir().resolve("foo").resolve("0");
final IndexStoreModule.Type[] values = IndexStoreModule.Type.values();
final IndexStoreModule.Type type = RandomPicks.randomFrom(random(), values);
Settings settings = Settings.settingsBuilder().put(IndexStoreModule.STORE_TYPE, type.name().toLowerCase(Locale.ROOT)).build();
FsDirectoryService service = new FsDirectoryService(settings, null, new ShardPath(tempDir, tempDir, "foo", new ShardId("foo", 0)));
FsDirectoryService service = new FsDirectoryService(settings, null, new ShardPath(false, tempDir, tempDir, "foo", new ShardId("foo", 0)));
try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
switch (type) {
case NIOFS:
@ -70,9 +70,9 @@ public class IndexStoreTests extends ESTestCase {
}
public void testStoreDirectoryDefault() throws IOException {
final Path tempDir = createTempDir();
final Path tempDir = createTempDir().resolve("foo").resolve("0");
Settings settings = Settings.EMPTY;
FsDirectoryService service = new FsDirectoryService(settings, null, new ShardPath(tempDir, tempDir, "foo", new ShardId("foo", 0)));
FsDirectoryService service = new FsDirectoryService(settings, null, new ShardPath(false, tempDir, tempDir, "foo", new ShardId("foo", 0)));
try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
if (Constants.WINDOWS) {
assertTrue(directory.toString(), directory instanceof MMapDirectory || directory instanceof SimpleFSDirectory);