honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

adapt tests to use index uuid as folder name

This commit is contained in:
Areek Zillur 2016-03-14 23:22:12 -04:00
parent 2b18a3ce1d
commit c3078f4d65
14 changed files with 150 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
core/src/test/java/org/elasticsearch/bwcompat/OldIndexBackwardsCompatibilityIT.java
View File

@ -41,6 +41,7 @@ import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.IndexFolderUpgrader;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
@ -105,6 +106,8 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
List<String> indexes;
List<String> unsupportedIndexes;
static String singleDataPathNodeName;
static String multiDataPathNodeName;
static Path singleDataPath;
static Path[] multiDataPath;
@ -127,6 +130,8 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
@AfterClass
public static void tearDownStatics() {
singleDataPathNodeName = null;
multiDataPathNodeName = null;
singleDataPath = null;
multiDataPath = null;
}
@ -157,7 +162,8 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
InternalTestCluster.Async<String> multiDataPathNode = internalCluster().startNodeAsync(nodeSettings.build());
// find single data path dir
Path[] nodePaths = internalCluster().getInstance(NodeEnvironment.class, singleDataPathNode.get()).nodeDataPaths();
singleDataPathNodeName = singleDataPathNode.get();
Path[] nodePaths = internalCluster().getInstance(NodeEnvironment.class, singleDataPathNodeName).nodeDataPaths();
assertEquals(1, nodePaths.length);
singleDataPath = nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER);
assertFalse(Files.exists(singleDataPath));
@ -165,7 +171,8 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
logger.info("--> Single data path: {}", singleDataPath);
// find multi data path dirs
nodePaths = internalCluster().getInstance(NodeEnvironment.class, multiDataPathNode.get()).nodeDataPaths();
multiDataPathNodeName = multiDataPathNode.get();
nodePaths = internalCluster().getInstance(NodeEnvironment.class, multiDataPathNodeName).nodeDataPaths();
assertEquals(2, nodePaths.length);
multiDataPath = new Path[] {nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER),
nodePaths[1].resolve(NodeEnvironment.INDICES_FOLDER)};
@ -178,6 +185,13 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
replicas.get(); // wait for replicas
}
void upgradeIndexFolder() throws Exception {
final NodeEnvironment nodeEnvironment = internalCluster().getInstance(NodeEnvironment.class, singleDataPathNodeName);
IndexFolderUpgrader.upgradeIndicesIfNeeded(Settings.EMPTY, nodeEnvironment);
final NodeEnvironment nodeEnv = internalCluster().getInstance(NodeEnvironment.class, multiDataPathNodeName);
IndexFolderUpgrader.upgradeIndicesIfNeeded(Settings.EMPTY, nodeEnv);
}
String loadIndex(String indexFile) throws Exception {
Path unzipDir = createTempDir();
Path unzipDataDir = unzipDir.resolve("data");
@ -296,6 +310,10 @@ public class OldIndexBackwardsCompatibilityIT extends ESIntegTestCase {
void assertOldIndexWorks(String index) throws Exception {
Version version = extractVersion(index);
String indexName = loadIndex(index);
// we explicitly upgrade the index folders as these indices
// are imported as dangling indices and not available on
// node startup
upgradeIndexFolder();
importIndex(indexName);
assertIndexSanity(indexName, version);
assertBasicSearchWorks(indexName);

10
core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
View File

@ -92,22 +92,22 @@ public class DiskUsageTests extends ESTestCase {
}
public void testFillShardLevelInfo() {
final Index index = new Index("test", "_na_");
final Index index = new Index("test", "0xdeadbeef");
ShardRouting test_0 = ShardRouting.newUnassigned(index, 0, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
ShardRoutingHelper.initialize(test_0, "node1");
ShardRoutingHelper.moveToStarted(test_0);
Path test0Path = createTempDir().resolve("indices").resolve("test").resolve("0");
Path test0Path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("0");
CommonStats commonStats0 = new CommonStats();
commonStats0.store = new StoreStats(100, 1);
ShardRouting test_1 = ShardRouting.newUnassigned(index, 1, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
ShardRoutingHelper.initialize(test_1, "node2");
ShardRoutingHelper.moveToStarted(test_1);
Path test1Path = createTempDir().resolve("indices").resolve("test").resolve("1");
Path test1Path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("1");
CommonStats commonStats1 = new CommonStats();
commonStats1.store = new StoreStats(1000, 1);
ShardStats[] stats = new ShardStats[] {
new ShardStats(test_0, new ShardPath(false, test0Path, test0Path, "0xdeadbeef", test_0.shardId()), commonStats0 , null),
new ShardStats(test_1, new ShardPath(false, test1Path, test1Path, "0xdeadbeef", test_1.shardId()), commonStats1 , null)
new ShardStats(test_0, new ShardPath(false, test0Path, test0Path, test_0.shardId()), commonStats0 , null),
new ShardStats(test_1, new ShardPath(false, test1Path, test1Path, test_1.shardId()), commonStats1 , null)
};
ImmutableOpenMap.Builder<String, Long> shardSizes = ImmutableOpenMap.builder();
ImmutableOpenMap.Builder<ShardRouting, String> routingToPath = ImmutableOpenMap.builder();

6
core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java
View File

@ -22,8 +22,10 @@ package org.elasticsearch.cluster.allocation;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardRoutingState;
@ -42,6 +44,7 @@ import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@ -226,9 +229,10 @@ public class ClusterRerouteIT extends ESIntegTestCase {
assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).id()).get(0).state(), equalTo(ShardRoutingState.STARTED));
client().prepareIndex("test", "type", "1").setSource("field", "value").setRefresh(true).execute().actionGet();
final Index index = resolveIndex("test");
logger.info("--> closing all nodes");
Path[] shardLocation = internalCluster().getInstance(NodeEnvironment.class, node_1).availableShardPaths(new ShardId("test", "_na_", 0));
Path[] shardLocation = internalCluster().getInstance(NodeEnvironment.class, node_1).availableShardPaths(new ShardId(index, 0));
assertThat(FileSystemUtils.exists(shardLocation), equalTo(true)); // make sure the data is there!
internalCluster().closeNonSharedNodes(false); // don't wipe data directories the index needs to be there!

14
core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.gateway;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
@ -68,14 +69,15 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase {
index(index, "doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
ensureGreen();
assertIndexInMetaState(node1, index);
assertIndexDirectoryDeleted(node2, index);
Index resolveIndex = resolveIndex(index);
assertIndexDirectoryDeleted(node2, resolveIndex);
assertIndexInMetaState(masterNode, index);
logger.debug("relocating index...");
client().admin().indices().prepareUpdateSettings(index).setSettings(Settings.builder().put(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING.getKey() + "_name", node2)).get();
client().admin().cluster().prepareHealth().setWaitForRelocatingShards(0).get();
ensureGreen();
assertIndexDirectoryDeleted(node1, index);
assertIndexDirectoryDeleted(node1, resolveIndex);
assertIndexInMetaState(node2, index);
assertIndexInMetaState(masterNode, index);
}
@ -146,10 +148,10 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase {
assertThat(indicesMetaData.get(index).getState(), equalTo(IndexMetaData.State.OPEN));
}
protected void assertIndexDirectoryDeleted(final String nodeName, final String indexName) throws Exception {
protected void assertIndexDirectoryDeleted(final String nodeName, final Index index) throws Exception {
assertBusy(() -> {
logger.info("checking if index directory exists...");
assertFalse("Expecting index directory of " + indexName + " to be deleted from node " + nodeName, indexDirectoryExists(nodeName, indexName));
assertFalse("Expecting index directory of " + index + " to be deleted from node " + nodeName, indexDirectoryExists(nodeName, index));
}
);
}
@ -168,9 +170,9 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase {
}
private boolean indexDirectoryExists(String nodeName, String indexName) {
private boolean indexDirectoryExists(String nodeName, Index index) {
NodeEnvironment nodeEnv = ((InternalTestCluster) cluster()).getInstance(NodeEnvironment.class, nodeName);
for (Path path : nodeEnv.indexPaths(indexName)) {
for (Path path : nodeEnv.indexPaths(index)) {
if (Files.exists(path)) {
return true;
}

9
core/src/test/java/org/elasticsearch/gateway/MetaStateServiceTests.java
View File

@ -24,6 +24,7 @@ import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.test.ESTestCase;
import static org.hamcrest.Matchers.equalTo;
@ -43,15 +44,15 @@ public class MetaStateServiceTests extends ESTestCase {
MetaStateService metaStateService = new MetaStateService(randomSettings(), env);
IndexMetaData index = IndexMetaData.builder("test1").settings(indexSettings).build();
metaStateService.writeIndex("test_write", index, null);
assertThat(metaStateService.loadIndexState("test1"), equalTo(index));
metaStateService.writeIndex("test_write", index);
assertThat(metaStateService.loadIndexState(index.getIndex()), equalTo(index));
}
}
public void testLoadMissingIndex() throws Exception {
try (NodeEnvironment env = newNodeEnvironment()) {
MetaStateService metaStateService = new MetaStateService(randomSettings(), env);
assertThat(metaStateService.loadIndexState("test1"), nullValue());
assertThat(metaStateService.loadIndexState(new Index("test1", "test1UUID")), nullValue());
}
}
@ -94,7 +95,7 @@ public class MetaStateServiceTests extends ESTestCase {
.build();
metaStateService.writeGlobalState("test_write", metaData);
metaStateService.writeIndex("test_write", index, null);
metaStateService.writeIndex("test_write", index);
MetaData loadedState = metaStateService.loadFullState();
assertThat(loadedState.persistentSettings(), equalTo(metaData.persistentSettings()));

18
core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
View File

@ -70,6 +70,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.env.ShardLock;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.NodeServicesProvider;
@ -97,6 +98,7 @@ import org.elasticsearch.test.FieldMaskingReader;
import org.elasticsearch.test.IndexSettingsModule;
import org.elasticsearch.test.InternalSettingsPlugin;
import org.elasticsearch.test.VersionUtils;
import org.elasticsearch.test.junit.annotations.TestLogging;
import java.io.IOException;
import java.nio.file.Files;
@ -141,25 +143,25 @@ public class IndexShardTests extends ESSingleNodeTestCase {
public void testWriteShardState() throws Exception {
try (NodeEnvironment env = newNodeEnvironment()) {
ShardId id = new ShardId("foo", "_na_", 1);
ShardId id = new ShardId("foo", "fooUUID", 1);
long version = between(1, Integer.MAX_VALUE / 2);
boolean primary = randomBoolean();
AllocationId allocationId = randomBoolean() ? null : randomAllocationId();
ShardStateMetaData state1 = new ShardStateMetaData(version, primary, "foo", allocationId);
ShardStateMetaData state1 = new ShardStateMetaData(version, primary, "fooUUID", allocationId);
write(state1, env.availableShardPaths(id));
ShardStateMetaData shardStateMetaData = load(logger, env.availableShardPaths(id));
assertEquals(shardStateMetaData, state1);
ShardStateMetaData state2 = new ShardStateMetaData(version, primary, "foo", allocationId);
ShardStateMetaData state2 = new ShardStateMetaData(version, primary, "fooUUID", allocationId);
write(state2, env.availableShardPaths(id));
shardStateMetaData = load(logger, env.availableShardPaths(id));
assertEquals(shardStateMetaData, state1);
ShardStateMetaData state3 = new ShardStateMetaData(version + 1, primary, "foo", allocationId);
ShardStateMetaData state3 = new ShardStateMetaData(version + 1, primary, "fooUUID", allocationId);
write(state3, env.availableShardPaths(id));
shardStateMetaData = load(logger, env.availableShardPaths(id));
assertEquals(shardStateMetaData, state3);
assertEquals("foo", state3.indexUUID);
assertEquals("fooUUID", state3.indexUUID);
}
}
@ -167,7 +169,9 @@ public class IndexShardTests extends ESSingleNodeTestCase {
createIndex("test");
ensureGreen();
NodeEnvironment env = getInstanceFromNode(NodeEnvironment.class);
Path[] shardPaths = env.availableShardPaths(new ShardId("test", "_na_", 0));
ClusterService cs = getInstanceFromNode(ClusterService.class);
final Index index = cs.state().metaData().index("test").getIndex();
Path[] shardPaths = env.availableShardPaths(new ShardId(index, 0));
logger.info("--> paths: [{}]", (Object)shardPaths);
// Should not be able to acquire the lock because it's already open
try {
@ -179,7 +183,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
// Test without the regular shard lock to assume we can acquire it
// (worst case, meaning that the shard lock could be acquired and
// we're green to delete the shard's directory)
ShardLock sLock = new DummyShardLock(new ShardId("test", "_na_", 0));
ShardLock sLock = new DummyShardLock(new ShardId(index, 0));
try {
env.deleteShardDirectoryUnderLock(sLock, IndexSettingsModule.newIndexSettings("test", Settings.EMPTY));
fail("should not have been able to delete the directory");

31
core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java
View File

@ -24,6 +24,7 @@ import org.elasticsearch.cluster.routing.AllocationId;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.IndexSettingsModule;
@ -42,13 +43,13 @@ public class ShardPathTests extends ESTestCase {
Settings.Builder builder = settingsBuilder().put(IndexMetaData.SETTING_INDEX_UUID, "0xDEADBEEF")
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
Settings settings = builder.build();
ShardId shardId = new ShardId("foo", "_na_", 0);
ShardId shardId = new ShardId("foo", "0xDEADBEEF", 0);
Path[] paths = env.availableShardPaths(shardId);
Path path = randomFrom(paths);
ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF", AllocationId.newInitializing()), 2, path);
ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings));
assertEquals(path, shardPath.getDataPath());
assertEquals("0xDEADBEEF", shardPath.getIndexUUID());
assertEquals("0xDEADBEEF", shardPath.getShardId().getIndex().getUUID());
assertEquals("foo", shardPath.getShardId().getIndexName());
assertEquals(path.resolve("translog"), shardPath.resolveTranslog());
assertEquals(path.resolve("index"), shardPath.resolveIndex());
@ -57,14 +58,15 @@ public class ShardPathTests extends ESTestCase {
public void testFailLoadShardPathOnMultiState() throws IOException {
try (final NodeEnvironment env = newNodeEnvironment(settingsBuilder().build())) {
Settings.Builder builder = settingsBuilder().put(IndexMetaData.SETTING_INDEX_UUID, "0xDEADBEEF")
final String indexUUID = "0xDEADBEEF";
Settings.Builder builder = settingsBuilder().put(IndexMetaData.SETTING_INDEX_UUID, indexUUID)
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
Settings settings = builder.build();
ShardId shardId = new ShardId("foo", "_na_", 0);
ShardId shardId = new ShardId("foo", indexUUID, 0);
Path[] paths = env.availableShardPaths(shardId);
assumeTrue("This test tests multi data.path but we only got one", paths.length > 1);
int id = randomIntBetween(1, 10);
ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, "0xDEADBEEF", AllocationId.newInitializing()), id, paths);
ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, indexUUID, AllocationId.newInitializing()), id, paths);
ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings));
fail("Expected IllegalStateException");
} catch (IllegalStateException e) {
@ -77,7 +79,7 @@ public class ShardPathTests extends ESTestCase {
Settings.Builder builder = settingsBuilder().put(IndexMetaData.SETTING_INDEX_UUID, "foobar")
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
Settings settings = builder.build();
ShardId shardId = new ShardId("foo", "_na_", 0);
ShardId shardId = new ShardId("foo", "foobar", 0);
Path[] paths = env.availableShardPaths(shardId);
Path path = randomFrom(paths);
int id = randomIntBetween(1, 10);
@ -90,9 +92,10 @@ public class ShardPathTests extends ESTestCase {
}
public void testIllegalCustomDataPath() {
final Path path = createTempDir().resolve("foo").resolve("0");
Index index = new Index("foo", "foo");
final Path path = createTempDir().resolve(index.getUUID()).resolve("0");
try {
new ShardPath(true, path, path, "foo", new ShardId("foo", "_na_", 0));
new ShardPath(true, path, path, new ShardId(index, 0));
fail("Expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), is("shard state path must be different to the data path when using custom data paths"));
@ -100,8 +103,9 @@ public class ShardPathTests extends ESTestCase {
}
public void testValidCtor() {
final Path path = createTempDir().resolve("foo").resolve("0");
ShardPath shardPath = new ShardPath(false, path, path, "foo", new ShardId("foo", "_na_", 0));
Index index = new Index("foo", "foo");
final Path path = createTempDir().resolve(index.getUUID()).resolve("0");
ShardPath shardPath = new ShardPath(false, path, path, new ShardId(index, 0));
assertFalse(shardPath.isCustomDataPath());
assertEquals(shardPath.getDataPath(), path);
assertEquals(shardPath.getShardStatePath(), path);
@ -111,8 +115,9 @@ public class ShardPathTests extends ESTestCase {
boolean useCustomDataPath = randomBoolean();
final Settings indexSettings;
final Settings nodeSettings;
final String indexUUID = "0xDEADBEEF";
Settings.Builder indexSettingsBuilder = settingsBuilder()
.put(IndexMetaData.SETTING_INDEX_UUID, "0xDEADBEEF")
.put(IndexMetaData.SETTING_INDEX_UUID, indexUUID)
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
final Path customPath;
if (useCustomDataPath) {
@ -132,10 +137,10 @@ public class ShardPathTests extends ESTestCase {
nodeSettings = Settings.EMPTY;
}
try (final NodeEnvironment env = newNodeEnvironment(nodeSettings)) {
ShardId shardId = new ShardId("foo", "_na_", 0);
ShardId shardId = new ShardId("foo", indexUUID, 0);
Path[] paths = env.availableShardPaths(shardId);
Path path = randomFrom(paths);
ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF", AllocationId.newInitializing()), 2, path);
ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, indexUUID, AllocationId.newInitializing()), 2, path);
ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), indexSettings));
boolean found = false;
for (Path p : env.nodeDataPaths()) {

16
core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
View File

@ -49,6 +49,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.gateway.PrimaryShardAllocator;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.MergePolicyConfig;
import org.elasticsearch.index.shard.IndexEventListener;
@ -571,8 +572,9 @@ public class CorruptedFileIT extends ESIntegTestCase {
private Map<String, List<Path>> findFilesToCorruptForReplica() throws IOException {
Map<String, List<Path>> filesToNodes = new HashMap<>();
ClusterState state = client().admin().cluster().prepareState().get().getState();
Index test = state.metaData().index("test").getIndex();
for (ShardRouting shardRouting : state.getRoutingTable().allShards("test")) {
if (shardRouting.primary() == true) {
if (shardRouting.primary()) {
continue;
}
assertTrue(shardRouting.assignedToNode());
@ -582,8 +584,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
filesToNodes.put(nodeStats.getNode().getName(), files);
for (FsInfo.Path info : nodeStats.getFs()) {
String path = info.getPath();
final String relativeDataLocationPath = "indices/test/" + Integer.toString(shardRouting.getId()) + "/index";
Path file = PathUtils.get(path).resolve(relativeDataLocationPath);
Path file = PathUtils.get(path).resolve("indices").resolve(test.getUUID()).resolve(Integer.toString(shardRouting.getId())).resolve("index");
if (Files.exists(file)) { // multi data path might only have one path in use
try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
for (Path item : stream) {
@ -604,6 +605,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
private ShardRouting corruptRandomPrimaryFile(final boolean includePerCommitFiles) throws IOException {
ClusterState state = client().admin().cluster().prepareState().get().getState();
Index test = state.metaData().index("test").getIndex();
GroupShardsIterator shardIterators = state.getRoutingNodes().getRoutingTable().activePrimaryShardsGrouped(new String[]{"test"}, false);
List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
ShardIterator shardIterator = RandomPicks.randomFrom(getRandom(), iterators);
@ -616,8 +618,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
Set<Path> files = new TreeSet<>(); // treeset makes sure iteration order is deterministic
for (FsInfo.Path info : nodeStatses.getNodes()[0].getFs()) {
String path = info.getPath();
final String relativeDataLocationPath = "indices/test/" + Integer.toString(shardRouting.getId()) + "/index";
Path file = PathUtils.get(path).resolve(relativeDataLocationPath);
Path file = PathUtils.get(path).resolve("indices").resolve(test.getUUID()).resolve(Integer.toString(shardRouting.getId())).resolve("index");
if (Files.exists(file)) { // multi data path might only have one path in use
try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
for (Path item : stream) {
@ -676,12 +677,13 @@ public class CorruptedFileIT extends ESIntegTestCase {
public List<Path> listShardFiles(ShardRouting routing) throws IOException {
NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(routing.currentNodeId()).setFs(true).get();
ClusterState state = client().admin().cluster().prepareState().get().getState();
final Index test = state.metaData().index("test").getIndex();
assertThat(routing.toString(), nodeStatses.getNodes().length, equalTo(1));
List<Path> files = new ArrayList<>();
for (FsInfo.Path info : nodeStatses.getNodes()[0].getFs()) {
String path = info.getPath();
Path file = PathUtils.get(path).resolve("indices/test/" + Integer.toString(routing.getId()) + "/index");
Path file = PathUtils.get(path).resolve("indices/" + test.getUUID() + "/" + Integer.toString(routing.getId()) + "/index");
if (Files.exists(file)) { // multi data path might only have one path in use
try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
for (Path item : stream) {

4
core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
View File

@ -33,6 +33,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.MockEngineFactoryPlugin;
import org.elasticsearch.monitor.fs.FsInfo;
@ -110,6 +111,7 @@ public class CorruptedTranslogIT extends ESIntegTestCase {
private void corruptRandomTranslogFiles() throws IOException {
ClusterState state = client().admin().cluster().prepareState().get().getState();
GroupShardsIterator shardIterators = state.getRoutingNodes().getRoutingTable().activePrimaryShardsGrouped(new String[]{"test"}, false);
final Index test = state.metaData().index("test").getIndex();
List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
ShardIterator shardIterator = RandomPicks.randomFrom(getRandom(), iterators);
ShardRouting shardRouting = shardIterator.nextOrNull();
@ -121,7 +123,7 @@ public class CorruptedTranslogIT extends ESIntegTestCase {
Set<Path> files = new TreeSet<>(); // treeset makes sure iteration order is deterministic
for (FsInfo.Path fsPath : nodeStatses.getNodes()[0].getFs()) {
String path = fsPath.getPath();
final String relativeDataLocationPath = "indices/test/" + Integer.toString(shardRouting.getId()) + "/translog";
final String relativeDataLocationPath = "indices/"+ test.getUUID() +"/" + Integer.toString(shardRouting.getId()) + "/translog";
Path file = PathUtils.get(path).resolve(relativeDataLocationPath);
if (Files.exists(file)) {
logger.info("--> path: {}", file);

8
core/src/test/java/org/elasticsearch/index/store/FsDirectoryServiceTests.java
View File

@ -46,9 +46,9 @@ public class FsDirectoryServiceTests extends ESTestCase {
IndexSettings settings = IndexSettingsModule.newIndexSettings("foo", build);
IndexStoreConfig config = new IndexStoreConfig(build);
IndexStore store = new IndexStore(settings, config);
Path tempDir = createTempDir().resolve("foo").resolve("0");
Path tempDir = createTempDir().resolve(settings.getUUID()).resolve("0");
Files.createDirectories(tempDir);
ShardPath path = new ShardPath(false, tempDir, tempDir, settings.getUUID(), new ShardId(settings.getIndex(), 0));
ShardPath path = new ShardPath(false, tempDir, tempDir, new ShardId(settings.getIndex(), 0));
FsDirectoryService fsDirectoryService = new FsDirectoryService(settings, store, path);
Directory directory = fsDirectoryService.newDirectory();
assertTrue(directory instanceof RateLimitedFSDirectory);
@ -62,9 +62,9 @@ public class FsDirectoryServiceTests extends ESTestCase {
IndexSettings settings = IndexSettingsModule.newIndexSettings("foo", build);
IndexStoreConfig config = new IndexStoreConfig(build);
IndexStore store = new IndexStore(settings, config);
Path tempDir = createTempDir().resolve("foo").resolve("0");
Path tempDir = createTempDir().resolve(settings.getUUID()).resolve("0");
Files.createDirectories(tempDir);
ShardPath path = new ShardPath(false, tempDir, tempDir, settings.getUUID(), new ShardId(settings.getIndex(), 0));
ShardPath path = new ShardPath(false, tempDir, tempDir, new ShardId(settings.getIndex(), 0));
FsDirectoryService fsDirectoryService = new FsDirectoryService(settings, store, path);
Directory directory = fsDirectoryService.newDirectory();
assertTrue(directory instanceof RateLimitedFSDirectory);

10
core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java
View File

@ -47,13 +47,14 @@ import java.util.Locale;
public class IndexStoreTests extends ESTestCase {
public void testStoreDirectory() throws IOException {
final Path tempDir = createTempDir().resolve("foo").resolve("0");
Index index = new Index("foo", "fooUUID");
final Path tempDir = createTempDir().resolve(index.getUUID()).resolve("0");
final IndexModule.Type[] values = IndexModule.Type.values();
final IndexModule.Type type = RandomPicks.randomFrom(random(), values);
Settings settings = Settings.settingsBuilder().put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), type.name().toLowerCase(Locale.ROOT))
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
FsDirectoryService service = new FsDirectoryService(indexSettings, null, new ShardPath(false, tempDir, tempDir, "foo", new ShardId("foo", "_na_", 0)));
FsDirectoryService service = new FsDirectoryService(indexSettings, null, new ShardPath(false, tempDir, tempDir, new ShardId(index, 0)));
try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
switch (type) {
case NIOFS:
@ -84,8 +85,9 @@ public class IndexStoreTests extends ESTestCase {
}
public void testStoreDirectoryDefault() throws IOException {
final Path tempDir = createTempDir().resolve("foo").resolve("0");
FsDirectoryService service = new FsDirectoryService(IndexSettingsModule.newIndexSettings("foo", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build()), null, new ShardPath(false, tempDir, tempDir, "foo", new ShardId("foo", "_na_", 0)));
Index index = new Index("bar", "foo");
final Path tempDir = createTempDir().resolve(index.getUUID()).resolve("0");
FsDirectoryService service = new FsDirectoryService(IndexSettingsModule.newIndexSettings("bar", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build()), null, new ShardPath(false, tempDir, tempDir, new ShardId(index, 0)));
try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
if (Constants.WINDOWS) {
assertTrue(directory.toString(), directory instanceof MMapDirectory || directory instanceof SimpleFSDirectory);

98
core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java
View File

@ -112,12 +112,14 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1))
);
ensureGreen("test");
ClusterState state = client().admin().cluster().prepareState().get().getState();
Index index = state.metaData().index("test").getIndex();
logger.info("--> making sure that shard and its replica are allocated on node_1 and node_2");
assertThat(Files.exists(shardDirectory(node_1, "test", 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_1, "test")), equalTo(true));
assertThat(Files.exists(shardDirectory(node_2, "test", 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_2, "test")), equalTo(true));
assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_1, index)), equalTo(true));
assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_2, index)), equalTo(true));
logger.info("--> starting node server3");
final String node_3 = internalCluster().startNode(Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false));
@ -128,12 +130,12 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
.get();
assertThat(clusterHealth.isTimedOut(), equalTo(false));
assertThat(Files.exists(shardDirectory(node_1, "test", 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_1, "test")), equalTo(true));
assertThat(Files.exists(shardDirectory(node_2, "test", 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_2, "test")), equalTo(true));
assertThat(Files.exists(shardDirectory(node_3, "test", 0)), equalTo(false));
assertThat(Files.exists(indexDirectory(node_3, "test")), equalTo(false));
assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_1, index)), equalTo(true));
assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_2, index)), equalTo(true));
assertThat(Files.exists(shardDirectory(node_3, index, 0)), equalTo(false));
assertThat(Files.exists(indexDirectory(node_3, index)), equalTo(false));
logger.info("--> move shard from node_1 to node_3, and wait for relocation to finish");
@ -161,12 +163,12 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
.get();
assertThat(clusterHealth.isTimedOut(), equalTo(false));
assertThat(waitForShardDeletion(node_1, "test", 0), equalTo(false));
assertThat(waitForIndexDeletion(node_1, "test"), equalTo(false));
assertThat(Files.exists(shardDirectory(node_2, "test", 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_2, "test")), equalTo(true));
assertThat(Files.exists(shardDirectory(node_3, "test", 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_3, "test")), equalTo(true));
assertThat(waitForShardDeletion(node_1, index, 0), equalTo(false));
assertThat(waitForIndexDeletion(node_1, index), equalTo(false));
assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_2, index)), equalTo(true));
assertThat(Files.exists(shardDirectory(node_3, index, 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_3, index)), equalTo(true));
}
@ -180,16 +182,18 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
);
ensureGreen("test");
assertThat(Files.exists(shardDirectory(node_1, "test", 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_1, "test")), equalTo(true));
ClusterState state = client().admin().cluster().prepareState().get().getState();
Index index = state.metaData().index("test").getIndex();
assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_1, index)), equalTo(true));
final String node_2 = internalCluster().startDataOnlyNode(Settings.builder().build());
assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut());
assertThat(Files.exists(shardDirectory(node_1, "test", 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_1, "test")), equalTo(true));
assertThat(Files.exists(shardDirectory(node_2, "test", 0)), equalTo(false));
assertThat(Files.exists(indexDirectory(node_2, "test")), equalTo(false));
assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
assertThat(Files.exists(indexDirectory(node_1, index)), equalTo(true));
assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(false));
assertThat(Files.exists(indexDirectory(node_2, index)), equalTo(false));
// add a transport delegate that will prevent the shard active request to succeed the first time after relocation has finished.
// node_1 will then wait for the next cluster state change before it tries a next attempt to delete the shard.
@ -220,14 +224,14 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
// it must still delete the shard, even if it cannot find it anymore in indicesservice
client().admin().indices().prepareDelete("test").get();
assertThat(waitForShardDeletion(node_1, "test", 0), equalTo(false));
assertThat(waitForIndexDeletion(node_1, "test"), equalTo(false));
assertThat(Files.exists(shardDirectory(node_1, "test", 0)), equalTo(false));
assertThat(Files.exists(indexDirectory(node_1, "test")), equalTo(false));
assertThat(waitForShardDeletion(node_2, "test", 0), equalTo(false));
assertThat(waitForIndexDeletion(node_2, "test"), equalTo(false));
assertThat(Files.exists(shardDirectory(node_2, "test", 0)), equalTo(false));
assertThat(Files.exists(indexDirectory(node_2, "test")), equalTo(false));
assertThat(waitForShardDeletion(node_1, index, 0), equalTo(false));
assertThat(waitForIndexDeletion(node_1, index), equalTo(false));
assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(false));
assertThat(Files.exists(indexDirectory(node_1, index)), equalTo(false));
assertThat(waitForShardDeletion(node_2, index, 0), equalTo(false));
assertThat(waitForIndexDeletion(node_2, index), equalTo(false));
assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(false));
assertThat(Files.exists(indexDirectory(node_2, index)), equalTo(false));
}
public void testShardsCleanup() throws Exception {
@ -241,9 +245,11 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
);
ensureGreen("test");
ClusterState state = client().admin().cluster().prepareState().get().getState();
Index index = state.metaData().index("test").getIndex();
logger.info("--> making sure that shard and its replica are allocated on node_1 and node_2");
assertThat(Files.exists(shardDirectory(node_1, "test", 0)), equalTo(true));
assertThat(Files.exists(shardDirectory(node_2, "test", 0)), equalTo(true));
assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(true));
logger.info("--> starting node server3");
String node_3 = internalCluster().startNode();
@ -255,10 +261,10 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
assertThat(clusterHealth.isTimedOut(), equalTo(false));
logger.info("--> making sure that shard is not allocated on server3");
assertThat(waitForShardDeletion(node_3, "test", 0), equalTo(false));
assertThat(waitForShardDeletion(node_3, index, 0), equalTo(false));
Path server2Shard = shardDirectory(node_2, "test", 0);
logger.info("--> stopping node {}", node_2);
Path server2Shard = shardDirectory(node_2, index, 0);
logger.info("--> stopping node " + node_2);
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node_2));
logger.info("--> running cluster_health");
@ -273,9 +279,9 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
assertThat(Files.exists(server2Shard), equalTo(true));
logger.info("--> making sure that shard and its replica exist on server1, server2 and server3");
assertThat(Files.exists(shardDirectory(node_1, "test", 0)), equalTo(true));
assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
assertThat(Files.exists(server2Shard), equalTo(true));
assertThat(Files.exists(shardDirectory(node_3, "test", 0)), equalTo(true));
assertThat(Files.exists(shardDirectory(node_3, index, 0)), equalTo(true));
logger.info("--> starting node node_4");
final String node_4 = internalCluster().startNode();
@ -284,9 +290,9 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
ensureGreen();
logger.info("--> making sure that shard and its replica are allocated on server1 and server3 but not on server2");
assertThat(Files.exists(shardDirectory(node_1, "test", 0)), equalTo(true));
assertThat(Files.exists(shardDirectory(node_3, "test", 0)), equalTo(true));
assertThat(waitForShardDeletion(node_4, "test", 0), equalTo(false));
assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
assertThat(Files.exists(shardDirectory(node_3, index, 0)), equalTo(true));
assertThat(waitForShardDeletion(node_4, index, 0), equalTo(false));
}
public void testShardActiveElsewhereDoesNotDeleteAnother() throws Exception {
@ -426,30 +432,30 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
waitNoPendingTasksOnAll();
logger.info("Checking if shards aren't removed");
for (int shard : node2Shards) {
assertTrue(waitForShardDeletion(nonMasterNode, "test", shard));
assertTrue(waitForShardDeletion(nonMasterNode, index, shard));
}
}
private Path indexDirectory(String server, String index) {
private Path indexDirectory(String server, Index index) {
NodeEnvironment env = internalCluster().getInstance(NodeEnvironment.class, server);
final Path[] paths = env.indexPaths(index);
assert paths.length == 1;
return paths[0];
}
private Path shardDirectory(String server, String index, int shard) {
private Path shardDirectory(String server, Index index, int shard) {
NodeEnvironment env = internalCluster().getInstance(NodeEnvironment.class, server);
final Path[] paths = env.availableShardPaths(new ShardId(index, "_na_", shard));
final Path[] paths = env.availableShardPaths(new ShardId(index, shard));
assert paths.length == 1;
return paths[0];
}
private boolean waitForShardDeletion(final String server, final String index, final int shard) throws InterruptedException {
private boolean waitForShardDeletion(final String server, final Index index, final int shard) throws InterruptedException {
awaitBusy(() -> !Files.exists(shardDirectory(server, index, shard)));
return Files.exists(shardDirectory(server, index, shard));
}
private boolean waitForIndexDeletion(final String server, final String index) throws InterruptedException {
private boolean waitForIndexDeletion(final String server, final Index index) throws InterruptedException {
awaitBusy(() -> !Files.exists(indexDirectory(server, index)));
return Files.exists(indexDirectory(server, index));
}

2
plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperUpgradeTests.java
View File

@ -52,6 +52,7 @@ public class Murmur3FieldMapperUpgradeTests extends ESIntegTestCase {
public void testUpgradeOldMapping() throws IOException, ExecutionException, InterruptedException {
final String indexName = "index-mapper-murmur3-2.0.0";
final String indexUUID = "1VzJds59TTK7lRu17W0mcg";
InternalTestCluster.Async<String> master = internalCluster().startNodeAsync();
Path unzipDir = createTempDir();
Path unzipDataDir = unzipDir.resolve("data");
@ -72,6 +73,7 @@ public class Murmur3FieldMapperUpgradeTests extends ESIntegTestCase {
assertFalse(Files.exists(dataPath));
Path src = unzipDataDir.resolve(indexName + "/nodes/0/indices");
Files.move(src, dataPath);
Files.move(dataPath.resolve(indexName), dataPath.resolve(indexUUID));
master.get();
// force reloading dangling indices with a cluster state republish

2
plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java
View File

@ -53,6 +53,7 @@ public class SizeFieldMapperUpgradeTests extends ESIntegTestCase {
public void testUpgradeOldMapping() throws IOException, ExecutionException, InterruptedException {
final String indexName = "index-mapper-size-2.0.0";
final String indexUUID = "ENCw7sG0SWuTPcH60bHheg";
InternalTestCluster.Async<String> master = internalCluster().startNodeAsync();
Path unzipDir = createTempDir();
Path unzipDataDir = unzipDir.resolve("data");
@ -73,6 +74,7 @@ public class SizeFieldMapperUpgradeTests extends ESIntegTestCase {
assertFalse(Files.exists(dataPath));
Path src = unzipDataDir.resolve(indexName + "/nodes/0/indices");
Files.move(src, dataPath);
Files.move(dataPath.resolve(indexName), dataPath.resolve(indexUUID));
master.get();
// force reloading dangling indices with a cluster state republish
client().admin().cluster().prepareReroute().get();