honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-03-09 14:34:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow shards to be allocated if leftover shard from different index exists.

Browse Source 
If an index name is reused but a leftover shard still exists on any node
we fail repeatedly to allocate the shard since we now check the index UUID
before reusing data. This commit allows to recover even if there is such a
leftover shard by deleting the leftover shard.

Closes #10677
This commit is contained in:
Simon Willnauer 2015-06-30 15:46:15 +02:00
parent 1e35bf3171
commit cdcfcf4b00
3 changed files with 66 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
core/src/main/java/org/elasticsearch/index/IndexService.java
View File

@ -284,8 +284,20 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone
boolean success = false;
Injector shardInjector = null;
try {
ShardPath path = ShardPath.loadShardPath(logger, nodeEnv, shardId, indexSettings);
lock = nodeEnv.shardLock(shardId, TimeUnit.SECONDS.toMillis(5));
ShardPath path;
try {
path = ShardPath.loadShardPath(logger, nodeEnv, shardId, indexSettings);
} catch (IllegalStateException ex) {
logger.warn("{} failed to load shard path, trying to archive leftover", shardId);
try {
ShardPath.deleteLeftoverShardDirectory(logger, nodeEnv, lock, indexSettings);
path = ShardPath.loadShardPath(logger, nodeEnv, shardId, indexSettings);
} catch (Throwable t) {
t.addSuppressed(ex);
throw t;
}
}
if (path == null) {
path = ShardPath.selectNewPathForShard(nodeEnv, shardId, indexSettings, getAvgShardSizeInBytes(), this);
logger.debug("{} creating using a new path [{}]", shardId, path);
@ -293,7 +305,6 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone
logger.debug("{} creating using an existing path [{}]", shardId, path);
}
lock = nodeEnv.shardLock(shardId, TimeUnit.SECONDS.toMillis(5));
if (shards.containsKey(shardId.id())) {
throw new IndexShardAlreadyExistsException(shardId + " already exists");
}

24
core/src/main/java/org/elasticsearch/index/shard/ShardPath.java
View File

@ -18,10 +18,12 @@
*/
package org.elasticsearch.index.shard;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.env.ShardLock;
import org.elasticsearch.index.settings.IndexSettings;
import java.io.IOException;
@ -88,7 +90,7 @@ public final class ShardPath {
for (Path path : paths) {
ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, path);
if (load != null) {
if ((load.indexUUID.equals(indexUUID) || IndexMetaData.INDEX_UUID_NA_VALUE.equals(load.indexUUID)) == false) {
if (load.indexUUID.equals(indexUUID) == false && IndexMetaData.INDEX_UUID_NA_VALUE.equals(load.indexUUID) == false) {
logger.warn("{} found shard on path: [{}] with a different index UUID - this shard seems to be leftover from a different index with the same name. Remove the leftover shard in order to reuse the path with the current index", shardId, path);
throw new IllegalStateException(shardId + " index UUID in shard state was: " + load.indexUUID + " expected: " + indexUUID + " on shard path: " + path);
}
@ -115,6 +117,26 @@ public final class ShardPath {
}
}
/**
* This method tries to delete left-over shards where the index name has been reused but the UUID is different
* to allow the new shard to be allocated.
*/
public static void deleteLeftoverShardDirectory(ESLogger logger, NodeEnvironment env, ShardLock lock, @IndexSettings Settings indexSettings) throws IOException {
final String indexUUID = indexSettings.get(IndexMetaData.SETTING_INDEX_UUID, IndexMetaData.INDEX_UUID_NA_VALUE);
final Path[] paths = env.availableShardPaths(lock.getShardId());
for (Path path : paths) {
ShardStateMetaData load = ShardStateMetaData.FORMAT.loadLatestState(logger, path);
if (load != null) {
if (load.indexUUID.equals(indexUUID) == false && IndexMetaData.INDEX_UUID_NA_VALUE.equals(load.indexUUID) == false) {
logger.warn("{} deleting leftover shard on path: [{}] with a different index UUID", lock.getShardId(), path);
assert Files.isDirectory(path) : path + " is not a directory";
NodeEnvironment.acquireFSLockForPaths(indexSettings, paths);
IOUtils.rm(path);
}
}
}
}
/** Maps each path.data path to a "guess" of how many bytes the shards allocated to that path might additionally use over their
* lifetime; we do this so a bunch of newly allocated shards won't just all go the path with the most free space at this moment. */
private static Map<Path,Long> getEstimatedReservedBytes(NodeEnvironment env, long avgShardSizeInBytes, Iterable<IndexShard> shards) throws IOException {

29
core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
View File

@ -23,11 +23,13 @@ import org.apache.lucene.store.LockObtainFailedException;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardRoutingState;
import org.elasticsearch.cluster.routing.TestShardRouting;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.NodeEnvironment;
@ -46,7 +48,9 @@ import org.elasticsearch.test.VersionUtils;
import org.junit.Test;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ExecutionException;
@ -54,6 +58,7 @@ import java.util.concurrent.ExecutionException;
import static org.elasticsearch.cluster.metadata.IndexMetaData.*;
import static org.elasticsearch.common.settings.Settings.settingsBuilder;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
import static org.hamcrest.Matchers.equalTo;
@ -406,4 +411,28 @@ public class IndexShardTests extends ElasticsearchSingleNodeTest {
client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_PRIORITY, 400).build()).get();
assertEquals(400, indexSettingsService.getSettings().getAsInt(IndexMetaData.SETTING_PRIORITY, 0).intValue());
}
public void testRecoverIntoLeftover() throws IOException {
createIndex("test");
ensureGreen("test");
client().prepareIndex("test", "bar", "1").setSource("{}").setRefresh(true).get();
client().admin().indices().prepareFlush("test").get();
SearchResponse response = client().prepareSearch("test").get();
assertHitCount(response, 1l);
IndicesService indicesService = getInstanceFromNode(IndicesService.class);
IndexService test = indicesService.indexService("test");
IndexShard shard = test.shard(0);
ShardPath shardPath = shard.shardPath();
Path dataPath = shardPath.getDataPath();
client().admin().indices().prepareClose("test").get();
Path tempDir = createTempDir();
Files.move(dataPath, tempDir.resolve("test"));
client().admin().indices().prepareDelete("test").get();
Files.createDirectories(dataPath.getParent());
Files.move(tempDir.resolve("test"), dataPath);
createIndex("test");
ensureGreen("test");
response = client().prepareSearch("test").get();
assertHitCount(response, 0l);
}
}