honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve Repository Consistency Check in Tests (#44204) 

* Improve Repository Consistency Check in Tests (#44099)

* Check that index metadata as well as snapshot metadata always exists
when referenced by other metadata

* Fix SnapshotResiliencyTests on ExtraFS (#44113)

* As a result of #44099 we're now checking more directories and have to
ignore the `extraN` folders for those like we do for indices already
* Closes #44112
This commit is contained in:
Armin Braun 2019-07-11 11:14:37 +02:00 committed by GitHub
parent 8a554f9737
commit c0ed64bb92
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 45 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
View File

@ -174,13 +174,15 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
private static final String TESTS_FILE = "tests-";
private static final String METADATA_NAME_FORMAT = "meta-%s.dat";
private static final String METADATA_PREFIX = "meta-";
public static final String METADATA_NAME_FORMAT = METADATA_PREFIX + "%s.dat";
private static final String METADATA_CODEC = "metadata";
private static final String INDEX_METADATA_CODEC = "index-metadata";
private static final String SNAPSHOT_NAME_FORMAT = SNAPSHOT_PREFIX + "%s.dat";
public static final String SNAPSHOT_NAME_FORMAT = SNAPSHOT_PREFIX + "%s.dat";
private static final String SNAPSHOT_INDEX_PREFIX = "index-";

1
server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
View File

@ -504,6 +504,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
}
public void testRestoreIndexWithMissingShards() throws Exception {
disableRepoConsistencyCheck("This test leaves behind a purposely broken repository");
logger.info("--> start 2 nodes");
internalCluster().startNode();
internalCluster().startNode();

44
test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java
View File

@ -31,6 +31,7 @@ import org.elasticsearch.repositories.IndexId;
import org.elasticsearch.repositories.RepositoriesService;
import org.elasticsearch.repositories.RepositoryData;
import org.elasticsearch.snapshots.SnapshotId;
import org.elasticsearch.snapshots.SnapshotInfo;
import org.elasticsearch.test.InternalTestCluster;
import org.elasticsearch.threadpool.ThreadPool;
@ -40,10 +41,13 @@ import java.io.InputStream;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.Executor;
import java.util.stream.Collectors;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.hasKey;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@ -84,7 +88,7 @@ public final class BlobStoreTestUtil {
repositoryData = RepositoryData.snapshotsFromXContent(parser, latestGen);
}
assertIndexUUIDs(blobContainer, repositoryData);
assertSnapshotUUIDs(blobContainer, repositoryData);
assertSnapshotUUIDs(repository, repositoryData);
listener.onResponse(null);
}
});
@ -107,20 +111,52 @@ public final class BlobStoreTestUtil {
if (indicesContainer == null) {
foundIndexUUIDs = Collections.emptyList();
} else {
// Skip Lucene MockFS extraN directory
foundIndexUUIDs = indicesContainer.children().keySet().stream().filter(
s -> s.startsWith("extra") == false).collect(Collectors.toList());
}
assertThat(foundIndexUUIDs, containsInAnyOrder(expectedIndexUUIDs.toArray(Strings.EMPTY_ARRAY)));
}
private static void assertSnapshotUUIDs(BlobContainer repoRoot, RepositoryData repositoryData) throws IOException {
final List<String> expectedSnapshotUUIDs =
repositoryData.getSnapshotIds().stream().map(SnapshotId::getUUID).collect(Collectors.toList());
private static void assertSnapshotUUIDs(BlobStoreRepository repository, RepositoryData repositoryData) throws IOException {
final BlobContainer repoRoot = repository.blobContainer();
final Collection<SnapshotId> snapshotIds = repositoryData.getSnapshotIds();
final List<String> expectedSnapshotUUIDs = snapshotIds.stream().map(SnapshotId::getUUID).collect(Collectors.toList());
for (String prefix : new String[]{"snap-", "meta-"}) {
final Collection<String> foundSnapshotUUIDs = repoRoot.listBlobs().keySet().stream().filter(p -> p.startsWith(prefix))
.map(p -> p.replace(prefix, "").replace(".dat", ""))
.collect(Collectors.toSet());
assertThat(foundSnapshotUUIDs, containsInAnyOrder(expectedSnapshotUUIDs.toArray(Strings.EMPTY_ARRAY)));
}
final BlobContainer indicesContainer = repository.getBlobContainer().children().get("indices");
final Map<String, BlobContainer> indices;
if (indicesContainer == null) {
indices = Collections.emptyMap();
} else {
indices = indicesContainer.children();
}
// Assert that for each snapshot, the relevant metadata was written to index and shard folders
for (SnapshotId snapshotId: snapshotIds) {
final SnapshotInfo snapshotInfo = repository.getSnapshotInfo(snapshotId);
for (String index : snapshotInfo.indices()) {
final IndexId indexId = repositoryData.resolveIndexId(index);
assertThat(indices, hasKey(indexId.getId()));
final BlobContainer indexContainer = indices.get(indexId.getId());
assertThat(indexContainer.listBlobs(),
hasKey(String.format(Locale.ROOT, BlobStoreRepository.METADATA_NAME_FORMAT, snapshotId.getUUID())));
for (Map.Entry<String, BlobContainer> entry : indexContainer.children().entrySet()) {
// Skip Lucene MockFS extraN directory
if (entry.getKey().startsWith("extra")) {
continue;
}
if (snapshotInfo.shardFailures().stream().noneMatch(shardFailure ->
shardFailure.index().equals(index) != false && shardFailure.shardId() == Integer.parseInt(entry.getKey()))) {
assertThat(entry.getValue().listBlobs(),
hasKey(String.format(Locale.ROOT, BlobStoreRepository.SNAPSHOT_NAME_FORMAT, snapshotId.getUUID())));
}
}
}
}
}
}