honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

RecoveryWhileUnderLoadIT: output specific missing doc ids and their shard routing on failure 

Also increase logging levels to see when a doc was indexed
This commit is contained in:
Boaz Leskes 2016-03-22 18:17:37 +01:00
parent 3ed4ff054f
commit 20644666e5
1 changed files with 21 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
View File

@ -22,8 +22,12 @@ package org.elasticsearch.recovery;
import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.admin.indices.stats.ShardStats;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.routing.Murmur3HashFunction;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
@ -31,6 +35,7 @@ import org.elasticsearch.common.math.MathUtils;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.shard.DocsStats;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.translog.Translog;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.sort.SortOrder;
@ -50,7 +55,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllS
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout;
@TestLogging("_root:DEBUG")
@TestLogging("_root:DEBUG,index.shard:TRACE")
public class RecoveryWhileUnderLoadIT extends ESIntegTestCase {
private final ESLogger logger = Loggers.getLogger(RecoveryWhileUnderLoadIT.class);
@ -268,14 +273,12 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase {
private void iterateAssertCount(final int numberOfShards, final long numberOfDocs, final int iterations) throws Exception {
SearchResponse[] iterationResults = new SearchResponse[iterations];
boolean error = false;
SearchResponse lastErroneousResponse = null;
for (int i = 0; i < iterations; i++) {
SearchResponse searchResponse = client().prepareSearch().setSize((int) numberOfDocs).setQuery(matchAllQuery()).addSort("id", SortOrder.ASC).get();
logSearchResponse(numberOfShards, numberOfDocs, i, searchResponse);
iterationResults[i] = searchResponse;
if (searchResponse.getHits().totalHits() != numberOfDocs) {
error = true;
lastErroneousResponse = searchResponse;
}
}
@ -287,12 +290,21 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase {
logger.info("shard [{}] - count {}, primary {}", shardStats.getShardRouting().id(), docsStats.getCount(), shardStats.getShardRouting().primary());
}
for (int doc = 1, hit = 0; hit < lastErroneousResponse.getHits().getHits().length; hit++, doc++) {
SearchHit searchHit = lastErroneousResponse.getHits().getAt(hit);
while (doc < Integer.parseInt(searchHit.id())) {
logger.info("missing doc [{}], indexed to shard [{}]", doc, MathUtils.mod(Murmur3HashFunction.hash(Integer.toString(doc)), numberOfShards));
doc++;
ClusterService clusterService = clusterService();
final ClusterState state = clusterService.state();
for (int shard = 0; shard < numberOfShards; shard++) {
// background indexer starts using ids on 1
for (int id = 1; id <= numberOfDocs; id++) {
ShardId docShard = clusterService.operationRouting().shardId(state, "test", Long.toString(id), null);
if (docShard.id() == shard) {
for (ShardRouting shardRouting : state.routingTable().shardRoutingTable("test", shard)) {
GetResponse response = client().prepareGet("test", "type", Long.toString(id))
.setPreference("_only_node:" + shardRouting.currentNodeId()).get();
if (response.isExists()) {
logger.info("missing id [{}] on shard {}", id, shardRouting);
}
}
}
}
}