Add more assertion for testRecoverLocallyUpToGlobalCheckpoint

Tracked at #54829
This commit is contained in:
Nhat Nguyen 2020-04-06 18:39:27 -04:00
parent 99846f47b7
commit 22be925f58
1 changed files with 13 additions and 0 deletions

View File

@ -57,6 +57,7 @@ import org.elasticsearch.cluster.routing.allocation.command.AllocateEmptyPrimary
import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
@ -72,6 +73,7 @@ import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.recovery.RecoveryStats;
import org.elasticsearch.index.seqno.ReplicationTracker;
import org.elasticsearch.index.seqno.RetentionLeases;
import org.elasticsearch.index.seqno.SequenceNumbers;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.ShardId;
@ -1176,8 +1178,16 @@ public class IndexRecoveryIT extends ESIntegTestCase {
MockTransportService transportService = (MockTransportService) internalCluster().getInstance(TransportService.class, node);
transportService.addSendBehavior((connection, requestId, action, request, options) -> {
if (action.equals(PeerRecoverySourceService.Actions.START_RECOVERY)) {
assertFalse("recovery request was set already", startRecoveryRequestFuture.isDone());
startRecoveryRequestFuture.onResponse((StartRecoveryRequest) request);
}
if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) {
RetentionLeases retentionLeases = internalCluster().getInstance(IndicesService.class, node)
.indexServiceSafe(resolveIndex(indexName))
.getShard(0).getRetentionLeases();
throw new AssertionError("expect an operation-based recovery:" +
"retention leases" + Strings.toString(retentionLeases) + "]");
}
connection.sendRequest(requestId, action, request, options);
});
}
@ -1192,6 +1202,8 @@ public class IndexRecoveryIT extends ESIntegTestCase {
final long maxSeqNo = shard.seqNoStats().getMaxSeqNo();
shard.failShard("test", new IOException("simulated"));
StartRecoveryRequest startRecoveryRequest = startRecoveryRequestFuture.actionGet();
logger.info("--> start recovery request: starting seq_no {}, commit {}", startRecoveryRequest.startingSeqNo(),
startRecoveryRequest.metadataSnapshot().getCommitUserData());
SequenceNumbers.CommitInfo commitInfoAfterLocalRecovery = SequenceNumbers.loadSeqNoInfoFromLuceneCommit(
startRecoveryRequest.metadataSnapshot().getCommitUserData().entrySet());
assertThat(commitInfoAfterLocalRecovery.localCheckpoint, equalTo(lastSyncedGlobalCheckpoint));
@ -1200,6 +1212,7 @@ public class IndexRecoveryIT extends ESIntegTestCase {
ensureGreen(indexName);
for (RecoveryState recoveryState : client().admin().indices().prepareRecoveries().get().shardRecoveryStates().get(indexName)) {
if (startRecoveryRequest.targetNode().equals(recoveryState.getTargetNode())) {
assertThat("expect an operation-based recovery", recoveryState.getIndex().fileDetails(), empty());
assertThat("total recovered translog operations must include both local and remote recovery",
recoveryState.getTranslog().recoveredOperations(),
greaterThanOrEqualTo(Math.toIntExact(maxSeqNo - localCheckpointOfSafeCommit)));