honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix SearchableSnapshotsIntegTests.testCreateAndRestoreSearchableSnapshot (#65343) (#65351) 

The recovery stats assertions in this test ran without any waiting for
the recoveries to actually finish. The fact that they ran after the concurrent
searches checks generally meant that they would pass (because of searches warming caches
+ general relative slowness of searches) but there is no hard guarantees this will work
reliably as the pre-fetch threads which will update the recovery state might still be slow
to do so randomly, causing the assertions to trip.

closes #65302
This commit is contained in:
Armin Braun 2020-11-23 12:30:18 +01:00 committed by GitHub
parent b0cea04f95
commit 7fbdcb5e00
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 25 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java
View File

@ -64,6 +64,7 @@ import java.util.Set;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
@ -700,32 +701,34 @@ public class SearchableSnapshotsIntegTests extends BaseSearchableSnapshotsIntegT
}
}
private void assertRecoveryStats(String indexName, boolean preWarmEnabled) {
private void assertRecoveryStats(String indexName, boolean preWarmEnabled) throws Exception {
int shardCount = getNumShards(indexName).totalNumShards;
final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries(indexName).get();
assertThat(recoveryResponse.toString(), recoveryResponse.shardRecoveryStates().get(indexName).size(), equalTo(shardCount));
assertBusy(() -> {
final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries(indexName).get();
assertThat(recoveryResponse.toString(), recoveryResponse.shardRecoveryStates().get(indexName).size(), equalTo(shardCount));
for (List<RecoveryState> recoveryStates : recoveryResponse.shardRecoveryStates().values()) {
for (RecoveryState recoveryState : recoveryStates) {
ByteSizeValue cacheSize = getCacheSizeForNode(recoveryState.getTargetNode().getName());
boolean unboundedCache = cacheSize.equals(new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES));
RecoveryState.Index index = recoveryState.getIndex();
assertThat(
Strings.toString(recoveryState, true, true),
index.recoveredFileCount(),
preWarmEnabled && unboundedCache ? equalTo(index.totalRecoverFiles()) : greaterThanOrEqualTo(0)
);
for (List<RecoveryState> recoveryStates : recoveryResponse.shardRecoveryStates().values()) {
for (RecoveryState recoveryState : recoveryStates) {
ByteSizeValue cacheSize = getCacheSizeForNode(recoveryState.getTargetNode().getName());
boolean unboundedCache = cacheSize.equals(new ByteSizeValue(Long.MAX_VALUE, ByteSizeUnit.BYTES));
RecoveryState.Index index = recoveryState.getIndex();
assertThat(
Strings.toString(recoveryState, true, true),
index.recoveredFileCount(),
preWarmEnabled && unboundedCache ? equalTo(index.totalRecoverFiles()) : greaterThanOrEqualTo(0)
);
// Since the cache size is variable, the pre-warm phase might fail as some of the files can be evicted
// while a part is pre-fetched, in that case the recovery state stage is left as FINALIZE.
assertThat(
recoveryState.getStage(),
unboundedCache
? equalTo(RecoveryState.Stage.DONE)
: anyOf(equalTo(RecoveryState.Stage.DONE), equalTo(RecoveryState.Stage.FINALIZE))
);
// Since the cache size is variable, the pre-warm phase might fail as some of the files can be evicted
// while a part is pre-fetched, in that case the recovery state stage is left as FINALIZE.
assertThat(
recoveryState.getStage(),
unboundedCache
? equalTo(RecoveryState.Stage.DONE)
: anyOf(equalTo(RecoveryState.Stage.DONE), equalTo(RecoveryState.Stage.FINALIZE))
);
}
}
}
}, 30L, TimeUnit.SECONDS);
}
private void assertSearchableSnapshotStats(String indexName, boolean cacheEnabled, List<String> nonCachedExtensions) {