maybe fix leadership integration test flakes (#11031)

This commit is contained in:
Clint Wylie 2021-03-26 03:43:06 -07:00 committed by GitHub
parent bf20f9e979
commit f160548231
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 35 additions and 22 deletions

View File

@ -66,6 +66,9 @@ public class ITHighAvailabilityTest
private static final String SYSTEM_QUERIES_RESOURCE = "/queries/high_availability_sys.json"; private static final String SYSTEM_QUERIES_RESOURCE = "/queries/high_availability_sys.json";
private static final int NUM_LEADERSHIP_SWAPS = 3; private static final int NUM_LEADERSHIP_SWAPS = 3;
private static final int NUM_RETRIES = 120;
private static final long RETRY_DELAY = TimeUnit.SECONDS.toMillis(5);
@Inject @Inject
private IntegrationTestingConfig config; private IntegrationTestingConfig config;
@ -126,26 +129,31 @@ public class ITHighAvailabilityTest
{ {
ITRetryUtil.retryUntil( ITRetryUtil.retryUntil(
() -> { () -> {
List<DruidNodeDiscovery> disco = ImmutableList.of( try {
druidNodeDiscovery.getForNodeRole(NodeRole.COORDINATOR), List<DruidNodeDiscovery> disco = ImmutableList.of(
druidNodeDiscovery.getForNodeRole(NodeRole.OVERLORD), druidNodeDiscovery.getForNodeRole(NodeRole.COORDINATOR),
druidNodeDiscovery.getForNodeRole(NodeRole.HISTORICAL), druidNodeDiscovery.getForNodeRole(NodeRole.OVERLORD),
druidNodeDiscovery.getForNodeRole(NodeRole.MIDDLE_MANAGER), druidNodeDiscovery.getForNodeRole(NodeRole.HISTORICAL),
druidNodeDiscovery.getForNodeRole(NodeRole.INDEXER), druidNodeDiscovery.getForNodeRole(NodeRole.MIDDLE_MANAGER),
druidNodeDiscovery.getForNodeRole(NodeRole.BROKER), druidNodeDiscovery.getForNodeRole(NodeRole.INDEXER),
druidNodeDiscovery.getForNodeRole(NodeRole.ROUTER) druidNodeDiscovery.getForNodeRole(NodeRole.BROKER),
); druidNodeDiscovery.getForNodeRole(NodeRole.ROUTER)
);
int servicesDiscovered = 0; int servicesDiscovered = 0;
for (DruidNodeDiscovery nodeRole : disco) { for (DruidNodeDiscovery nodeRole : disco) {
Collection<DiscoveryDruidNode> nodes = nodeRole.getAllNodes(); Collection<DiscoveryDruidNode> nodes = nodeRole.getAllNodes();
servicesDiscovered += testSelfDiscovery(nodes); servicesDiscovered += testSelfDiscovery(nodes);
}
return servicesDiscovered > 5;
}
catch (Throwable t) {
return false;
} }
return servicesDiscovered > 5;
}, },
true, true,
TimeUnit.SECONDS.toMillis(5), RETRY_DELAY,
60, NUM_RETRIES,
"Standard services discovered" "Standard services discovered"
); );
} }
@ -155,14 +163,19 @@ public class ITHighAvailabilityTest
{ {
ITRetryUtil.retryUntil( ITRetryUtil.retryUntil(
() -> { () -> {
DruidNodeDiscovery customDisco = try {
druidNodeDiscovery.getForNodeRole(new NodeRole(CliCustomNodeRole.SERVICE_NAME)); DruidNodeDiscovery customDisco =
int count = testSelfDiscovery(customDisco.getAllNodes()); druidNodeDiscovery.getForNodeRole(new NodeRole(CliCustomNodeRole.SERVICE_NAME));
return count > 0; int count = testSelfDiscovery(customDisco.getAllNodes());
return count > 0;
}
catch (Throwable t) {
return false;
}
}, },
true, true,
TimeUnit.SECONDS.toMillis(5), RETRY_DELAY,
60, NUM_RETRIES,
"Custom service discovered" "Custom service discovered"
); );
} }