Ryan Ernst 8c01d6ea69 Tests: Add cluster health check to xpack integ wait conditions (elastic/x-pack-elasticsearch#740)
The wait condition used for integ tests by default calls the cluster
health api with wait_for_nodes nd wait_for_status. However, xpack
overrides the wait condition to add auth, but most of these conditions
still looked at the root ES url, which means the tests are susceptible
to race conditions with the check and node startup. This change modifies
the url for the authenticated wait condtion to check the health api,
with the appropriate wait_for_nodes and wait_for_status.

Original commit: elastic/x-pack-elasticsearch@0b23ef528f
2017-03-15 10:23:26 -07:00

163 lines
6.1 KiB
Groovy

import org.elasticsearch.gradle.test.NodeInfo
import org.elasticsearch.gradle.test.RestIntegTestTask
import java.nio.charset.StandardCharsets
apply plugin: 'elasticsearch.standalone-test'
Closure waitWithAuth = { NodeInfo node, AntBuilder ant ->
File tmpFile = new File(node.cwd, 'wait.success')
// wait up to twenty seconds
final long stopTime = System.currentTimeMillis() + 20000L;
Exception lastException = null;
while (System.currentTimeMillis() < stopTime) {
lastException = null;
// we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned
HttpURLConnection httpURLConnection = null;
try {
// TODO this sucks having to hardcode number of nodes, but node.config.numNodes isn't necessarily accurate for rolling
httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health?wait_for_nodes=2&wait_for_status=yellow").openConnection();
httpURLConnection.setRequestProperty("Authorization", "Basic " +
Base64.getEncoder().encodeToString("elastic:changeme".getBytes(StandardCharsets.UTF_8)));
httpURLConnection.setRequestMethod("GET");
httpURLConnection.setConnectTimeout(1000);
httpURLConnection.setReadTimeout(30000); // read needs to wait for nodes!
httpURLConnection.connect();
if (httpURLConnection.getResponseCode() == 200) {
tmpFile.withWriter StandardCharsets.UTF_8.name(), {
it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name()))
}
break;
}
} catch (Exception e) {
logger.debug("failed to call cluster health", e)
lastException = e
} finally {
if (httpURLConnection != null) {
httpURLConnection.disconnect();
}
}
// did not start, so wait a bit before trying again
Thread.sleep(500L);
}
if (tmpFile.exists() == false && lastException != null) {
logger.error("final attempt of calling cluster health failed", lastException)
}
return tmpFile.exists()
}
String outputDir = "generated-resources/${project.name}"
task oldClusterTest(type: RestIntegTestTask) {
mustRunAfter(precommit)
}
oldClusterTestCluster {
plugin ':x-pack-elasticsearch:plugin'
distribution = 'zip'
bwcVersion = '5.4.0-SNAPSHOT' // TODO: either randomize, or make this settable with sysprop
numBwcNodes = 2
numNodes = 2
clusterName = 'rolling-upgrade'
waitCondition = waitWithAuth
setting 'logger.org.elasticsearch.xpack.security', 'TRACE'
setting 'xpack.security.transport.ssl.enabled', 'true'
setting 'xpack.ssl.keystore.path', 'testnode.jks'
setting 'xpack.ssl.keystore.password', 'testnode'
extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
}
oldClusterTestRunner {
systemProperty 'tests.rest.suite', 'old_cluster'
}
task mixedClusterTest(type: RestIntegTestTask) {
dependsOn(oldClusterTestRunner, 'oldClusterTestCluster#node1.stop')
}
mixedClusterTestCluster {
plugin ':x-pack-elasticsearch:plugin'
distribution = 'zip'
clusterName = 'rolling-upgrade'
unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() }
dataDir = "${-> oldClusterTest.nodes[1].dataDir}"
waitCondition = waitWithAuth
setting 'xpack.ssl.keystore.path', 'testnode.jks'
setting 'xpack.ssl.keystore.password', 'testnode'
extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
}
mixedClusterTestRunner {
systemProperty 'tests.rest.suite', 'mixed_cluster'
finalizedBy 'oldClusterTestCluster#node0.stop'
}
task upgradedClusterTest(type: RestIntegTestTask) {
dependsOn(mixedClusterTestRunner, 'oldClusterTestCluster#node0.stop')
}
upgradedClusterTestCluster {
plugin ':x-pack-elasticsearch:plugin'
distribution = 'zip'
clusterName = 'rolling-upgrade'
unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() }
dataDir = "${-> oldClusterTest.nodes[0].dataDir}"
waitCondition = waitWithAuth
setting 'xpack.ssl.keystore.path', 'testnode.jks'
setting 'xpack.ssl.keystore.password', 'testnode'
extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
}
upgradedClusterTestRunner {
systemProperty 'tests.rest.suite', 'upgraded_cluster'
// only need to kill the mixed cluster tests node here because we explicitly told it to not stop nodes upon completion
finalizedBy 'mixedClusterTestCluster#stop'
}
task integTest {
dependsOn = [upgradedClusterTest]
}
test.enabled = false // no unit tests for rolling upgrades, only the rest integration test
check.dependsOn(integTest)
dependencies {
testCompile project(path: ':x-pack-elasticsearch:plugin', configuration: 'runtime')
}
// copy x-pack plugin info so it is on the classpath and security manager has the right permissions
task copyXPackRestSpec(type: Copy) {
dependsOn(project.configurations.restSpec, 'processTestResources')
from project(':x-pack-elasticsearch:plugin').sourceSets.test.resources
include 'rest-api-spec/api/**'
into project.sourceSets.test.output.resourcesDir
}
task copyTestNodeKeystore(type: Copy) {
dependsOn(copyXPackRestSpec)
from project(':x-pack-elasticsearch:plugin')
.file('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
into outputDir
}
task copyXPackPluginProps(type: Copy) {
dependsOn(copyTestNodeKeystore)
from project(':x-pack-elasticsearch:plugin').file('src/main/plugin-metadata')
from project(':x-pack-elasticsearch:plugin').tasks.pluginProperties
into outputDir
}
project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackPluginProps)
repositories {
maven {
url "https://oss.sonatype.org/content/repositories/snapshots/"
}
maven {
url "https://artifacts.elastic.co/maven"
}
maven {
url "https://snapshots.elastic.co/maven"
}
}