honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Testclsuters: convert plugins qa projects (#41496) 

Add testclusters support for files in keystore and convert qa subprojects within plugins.
This commit is contained in:
Alpar Torok 2019-04-26 18:34:03 +03:00 committed by Ryan Ernst
parent 06827a495f
commit 335f2bf102
12 changed files with 107 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
View File

@ -89,6 +89,7 @@ class PluginBuildPlugin extends BuildPlugin {
project.extensions.getByType(PluginPropertiesExtension).extendedPlugins.each { pluginName ->
// Auto add dependent modules to the test cluster
if (project.findProject(":modules:${pluginName}") != null) {
project.integTest.dependsOn(project.project(":modules:${pluginName}").tasks.bundlePlugin)
project.testClusters.integTest.module(
project.file(project.project(":modules:${pluginName}").tasks.bundlePlugin.archiveFile)
)

1
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
View File

@ -120,7 +120,6 @@ class RestIntegTestTask extends DefaultTask {
if (usesTestclusters == true) {
ElasticsearchCluster cluster = project.testClusters."${name}"
nonInputProperties.systemProperty('tests.rest.cluster', "${-> cluster.allHttpSocketURI.join(",") }")
nonInputProperties.systemProperty('tests.config.dir', "${-> cluster.singleNode().getConfigDir() }")
nonInputProperties.systemProperty('tests.cluster', "${-> cluster.transportPortURI }")
} else {
// we pass all nodes to the rest cluster to allow the clients to round-robin between them

7
buildSrc/src/main/java/org/elasticsearch/gradle/FileSupplier.java Normal file
View File

@ -0,0 +1,7 @@
package org.elasticsearch.gradle;
import java.io.File;
import java.util.function.Supplier;
public interface FileSupplier extends Supplier<File> {
}

11
buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java
View File

@ -20,6 +20,7 @@ package org.elasticsearch.gradle.testclusters;
import org.elasticsearch.GradleServicesAdapter;
import org.elasticsearch.gradle.Distribution;
import org.elasticsearch.gradle.FileSupplier;
import org.elasticsearch.gradle.Version;
import org.gradle.api.NamedDomainObjectContainer;
import org.gradle.api.Project;
@ -143,6 +144,16 @@ public class ElasticsearchCluster implements TestClusterConfiguration {
nodes.all(each -> each.keystore(key, valueSupplier));
}
@Override
public void keystore(String key, File value) {
nodes.all(each -> each.keystore(key, value));
}
@Override
public void keystore(String key, FileSupplier valueSupplier) {
nodes.all(each -> each.keystore(key, valueSupplier));
}
@Override
public void setting(String key, String value) {
nodes.all(each -> each.setting(key, value));

38
buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
View File

@ -20,6 +20,7 @@ package org.elasticsearch.gradle.testclusters;
import org.elasticsearch.GradleServicesAdapter;
import org.elasticsearch.gradle.Distribution;
import org.elasticsearch.gradle.FileSupplier;
import org.elasticsearch.gradle.OS;
import org.elasticsearch.gradle.Version;
import org.gradle.api.logging.Logger;
@ -63,7 +64,8 @@ public class ElasticsearchNode implements TestClusterConfiguration {
private static final int NODE_UP_TIMEOUT = 60;
private static final TimeUnit NODE_UP_TIMEOUT_UNIT = TimeUnit.SECONDS;
private static final List<String> OVERRIDABLE_SETTINGS = Arrays.asList(
"path.repo"
"path.repo",
"discovery.seed_providers"
);
private final String path;
@ -79,6 +81,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
private final List<File> modules = new ArrayList<>();
private final Map<String, Supplier<CharSequence>> settings = new LinkedHashMap<>();
private final Map<String, Supplier<CharSequence>> keystoreSettings = new LinkedHashMap<>();
private final Map<String, FileSupplier> keystoreFiles = new LinkedHashMap<>();
private final Map<String, Supplier<CharSequence>> systemProperties = new LinkedHashMap<>();
private final Map<String, Supplier<CharSequence>> environment = new LinkedHashMap<>();
private final Map<String, File> extraConfigFiles = new HashMap<>();
@ -171,6 +174,19 @@ public class ElasticsearchNode implements TestClusterConfiguration {
addSupplier("Keystore", keystoreSettings, key, valueSupplier);
}
@Override
public void keystore(String key, File value) {
requireNonNull(value, "keystore value was null when configuring test cluster`" + this + "`");
keystore(key, () -> value);
}
@Override
public void keystore(String key, FileSupplier valueSupplier) {
requireNonNull(key, "Keystore" + " key was null when configuring test cluster `" + this + "`");
requireNonNull(valueSupplier, "Keystore" + " value supplier was null when configuring test cluster `" + this + "`");
keystoreFiles.put(key, valueSupplier);
}
@Override
public void setting(String key, String value) {
addSupplier("Settings", settings, key, value);
@ -281,12 +297,22 @@ public class ElasticsearchNode implements TestClusterConfiguration {
"install", "--batch", plugin.toString())
);
if (keystoreSettings.isEmpty() == false) {
checkSuppliers("Keystore", keystoreSettings);
if (keystoreSettings.isEmpty() == false || keystoreFiles.isEmpty() == false) {
runElaticsearchBinScript("elasticsearch-keystore", "create");
keystoreSettings.forEach((key, value) -> {
runElaticsearchBinScriptWithInput(value.get().toString(), "elasticsearch-keystore", "add", "-x", key);
});
checkSuppliers("Keystore", keystoreSettings);
keystoreSettings.forEach((key, value) ->
runElaticsearchBinScriptWithInput(value.get().toString(), "elasticsearch-keystore", "add", "-x", key)
);
for (Map.Entry<String, FileSupplier> entry : keystoreFiles.entrySet()) {
File file = entry.getValue().get();
requireNonNull(file, "supplied keystoreFile was null when configuring " + this);
if (file.exists() == false) {
throw new TestClustersException("supplied keystore file " + file + " does not exist, require for " + this);
}
runElaticsearchBinScript("elasticsearch-keystore", "add-file", entry.getKey(), file.getAbsolutePath());
}
}
installModules();

5
buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.gradle.testclusters;
import org.elasticsearch.gradle.Distribution;
import org.elasticsearch.gradle.FileSupplier;
import org.gradle.api.logging.Logging;
import org.slf4j.Logger;
@ -47,6 +48,10 @@ public interface TestClusterConfiguration {
void keystore(String key, Supplier<CharSequence> valueSupplier);
void keystore(String key, File value);
void keystore(String key, FileSupplier valueSupplier);
void setting(String key, String value);
void setting(String key, Supplier<CharSequence> valueSupplier);

5
plugins/build.gradle
View File

@ -17,10 +17,13 @@
* under the License.
*/
subprojects {
apply plugin: 'elasticsearch.testclusters'
}
// only configure immediate children of plugins dir
configure(subprojects.findAll { it.parent.path == project.path }) {
group = 'org.elasticsearch.plugin'
apply plugin: 'elasticsearch.testclusters'
apply plugin: 'elasticsearch.esplugin'
esplugin {

37
plugins/discovery-ec2/qa/amazon-ec2/build.gradle
View File

@ -29,14 +29,13 @@ dependencies {
}
final int ec2NumberOfNodes = 3
File ec2DiscoveryFile = new File(project.buildDir, 'generated-resources/nodes.uri')
/** A task to start the AmazonEC2Fixture which emulates an EC2 service **/
task ec2Fixture(type: AntFixture) {
dependsOn compileTestJava
env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
executable = new File(project.runtimeJavaHome, 'bin/java')
args 'org.elasticsearch.discovery.ec2.AmazonEC2Fixture', baseDir, ec2DiscoveryFile.absolutePath
args 'org.elasticsearch.discovery.ec2.AmazonEC2Fixture', baseDir, "${buildDir}/testclusters/integTest-1/config/unicast_hosts.txt"
}
Map<String, Object> expansions = [
@ -48,28 +47,20 @@ processTestResources {
MavenFilteringHack.filter(it, expansions)
}
integTestCluster {
dependsOn ec2Fixture
numNodes = ec2NumberOfNodes
plugin ':plugins:discovery-ec2'
keystoreSetting 'discovery.ec2.access_key', 'ec2_integration_test_access_key'
keystoreSetting 'discovery.ec2.secret_key', 'ec2_integration_test_secret_key'
integTest {
dependsOn ec2Fixture, project(':plugins:discovery-ec2').bundlePlugin
}
testClusters.integTest {
numberOfNodes = ec2NumberOfNodes
plugin file(project(':plugins:discovery-ec2').bundlePlugin.archiveFile)
keystore 'discovery.ec2.access_key', 'ec2_integration_test_access_key'
keystore 'discovery.ec2.secret_key', 'ec2_integration_test_secret_key'
setting 'discovery.seed_providers', 'ec2'
setting 'network.host', '_ec2_'
setting 'discovery.ec2.endpoint', "http://${-> ec2Fixture.addressAndPort}"
systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", "http://${-> ec2Fixture.addressAndPort}"
setting 'discovery.ec2.endpoint', { "http://${ec2Fixture.addressAndPort}" }
unicastTransportUri = { seedNode, node, ant -> return null }
waitCondition = { node, ant ->
ec2DiscoveryFile.parentFile.mkdirs()
ec2DiscoveryFile.setText(integTest.nodes.collect { n -> "${n.transportUri()}" }.join('\n'), 'UTF-8')
File tmpFile = new File(node.cwd, 'wait.success')
ant.get(src: "http://${node.httpUri()}/",
dest: tmpFile.toString(),
ignoreerrors: true,
retries: 10)
return tmpFile.exists()
}
systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "http://${ec2Fixture.addressAndPort}" }
}

36
plugins/discovery-gce/qa/gce/build.gradle
View File

@ -24,7 +24,6 @@ apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
final int gceNumberOfNodes = 3
File gceDiscoveryFile = new File(project.buildDir, 'generated-resources/nodes.uri')
dependencies {
testCompile project(path: ':plugins:discovery-gce', configuration: 'runtime')
@ -35,7 +34,7 @@ task gceFixture(type: AntFixture) {
dependsOn compileTestJava
env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
executable = new File(project.runtimeJavaHome, 'bin/java')
args 'org.elasticsearch.cloud.gce.GCEFixture', baseDir, gceDiscoveryFile.getAbsolutePath()
args 'org.elasticsearch.cloud.gce.GCEFixture', baseDir, "${buildDir}/testclusters/integTest-1/config/unicast_hosts.txt"
}
Map<String, Object> expansions = [
@ -47,34 +46,21 @@ processTestResources {
MavenFilteringHack.filter(it, expansions)
}
integTestCluster {
dependsOn gceFixture
numNodes = gceNumberOfNodes
plugin ':plugins:discovery-gce'
setting 'discovery.seed_providers', 'gce'
integTest {
dependsOn gceFixture, project(':plugins:discovery-gce').bundlePlugin
}
testClusters.integTest {
numberOfNodes = gceNumberOfNodes
plugin file(project(':plugins:discovery-gce').bundlePlugin.archiveFile)
// use gce fixture for Auth calls instead of http://metadata.google.internal
integTestCluster.environment 'GCE_METADATA_HOST', "http://${-> gceFixture.addressAndPort}"
environment 'GCE_METADATA_HOST', { "http://${gceFixture.addressAndPort}" }
// allows to configure hidden settings (`cloud.gce.host` and `cloud.gce.root_url`)
systemProperty 'es.allow_reroute_gce_settings', 'true'
setting 'discovery.seed_providers', 'gce'
// use gce fixture for metadata server calls instead of http://metadata.google.internal
setting 'cloud.gce.host', "http://${-> gceFixture.addressAndPort}"
setting 'cloud.gce.host', { "http://${gceFixture.addressAndPort}" }
// use gce fixture for API calls instead of https://www.googleapis.com
setting 'cloud.gce.root_url', "http://${-> gceFixture.addressAndPort}"
unicastTransportUri = { seedNode, node, ant -> return null }
waitCondition = { node, ant ->
gceDiscoveryFile.parentFile.mkdirs()
gceDiscoveryFile.setText(integTest.nodes.collect { n -> "${n.transportUri()}" }.join('\n'), 'UTF-8')
File tmpFile = new File(node.cwd, 'wait.success')
ant.get(src: "http://${node.httpUri()}/",
dest: tmpFile.toString(),
ignoreerrors: true,
retries: 10)
return tmpFile.exists()
}
setting 'cloud.gce.root_url', { "http://${gceFixture.addressAndPort}" }
}

19
plugins/repository-azure/qa/microsoft-azure-storage/build.gradle
View File

@ -23,10 +23,6 @@ import org.elasticsearch.gradle.test.AntFixture
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
integTestCluster {
plugin ':plugins:repository-azure'
}
boolean useFixture = false
String azureAccount = System.getenv("azure_storage_account")
@ -60,16 +56,21 @@ processTestResources {
MavenFilteringHack.filter(it, expansions)
}
integTestCluster {
keystoreSetting 'azure.client.integration_test.account', azureAccount
keystoreSetting 'azure.client.integration_test.key', azureKey
integTest {
dependsOn project(':plugins:repository-azure').bundlePlugin
}
testClusters.integTest {
plugin file(project(':plugins:repository-azure').bundlePlugin.archiveFile)
keystore 'azure.client.integration_test.account', azureAccount
keystore 'azure.client.integration_test.key', azureKey
if (useFixture) {
dependsOn azureStorageFixture
tasks.integTest.dependsOn azureStorageFixture
// Use a closure on the string to delay evaluation until tests are executed. The endpoint_suffix is used
// in a hacky way to change the protocol and endpoint. We must fix that.
setting 'azure.client.integration_test.endpoint_suffix',
"ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://${ -> azureStorageFixture.addressAndPort }"
{ "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://${azureStorageFixture.addressAndPort }" }
} else {
println "Using an external service to test the repository-azure plugin"
}

20
plugins/repository-gcs/qa/google-cloud-storage/build.gradle
View File

@ -26,10 +26,6 @@ import java.security.KeyPairGenerator
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
integTestCluster {
plugin ':plugins:repository-gcs'
}
boolean useFixture = false
String gcsServiceAccount = System.getenv("google_storage_service_account")
@ -87,14 +83,20 @@ processTestResources {
MavenFilteringHack.filter(it, expansions)
}
integTestCluster {
keystoreFile 'gcs.client.integration_test.credentials_file', "${serviceAccountFile.absolutePath}"
integTest {
dependsOn project(':plugins:repository-gcs').bundlePlugin
}
testClusters.integTest {
plugin file(project(':plugins:repository-gcs').bundlePlugin.archiveFile)
keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile
if (useFixture) {
dependsOn createServiceAccountFile, googleCloudStorageFixture
tasks.integTest.dependsOn createServiceAccountFile, googleCloudStorageFixture
/* Use a closure on the string to delay evaluation until tests are executed */
setting 'gcs.client.integration_test.endpoint', "http://${ -> googleCloudStorageFixture.addressAndPort }"
setting 'gcs.client.integration_test.token_uri', "http://${ -> googleCloudStorageFixture.addressAndPort }/o/oauth2/token"
setting 'gcs.client.integration_test.endpoint', { "http://${googleCloudStorageFixture.addressAndPort}" }
setting 'gcs.client.integration_test.token_uri', { "http://${googleCloudStorageFixture.addressAndPort}/o/oauth2/token" }
} else {
println "Using an external service to test the repository-gcs plugin"
}

1
x-pack/qa/security-migrate-tests/build.gradle
View File

@ -30,6 +30,7 @@ integTestCluster {
retries: 10)
return tmpFile.exists()
}
// TODO: systemProperty('tests.cluster', "${-> cluster.transportPortURI }") when migerating to testclusters
}
testingConventions {