honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-18 19:05:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove with(out)-system-key tests (#40547) (#40603)

Browse Source 
This change removes the variants of the rolling upgrade and full
cluster restart tests that use or do not use a system key. These tests
were added during 5.x when the system key was still used for security
and now the system key is only used as the watcher encryption key so
duplicating rolling upgrade and full cluster restarts is not needed.

The change here removes the subprojects for testing these scenarios and
defaults to always run with the watcher sensitive values encrypted for
these tests.
This commit is contained in:
Jay Modi 2019-04-01 14:23:24 -06:00 committed by GitHub
parent e88b862192
commit 12bf3b2025
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 255 additions and 399 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

271
x-pack/qa/full-cluster-restart/build.gradle
View File

@ -3,10 +3,10 @@ import org.elasticsearch.gradle.test.RestIntegTestTask
import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.Version
import java.nio.charset.StandardCharsets import java.nio.charset.StandardCharsets
import java.util.regex.Matcher
// Apply the java plugin to this project so the sources can be edited in an IDE // Apply the java plugin to this project so the sources can be edited in an IDE
apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.standalone-test'
unitTest.enabled = false unitTest.enabled = false
dependencies { dependencies {
@ -70,8 +70,6 @@ Closure waitWithAuth = { NodeInfo node, AntBuilder ant ->
return tmpFile.exists() return tmpFile.exists()
} }
Project mainProject = project
String coreFullClusterRestartPath = project(':qa:full-cluster-restart').projectDir.toPath().resolve('src/test/java').toString() String coreFullClusterRestartPath = project(':qa:full-cluster-restart').projectDir.toPath().resolve('src/test/java').toString()
sourceSets { sourceSets {
test { test {
@ -89,224 +87,157 @@ forbiddenPatterns {
exclude '**/system_key' exclude '**/system_key'
} }
// tests are pushed down to subprojects String outputDir = "${buildDir}/generated-resources/${project.name}"
testingConventions.enabled = false
/** // This is a top level task which we will add dependencies to below.
* Subdirectories of this project are test rolling upgrades with various // It is a single task that can be used to backcompat tests against all versions.
* configuration options based on their name. task bwcTest {
*/
subprojects {
Matcher m = project.name =~ /with(out)?-system-key/
if (false == m.matches()) {
throw new InvalidUserDataException("Invalid project name [${project.name}]")
}
boolean withSystemKey = m.group(1) == null
apply plugin: 'elasticsearch.standalone-test'
// Use resources from the rolling-upgrade project in subdirectories
sourceSets {
test {
java {
srcDirs = ["${mainProject.projectDir}/src/test/java", coreFullClusterRestartPath]
}
resources {
srcDirs = ["${mainProject.projectDir}/src/test/resources"]
}
}
}
licenseHeaders {
approvedLicenses << 'Apache'
}
forbiddenPatterns {
exclude '**/system_key'
}
String outputDir = "${buildDir}/generated-resources/${project.name}"
// This is a top level task which we will add dependencies to below.
// It is a single task that can be used to backcompat tests against all versions.
task bwcTest {
description = 'Runs backwards compatibility tests.' description = 'Runs backwards compatibility tests.'
group = 'verification' group = 'verification'
} }
String output = "${buildDir}/generated-resources/${project.name}" task copyTestNodeKeyMaterial(type: Copy) {
task copyTestNodeKeyMaterial(type: Copy) {
from project(':x-pack:plugin:core').files('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem', from project(':x-pack:plugin:core').files('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem',
'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt', 'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt',
'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks') 'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
into outputDir into outputDir
} }
for (Version version : bwcVersions.indexCompatible) { for (Version version : bwcVersions.indexCompatible) {
String baseName = "v${version}" String baseName = "v${version}"
Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) { Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) {
mustRunAfter(precommit) mustRunAfter(precommit)
} }
Object extension = extensions.findByName("${baseName}#oldClusterTestCluster") Object extension = extensions.findByName("${baseName}#oldClusterTestCluster")
configure(extensions.findByName("${baseName}#oldClusterTestCluster")) { configure(extensions.findByName("${baseName}#oldClusterTestCluster")) {
dependsOn copyTestNodeKeyMaterial dependsOn copyTestNodeKeyMaterial
if (version.before('6.3.0')) { if (version.before('6.3.0')) {
String depVersion = version; String depVersion = version;
if (project.bwcVersions.unreleased.contains(version)) { if (project.bwcVersions.unreleased.contains(version)) {
depVersion += "-SNAPSHOT" depVersion += "-SNAPSHOT"
} }
mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${depVersion}" mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${depVersion}"
}
bwcVersion = version
numBwcNodes = 2
numNodes = 2
clusterName = 'full-cluster-restart'
String usersCli = version.before('6.3.0') ? 'bin/x-pack/users' : 'bin/elasticsearch-users'
setupCommand 'setupTestUser', usersCli, 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
waitCondition = waitWithAuth
// some tests rely on the translog not being flushed
setting 'indices.memory.shard_inactive_time', '20m'
setting 'xpack.security.enabled', 'true'
setting 'xpack.security.transport.ssl.enabled', 'true'
if (project.inFipsJvm) {
setting 'xpack.security.transport.ssl.key', 'testnode.pem'
setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
} else {
setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks'
setting 'xpack.security.transport.ssl.keystore.password', 'testnode'
}
setting 'xpack.license.self_generated.type', 'trial'
dependsOn copyTestNodeKeyMaterial
extraConfigFile 'testnode.pem', new File(outputDir + '/testnode.pem')
extraConfigFile 'testnode.crt', new File(outputDir + '/testnode.crt')
extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
if (withSystemKey) {
if (version.onOrAfter('5.1.0') && version.before('6.0.0')) {
// The setting didn't exist until 5.1.0
setting 'xpack.security.system_key.required', 'true'
} }
if (version.onOrAfter('6.0.0')) { bwcVersion = version
keystoreFile 'xpack.watcher.encryption_key', "${mainProject.projectDir}/src/test/resources/system_key" numBwcNodes = 2
numNodes = 2
clusterName = 'full-cluster-restart'
String usersCli = version.before('6.3.0') ? 'bin/x-pack/users' : 'bin/elasticsearch-users'
setupCommand 'setupTestUser', usersCli, 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
waitCondition = waitWithAuth
// some tests rely on the translog not being flushed
setting 'indices.memory.shard_inactive_time', '20m'
setting 'xpack.security.enabled', 'true'
setting 'xpack.security.transport.ssl.enabled', 'true'
if (project.inFipsJvm) {
setting 'xpack.security.transport.ssl.key', 'testnode.pem'
setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
} else { } else {
extraConfigFile 'x-pack/system_key', "${mainProject.projectDir}/src/test/resources/system_key" setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks'
setting 'xpack.security.transport.ssl.keystore.password', 'testnode'
} }
setting 'xpack.license.self_generated.type', 'trial'
dependsOn copyTestNodeKeyMaterial
extraConfigFile 'testnode.pem', new File(outputDir + '/testnode.pem')
extraConfigFile 'testnode.crt', new File(outputDir + '/testnode.crt')
extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
keystoreFile 'xpack.watcher.encryption_key', "${project.projectDir}/src/test/resources/system_key"
setting 'xpack.watcher.encrypt_sensitive_data', 'true' setting 'xpack.watcher.encrypt_sensitive_data', 'true'
}
} }
Task oldClusterTestRunner = tasks.getByName("${baseName}#oldClusterTestRunner") Task oldClusterTestRunner = tasks.getByName("${baseName}#oldClusterTestRunner")
oldClusterTestRunner.configure { oldClusterTestRunner.configure {
systemProperty 'tests.is_old_cluster', 'true' systemProperty 'tests.is_old_cluster', 'true'
systemProperty 'tests.old_cluster_version', version.toString().minus("-SNAPSHOT") systemProperty 'tests.old_cluster_version', version.toString().minus("-SNAPSHOT")
systemProperty 'tests.path.repo', new File(buildDir, "cluster/shared/repo") systemProperty 'tests.path.repo', new File(buildDir, "cluster/shared/repo")
exclude 'org/elasticsearch/upgrades/FullClusterRestartIT.class' exclude 'org/elasticsearch/upgrades/FullClusterRestartIT.class'
exclude 'org/elasticsearch/upgrades/FullClusterRestartSettingsUpgradeIT.class' exclude 'org/elasticsearch/upgrades/FullClusterRestartSettingsUpgradeIT.class'
exclude 'org/elasticsearch/upgrades/QueryBuilderBWCIT.class' exclude 'org/elasticsearch/upgrades/QueryBuilderBWCIT.class'
} }
Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask) Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask)
configure(extensions.findByName("${baseName}#upgradedClusterTestCluster")) { configure(extensions.findByName("${baseName}#upgradedClusterTestCluster")) {
dependsOn oldClusterTestRunner, dependsOn oldClusterTestRunner,
"${baseName}#oldClusterTestCluster#node0.stop", "${baseName}#oldClusterTestCluster#node0.stop",
"${baseName}#oldClusterTestCluster#node1.stop" "${baseName}#oldClusterTestCluster#node1.stop"
numNodes = 2 numNodes = 2
clusterName = 'full-cluster-restart' clusterName = 'full-cluster-restart'
dataDir = { nodeNum -> oldClusterTest.nodes[nodeNum].dataDir } dataDir = { nodeNum -> oldClusterTest.nodes[nodeNum].dataDir }
cleanShared = false // We want to keep snapshots made by the old cluster! cleanShared = false // We want to keep snapshots made by the old cluster!
setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser' setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
waitCondition = waitWithAuth waitCondition = waitWithAuth
// some tests rely on the translog not being flushed // some tests rely on the translog not being flushed
setting 'indices.memory.shard_inactive_time', '20m' setting 'indices.memory.shard_inactive_time', '20m'
setting 'xpack.security.enabled', 'true' setting 'xpack.security.enabled', 'true'
if (project.inFipsJvm) { if (project.inFipsJvm) {
setting 'xpack.security.transport.ssl.key', 'testnode.pem' setting 'xpack.security.transport.ssl.key', 'testnode.pem'
setting 'xpack.security.transport.ssl.certificate', 'testnode.crt' setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode' keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
} else { } else {
setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks' setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks'
setting 'xpack.security.transport.ssl.keystore.password', 'testnode' setting 'xpack.security.transport.ssl.keystore.password', 'testnode'
} }
setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.license.self_generated.type', 'trial'
dependsOn copyTestNodeKeyMaterial dependsOn copyTestNodeKeyMaterial
extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks') extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
extraConfigFile 'testnode.pem', new File(outputDir + '/testnode.pem') extraConfigFile 'testnode.pem', new File(outputDir + '/testnode.pem')
extraConfigFile 'testnode.crt', new File(outputDir + '/testnode.crt') extraConfigFile 'testnode.crt', new File(outputDir + '/testnode.crt')
if (withSystemKey) {
setting 'xpack.watcher.encrypt_sensitive_data', 'true' setting 'xpack.watcher.encrypt_sensitive_data', 'true'
keystoreFile 'xpack.watcher.encryption_key', "${mainProject.projectDir}/src/test/resources/system_key" keystoreFile 'xpack.watcher.encryption_key', "${project.projectDir}/src/test/resources/system_key"
}
} }
Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner") Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner")
upgradedClusterTestRunner.configure { upgradedClusterTestRunner.configure {
systemProperty 'tests.is_old_cluster', 'false' systemProperty 'tests.is_old_cluster', 'false'
systemProperty 'tests.old_cluster_version', version.toString().minus("-SNAPSHOT") systemProperty 'tests.old_cluster_version', version.toString().minus("-SNAPSHOT")
systemProperty 'tests.path.repo', new File(buildDir, "cluster/shared/repo") systemProperty 'tests.path.repo', new File(buildDir, "cluster/shared/repo")
exclude 'org/elasticsearch/upgrades/FullClusterRestartIT.class' exclude 'org/elasticsearch/upgrades/FullClusterRestartIT.class'
exclude 'org/elasticsearch/upgrades/FullClusterRestartSettingsUpgradeIT.class' exclude 'org/elasticsearch/upgrades/FullClusterRestartSettingsUpgradeIT.class'
exclude 'org/elasticsearch/upgrades/QueryBuilderBWCIT.class' exclude 'org/elasticsearch/upgrades/QueryBuilderBWCIT.class'
} }
Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") { Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") {
dependsOn = [upgradedClusterTest] dependsOn = [upgradedClusterTest]
} }
if (project.bwc_tests_enabled) { if (project.bwc_tests_enabled) {
bwcTest.dependsOn(versionBwcTest) bwcTest.dependsOn(versionBwcTest)
} }
} }
unitTest.enabled = false // no unit tests for full cluster restarts, only the rest integration test
// basic integ tests includes testing bwc against the most recent version // basic integ tests includes testing bwc against the most recent version
task bwcTestSnapshots { task bwcTestSnapshots {
if (project.bwc_tests_enabled) { if (project.bwc_tests_enabled) {
for (final def version : bwcVersions.unreleasedIndexCompatible) { for (final def version : bwcVersions.unreleasedIndexCompatible) {
dependsOn "v${version}#bwcTest" dependsOn "v${version}#bwcTest"
} }
} }
} }
check.dependsOn(bwcTestSnapshots) check.dependsOn(bwcTestSnapshots)
dependencies { // copy x-pack plugin info so it is on the classpath and security manager has the right permissions
// "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here task copyXPackRestSpec(type: Copy) {
testCompile project(path: xpackModule('core'), configuration: 'default')
testCompile project(path: xpackModule('watcher'), configuration: 'runtime')
testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
}
// copy x-pack plugin info so it is on the classpath and security manager has the right permissions
task copyXPackRestSpec(type: Copy) {
dependsOn(project.configurations.restSpec, 'processTestResources') dependsOn(project.configurations.restSpec, 'processTestResources')
from project(xpackModule('core')).sourceSets.test.resources from project(xpackModule('core')).sourceSets.test.resources
include 'rest-api-spec/api/**' include 'rest-api-spec/api/**'
into project.sourceSets.test.output.resourcesDir into project.sourceSets.test.output.resourcesDir
} }
task copyXPackPluginProps(type: Copy) { task copyXPackPluginProps(type: Copy) {
dependsOn(copyXPackRestSpec) dependsOn(copyXPackRestSpec)
from project(xpackModule('core')).file('src/main/plugin-metadata') from project(xpackModule('core')).file('src/main/plugin-metadata')
from project(xpackModule('core')).tasks.pluginProperties from project(xpackModule('core')).tasks.pluginProperties
into outputDir into outputDir
}
project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackPluginProps)
repositories {
maven {
url "https://artifacts.elastic.co/maven"
}
maven {
url "https://snapshots.elastic.co/maven"
}
}
} }
project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackPluginProps)

0
x-pack/qa/full-cluster-restart/with-system-key/build.gradle
View File

0
x-pack/qa/full-cluster-restart/without-system-key/build.gradle
View File

381
x-pack/qa/rolling-upgrade/build.gradle
View File

@ -3,10 +3,10 @@ import org.elasticsearch.gradle.test.RestIntegTestTask
import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.Version
import java.nio.charset.StandardCharsets import java.nio.charset.StandardCharsets
import java.util.regex.Matcher
// Apply the java plugin to this project so the sources can be edited in an IDE // Apply the java plugin to this project so the sources can be edited in an IDE
apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.standalone-test'
unitTest.enabled = false unitTest.enabled = false
dependencies { dependencies {
@ -68,163 +68,50 @@ Closure waitWithAuth = { NodeInfo node, AntBuilder ant ->
return tmpFile.exists() return tmpFile.exists()
} }
Project mainProject = project
compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked" compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
forbiddenPatterns { forbiddenPatterns {
exclude '**/system_key' exclude '**/system_key'
} }
// Tests are pushed down to subprojects String outputDir = "${buildDir}/generated-resources/${project.name}"
testingConventions.enabled = false
/** // This is a top level task which we will add dependencies to below.
* Subdirectories of this project are test rolling upgrades with various // It is a single task that can be used to backcompat tests against all versions.
* configuration options based on their name. task bwcTest {
*/
subprojects {
Matcher m = project.name =~ /with(out)?-system-key/
if (false == m.matches()) {
throw new InvalidUserDataException("Invalid project name [${project.name}]")
}
boolean withSystemKey = m.group(1) == null
apply plugin: 'elasticsearch.standalone-test'
// Use resources from the rolling-upgrade project in subdirectories
sourceSets {
test {
java {
srcDirs = ["${mainProject.projectDir}/src/test/java"]
}
resources {
srcDirs = ["${mainProject.projectDir}/src/test/resources"]
}
}
}
forbiddenPatterns {
exclude '**/system_key'
}
String outputDir = "${buildDir}/generated-resources/${project.name}"
// This is a top level task which we will add dependencies to below.
// It is a single task that can be used to backcompat tests against all versions.
task bwcTest {
description = 'Runs backwards compatibility tests.' description = 'Runs backwards compatibility tests.'
group = 'verification' group = 'verification'
} }
String output = "${buildDir}/generated-resources/${project.name}" task copyTestNodeKeyMaterial(type: Copy) {
task copyTestNodeKeyMaterial(type: Copy) {
from project(':x-pack:plugin:core').files('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem', from project(':x-pack:plugin:core').files('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem',
'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt', 'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt',
'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks') 'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
into outputDir into outputDir
} }
for (Version version : bwcVersions.wireCompatible) { for (Version version : bwcVersions.wireCompatible) {
String baseName = "v${version}" String baseName = "v${version}"
Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) { Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) {
mustRunAfter(precommit) mustRunAfter(precommit)
} }
configure(extensions.findByName("${baseName}#oldClusterTestCluster")) { configure(extensions.findByName("${baseName}#oldClusterTestCluster")) {
dependsOn copyTestNodeKeyMaterial dependsOn copyTestNodeKeyMaterial
if (version.before('6.3.0')) { if (version.before('6.3.0')) {
String depVersion = version; String depVersion = version;
if (project.bwcVersions.unreleased.contains(version)) { if (project.bwcVersions.unreleased.contains(version)) {
depVersion += "-SNAPSHOT" depVersion += "-SNAPSHOT"
} }
mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${depVersion}" mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${depVersion}"
}
String usersCli = version.before('6.3.0') ? 'bin/x-pack/users' : 'bin/elasticsearch-users'
setupCommand 'setupTestUser', usersCli, 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
bwcVersion = version
numBwcNodes = 3
numNodes = 3
clusterName = 'rolling-upgrade'
waitCondition = waitWithAuth
setting 'xpack.monitoring.exporters._http.type', 'http'
setting 'xpack.monitoring.exporters._http.enabled', 'false'
setting 'xpack.monitoring.exporters._http.auth.username', 'test_user'
setting 'xpack.monitoring.exporters._http.auth.password', 'x-pack-test-password'
setting 'xpack.license.self_generated.type', 'trial'
setting 'xpack.security.enabled', 'true'
setting 'xpack.security.transport.ssl.enabled', 'true'
setting 'xpack.security.authc.token.enabled', 'true'
setting 'xpack.security.audit.enabled', 'true'
if (project.inFipsJvm) {
setting 'xpack.security.transport.ssl.key', 'testnode.pem'
setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
} else {
setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks'
setting 'xpack.security.transport.ssl.keystore.password', 'testnode'
}
dependsOn copyTestNodeKeyMaterial
extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
extraConfigFile 'testnode.pem', new File(outputDir + '/testnode.pem')
extraConfigFile 'testnode.crt', new File(outputDir + '/testnode.crt')
if (version.onOrAfter('7.0.0')) {
setting 'xpack.security.authc.realms.file.file1.order', '0'
setting 'xpack.security.authc.realms.native.native1.order', '1'
} else {
setting 'xpack.security.authc.realms.file1.type', 'file'
setting 'xpack.security.authc.realms.file1.order', '0'
setting 'xpack.security.authc.realms.native1.type', 'native'
setting 'xpack.security.authc.realms.native1.order', '1'
}
if (withSystemKey) {
if (version.onOrAfter('5.1.0') && version.before('6.0.0')) {
// The setting didn't exist until 5.1.0
setting 'xpack.security.system_key.required', 'true'
} }
if (version.onOrAfter('6.0.0')) { String usersCli = version.before('6.3.0') ? 'bin/x-pack/users' : 'bin/elasticsearch-users'
keystoreFile 'xpack.watcher.encryption_key', "${mainProject.projectDir}/src/test/resources/system_key" setupCommand 'setupTestUser', usersCli, 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
} else { bwcVersion = version
String systemKeyFile = version.before('6.3.0') ? 'x-pack/system_key' : 'system_key' numBwcNodes = 3
extraConfigFile systemKeyFile, "${mainProject.projectDir}/src/test/resources/system_key" numNodes = 3
keystoreSetting 'xpack.security.authc.token.passphrase', 'token passphrase'
}
setting 'xpack.watcher.encrypt_sensitive_data', 'true'
}
if (version.onOrAfter('6.6.0')) {
setting 'ccr.auto_follow.wait_for_metadata_timeout', '1s'
}
// Old versions of the code contain an invalid assertion that trips
// during tests. Versions 5.6.9 and 6.2.4 have been fixed by removing
// the assertion, but this is impossible for released versions.
// However, released versions run without assertions, so end users won't
// be suffering the effects. This argument effectively removes the
// incorrect assertion from the older versions used in the BWC tests.
if (version.before('5.6.9') || (version.onOrAfter('6.0.0') && version.before('6.2.4'))) {
jvmArgs '-da:org.elasticsearch.xpack.monitoring.exporter.http.HttpExportBulk'
}
}
Task oldClusterTestRunner = tasks.getByName("${baseName}#oldClusterTestRunner")
oldClusterTestRunner.configure {
systemProperty 'tests.rest.suite', 'old_cluster'
}
Closure configureUpgradeCluster = {String name, Task lastRunner, int stopNode, Closure getOtherUnicastHostAddresses ->
configure(extensions.findByName("${baseName}#${name}")) {
dependsOn lastRunner, "${baseName}#oldClusterTestCluster#node${stopNode}.stop"
setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
clusterName = 'rolling-upgrade' clusterName = 'rolling-upgrade'
otherUnicastHostAddresses = { getOtherUnicastHostAddresses() }
minimumMasterNodes = { 2 }
autoSetInitialMasterNodes = false
/* Override the data directory so the new node always gets the node we
* just stopped's data directory. */
dataDir = { nodeNumber -> oldClusterTest.nodes[stopNode].dataDir }
waitCondition = waitWithAuth waitCondition = waitWithAuth
setting 'xpack.monitoring.exporters._http.type', 'http' setting 'xpack.monitoring.exporters._http.type', 'http'
setting 'xpack.monitoring.exporters._http.enabled', 'false' setting 'xpack.monitoring.exporters._http.enabled', 'false'
@ -233,147 +120,187 @@ subprojects {
setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.license.self_generated.type', 'trial'
setting 'xpack.security.enabled', 'true' setting 'xpack.security.enabled', 'true'
setting 'xpack.security.transport.ssl.enabled', 'true' setting 'xpack.security.transport.ssl.enabled', 'true'
if (project.inFipsJvm) {
setting 'xpack.security.transport.ssl.key', 'testnode.pem'
setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
} else {
setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks'
setting 'xpack.security.transport.ssl.keystore.password', 'testnode'
}
setting 'node.attr.upgraded', 'true'
setting 'xpack.security.authc.token.enabled', 'true' setting 'xpack.security.authc.token.enabled', 'true'
setting 'xpack.security.audit.enabled', 'true' setting 'xpack.security.audit.enabled', 'true'
setting 'node.name', "upgraded-node-${stopNode}" if (project.inFipsJvm) {
setting 'xpack.security.transport.ssl.key', 'testnode.pem'
setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
} else {
setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks'
setting 'xpack.security.transport.ssl.keystore.password', 'testnode'
}
dependsOn copyTestNodeKeyMaterial dependsOn copyTestNodeKeyMaterial
extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks') extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
extraConfigFile 'testnode.pem', new File(outputDir + '/testnode.pem') extraConfigFile 'testnode.pem', new File(outputDir + '/testnode.pem')
extraConfigFile 'testnode.crt', new File(outputDir + '/testnode.crt') extraConfigFile 'testnode.crt', new File(outputDir + '/testnode.crt')
setting 'xpack.security.authc.realms.file.file1.order', '0' if (version.onOrAfter('7.0.0')) {
setting 'xpack.security.authc.realms.native.native1.order', '1' setting 'xpack.security.authc.realms.file.file1.order', '0'
if (withSystemKey) { setting 'xpack.security.authc.realms.native.native1.order', '1'
setting 'xpack.watcher.encrypt_sensitive_data', 'true' } else {
keystoreFile 'xpack.watcher.encryption_key', "${mainProject.projectDir}/src/test/resources/system_key" setting 'xpack.security.authc.realms.file1.type', 'file'
setting 'xpack.security.authc.realms.file1.order', '0'
setting 'xpack.security.authc.realms.native1.type', 'native'
setting 'xpack.security.authc.realms.native1.order', '1'
} }
if (version.before('6.0.0')) {
keystoreSetting 'xpack.security.authc.token.passphrase', 'token passphrase' keystoreFile 'xpack.watcher.encryption_key', "${project.projectDir}/src/test/resources/system_key"
setting 'xpack.watcher.encrypt_sensitive_data', 'true'
if (version.onOrAfter('6.6.0')) {
setting 'ccr.auto_follow.wait_for_metadata_timeout', '1s'
}
// Old versions of the code contain an invalid assertion that trips
// during tests. Versions 5.6.9 and 6.2.4 have been fixed by removing
// the assertion, but this is impossible for released versions.
// However, released versions run without assertions, so end users won't
// be suffering the effects. This argument effectively removes the
// incorrect assertion from the older versions used in the BWC tests.
if (version.before('5.6.9') || (version.onOrAfter('6.0.0') && version.before('6.2.4'))) {
jvmArgs '-da:org.elasticsearch.xpack.monitoring.exporter.http.HttpExportBulk'
}
}
Task oldClusterTestRunner = tasks.getByName("${baseName}#oldClusterTestRunner")
oldClusterTestRunner.configure {
systemProperty 'tests.rest.suite', 'old_cluster'
}
Closure configureUpgradeCluster = {String name, Task lastRunner, int stopNode, Closure getOtherUnicastHostAddresses ->
configure(extensions.findByName("${baseName}#${name}")) {
dependsOn lastRunner, "${baseName}#oldClusterTestCluster#node${stopNode}.stop"
setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
clusterName = 'rolling-upgrade'
otherUnicastHostAddresses = { getOtherUnicastHostAddresses() }
minimumMasterNodes = { 2 }
autoSetInitialMasterNodes = false
/* Override the data directory so the new node always gets the node we
* just stopped's data directory. */
dataDir = { nodeNumber -> oldClusterTest.nodes[stopNode].dataDir }
waitCondition = waitWithAuth
setting 'xpack.monitoring.exporters._http.type', 'http'
setting 'xpack.monitoring.exporters._http.enabled', 'false'
setting 'xpack.monitoring.exporters._http.auth.username', 'test_user'
setting 'xpack.monitoring.exporters._http.auth.password', 'x-pack-test-password'
setting 'xpack.license.self_generated.type', 'trial'
setting 'xpack.security.enabled', 'true'
setting 'xpack.security.transport.ssl.enabled', 'true'
if (project.inFipsJvm) {
setting 'xpack.security.transport.ssl.key', 'testnode.pem'
setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
} else {
setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks'
setting 'xpack.security.transport.ssl.keystore.password', 'testnode'
}
setting 'node.attr.upgraded', 'true'
setting 'xpack.security.authc.token.enabled', 'true'
setting 'xpack.security.audit.enabled', 'true'
setting 'node.name', "upgraded-node-${stopNode}"
dependsOn copyTestNodeKeyMaterial
extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
extraConfigFile 'testnode.pem', new File(outputDir + '/testnode.pem')
extraConfigFile 'testnode.crt', new File(outputDir + '/testnode.crt')
setting 'xpack.security.authc.realms.file.file1.order', '0'
setting 'xpack.security.authc.realms.native.native1.order', '1'
setting 'xpack.watcher.encrypt_sensitive_data', 'true'
keystoreFile 'xpack.watcher.encryption_key', "${project.projectDir}/src/test/resources/system_key"
} }
}
} }
Task oneThirdUpgradedTest = tasks.create(name: "${baseName}#oneThirdUpgradedTest", type: RestIntegTestTask) Task oneThirdUpgradedTest = tasks.create(name: "${baseName}#oneThirdUpgradedTest", type: RestIntegTestTask)
configureUpgradeCluster("oneThirdUpgradedTestCluster", oldClusterTestRunner, 0, configureUpgradeCluster("oneThirdUpgradedTestCluster", oldClusterTestRunner, 0,
// Use all running nodes as seed nodes so there is no race between pinging and the tests // Use all running nodes as seed nodes so there is no race between pinging and the tests
{ [oldClusterTest.nodes.get(1).transportUri(), oldClusterTest.nodes.get(2).transportUri()] }) { [oldClusterTest.nodes.get(1).transportUri(), oldClusterTest.nodes.get(2).transportUri()] })
Task oneThirdUpgradedTestRunner = tasks.getByName("${baseName}#oneThirdUpgradedTestRunner") Task oneThirdUpgradedTestRunner = tasks.getByName("${baseName}#oneThirdUpgradedTestRunner")
oneThirdUpgradedTestRunner.configure { oneThirdUpgradedTestRunner.configure {
systemProperty 'tests.rest.suite', 'mixed_cluster' systemProperty 'tests.rest.suite', 'mixed_cluster'
systemProperty 'tests.first_round', 'true' systemProperty 'tests.first_round', 'true'
// We only need to run these tests once so we may as well do it when we're two thirds upgraded // We only need to run these tests once so we may as well do it when we're two thirds upgraded
systemProperty 'tests.rest.blacklist', [ systemProperty 'tests.rest.blacklist', [
'mixed_cluster/10_basic/Start scroll in mixed cluster on upgraded node that we will continue after upgrade', 'mixed_cluster/10_basic/Start scroll in mixed cluster on upgraded node that we will continue after upgrade',
'mixed_cluster/30_ml_jobs_crud/Create a job in the mixed cluster and write some data', 'mixed_cluster/30_ml_jobs_crud/Create a job in the mixed cluster and write some data',
'mixed_cluster/40_ml_datafeed_crud/Put job and datafeed in mixed cluster', 'mixed_cluster/40_ml_datafeed_crud/Put job and datafeed in mixed cluster',
].join(',') ].join(',')
finalizedBy "${baseName}#oldClusterTestCluster#node1.stop" finalizedBy "${baseName}#oldClusterTestCluster#node1.stop"
} }
Task twoThirdsUpgradedTest = tasks.create(name: "${baseName}#twoThirdsUpgradedTest", type: RestIntegTestTask) Task twoThirdsUpgradedTest = tasks.create(name: "${baseName}#twoThirdsUpgradedTest", type: RestIntegTestTask)
configureUpgradeCluster("twoThirdsUpgradedTestCluster", oneThirdUpgradedTestRunner, 1, configureUpgradeCluster("twoThirdsUpgradedTestCluster", oneThirdUpgradedTestRunner, 1,
// Use all running nodes as seed nodes so there is no race between pinging and the tests // Use all running nodes as seed nodes so there is no race between pinging and the tests
{ [oldClusterTest.nodes.get(2).transportUri(), oneThirdUpgradedTest.nodes.get(0).transportUri()] }) { [oldClusterTest.nodes.get(2).transportUri(), oneThirdUpgradedTest.nodes.get(0).transportUri()] })
Task twoThirdsUpgradedTestRunner = tasks.getByName("${baseName}#twoThirdsUpgradedTestRunner") Task twoThirdsUpgradedTestRunner = tasks.getByName("${baseName}#twoThirdsUpgradedTestRunner")
twoThirdsUpgradedTestRunner.configure { twoThirdsUpgradedTestRunner.configure {
systemProperty 'tests.rest.suite', 'mixed_cluster' systemProperty 'tests.rest.suite', 'mixed_cluster'
systemProperty 'tests.first_round', 'false' systemProperty 'tests.first_round', 'false'
finalizedBy "${baseName}#oldClusterTestCluster#node2.stop" finalizedBy "${baseName}#oldClusterTestCluster#node2.stop"
} }
Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask) Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask)
configureUpgradeCluster("upgradedClusterTestCluster", twoThirdsUpgradedTestRunner, 2, configureUpgradeCluster("upgradedClusterTestCluster", twoThirdsUpgradedTestRunner, 2,
// Use all running nodes as seed nodes so there is no race between pinging and the tests // Use all running nodes as seed nodes so there is no race between pinging and the tests
{ [oneThirdUpgradedTest.nodes.get(0).transportUri(), twoThirdsUpgradedTest.nodes.get(0).transportUri()] }) { [oneThirdUpgradedTest.nodes.get(0).transportUri(), twoThirdsUpgradedTest.nodes.get(0).transportUri()] })
Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner") Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner")
upgradedClusterTestRunner.configure { upgradedClusterTestRunner.configure {
systemProperty 'tests.rest.suite', 'upgraded_cluster' systemProperty 'tests.rest.suite', 'upgraded_cluster'
/* /*
* Force stopping all the upgraded nodes after the test runner * Force stopping all the upgraded nodes after the test runner
* so they are alive during the test. * so they are alive during the test.
*/ */
finalizedBy "${baseName}#oneThirdUpgradedTestCluster#stop" finalizedBy "${baseName}#oneThirdUpgradedTestCluster#stop"
finalizedBy "${baseName}#twoThirdsUpgradedTestCluster#stop" finalizedBy "${baseName}#twoThirdsUpgradedTestCluster#stop"
// migration tests should only run when the original/old cluster nodes where versions < 5.2.0. // migration tests should only run when the original/old cluster nodes where versions < 5.2.0.
// this stinks but we do the check here since our rest tests do not support conditionals // this stinks but we do the check here since our rest tests do not support conditionals
// otherwise we could check the index created version // otherwise we could check the index created version
String versionStr = project.extensions.findByName("${baseName}#oldClusterTestCluster").properties.get('bwcVersion') String versionStr = project.extensions.findByName("${baseName}#oldClusterTestCluster").properties.get('bwcVersion')
String[] versionParts = versionStr.split('\\.') String[] versionParts = versionStr.split('\\.')
if (versionParts[0].equals("5")) { if (versionParts[0].equals("5")) {
Integer minor = Integer.parseInt(versionParts[1]) Integer minor = Integer.parseInt(versionParts[1])
if (minor >= 2) { if (minor >= 2) {
systemProperty 'tests.rest.blacklist', '/20_security/Verify default password migration results in upgraded cluster' systemProperty 'tests.rest.blacklist', '/20_security/Verify default password migration results in upgraded cluster'
} }
} }
} }
Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") { Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") {
dependsOn = [upgradedClusterTest] dependsOn = [upgradedClusterTest]
} }
if (project.bwc_tests_enabled) { if (project.bwc_tests_enabled) {
bwcTest.dependsOn(versionBwcTest) bwcTest.dependsOn(versionBwcTest)
} }
} }
unitTest.enabled = false // no unit tests for rolling upgrades, only the rest integration test // basic integ tests includes testing bwc against the most recent version
task bwcTestSnapshots {
// basic integ tests includes testing bwc against the most recent version
task bwcTestSnapshots {
if (project.bwc_tests_enabled) { if (project.bwc_tests_enabled) {
for (final def version : bwcVersions.unreleasedWireCompatible) { for (final def version : bwcVersions.unreleasedWireCompatible) {
dependsOn "v${version}#bwcTest" dependsOn "v${version}#bwcTest"
} }
} }
} }
check.dependsOn(bwcTestSnapshots) check.dependsOn(bwcTestSnapshots)
dependencies { // copy x-pack plugin info so it is on the classpath and security manager has the right permissions
// "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here task copyXPackRestSpec(type: Copy) {
testCompile project(path: xpackModule('core'), configuration: 'default')
testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
testCompile project(path: xpackModule('watcher'))
}
compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
// copy x-pack plugin info so it is on the classpath and security manager has the right permissions
task copyXPackRestSpec(type: Copy) {
dependsOn(project.configurations.restSpec, 'processTestResources') dependsOn(project.configurations.restSpec, 'processTestResources')
from project(xpackProject('plugin').path).sourceSets.test.resources from project(xpackProject('plugin').path).sourceSets.test.resources
include 'rest-api-spec/api/**' include 'rest-api-spec/api/**'
into project.sourceSets.test.output.resourcesDir into project.sourceSets.test.output.resourcesDir
} }
task copyXPackPluginProps(type: Copy) { task copyXPackPluginProps(type: Copy) {
dependsOn(copyXPackRestSpec) dependsOn(copyXPackRestSpec)
from project(xpackModule('core')).file('src/main/plugin-metadata') from project(xpackModule('core')).file('src/main/plugin-metadata')
from project(xpackModule('core')).tasks.pluginProperties from project(xpackModule('core')).tasks.pluginProperties
into outputDir into outputDir
}
project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackPluginProps)
repositories {
maven {
url "https://artifacts.elastic.co/maven"
}
maven {
url "https://snapshots.elastic.co/maven"
}
}
} }
project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackPluginProps)

1
x-pack/qa/rolling-upgrade/with-system-key/build.gradle
View File

@ -1 +0,0 @@
group = "${group}.x-pack.qa.rolling-upgrade.with-system-key"

1
x-pack/qa/rolling-upgrade/without-system-key/build.gradle
View File

@ -1 +0,0 @@
group = "${group}.x-pack.qa.rolling-upgrade.without-system-key"