This change removes the variants of the rolling upgrade and full cluster restart tests that use or do not use a system key. These tests were added during 5.x when the system key was still used for security and now the system key is only used as the watcher encryption key so duplicating rolling upgrade and full cluster restarts is not needed. The change here removes the subprojects for testing these scenarios and defaults to always run with the watcher sensitive values encrypted for these tests.
This commit is contained in:
Jay Modi
GitHub
parent
e88b862192
commit
12bf3b2025
|
|
@ -3,10 +3,10 @@ import org.elasticsearch.gradle.test.RestIntegTestTask
|
|||
import org.elasticsearch.gradle.Version
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.util.regex.Matcher
|
||||
|
||||
// Apply the java plugin to this project so the sources can be edited in an IDE
|
||||
apply plugin: 'elasticsearch.build'
|
||||
apply plugin: 'elasticsearch.standalone-test'
|
||||
|
||||
unitTest.enabled = false
|
||||
|
||||
dependencies {
|
||||
|
|
@ -70,8 +70,6 @@ Closure waitWithAuth = { NodeInfo node, AntBuilder ant ->
|
|||
return tmpFile.exists()
|
||||
}
|
||||
|
||||
Project mainProject = project
|
||||
|
||||
String coreFullClusterRestartPath = project(':qa:full-cluster-restart').projectDir.toPath().resolve('src/test/java').toString()
|
||||
sourceSets {
|
||||
test {
|
||||
|
|
@ -89,224 +87,157 @@ forbiddenPatterns {
|
|||
exclude '**/system_key'
|
||||
}
|
||||
|
||||
// tests are pushed down to subprojects
|
||||
testingConventions.enabled = false
|
||||
String outputDir = "${buildDir}/generated-resources/${project.name}"
|
||||
|
||||
/**
|
||||
* Subdirectories of this project are test rolling upgrades with various
|
||||
* configuration options based on their name.
|
||||
*/
|
||||
subprojects {
|
||||
Matcher m = project.name =~ /with(out)?-system-key/
|
||||
if (false == m.matches()) {
|
||||
throw new InvalidUserDataException("Invalid project name [${project.name}]")
|
||||
}
|
||||
boolean withSystemKey = m.group(1) == null
|
||||
|
||||
apply plugin: 'elasticsearch.standalone-test'
|
||||
|
||||
// Use resources from the rolling-upgrade project in subdirectories
|
||||
sourceSets {
|
||||
test {
|
||||
java {
|
||||
srcDirs = ["${mainProject.projectDir}/src/test/java", coreFullClusterRestartPath]
|
||||
}
|
||||
resources {
|
||||
srcDirs = ["${mainProject.projectDir}/src/test/resources"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
licenseHeaders {
|
||||
approvedLicenses << 'Apache'
|
||||
}
|
||||
|
||||
forbiddenPatterns {
|
||||
exclude '**/system_key'
|
||||
}
|
||||
|
||||
String outputDir = "${buildDir}/generated-resources/${project.name}"
|
||||
|
||||
// This is a top level task which we will add dependencies to below.
|
||||
// It is a single task that can be used to backcompat tests against all versions.
|
||||
task bwcTest {
|
||||
// This is a top level task which we will add dependencies to below.
|
||||
// It is a single task that can be used to backcompat tests against all versions.
|
||||
task bwcTest {
|
||||
description = 'Runs backwards compatibility tests.'
|
||||
group = 'verification'
|
||||
}
|
||||
}
|
||||
|
||||
String output = "${buildDir}/generated-resources/${project.name}"
|
||||
task copyTestNodeKeyMaterial(type: Copy) {
|
||||
task copyTestNodeKeyMaterial(type: Copy) {
|
||||
from project(':x-pack:plugin:core').files('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem',
|
||||
'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt',
|
||||
'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
|
||||
'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt',
|
||||
'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
|
||||
into outputDir
|
||||
}
|
||||
}
|
||||
|
||||
for (Version version : bwcVersions.indexCompatible) {
|
||||
for (Version version : bwcVersions.indexCompatible) {
|
||||
String baseName = "v${version}"
|
||||
|
||||
Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) {
|
||||
mustRunAfter(precommit)
|
||||
mustRunAfter(precommit)
|
||||
}
|
||||
|
||||
Object extension = extensions.findByName("${baseName}#oldClusterTestCluster")
|
||||
configure(extensions.findByName("${baseName}#oldClusterTestCluster")) {
|
||||
dependsOn copyTestNodeKeyMaterial
|
||||
if (version.before('6.3.0')) {
|
||||
String depVersion = version;
|
||||
if (project.bwcVersions.unreleased.contains(version)) {
|
||||
depVersion += "-SNAPSHOT"
|
||||
}
|
||||
mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${depVersion}"
|
||||
dependsOn copyTestNodeKeyMaterial
|
||||
if (version.before('6.3.0')) {
|
||||
String depVersion = version;
|
||||
if (project.bwcVersions.unreleased.contains(version)) {
|
||||
depVersion += "-SNAPSHOT"
|
||||
}
|
||||
mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${depVersion}"
|
||||
|
||||
}
|
||||
bwcVersion = version
|
||||
numBwcNodes = 2
|
||||
numNodes = 2
|
||||
clusterName = 'full-cluster-restart'
|
||||
String usersCli = version.before('6.3.0') ? 'bin/x-pack/users' : 'bin/elasticsearch-users'
|
||||
setupCommand 'setupTestUser', usersCli, 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
|
||||
waitCondition = waitWithAuth
|
||||
|
||||
// some tests rely on the translog not being flushed
|
||||
setting 'indices.memory.shard_inactive_time', '20m'
|
||||
|
||||
setting 'xpack.security.enabled', 'true'
|
||||
setting 'xpack.security.transport.ssl.enabled', 'true'
|
||||
if (project.inFipsJvm) {
|
||||
setting 'xpack.security.transport.ssl.key', 'testnode.pem'
|
||||
setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
|
||||
keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
|
||||
} else {
|
||||
setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks'
|
||||
setting 'xpack.security.transport.ssl.keystore.password', 'testnode'
|
||||
}
|
||||
setting 'xpack.license.self_generated.type', 'trial'
|
||||
dependsOn copyTestNodeKeyMaterial
|
||||
extraConfigFile 'testnode.pem', new File(outputDir + '/testnode.pem')
|
||||
extraConfigFile 'testnode.crt', new File(outputDir + '/testnode.crt')
|
||||
extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
|
||||
if (withSystemKey) {
|
||||
if (version.onOrAfter('5.1.0') && version.before('6.0.0')) {
|
||||
// The setting didn't exist until 5.1.0
|
||||
setting 'xpack.security.system_key.required', 'true'
|
||||
}
|
||||
if (version.onOrAfter('6.0.0')) {
|
||||
keystoreFile 'xpack.watcher.encryption_key', "${mainProject.projectDir}/src/test/resources/system_key"
|
||||
bwcVersion = version
|
||||
numBwcNodes = 2
|
||||
numNodes = 2
|
||||
clusterName = 'full-cluster-restart'
|
||||
String usersCli = version.before('6.3.0') ? 'bin/x-pack/users' : 'bin/elasticsearch-users'
|
||||
setupCommand 'setupTestUser', usersCli, 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
|
||||
waitCondition = waitWithAuth
|
||||
|
||||
// some tests rely on the translog not being flushed
|
||||
setting 'indices.memory.shard_inactive_time', '20m'
|
||||
|
||||
setting 'xpack.security.enabled', 'true'
|
||||
setting 'xpack.security.transport.ssl.enabled', 'true'
|
||||
if (project.inFipsJvm) {
|
||||
setting 'xpack.security.transport.ssl.key', 'testnode.pem'
|
||||
setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
|
||||
keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
|
||||
} else {
|
||||
extraConfigFile 'x-pack/system_key', "${mainProject.projectDir}/src/test/resources/system_key"
|
||||
setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks'
|
||||
setting 'xpack.security.transport.ssl.keystore.password', 'testnode'
|
||||
}
|
||||
setting 'xpack.license.self_generated.type', 'trial'
|
||||
dependsOn copyTestNodeKeyMaterial
|
||||
extraConfigFile 'testnode.pem', new File(outputDir + '/testnode.pem')
|
||||
extraConfigFile 'testnode.crt', new File(outputDir + '/testnode.crt')
|
||||
extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
|
||||
|
||||
keystoreFile 'xpack.watcher.encryption_key', "${project.projectDir}/src/test/resources/system_key"
|
||||
setting 'xpack.watcher.encrypt_sensitive_data', 'true'
|
||||
}
|
||||
}
|
||||
|
||||
Task oldClusterTestRunner = tasks.getByName("${baseName}#oldClusterTestRunner")
|
||||
oldClusterTestRunner.configure {
|
||||
systemProperty 'tests.is_old_cluster', 'true'
|
||||
systemProperty 'tests.old_cluster_version', version.toString().minus("-SNAPSHOT")
|
||||
systemProperty 'tests.path.repo', new File(buildDir, "cluster/shared/repo")
|
||||
exclude 'org/elasticsearch/upgrades/FullClusterRestartIT.class'
|
||||
exclude 'org/elasticsearch/upgrades/FullClusterRestartSettingsUpgradeIT.class'
|
||||
exclude 'org/elasticsearch/upgrades/QueryBuilderBWCIT.class'
|
||||
systemProperty 'tests.is_old_cluster', 'true'
|
||||
systemProperty 'tests.old_cluster_version', version.toString().minus("-SNAPSHOT")
|
||||
systemProperty 'tests.path.repo', new File(buildDir, "cluster/shared/repo")
|
||||
exclude 'org/elasticsearch/upgrades/FullClusterRestartIT.class'
|
||||
exclude 'org/elasticsearch/upgrades/FullClusterRestartSettingsUpgradeIT.class'
|
||||
exclude 'org/elasticsearch/upgrades/QueryBuilderBWCIT.class'
|
||||
}
|
||||
|
||||
Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask)
|
||||
|
||||
configure(extensions.findByName("${baseName}#upgradedClusterTestCluster")) {
|
||||
dependsOn oldClusterTestRunner,
|
||||
"${baseName}#oldClusterTestCluster#node0.stop",
|
||||
"${baseName}#oldClusterTestCluster#node1.stop"
|
||||
numNodes = 2
|
||||
clusterName = 'full-cluster-restart'
|
||||
dataDir = { nodeNum -> oldClusterTest.nodes[nodeNum].dataDir }
|
||||
cleanShared = false // We want to keep snapshots made by the old cluster!
|
||||
setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
|
||||
waitCondition = waitWithAuth
|
||||
dependsOn oldClusterTestRunner,
|
||||
"${baseName}#oldClusterTestCluster#node0.stop",
|
||||
"${baseName}#oldClusterTestCluster#node1.stop"
|
||||
numNodes = 2
|
||||
clusterName = 'full-cluster-restart'
|
||||
dataDir = { nodeNum -> oldClusterTest.nodes[nodeNum].dataDir }
|
||||
cleanShared = false // We want to keep snapshots made by the old cluster!
|
||||
setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
|
||||
waitCondition = waitWithAuth
|
||||
|
||||
// some tests rely on the translog not being flushed
|
||||
setting 'indices.memory.shard_inactive_time', '20m'
|
||||
setting 'xpack.security.enabled', 'true'
|
||||
if (project.inFipsJvm) {
|
||||
setting 'xpack.security.transport.ssl.key', 'testnode.pem'
|
||||
setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
|
||||
keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
|
||||
} else {
|
||||
setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks'
|
||||
setting 'xpack.security.transport.ssl.keystore.password', 'testnode'
|
||||
}
|
||||
setting 'xpack.license.self_generated.type', 'trial'
|
||||
dependsOn copyTestNodeKeyMaterial
|
||||
extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
|
||||
extraConfigFile 'testnode.pem', new File(outputDir + '/testnode.pem')
|
||||
extraConfigFile 'testnode.crt', new File(outputDir + '/testnode.crt')
|
||||
if (withSystemKey) {
|
||||
setting 'xpack.watcher.encrypt_sensitive_data', 'true'
|
||||
keystoreFile 'xpack.watcher.encryption_key', "${mainProject.projectDir}/src/test/resources/system_key"
|
||||
}
|
||||
// some tests rely on the translog not being flushed
|
||||
setting 'indices.memory.shard_inactive_time', '20m'
|
||||
setting 'xpack.security.enabled', 'true'
|
||||
if (project.inFipsJvm) {
|
||||
setting 'xpack.security.transport.ssl.key', 'testnode.pem'
|
||||
setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
|
||||
keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
|
||||
} else {
|
||||
setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks'
|
||||
setting 'xpack.security.transport.ssl.keystore.password', 'testnode'
|
||||
}
|
||||
setting 'xpack.license.self_generated.type', 'trial'
|
||||
dependsOn copyTestNodeKeyMaterial
|
||||
extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
|
||||
extraConfigFile 'testnode.pem', new File(outputDir + '/testnode.pem')
|
||||
extraConfigFile 'testnode.crt', new File(outputDir + '/testnode.crt')
|
||||
|
||||
setting 'xpack.watcher.encrypt_sensitive_data', 'true'
|
||||
keystoreFile 'xpack.watcher.encryption_key', "${project.projectDir}/src/test/resources/system_key"
|
||||
}
|
||||
|
||||
Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner")
|
||||
upgradedClusterTestRunner.configure {
|
||||
systemProperty 'tests.is_old_cluster', 'false'
|
||||
systemProperty 'tests.old_cluster_version', version.toString().minus("-SNAPSHOT")
|
||||
systemProperty 'tests.path.repo', new File(buildDir, "cluster/shared/repo")
|
||||
exclude 'org/elasticsearch/upgrades/FullClusterRestartIT.class'
|
||||
exclude 'org/elasticsearch/upgrades/FullClusterRestartSettingsUpgradeIT.class'
|
||||
exclude 'org/elasticsearch/upgrades/QueryBuilderBWCIT.class'
|
||||
systemProperty 'tests.is_old_cluster', 'false'
|
||||
systemProperty 'tests.old_cluster_version', version.toString().minus("-SNAPSHOT")
|
||||
systemProperty 'tests.path.repo', new File(buildDir, "cluster/shared/repo")
|
||||
exclude 'org/elasticsearch/upgrades/FullClusterRestartIT.class'
|
||||
exclude 'org/elasticsearch/upgrades/FullClusterRestartSettingsUpgradeIT.class'
|
||||
exclude 'org/elasticsearch/upgrades/QueryBuilderBWCIT.class'
|
||||
}
|
||||
|
||||
Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") {
|
||||
dependsOn = [upgradedClusterTest]
|
||||
dependsOn = [upgradedClusterTest]
|
||||
}
|
||||
|
||||
if (project.bwc_tests_enabled) {
|
||||
bwcTest.dependsOn(versionBwcTest)
|
||||
bwcTest.dependsOn(versionBwcTest)
|
||||
}
|
||||
}
|
||||
|
||||
unitTest.enabled = false // no unit tests for full cluster restarts, only the rest integration test
|
||||
}
|
||||
|
||||
// basic integ tests includes testing bwc against the most recent version
|
||||
task bwcTestSnapshots {
|
||||
// basic integ tests includes testing bwc against the most recent version
|
||||
task bwcTestSnapshots {
|
||||
if (project.bwc_tests_enabled) {
|
||||
for (final def version : bwcVersions.unreleasedIndexCompatible) {
|
||||
dependsOn "v${version}#bwcTest"
|
||||
}
|
||||
for (final def version : bwcVersions.unreleasedIndexCompatible) {
|
||||
dependsOn "v${version}#bwcTest"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
check.dependsOn(bwcTestSnapshots)
|
||||
check.dependsOn(bwcTestSnapshots)
|
||||
|
||||
dependencies {
|
||||
// "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
|
||||
testCompile project(path: xpackModule('core'), configuration: 'default')
|
||||
testCompile project(path: xpackModule('watcher'), configuration: 'runtime')
|
||||
testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
|
||||
testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
|
||||
}
|
||||
|
||||
// copy x-pack plugin info so it is on the classpath and security manager has the right permissions
|
||||
task copyXPackRestSpec(type: Copy) {
|
||||
// copy x-pack plugin info so it is on the classpath and security manager has the right permissions
|
||||
task copyXPackRestSpec(type: Copy) {
|
||||
dependsOn(project.configurations.restSpec, 'processTestResources')
|
||||
from project(xpackModule('core')).sourceSets.test.resources
|
||||
include 'rest-api-spec/api/**'
|
||||
into project.sourceSets.test.output.resourcesDir
|
||||
}
|
||||
}
|
||||
|
||||
task copyXPackPluginProps(type: Copy) {
|
||||
task copyXPackPluginProps(type: Copy) {
|
||||
dependsOn(copyXPackRestSpec)
|
||||
from project(xpackModule('core')).file('src/main/plugin-metadata')
|
||||
from project(xpackModule('core')).tasks.pluginProperties
|
||||
into outputDir
|
||||
}
|
||||
project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackPluginProps)
|
||||
|
||||
repositories {
|
||||
maven {
|
||||
url "https://artifacts.elastic.co/maven"
|
||||
}
|
||||
maven {
|
||||
url "https://snapshots.elastic.co/maven"
|
||||
}
|
||||
}
|
||||
}
|
||||
project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackPluginProps)
|
||||
|
|
|
|||
|
|
@ -3,10 +3,10 @@ import org.elasticsearch.gradle.test.RestIntegTestTask
|
|||
import org.elasticsearch.gradle.Version
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.util.regex.Matcher
|
||||
|
||||
// Apply the java plugin to this project so the sources can be edited in an IDE
|
||||
apply plugin: 'elasticsearch.build'
|
||||
apply plugin: 'elasticsearch.standalone-test'
|
||||
|
||||
unitTest.enabled = false
|
||||
|
||||
dependencies {
|
||||
|
|
@ -68,163 +68,50 @@ Closure waitWithAuth = { NodeInfo node, AntBuilder ant ->
|
|||
return tmpFile.exists()
|
||||
}
|
||||
|
||||
Project mainProject = project
|
||||
|
||||
compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
|
||||
|
||||
forbiddenPatterns {
|
||||
exclude '**/system_key'
|
||||
}
|
||||
|
||||
// Tests are pushed down to subprojects
|
||||
testingConventions.enabled = false
|
||||
String outputDir = "${buildDir}/generated-resources/${project.name}"
|
||||
|
||||
/**
|
||||
* Subdirectories of this project are test rolling upgrades with various
|
||||
* configuration options based on their name.
|
||||
*/
|
||||
subprojects {
|
||||
Matcher m = project.name =~ /with(out)?-system-key/
|
||||
if (false == m.matches()) {
|
||||
throw new InvalidUserDataException("Invalid project name [${project.name}]")
|
||||
}
|
||||
boolean withSystemKey = m.group(1) == null
|
||||
|
||||
apply plugin: 'elasticsearch.standalone-test'
|
||||
|
||||
// Use resources from the rolling-upgrade project in subdirectories
|
||||
sourceSets {
|
||||
test {
|
||||
java {
|
||||
srcDirs = ["${mainProject.projectDir}/src/test/java"]
|
||||
}
|
||||
resources {
|
||||
srcDirs = ["${mainProject.projectDir}/src/test/resources"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
forbiddenPatterns {
|
||||
exclude '**/system_key'
|
||||
}
|
||||
|
||||
String outputDir = "${buildDir}/generated-resources/${project.name}"
|
||||
|
||||
// This is a top level task which we will add dependencies to below.
|
||||
// It is a single task that can be used to backcompat tests against all versions.
|
||||
task bwcTest {
|
||||
// This is a top level task which we will add dependencies to below.
|
||||
// It is a single task that can be used to backcompat tests against all versions.
|
||||
task bwcTest {
|
||||
description = 'Runs backwards compatibility tests.'
|
||||
group = 'verification'
|
||||
}
|
||||
}
|
||||
|
||||
String output = "${buildDir}/generated-resources/${project.name}"
|
||||
task copyTestNodeKeyMaterial(type: Copy) {
|
||||
task copyTestNodeKeyMaterial(type: Copy) {
|
||||
from project(':x-pack:plugin:core').files('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem',
|
||||
'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt',
|
||||
'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
|
||||
'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt',
|
||||
'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
|
||||
into outputDir
|
||||
}
|
||||
}
|
||||
|
||||
for (Version version : bwcVersions.wireCompatible) {
|
||||
for (Version version : bwcVersions.wireCompatible) {
|
||||
String baseName = "v${version}"
|
||||
|
||||
Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) {
|
||||
mustRunAfter(precommit)
|
||||
mustRunAfter(precommit)
|
||||
}
|
||||
|
||||
configure(extensions.findByName("${baseName}#oldClusterTestCluster")) {
|
||||
dependsOn copyTestNodeKeyMaterial
|
||||
if (version.before('6.3.0')) {
|
||||
String depVersion = version;
|
||||
if (project.bwcVersions.unreleased.contains(version)) {
|
||||
depVersion += "-SNAPSHOT"
|
||||
}
|
||||
mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${depVersion}"
|
||||
}
|
||||
String usersCli = version.before('6.3.0') ? 'bin/x-pack/users' : 'bin/elasticsearch-users'
|
||||
setupCommand 'setupTestUser', usersCli, 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
|
||||
bwcVersion = version
|
||||
numBwcNodes = 3
|
||||
numNodes = 3
|
||||
clusterName = 'rolling-upgrade'
|
||||
waitCondition = waitWithAuth
|
||||
setting 'xpack.monitoring.exporters._http.type', 'http'
|
||||
setting 'xpack.monitoring.exporters._http.enabled', 'false'
|
||||
setting 'xpack.monitoring.exporters._http.auth.username', 'test_user'
|
||||
setting 'xpack.monitoring.exporters._http.auth.password', 'x-pack-test-password'
|
||||
setting 'xpack.license.self_generated.type', 'trial'
|
||||
setting 'xpack.security.enabled', 'true'
|
||||
setting 'xpack.security.transport.ssl.enabled', 'true'
|
||||
setting 'xpack.security.authc.token.enabled', 'true'
|
||||
setting 'xpack.security.audit.enabled', 'true'
|
||||
if (project.inFipsJvm) {
|
||||
setting 'xpack.security.transport.ssl.key', 'testnode.pem'
|
||||
setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
|
||||
keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
|
||||
} else {
|
||||
setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks'
|
||||
setting 'xpack.security.transport.ssl.keystore.password', 'testnode'
|
||||
}
|
||||
dependsOn copyTestNodeKeyMaterial
|
||||
extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
|
||||
extraConfigFile 'testnode.pem', new File(outputDir + '/testnode.pem')
|
||||
extraConfigFile 'testnode.crt', new File(outputDir + '/testnode.crt')
|
||||
if (version.onOrAfter('7.0.0')) {
|
||||
setting 'xpack.security.authc.realms.file.file1.order', '0'
|
||||
setting 'xpack.security.authc.realms.native.native1.order', '1'
|
||||
} else {
|
||||
setting 'xpack.security.authc.realms.file1.type', 'file'
|
||||
setting 'xpack.security.authc.realms.file1.order', '0'
|
||||
setting 'xpack.security.authc.realms.native1.type', 'native'
|
||||
setting 'xpack.security.authc.realms.native1.order', '1'
|
||||
}
|
||||
|
||||
if (withSystemKey) {
|
||||
if (version.onOrAfter('5.1.0') && version.before('6.0.0')) {
|
||||
// The setting didn't exist until 5.1.0
|
||||
setting 'xpack.security.system_key.required', 'true'
|
||||
dependsOn copyTestNodeKeyMaterial
|
||||
if (version.before('6.3.0')) {
|
||||
String depVersion = version;
|
||||
if (project.bwcVersions.unreleased.contains(version)) {
|
||||
depVersion += "-SNAPSHOT"
|
||||
}
|
||||
mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${depVersion}"
|
||||
}
|
||||
if (version.onOrAfter('6.0.0')) {
|
||||
keystoreFile 'xpack.watcher.encryption_key', "${mainProject.projectDir}/src/test/resources/system_key"
|
||||
} else {
|
||||
String systemKeyFile = version.before('6.3.0') ? 'x-pack/system_key' : 'system_key'
|
||||
extraConfigFile systemKeyFile, "${mainProject.projectDir}/src/test/resources/system_key"
|
||||
keystoreSetting 'xpack.security.authc.token.passphrase', 'token passphrase'
|
||||
}
|
||||
setting 'xpack.watcher.encrypt_sensitive_data', 'true'
|
||||
}
|
||||
|
||||
if (version.onOrAfter('6.6.0')) {
|
||||
setting 'ccr.auto_follow.wait_for_metadata_timeout', '1s'
|
||||
}
|
||||
|
||||
// Old versions of the code contain an invalid assertion that trips
|
||||
// during tests. Versions 5.6.9 and 6.2.4 have been fixed by removing
|
||||
// the assertion, but this is impossible for released versions.
|
||||
// However, released versions run without assertions, so end users won't
|
||||
// be suffering the effects. This argument effectively removes the
|
||||
// incorrect assertion from the older versions used in the BWC tests.
|
||||
if (version.before('5.6.9') || (version.onOrAfter('6.0.0') && version.before('6.2.4'))) {
|
||||
jvmArgs '-da:org.elasticsearch.xpack.monitoring.exporter.http.HttpExportBulk'
|
||||
}
|
||||
}
|
||||
|
||||
Task oldClusterTestRunner = tasks.getByName("${baseName}#oldClusterTestRunner")
|
||||
oldClusterTestRunner.configure {
|
||||
systemProperty 'tests.rest.suite', 'old_cluster'
|
||||
}
|
||||
|
||||
Closure configureUpgradeCluster = {String name, Task lastRunner, int stopNode, Closure getOtherUnicastHostAddresses ->
|
||||
configure(extensions.findByName("${baseName}#${name}")) {
|
||||
dependsOn lastRunner, "${baseName}#oldClusterTestCluster#node${stopNode}.stop"
|
||||
setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
|
||||
String usersCli = version.before('6.3.0') ? 'bin/x-pack/users' : 'bin/elasticsearch-users'
|
||||
setupCommand 'setupTestUser', usersCli, 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
|
||||
bwcVersion = version
|
||||
numBwcNodes = 3
|
||||
numNodes = 3
|
||||
clusterName = 'rolling-upgrade'
|
||||
otherUnicastHostAddresses = { getOtherUnicastHostAddresses() }
|
||||
minimumMasterNodes = { 2 }
|
||||
autoSetInitialMasterNodes = false
|
||||
/* Override the data directory so the new node always gets the node we
|
||||
* just stopped's data directory. */
|
||||
dataDir = { nodeNumber -> oldClusterTest.nodes[stopNode].dataDir }
|
||||
waitCondition = waitWithAuth
|
||||
setting 'xpack.monitoring.exporters._http.type', 'http'
|
||||
setting 'xpack.monitoring.exporters._http.enabled', 'false'
|
||||
|
|
@ -233,147 +120,187 @@ subprojects {
|
|||
setting 'xpack.license.self_generated.type', 'trial'
|
||||
setting 'xpack.security.enabled', 'true'
|
||||
setting 'xpack.security.transport.ssl.enabled', 'true'
|
||||
if (project.inFipsJvm) {
|
||||
setting 'xpack.security.transport.ssl.key', 'testnode.pem'
|
||||
setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
|
||||
keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
|
||||
} else {
|
||||
setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks'
|
||||
setting 'xpack.security.transport.ssl.keystore.password', 'testnode'
|
||||
}
|
||||
setting 'node.attr.upgraded', 'true'
|
||||
setting 'xpack.security.authc.token.enabled', 'true'
|
||||
setting 'xpack.security.audit.enabled', 'true'
|
||||
setting 'node.name', "upgraded-node-${stopNode}"
|
||||
if (project.inFipsJvm) {
|
||||
setting 'xpack.security.transport.ssl.key', 'testnode.pem'
|
||||
setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
|
||||
keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
|
||||
} else {
|
||||
setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks'
|
||||
setting 'xpack.security.transport.ssl.keystore.password', 'testnode'
|
||||
}
|
||||
dependsOn copyTestNodeKeyMaterial
|
||||
extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
|
||||
extraConfigFile 'testnode.pem', new File(outputDir + '/testnode.pem')
|
||||
extraConfigFile 'testnode.crt', new File(outputDir + '/testnode.crt')
|
||||
setting 'xpack.security.authc.realms.file.file1.order', '0'
|
||||
setting 'xpack.security.authc.realms.native.native1.order', '1'
|
||||
if (withSystemKey) {
|
||||
setting 'xpack.watcher.encrypt_sensitive_data', 'true'
|
||||
keystoreFile 'xpack.watcher.encryption_key', "${mainProject.projectDir}/src/test/resources/system_key"
|
||||
if (version.onOrAfter('7.0.0')) {
|
||||
setting 'xpack.security.authc.realms.file.file1.order', '0'
|
||||
setting 'xpack.security.authc.realms.native.native1.order', '1'
|
||||
} else {
|
||||
setting 'xpack.security.authc.realms.file1.type', 'file'
|
||||
setting 'xpack.security.authc.realms.file1.order', '0'
|
||||
setting 'xpack.security.authc.realms.native1.type', 'native'
|
||||
setting 'xpack.security.authc.realms.native1.order', '1'
|
||||
}
|
||||
if (version.before('6.0.0')) {
|
||||
keystoreSetting 'xpack.security.authc.token.passphrase', 'token passphrase'
|
||||
|
||||
keystoreFile 'xpack.watcher.encryption_key', "${project.projectDir}/src/test/resources/system_key"
|
||||
setting 'xpack.watcher.encrypt_sensitive_data', 'true'
|
||||
|
||||
if (version.onOrAfter('6.6.0')) {
|
||||
setting 'ccr.auto_follow.wait_for_metadata_timeout', '1s'
|
||||
}
|
||||
|
||||
// Old versions of the code contain an invalid assertion that trips
|
||||
// during tests. Versions 5.6.9 and 6.2.4 have been fixed by removing
|
||||
// the assertion, but this is impossible for released versions.
|
||||
// However, released versions run without assertions, so end users won't
|
||||
// be suffering the effects. This argument effectively removes the
|
||||
// incorrect assertion from the older versions used in the BWC tests.
|
||||
if (version.before('5.6.9') || (version.onOrAfter('6.0.0') && version.before('6.2.4'))) {
|
||||
jvmArgs '-da:org.elasticsearch.xpack.monitoring.exporter.http.HttpExportBulk'
|
||||
}
|
||||
}
|
||||
|
||||
Task oldClusterTestRunner = tasks.getByName("${baseName}#oldClusterTestRunner")
|
||||
oldClusterTestRunner.configure {
|
||||
systemProperty 'tests.rest.suite', 'old_cluster'
|
||||
}
|
||||
|
||||
Closure configureUpgradeCluster = {String name, Task lastRunner, int stopNode, Closure getOtherUnicastHostAddresses ->
|
||||
configure(extensions.findByName("${baseName}#${name}")) {
|
||||
dependsOn lastRunner, "${baseName}#oldClusterTestCluster#node${stopNode}.stop"
|
||||
setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
|
||||
clusterName = 'rolling-upgrade'
|
||||
otherUnicastHostAddresses = { getOtherUnicastHostAddresses() }
|
||||
minimumMasterNodes = { 2 }
|
||||
autoSetInitialMasterNodes = false
|
||||
/* Override the data directory so the new node always gets the node we
|
||||
* just stopped's data directory. */
|
||||
dataDir = { nodeNumber -> oldClusterTest.nodes[stopNode].dataDir }
|
||||
waitCondition = waitWithAuth
|
||||
setting 'xpack.monitoring.exporters._http.type', 'http'
|
||||
setting 'xpack.monitoring.exporters._http.enabled', 'false'
|
||||
setting 'xpack.monitoring.exporters._http.auth.username', 'test_user'
|
||||
setting 'xpack.monitoring.exporters._http.auth.password', 'x-pack-test-password'
|
||||
setting 'xpack.license.self_generated.type', 'trial'
|
||||
setting 'xpack.security.enabled', 'true'
|
||||
setting 'xpack.security.transport.ssl.enabled', 'true'
|
||||
if (project.inFipsJvm) {
|
||||
setting 'xpack.security.transport.ssl.key', 'testnode.pem'
|
||||
setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
|
||||
keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
|
||||
} else {
|
||||
setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks'
|
||||
setting 'xpack.security.transport.ssl.keystore.password', 'testnode'
|
||||
}
|
||||
setting 'node.attr.upgraded', 'true'
|
||||
setting 'xpack.security.authc.token.enabled', 'true'
|
||||
setting 'xpack.security.audit.enabled', 'true'
|
||||
setting 'node.name', "upgraded-node-${stopNode}"
|
||||
dependsOn copyTestNodeKeyMaterial
|
||||
extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
|
||||
extraConfigFile 'testnode.pem', new File(outputDir + '/testnode.pem')
|
||||
extraConfigFile 'testnode.crt', new File(outputDir + '/testnode.crt')
|
||||
setting 'xpack.security.authc.realms.file.file1.order', '0'
|
||||
setting 'xpack.security.authc.realms.native.native1.order', '1'
|
||||
setting 'xpack.watcher.encrypt_sensitive_data', 'true'
|
||||
keystoreFile 'xpack.watcher.encryption_key', "${project.projectDir}/src/test/resources/system_key"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Task oneThirdUpgradedTest = tasks.create(name: "${baseName}#oneThirdUpgradedTest", type: RestIntegTestTask)
|
||||
|
||||
configureUpgradeCluster("oneThirdUpgradedTestCluster", oldClusterTestRunner, 0,
|
||||
// Use all running nodes as seed nodes so there is no race between pinging and the tests
|
||||
{ [oldClusterTest.nodes.get(1).transportUri(), oldClusterTest.nodes.get(2).transportUri()] })
|
||||
// Use all running nodes as seed nodes so there is no race between pinging and the tests
|
||||
{ [oldClusterTest.nodes.get(1).transportUri(), oldClusterTest.nodes.get(2).transportUri()] })
|
||||
|
||||
Task oneThirdUpgradedTestRunner = tasks.getByName("${baseName}#oneThirdUpgradedTestRunner")
|
||||
oneThirdUpgradedTestRunner.configure {
|
||||
systemProperty 'tests.rest.suite', 'mixed_cluster'
|
||||
systemProperty 'tests.first_round', 'true'
|
||||
// We only need to run these tests once so we may as well do it when we're two thirds upgraded
|
||||
systemProperty 'tests.rest.blacklist', [
|
||||
'mixed_cluster/10_basic/Start scroll in mixed cluster on upgraded node that we will continue after upgrade',
|
||||
'mixed_cluster/30_ml_jobs_crud/Create a job in the mixed cluster and write some data',
|
||||
'mixed_cluster/40_ml_datafeed_crud/Put job and datafeed in mixed cluster',
|
||||
systemProperty 'tests.rest.suite', 'mixed_cluster'
|
||||
systemProperty 'tests.first_round', 'true'
|
||||
// We only need to run these tests once so we may as well do it when we're two thirds upgraded
|
||||
systemProperty 'tests.rest.blacklist', [
|
||||
'mixed_cluster/10_basic/Start scroll in mixed cluster on upgraded node that we will continue after upgrade',
|
||||
'mixed_cluster/30_ml_jobs_crud/Create a job in the mixed cluster and write some data',
|
||||
'mixed_cluster/40_ml_datafeed_crud/Put job and datafeed in mixed cluster',
|
||||
].join(',')
|
||||
finalizedBy "${baseName}#oldClusterTestCluster#node1.stop"
|
||||
finalizedBy "${baseName}#oldClusterTestCluster#node1.stop"
|
||||
}
|
||||
|
||||
Task twoThirdsUpgradedTest = tasks.create(name: "${baseName}#twoThirdsUpgradedTest", type: RestIntegTestTask)
|
||||
|
||||
configureUpgradeCluster("twoThirdsUpgradedTestCluster", oneThirdUpgradedTestRunner, 1,
|
||||
// Use all running nodes as seed nodes so there is no race between pinging and the tests
|
||||
{ [oldClusterTest.nodes.get(2).transportUri(), oneThirdUpgradedTest.nodes.get(0).transportUri()] })
|
||||
// Use all running nodes as seed nodes so there is no race between pinging and the tests
|
||||
{ [oldClusterTest.nodes.get(2).transportUri(), oneThirdUpgradedTest.nodes.get(0).transportUri()] })
|
||||
|
||||
Task twoThirdsUpgradedTestRunner = tasks.getByName("${baseName}#twoThirdsUpgradedTestRunner")
|
||||
twoThirdsUpgradedTestRunner.configure {
|
||||
systemProperty 'tests.rest.suite', 'mixed_cluster'
|
||||
systemProperty 'tests.first_round', 'false'
|
||||
finalizedBy "${baseName}#oldClusterTestCluster#node2.stop"
|
||||
systemProperty 'tests.rest.suite', 'mixed_cluster'
|
||||
systemProperty 'tests.first_round', 'false'
|
||||
finalizedBy "${baseName}#oldClusterTestCluster#node2.stop"
|
||||
}
|
||||
|
||||
Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask)
|
||||
|
||||
configureUpgradeCluster("upgradedClusterTestCluster", twoThirdsUpgradedTestRunner, 2,
|
||||
// Use all running nodes as seed nodes so there is no race between pinging and the tests
|
||||
{ [oneThirdUpgradedTest.nodes.get(0).transportUri(), twoThirdsUpgradedTest.nodes.get(0).transportUri()] })
|
||||
// Use all running nodes as seed nodes so there is no race between pinging and the tests
|
||||
{ [oneThirdUpgradedTest.nodes.get(0).transportUri(), twoThirdsUpgradedTest.nodes.get(0).transportUri()] })
|
||||
|
||||
Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner")
|
||||
upgradedClusterTestRunner.configure {
|
||||
systemProperty 'tests.rest.suite', 'upgraded_cluster'
|
||||
/*
|
||||
* Force stopping all the upgraded nodes after the test runner
|
||||
* so they are alive during the test.
|
||||
*/
|
||||
finalizedBy "${baseName}#oneThirdUpgradedTestCluster#stop"
|
||||
finalizedBy "${baseName}#twoThirdsUpgradedTestCluster#stop"
|
||||
systemProperty 'tests.rest.suite', 'upgraded_cluster'
|
||||
/*
|
||||
* Force stopping all the upgraded nodes after the test runner
|
||||
* so they are alive during the test.
|
||||
*/
|
||||
finalizedBy "${baseName}#oneThirdUpgradedTestCluster#stop"
|
||||
finalizedBy "${baseName}#twoThirdsUpgradedTestCluster#stop"
|
||||
|
||||
// migration tests should only run when the original/old cluster nodes where versions < 5.2.0.
|
||||
// this stinks but we do the check here since our rest tests do not support conditionals
|
||||
// otherwise we could check the index created version
|
||||
String versionStr = project.extensions.findByName("${baseName}#oldClusterTestCluster").properties.get('bwcVersion')
|
||||
String[] versionParts = versionStr.split('\\.')
|
||||
if (versionParts[0].equals("5")) {
|
||||
Integer minor = Integer.parseInt(versionParts[1])
|
||||
if (minor >= 2) {
|
||||
systemProperty 'tests.rest.blacklist', '/20_security/Verify default password migration results in upgraded cluster'
|
||||
}
|
||||
}
|
||||
// migration tests should only run when the original/old cluster nodes where versions < 5.2.0.
|
||||
// this stinks but we do the check here since our rest tests do not support conditionals
|
||||
// otherwise we could check the index created version
|
||||
String versionStr = project.extensions.findByName("${baseName}#oldClusterTestCluster").properties.get('bwcVersion')
|
||||
String[] versionParts = versionStr.split('\\.')
|
||||
if (versionParts[0].equals("5")) {
|
||||
Integer minor = Integer.parseInt(versionParts[1])
|
||||
if (minor >= 2) {
|
||||
systemProperty 'tests.rest.blacklist', '/20_security/Verify default password migration results in upgraded cluster'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") {
|
||||
dependsOn = [upgradedClusterTest]
|
||||
dependsOn = [upgradedClusterTest]
|
||||
}
|
||||
|
||||
if (project.bwc_tests_enabled) {
|
||||
bwcTest.dependsOn(versionBwcTest)
|
||||
bwcTest.dependsOn(versionBwcTest)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unitTest.enabled = false // no unit tests for rolling upgrades, only the rest integration test
|
||||
|
||||
// basic integ tests includes testing bwc against the most recent version
|
||||
task bwcTestSnapshots {
|
||||
// basic integ tests includes testing bwc against the most recent version
|
||||
task bwcTestSnapshots {
|
||||
if (project.bwc_tests_enabled) {
|
||||
for (final def version : bwcVersions.unreleasedWireCompatible) {
|
||||
dependsOn "v${version}#bwcTest"
|
||||
}
|
||||
for (final def version : bwcVersions.unreleasedWireCompatible) {
|
||||
dependsOn "v${version}#bwcTest"
|
||||
}
|
||||
}
|
||||
}
|
||||
check.dependsOn(bwcTestSnapshots)
|
||||
}
|
||||
check.dependsOn(bwcTestSnapshots)
|
||||
|
||||
dependencies {
|
||||
// "org.elasticsearch.plugin:x-pack-core:${version}" doesn't work with idea because the testArtifacts are also here
|
||||
testCompile project(path: xpackModule('core'), configuration: 'default')
|
||||
testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
|
||||
testCompile project(path: xpackModule('watcher'))
|
||||
}
|
||||
|
||||
compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
|
||||
|
||||
// copy x-pack plugin info so it is on the classpath and security manager has the right permissions
|
||||
task copyXPackRestSpec(type: Copy) {
|
||||
// copy x-pack plugin info so it is on the classpath and security manager has the right permissions
|
||||
task copyXPackRestSpec(type: Copy) {
|
||||
dependsOn(project.configurations.restSpec, 'processTestResources')
|
||||
from project(xpackProject('plugin').path).sourceSets.test.resources
|
||||
include 'rest-api-spec/api/**'
|
||||
into project.sourceSets.test.output.resourcesDir
|
||||
}
|
||||
}
|
||||
|
||||
task copyXPackPluginProps(type: Copy) {
|
||||
task copyXPackPluginProps(type: Copy) {
|
||||
dependsOn(copyXPackRestSpec)
|
||||
from project(xpackModule('core')).file('src/main/plugin-metadata')
|
||||
from project(xpackModule('core')).tasks.pluginProperties
|
||||
into outputDir
|
||||
}
|
||||
project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackPluginProps)
|
||||
|
||||
repositories {
|
||||
maven {
|
||||
url "https://artifacts.elastic.co/maven"
|
||||
}
|
||||
maven {
|
||||
url "https://snapshots.elastic.co/maven"
|
||||
}
|
||||
}
|
||||
}
|
||||
project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackPluginProps)
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
group = "${group}.x-pack.qa.rolling-upgrade.with-system-key"
|
||||
|
|
@ -1 +0,0 @@
|
|||
group = "${group}.x-pack.qa.rolling-upgrade.without-system-key"
|
||||
Loading…
Reference in New Issue