OpenSearch/x-pack/qa/rolling-upgrade/build.gradle

import org.elasticsearch.gradle.test.NodeInfo
import org.elasticsearch.gradle.test.RestIntegTestTask
import org.elasticsearch.gradle.Version

import java.nio.charset.StandardCharsets

// Apply the java plugin to this project so the sources can be edited in an IDE
apply plugin: 'elasticsearch.standalone-test'

test.enabled = false

dependencies {
  testCompile project(':x-pack:qa')
  testCompile project(':client:rest-high-level')
}

Closure waitWithAuth = { NodeInfo node, AntBuilder ant ->
    File tmpFile = new File(node.cwd, 'wait.success')

    // wait up to two minutes
    final long stopTime = System.currentTimeMillis() + (2 * 60000L);
    Exception lastException = null;
    int lastResponseCode = 0

    while (System.currentTimeMillis() < stopTime) {

        lastException = null;
        // we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned
        HttpURLConnection httpURLConnection = null;
        try {
            // TODO this sucks having to hardcode number of nodes, but node.config.numNodes isn't necessarily accurate for rolling
            httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health?wait_for_nodes=3&wait_for_status=yellow").openConnection();
            httpURLConnection.setRequestProperty("Authorization", "Basic " +
                    Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8)));
            httpURLConnection.setRequestMethod("GET");
            httpURLConnection.setConnectTimeout(1000);
            httpURLConnection.setReadTimeout(30000); // read needs to wait for nodes!
            httpURLConnection.connect();
            lastResponseCode = httpURLConnection.getResponseCode()
            if (lastResponseCode == 200) {
                tmpFile.withWriter StandardCharsets.UTF_8.name(), {
                    it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name()))
                }
                break;
            }
        } catch (Exception e) {
            logger.debug("failed to call cluster health", e)
            lastException = e
        } finally {
            if (httpURLConnection != null) {
                httpURLConnection.disconnect();
            }
        }

        // did not start, so wait a bit before trying again
        Thread.sleep(500L);
    }
    if (tmpFile.exists() == false) {
        final String message = "final attempt of calling cluster health failed [lastResponseCode=${lastResponseCode}]"
        if (lastException != null) {
            logger.error(message, lastException)
        } else {
            logger.error(message + " [no exception]")
        }
    }
    return tmpFile.exists()
}

compileTestJava.options.compilerArgs << "-Xlint:-cast,-rawtypes,-unchecked"

forbiddenPatterns {
  exclude '**/system_key'
}

String outputDir = "${buildDir}/generated-resources/${project.name}"

// This is a top level task which we will add dependencies to below.
// It is a single task that can be used to backcompat tests against all versions.
task bwcTest {
    description = 'Runs backwards compatibility tests.'
    group = 'verification'
}

task copyTestNodeKeyMaterial(type: Copy) {
    from project(':x-pack:plugin:core').files('src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.pem',
            'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.crt',
            'src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks')
    into outputDir
}

for (Version version : bwcVersions.wireCompatible) {
    String baseName = "v${version}"
    

    Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) {
        mustRunAfter(precommit)
    }

    configure(extensions.findByName("${baseName}#oldClusterTestCluster")) {
        dependsOn copyTestNodeKeyMaterial
        if (version.before('6.3.0')) {
            String depVersion = version;
            if (project.bwcVersions.unreleased.contains(version)) {
                depVersion += "-SNAPSHOT"
            }
            mavenPlugin 'x-pack', "org.elasticsearch.plugin:x-pack:${depVersion}"
        }
        String usersCli = version.before('6.3.0') ? 'bin/x-pack/users' : 'bin/elasticsearch-users'
        setupCommand 'setupTestUser', usersCli, 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
        bwcVersion = version
        numBwcNodes = 3
        numNodes = 3
        clusterName = 'rolling-upgrade'
        waitCondition = waitWithAuth
        setting 'xpack.monitoring.exporters._http.type', 'http'
        setting 'xpack.monitoring.exporters._http.enabled', 'false'
        setting 'xpack.monitoring.exporters._http.auth.username', 'test_user'
        setting 'xpack.monitoring.exporters._http.auth.password', 'x-pack-test-password'
        setting 'xpack.license.self_generated.type', 'trial'
        setting 'xpack.security.enabled', 'true'
        setting 'xpack.security.transport.ssl.enabled', 'true'
        setting 'xpack.security.authc.token.enabled', 'true'
        setting 'xpack.security.authc.token.timeout', '60m'
        setting 'xpack.security.audit.enabled', 'true'
        rootProject.globalInfo.ready {
            if (project.inFipsJvm) {
                setting 'xpack.security.transport.ssl.key', 'testnode.pem'
                setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
                keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
            } else {
                setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks'
                setting 'xpack.security.transport.ssl.keystore.password', 'testnode'
            }
        }
        dependsOn copyTestNodeKeyMaterial
        extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
        extraConfigFile 'testnode.pem', new File(outputDir + '/testnode.pem')
        extraConfigFile 'testnode.crt', new File(outputDir + '/testnode.crt')
        if (version.onOrAfter('7.0.0')) {
            setting 'xpack.security.authc.realms.file.file1.order', '0'
            setting 'xpack.security.authc.realms.native.native1.order', '1'
        } else {
            setting 'xpack.security.authc.realms.file1.type', 'file'
            setting 'xpack.security.authc.realms.file1.order', '0'
            setting 'xpack.security.authc.realms.native1.type', 'native'
            setting 'xpack.security.authc.realms.native1.order', '1'
        }

        keystoreFile 'xpack.watcher.encryption_key', "${project.projectDir}/src/test/resources/system_key"
        setting 'xpack.watcher.encrypt_sensitive_data', 'true'

        if (version.onOrAfter('6.6.0')) {
            setting 'ccr.auto_follow.wait_for_metadata_timeout', '1s'
        }

        // Old versions of the code contain an invalid assertion that trips
        // during tests.  Versions 5.6.9 and 6.2.4 have been fixed by removing
        // the assertion, but this is impossible for released versions.
        // However, released versions run without assertions, so end users won't
        // be suffering the effects.  This argument effectively removes the
        // incorrect assertion from the older versions used in the BWC tests.
        if (version.before('5.6.9') || (version.onOrAfter('6.0.0') && version.before('6.2.4'))) {
            jvmArgs '-da:org.elasticsearch.xpack.monitoring.exporter.http.HttpExportBulk'
        }
    }

    Task oldClusterTestRunner = tasks.getByName("${baseName}#oldClusterTestRunner")
    oldClusterTestRunner.configure {
        systemProperty 'tests.rest.suite', 'old_cluster'
        systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
        // Dataframe transforms were not added until 7.2.0
        if (version.before('7.2.0')) {
            systemProperty 'tests.rest.blacklist', [
                    'old_cluster/80_data_frame_jobs_crud/Test put batch data frame transforms on old cluster'
            ].join(',')
        }
    }

    Closure configureUpgradeCluster = {String name, Task lastRunner, int stopNode, Closure getOtherUnicastHostAddresses ->
        configure(extensions.findByName("${baseName}#${name}")) {
            dependsOn lastRunner, "${baseName}#oldClusterTestCluster#node${stopNode}.stop"
            setupCommand 'setupTestUser', 'bin/elasticsearch-users', 'useradd', 'test_user', '-p', 'x-pack-test-password', '-r', 'superuser'
            clusterName = 'rolling-upgrade'
            otherUnicastHostAddresses = { getOtherUnicastHostAddresses() }
            minimumMasterNodes = { 2 }
            autoSetInitialMasterNodes = false
            /* Override the data directory so the new node always gets the node we
            * just stopped's data directory. */
            dataDir = { nodeNumber -> oldClusterTest.nodes[stopNode].dataDir }
            waitCondition = waitWithAuth
            setting 'xpack.monitoring.exporters._http.type', 'http'
            setting 'xpack.monitoring.exporters._http.enabled', 'false'
            setting 'xpack.monitoring.exporters._http.auth.username', 'test_user'
            setting 'xpack.monitoring.exporters._http.auth.password', 'x-pack-test-password'
            setting 'xpack.license.self_generated.type', 'trial'
            setting 'xpack.security.enabled', 'true'
            setting 'xpack.security.transport.ssl.enabled', 'true'
            setting 'xpack.security.authc.token.timeout', '60m'
            rootProject.globalInfo.ready {
                if (project.inFipsJvm) {
                    setting 'xpack.security.transport.ssl.key', 'testnode.pem'
                    setting 'xpack.security.transport.ssl.certificate', 'testnode.crt'
                    keystoreSetting 'xpack.security.transport.ssl.secure_key_passphrase', 'testnode'
                } else {
                    setting 'xpack.security.transport.ssl.keystore.path', 'testnode.jks'
                    setting 'xpack.security.transport.ssl.keystore.password', 'testnode'
                }
            }
            setting 'node.attr.upgraded', 'true'
            setting 'xpack.security.authc.token.enabled', 'true'
            setting 'xpack.security.audit.enabled', 'true'
            setting 'node.name', "upgraded-node-${stopNode}"
            dependsOn copyTestNodeKeyMaterial
            extraConfigFile 'testnode.jks', new File(outputDir + '/testnode.jks')
            extraConfigFile 'testnode.pem', new File(outputDir + '/testnode.pem')
            extraConfigFile 'testnode.crt', new File(outputDir + '/testnode.crt')
            setting 'xpack.security.authc.realms.file.file1.order', '0'
            setting 'xpack.security.authc.realms.native.native1.order', '1'
            setting 'xpack.watcher.encrypt_sensitive_data', 'true'
            keystoreFile 'xpack.watcher.encryption_key', "${project.projectDir}/src/test/resources/system_key"
        }
    }

    Task oneThirdUpgradedTest = tasks.create(name: "${baseName}#oneThirdUpgradedTest", type: RestIntegTestTask)

    configureUpgradeCluster("oneThirdUpgradedTestCluster", oldClusterTestRunner, 0,
            // Use all running nodes as seed nodes so there is no race between pinging and the tests
            { [oldClusterTest.nodes.get(1).transportUri(), oldClusterTest.nodes.get(2).transportUri()] })

    Task oneThirdUpgradedTestRunner = tasks.getByName("${baseName}#oneThirdUpgradedTestRunner")
    oneThirdUpgradedTestRunner.configure {
        systemProperty 'tests.rest.suite', 'mixed_cluster'
        systemProperty 'tests.first_round', 'true'
        systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
        // We only need to run these tests once so we may as well do it when we're two thirds upgraded
        def toBlackList = [
                'mixed_cluster/10_basic/Start scroll in mixed cluster on upgraded node that we will continue after upgrade',
                'mixed_cluster/30_ml_jobs_crud/Create a job in the mixed cluster and write some data',
                'mixed_cluster/40_ml_datafeed_crud/Put job and datafeed without aggs in mixed cluster',
                'mixed_cluster/40_ml_datafeed_crud/Put job and datafeed with aggs in mixed cluster',
                'mixed_cluster/80_data_frame_jobs_crud/Test put batch data frame transforms on mixed cluster'
        ]
        // Dataframe transforms were not added until 7.2.0
        if (version.before('7.2.0')) {
            toBlackList << 'mixed_cluster/80_data_frame_jobs_crud/Test GET, start, and stop old cluster batch transforms'
        }
        systemProperty 'tests.rest.blacklist', toBlackList.join(',')
        finalizedBy "${baseName}#oldClusterTestCluster#node1.stop"
    }

    Task twoThirdsUpgradedTest = tasks.create(name: "${baseName}#twoThirdsUpgradedTest", type: RestIntegTestTask)

    configureUpgradeCluster("twoThirdsUpgradedTestCluster", oneThirdUpgradedTestRunner, 1,
            // Use all running nodes as seed nodes so there is no race between pinging and the tests
            { [oldClusterTest.nodes.get(2).transportUri(), oneThirdUpgradedTest.nodes.get(0).transportUri()] })

    Task twoThirdsUpgradedTestRunner = tasks.getByName("${baseName}#twoThirdsUpgradedTestRunner")
    twoThirdsUpgradedTestRunner.configure {
        systemProperty 'tests.rest.suite', 'mixed_cluster'
        systemProperty 'tests.first_round', 'false'
        systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
        finalizedBy "${baseName}#oldClusterTestCluster#node2.stop"
        // Dataframe transforms were not added until 7.2.0
        if (version.before('7.2.0')) {
            systemProperty 'tests.rest.blacklist', [
                    'mixed_cluster/80_data_frame_jobs_crud/Test put batch data frame transforms on mixed cluster',
                    'mixed_cluster/80_data_frame_jobs_crud/Test GET, start, and stop old cluster batch transforms'

            ].join(',')
        }
    }

    Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask)

    configureUpgradeCluster("upgradedClusterTestCluster", twoThirdsUpgradedTestRunner, 2,
            // Use all running nodes as seed nodes so there is no race between pinging and the tests
            { [oneThirdUpgradedTest.nodes.get(0).transportUri(), twoThirdsUpgradedTest.nodes.get(0).transportUri()] })

    Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner")
    upgradedClusterTestRunner.configure {
        systemProperty 'tests.rest.suite', 'upgraded_cluster'
        systemProperty 'tests.upgrade_from_version', version.toString().replace('-SNAPSHOT', '')
        /*
         * Force stopping all the upgraded nodes after the test runner
         * so they are alive during the test.
         */
        finalizedBy "${baseName}#oneThirdUpgradedTestCluster#stop"
        finalizedBy "${baseName}#twoThirdsUpgradedTestCluster#stop"

        // migration tests should only run when the original/old cluster nodes where versions < 5.2.0.
        // this stinks but we do the check here since our rest tests do not support conditionals
        // otherwise we could check the index created version
        String versionStr = project.extensions.findByName("${baseName}#oldClusterTestCluster").properties.get('bwcVersion')
        String[] versionParts = versionStr.split('\\.')
        def toBlackList = []
        if (versionParts[0].equals("5")) {
            Integer minor = Integer.parseInt(versionParts[1])
            if (minor >= 2) {
                toBlackList << '/20_security/Verify default password migration results in upgraded cluster'
            }
        }
        // Dataframe transforms were not added until 7.2.0
        if (version.before('7.2.0')) {
            toBlackList << 'upgraded_cluster/80_data_frame_jobs_crud/Get start, stop, and delete old and mixed cluster batch data frame transforms'
        }
        if (!toBlackList.empty) {
            systemProperty 'tests.rest.blacklist', toBlackList.join(',')
        }
    }

    Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") {
        dependsOn = [upgradedClusterTest]
    }

    if (project.bwc_tests_enabled) {
        bwcTest.dependsOn(versionBwcTest)
    }
}

// basic integ tests includes testing bwc against the most recent version
task bwcTestSnapshots {
    if (project.bwc_tests_enabled) {
        for (final def version : bwcVersions.unreleasedWireCompatible) {
            dependsOn "v${version}#bwcTest"
        }
    }
}
check.dependsOn(bwcTestSnapshots)

// copy x-pack plugin info so it is on the classpath and security manager has the right permissions
task copyXPackRestSpec(type: Copy) {
    dependsOn(project.configurations.restSpec, 'processTestResources')
    from project(xpackProject('plugin').path).sourceSets.test.resources
    include 'rest-api-spec/api/**'
    into project.sourceSets.test.output.resourcesDir
}
project.sourceSets.test.output.dir(outputDir, builtBy: copyXPackRestSpec)