honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

QA: Switch rolling upgrade to 3 nodes (#30728) 

Switches the rolling upgrade tests from upgrading two nodes to upgrading
three nodes which is much more realistic and much better able to find
unexpected bugs. It upgrades the nodes one at a time and runs tests
between each upgrade. As such this now has four test runs:

1. Old
2. One third upgraded
3. Two thirds upgraded
4. Upgraded

It sets system properties so the tests can figure out which stage they
are in. It reuses the same yml tests for the "one third" and "two
thirds" cases because they are *almost* the same case.

This rewrites the yml-based indexing tests to be Java based because the
yml-based tests can't handle different expected values for the counts.
And the indexing tests need that when they are run twice.
This commit is contained in:
Nik Everett 2018-05-21 13:15:13 -04:00 committed by GitHub
parent 42b0b45659
commit be8c0f27be
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 323 additions and 269 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
qa/rolling-upgrade/build.gradle
View File

@ -30,6 +30,26 @@ task bwcTest {
}
for (Version version : bwcVersions.wireCompatible) {
/*
* The goal here is to:
* <ul>
* <li>start three nodes on the old version
* <li>run tests with systemProperty 'tests.rest.suite', 'old_cluster'
* <li>shut down one node
* <li>start a node with the new version
* <li>run tests with systemProperty 'tests.rest.suite', 'mixed_cluster'
* <li>shut down one node on the old version
* <li>start a node with the new version
* <li>run tests with systemProperty 'tests.rest.suite', 'mixed_cluster' again
* <li>shut down the last node with the old version
* <li>start a node with the new version
* <li>run tests with systemProperty 'tests.rest.suite', 'upgraded_cluster'
* <li>shut down the entire cluster
* </ul>
*
* Be careful: gradle dry run spits out tasks in the wrong order but,
* strangely, running the tasks works properly.
*/
String baseName = "v${version}"
Task oldClusterTest = tasks.create(name: "${baseName}#oldClusterTest", type: RestIntegTestTask) {
@ -39,8 +59,8 @@ for (Version version : bwcVersions.wireCompatible) {
Object extension = extensions.findByName("${baseName}#oldClusterTestCluster")
configure(extensions.findByName("${baseName}#oldClusterTestCluster")) {
bwcVersion = version
numBwcNodes = 2
numNodes = 2
numBwcNodes = 3
numNodes = 3
clusterName = 'rolling-upgrade'
setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
if (version.onOrAfter('5.3.0')) {
@ -53,43 +73,57 @@ for (Version version : bwcVersions.wireCompatible) {
systemProperty 'tests.rest.suite', 'old_cluster'
}
Task mixedClusterTest = tasks.create(name: "${baseName}#mixedClusterTest", type: RestIntegTestTask)
configure(extensions.findByName("${baseName}#mixedClusterTestCluster")) {
dependsOn oldClusterTestRunner, "${baseName}#oldClusterTestCluster#node1.stop"
clusterName = 'rolling-upgrade'
unicastTransportUri = { seedNode, node, ant -> oldClusterTest.nodes.get(0).transportUri() }
minimumMasterNodes = { 2 }
/* Override the data directory so the new node always gets the node we
* just stopped's data directory. */
dataDir = { nodeNumber -> oldClusterTest.nodes[1].dataDir }
setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
Closure configureUpgradeCluster = {String name, Task lastRunner, int stopNode, Closure unicastSeed ->
configure(extensions.findByName("${baseName}#${name}")) {
dependsOn lastRunner, "${baseName}#oldClusterTestCluster#node${stopNode}.stop"
clusterName = 'rolling-upgrade'
unicastTransportUri = { seedNode, node, ant -> unicastSeed() }
minimumMasterNodes = { 3 }
/* Override the data directory so the new node always gets the node we
* just stopped's data directory. */
dataDir = { nodeNumber -> oldClusterTest.nodes[stopNode].dataDir }
setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
}
}
Task mixedClusterTestRunner = tasks.getByName("${baseName}#mixedClusterTestRunner")
mixedClusterTestRunner.configure {
Task oneThirdUpgradedTest = tasks.create(name: "${baseName}#oneThirdUpgradedTest", type: RestIntegTestTask)
configureUpgradeCluster("oneThirdUpgradedTestCluster", oldClusterTestRunner,
0, { oldClusterTest.nodes.get(1).transportUri() })
Task oneThirdUpgradedTestRunner = tasks.getByName("${baseName}#oneThirdUpgradedTestRunner")
oneThirdUpgradedTestRunner.configure {
systemProperty 'tests.rest.suite', 'mixed_cluster'
finalizedBy "${baseName}#oldClusterTestCluster#node0.stop"
systemProperty 'tests.first_round', 'true'
finalizedBy "${baseName}#oldClusterTestCluster#node1.stop"
}
Task twoThirdsUpgradedTest = tasks.create(name: "${baseName}#twoThirdsUpgradedTest", type: RestIntegTestTask)
configureUpgradeCluster("twoThirdsUpgradedTestCluster", oneThirdUpgradedTestRunner,
1, { oneThirdUpgradedTest.nodes.get(0).transportUri() })
Task twoThirdsUpgradedTestRunner = tasks.getByName("${baseName}#twoThirdsUpgradedTestRunner")
twoThirdsUpgradedTestRunner.configure {
systemProperty 'tests.rest.suite', 'mixed_cluster'
systemProperty 'tests.first_round', 'false'
finalizedBy "${baseName}#oldClusterTestCluster#node2.stop"
}
Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask)
configure(extensions.findByName("${baseName}#upgradedClusterTestCluster")) {
dependsOn mixedClusterTestRunner, "${baseName}#oldClusterTestCluster#node0.stop"
clusterName = 'rolling-upgrade'
unicastTransportUri = { seedNode, node, ant -> mixedClusterTest.nodes.get(0).transportUri() }
minimumMasterNodes = { 2 }
/* Override the data directory so the new node always gets the node we
* just stopped's data directory. */
dataDir = { nodeNumber -> oldClusterTest.nodes[0].dataDir}
setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
}
configureUpgradeCluster("upgradedClusterTestCluster", twoThirdsUpgradedTestRunner,
2, { twoThirdsUpgradedTest.nodes.get(0).transportUri() })
Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner")
upgradedClusterTestRunner.configure {
systemProperty 'tests.rest.suite', 'upgraded_cluster'
// only need to kill the mixed cluster tests node here because we explicitly told it to not stop nodes upon completion
finalizedBy "${baseName}#mixedClusterTestCluster#stop"
/*
* Force stopping all the upgraded nodes after the test runner
* so they are alive during the test.
*/
finalizedBy "${baseName}#oneThirdUpgradedTestCluster#stop"
finalizedBy "${baseName}#twoThirdsUpgradedTestCluster#stop"
}
Task versionBwcTest = tasks.create(name: "${baseName}#bwcTest") {

91
qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/AbstractRollingTestCase.java Normal file
View File

@ -0,0 +1,91 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.upgrades;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.elasticsearch.Version;
import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.client.Response;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.test.rest.yaml.ObjectPath;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Future;
import java.util.function.Predicate;
import static com.carrotsearch.randomizedtesting.RandomizedTest.randomAsciiOfLength;
import static java.util.Collections.emptyMap;
import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING;
import static org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING;
import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.notNullValue;
public abstract class AbstractRollingTestCase extends ESRestTestCase {
protected enum ClusterType {
OLD,
MIXED,
UPGRADED;
public static ClusterType parse(String value) {
switch (value) {
case "old_cluster":
return OLD;
case "mixed_cluster":
return MIXED;
case "upgraded_cluster":
return UPGRADED;
default:
throw new AssertionError("unknown cluster type: " + value);
}
}
}
protected static final ClusterType CLUSTER_TYPE = ClusterType.parse(System.getProperty("tests.rest.suite"));
@Override
protected final boolean preserveIndicesUponCompletion() {
return true;
}
@Override
protected final boolean preserveReposUponCompletion() {
return true;
}
@Override
protected final Settings restClientSettings() {
return Settings.builder().put(super.restClientSettings())
// increase the timeout here to 90 seconds to handle long waits for a green
// cluster health. the waits for green need to be longer than a minute to
// account for delayed shards
.put(ESRestTestCase.CLIENT_RETRY_TIMEOUT, "90s")
.put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s")
.build();
}
}

135
qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java Normal file
View File

@ -0,0 +1,135 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.upgrades;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.common.Booleans;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
/**
* Basic test that indexed documents survive the rolling restart. See
* {@link RecoveryIT} for much more in depth testing of the mechanism
* by which they survive.
*/
public class IndexingIT extends AbstractRollingTestCase {
public void testIndexing() throws IOException {
switch (CLUSTER_TYPE) {
case OLD:
break;
case MIXED:
Request waitForYellow = new Request("GET", "/_cluster/health");
waitForYellow.addParameter("wait_for_nodes", "3");
waitForYellow.addParameter("wait_for_status", "yellow");
client().performRequest(waitForYellow);
break;
case UPGRADED:
Request waitForGreen = new Request("GET", "/_cluster/health/test_index,index_with_replicas,empty_index");
waitForGreen.addParameter("wait_for_nodes", "3");
waitForGreen.addParameter("wait_for_status", "green");
// wait for long enough that we give delayed unassigned shards to stop being delayed
waitForGreen.addParameter("timeout", "70s");
waitForGreen.addParameter("level", "shards");
client().performRequest(waitForGreen);
break;
default:
throw new UnsupportedOperationException("Unknown cluster type [" + CLUSTER_TYPE + "]");
}
if (CLUSTER_TYPE == ClusterType.OLD) {
Request createTestIndex = new Request("PUT", "/test_index");
createTestIndex.setJsonEntity("{\"settings\": {\"index.number_of_replicas\": 0}}");
client().performRequest(createTestIndex);
String recoverQuickly = "{\"settings\": {\"index.unassigned.node_left.delayed_timeout\": \"100ms\"}}";
Request createIndexWithReplicas = new Request("PUT", "/index_with_replicas");
createIndexWithReplicas.setJsonEntity(recoverQuickly);
client().performRequest(createIndexWithReplicas);
Request createEmptyIndex = new Request("PUT", "/empty_index");
// Ask for recovery to be quick
createEmptyIndex.setJsonEntity(recoverQuickly);
client().performRequest(createEmptyIndex);
bulk("test_index", "_OLD", 5);
bulk("index_with_replicas", "_OLD", 5);
}
int expectedCount;
switch (CLUSTER_TYPE) {
case OLD:
expectedCount = 5;
break;
case MIXED:
if (Booleans.parseBoolean(System.getProperty("tests.first_round"))) {
expectedCount = 5;
} else {
expectedCount = 10;
}
break;
case UPGRADED:
expectedCount = 15;
break;
default:
throw new UnsupportedOperationException("Unknown cluster type [" + CLUSTER_TYPE + "]");
}
assertCount("test_index", expectedCount);
assertCount("index_with_replicas", 5);
assertCount("empty_index", 0);
if (CLUSTER_TYPE != ClusterType.OLD) {
bulk("test_index", "_" + CLUSTER_TYPE, 5);
Request toBeDeleted = new Request("PUT", "/test_index/doc/to_be_deleted");
toBeDeleted.addParameter("refresh", "true");
toBeDeleted.setJsonEntity("{\"f1\": \"delete-me\"}");
client().performRequest(toBeDeleted);
assertCount("test_index", expectedCount + 6);
Request delete = new Request("DELETE", "/test_index/doc/to_be_deleted");
delete.addParameter("refresh", "true");
client().performRequest(delete);
assertCount("test_index", expectedCount + 5);
}
}
private void bulk(String index, String valueSuffix, int count) throws IOException {
StringBuilder b = new StringBuilder();
for (int i = 0; i < count; i++) {
b.append("{\"index\": {\"_index\": \"").append(index).append("\", \"_type\": \"doc\"}}\n");
b.append("{\"f1\": \"v").append(i).append(valueSuffix).append("\", \"f2\": ").append(i).append("}\n");
}
Request bulk = new Request("POST", "/_bulk");
bulk.addParameter("refresh", "true");
bulk.setJsonEntity(b.toString());
client().performRequest(bulk);
}
private void assertCount(String index, int count) throws IOException {
Request searchTestIndexRequest = new Request("POST", "/" + index + "/_search");
searchTestIndexRequest.addParameter("filter_path", "hits.total");
Response searchTestIndexResponse = client().performRequest(searchTestIndexRequest);
assertEquals("{\"hits\":{\"total\":" + count + "}}",
EntityUtils.toString(searchTestIndexResponse.getEntity(), StandardCharsets.UTF_8));
}
}

69
qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java
View File

@ -46,53 +46,13 @@ import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.notNullValue;
public class RecoveryIT extends ESRestTestCase {
@Override
protected boolean preserveIndicesUponCompletion() {
return true;
}
@Override
protected boolean preserveReposUponCompletion() {
return true;
}
private enum CLUSTER_TYPE {
OLD,
MIXED,
UPGRADED;
public static CLUSTER_TYPE parse(String value) {
switch (value) {
case "old_cluster":
return OLD;
case "mixed_cluster":
return MIXED;
case "upgraded_cluster":
return UPGRADED;
default:
throw new AssertionError("unknown cluster type: " + value);
}
}
}
private final CLUSTER_TYPE clusterType = CLUSTER_TYPE.parse(System.getProperty("tests.rest.suite"));
@Override
protected Settings restClientSettings() {
return Settings.builder().put(super.restClientSettings())
// increase the timeout here to 90 seconds to handle long waits for a green
// cluster health. the waits for green need to be longer than a minute to
// account for delayed shards
.put(ESRestTestCase.CLIENT_RETRY_TIMEOUT, "90s")
.put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s")
.build();
}
/**
* In depth testing of the recovery mechanism during a rolling restart.
*/
public class RecoveryIT extends AbstractRollingTestCase {
public void testHistoryUUIDIsGenerated() throws Exception {
final String index = "index_history_uuid";
if (clusterType == CLUSTER_TYPE.OLD) {
if (CLUSTER_TYPE == ClusterType.OLD) {
Settings.Builder settings = Settings.builder()
.put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
.put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
@ -102,7 +62,7 @@ public class RecoveryIT extends ESRestTestCase {
// before timing out
.put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms");
createIndex(index, settings.build());
} else if (clusterType == CLUSTER_TYPE.UPGRADED) {
} else if (CLUSTER_TYPE == ClusterType.UPGRADED) {
ensureGreen(index);
Response response = client().performRequest("GET", index + "/_stats", Collections.singletonMap("level", "shards"));
assertOK(response);
@ -157,11 +117,11 @@ public class RecoveryIT extends ESRestTestCase {
final Map<String, Object> nodeMap = objectPath.evaluate("nodes");
List<String> nodes = new ArrayList<>(nodeMap.keySet());
switch (clusterType) {
switch (CLUSTER_TYPE) {
case OLD:
Settings.Builder settings = Settings.builder()
.put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
.put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
.put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2)
// if the node with the replica is the first to be restarted, while a replica is still recovering
// then delayed allocation will kick in. When the node comes back, the master will search for a copy
// but the recovering copy will be seen as invalid and the cluster health won't return to GREEN
@ -181,6 +141,7 @@ public class RecoveryIT extends ESRestTestCase {
assertOK(client().performRequest("POST", index + "/_refresh"));
assertCount(index, "_only_nodes:" + nodes.get(0), 60);
assertCount(index, "_only_nodes:" + nodes.get(1), 60);
assertCount(index, "_only_nodes:" + nodes.get(2), 60);
// make sure that we can index while the replicas are recovering
updateIndexSettings(index, Settings.builder().put(INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "primaries"));
break;
@ -191,9 +152,10 @@ public class RecoveryIT extends ESRestTestCase {
assertOK(client().performRequest("POST", index + "/_refresh"));
assertCount(index, "_only_nodes:" + nodes.get(0), 110);
assertCount(index, "_only_nodes:" + nodes.get(1), 110);
assertCount(index, "_only_nodes:" + nodes.get(2), 110);
break;
default:
throw new IllegalStateException("unknown type " + clusterType);
throw new IllegalStateException("unknown type " + CLUSTER_TYPE);
}
}
@ -221,11 +183,11 @@ public class RecoveryIT extends ESRestTestCase {
public void testRelocationWithConcurrentIndexing() throws Exception {
final String index = "relocation_with_concurrent_indexing";
switch (clusterType) {
switch (CLUSTER_TYPE) {
case OLD:
Settings.Builder settings = Settings.builder()
.put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
.put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
.put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2)
// if the node with the replica is the first to be restarted, while a replica is still recovering
// then delayed allocation will kick in. When the node comes back, the master will search for a copy
// but the recovering copy will be seen as invalid and the cluster health won't return to GREEN
@ -258,7 +220,7 @@ public class RecoveryIT extends ESRestTestCase {
break;
case UPGRADED:
updateIndexSettings(index, Settings.builder()
.put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
.put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2)
.put("index.routing.allocation.include._id", (String)null)
);
asyncIndexDocs(index, 60, 50).get();
@ -271,9 +233,10 @@ public class RecoveryIT extends ESRestTestCase {
assertCount(index, "_only_nodes:" + nodes.get(0), 110);
assertCount(index, "_only_nodes:" + nodes.get(1), 110);
assertCount(index, "_only_nodes:" + nodes.get(2), 110);
break;
default:
throw new IllegalStateException("unknown type " + clusterType);
throw new IllegalStateException("unknown type " + CLUSTER_TYPE);
}
}

1
qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java
View File

@ -60,4 +60,3 @@ public class UpgradeClusterClientYamlTestSuiteIT extends ESClientYamlSuiteTestCa
.build();
}
}

68
qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml
View File

@ -1,74 +1,8 @@
---
"Index data and search on the mixed cluster":
- do:
cluster.health:
wait_for_status: yellow
wait_for_nodes: 2
- do:
search:
index: test_index
- match: { hits.total: 5 } # no new indexed data, so expect the original 5 documents from the old cluster
- do:
search:
index: index_with_replicas
- match: { hits.total: 5 } # just check we recovered fine
- do:
bulk:
refresh: true
body:
- '{"index": {"_index": "test_index", "_type": "doc"}}'
- '{"f1": "v1_mixed", "f2": 5}'
- '{"index": {"_index": "test_index", "_type": "doc"}}'
- '{"f1": "v2_mixed", "f2": 6}'
- '{"index": {"_index": "test_index", "_type": "doc"}}'
- '{"f1": "v3_mixed", "f2": 7}'
- '{"index": {"_index": "test_index", "_type": "doc"}}'
- '{"f1": "v4_mixed", "f2": 8}'
- '{"index": {"_index": "test_index", "_type": "doc"}}'
- '{"f1": "v5_mixed", "f2": 9}'
- do:
index:
index: test_index
type: doc
id: d10
body: {"f1": "v6_mixed", "f2": 10}
- do:
indices.refresh:
index: test_index
- do:
search:
index: test_index
- match: { hits.total: 11 } # 5 docs from old cluster, 6 docs from mixed cluster
- do:
delete:
index: test_index
type: doc
id: d10
- do:
indices.refresh:
index: test_index
- do:
search:
index: test_index
- match: { hits.total: 10 }
---
"Verify that we can still find things with the template":
- do:
search_template:
index: test_search_template
body:
id: test_search_template
params:

89
qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml
View File

@ -1,76 +1,5 @@
---
"Index data, search, and create things in the cluster state that we'll validate are there after the ugprade":
- do:
indices.create:
index: test_index
body:
settings:
index:
number_of_replicas: 0
- do:
indices.create:
index: index_with_replicas # dummy index to ensure we can recover indices with replicas just fine
body:
# if the node with the replica is the first to be restarted, then delayed
# allocation will kick in, and the cluster health won't return to GREEN
# before timing out
index.unassigned.node_left.delayed_timeout: "100ms"
- do:
indices.create:
index: empty_index # index to ensure we can recover empty indices
body:
# if the node with the replica is the first to be restarted, then delayed
# allocation will kick in, and the cluster health won't return to GREEN
# before timing out
index.unassigned.node_left.delayed_timeout: "100ms"
- do:
bulk:
refresh: true
body:
- '{"index": {"_index": "test_index", "_type": "doc"}}'
- '{"f1": "v1_old", "f2": 0}'
- '{"index": {"_index": "test_index", "_type": "doc"}}'
- '{"f1": "v2_old", "f2": 1}'
- '{"index": {"_index": "test_index", "_type": "doc"}}'
- '{"f1": "v3_old", "f2": 2}'
- '{"index": {"_index": "test_index", "_type": "doc"}}'
- '{"f1": "v4_old", "f2": 3}'
- '{"index": {"_index": "test_index", "_type": "doc"}}'
- '{"f1": "v5_old", "f2": 4}'
- do:
bulk:
refresh: true
body:
- '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
- '{"f1": "d_old"}'
- '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
- '{"f1": "d_old"}'
- '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
- '{"f1": "d_old"}'
- '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
- '{"f1": "d_old"}'
- '{"index": {"_index": "index_with_replicas", "_type": "doc"}}'
- '{"f1": "d_old"}'
- do:
indices.refresh:
index: test_index,index_with_replicas
- do:
search:
index: test_index
- match: { hits.total: 5 }
- do:
search:
index: index_with_replicas
- match: { hits.total: 5 }
"Create things in the cluster state that we'll validate are there after the ugprade":
- do:
snapshot.create_repository:
repository: my_repo
@ -91,6 +20,21 @@
}
- match: { "acknowledged": true }
- do:
bulk:
refresh: true
body:
- '{"index": {"_index": "test_search_template", "_type": "doc"}}'
- '{"f1": "v1_old"}'
- '{"index": {"_index": "test_search_template", "_type": "doc"}}'
- '{"f1": "v2_old"}'
- '{"index": {"_index": "test_search_template", "_type": "doc"}}'
- '{"f1": "v3_old"}'
- '{"index": {"_index": "test_search_template", "_type": "doc"}}'
- '{"f1": "v4_old"}'
- '{"index": {"_index": "test_search_template", "_type": "doc"}}'
- '{"f1": "v5_old"}'
- do:
put_script:
id: test_search_template
@ -105,6 +49,7 @@
- do:
search_template:
index: test_search_template
body:
id: test_search_template
params:

49
qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml
View File

@ -1,55 +1,8 @@
---
"Index data and search on the upgraded cluster":
- do:
cluster.health:
wait_for_status: green
wait_for_nodes: 2
# wait for long enough that we give delayed unassigned shards to stop being delayed
timeout: 70s
level: shards
index: test_index,index_with_replicas,empty_index
- do:
search:
index: test_index
- match: { hits.total: 10 } # no new indexed data, so expect the original 10 documents from the old and mixed clusters
- do:
search:
index: index_with_replicas
- match: { hits.total: 5 } # just check we recovered fine
- do:
bulk:
refresh: true
body:
- '{"index": {"_index": "test_index", "_type": "doc"}}'
- '{"f1": "v1_upgraded", "f2": 10}'
- '{"index": {"_index": "test_index", "_type": "doc"}}'
- '{"f1": "v2_upgraded", "f2": 11}'
- '{"index": {"_index": "test_index", "_type": "doc"}}'
- '{"f1": "v3_upgraded", "f2": 12}'
- '{"index": {"_index": "test_index", "_type": "doc"}}'
- '{"f1": "v4_upgraded", "f2": 13}'
- '{"index": {"_index": "test_index", "_type": "doc"}}'
- '{"f1": "v5_upgraded", "f2": 14}'
- do:
indices.refresh:
index: test_index
- do:
search:
index: test_index
- match: { hits.total: 15 } # 10 docs from previous clusters plus 5 new docs
---
"Verify that we can still find things with the template":
- do:
search_template:
index: test_search_template
body:
id: test_search_template
params: