honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-17 02:14:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix bwc cluster formation in order to run BWC tests against a mixed version cluster (#21145)

Browse Source 
This fixes our cluster formation task to run REST tests against a mixed version cluster.
Yet, due to some limitations in our test framework `indices.rollover` tests are currently
disabled for the BWC case since they select the current master as the merge node which
happens to be a BWC node and we can't relocate all shards to it since the primaries are on
a higher version node. This will be fixed in a followup.

Closes #21142

Note: This has been cherry-picked from 5.0 and fixes several rest tests
as well as a BWC break in `OsStats.java`
This commit is contained in:
Simon Willnauer 2016-10-27 13:38:40 +02:00 committed by Simon Willnauer
parent 9944a594b1
commit 97cc426a89
12 changed files with 84 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
View File

@ -73,8 +73,8 @@ class ClusterFormationTasks {
}
// this is our current version distribution configuration we use for all kinds of REST tests etc.
String distroConfigName = "${task.name}_elasticsearchDistro"
Configuration distro = project.configurations.create(distroConfigName)
configureDistributionDependency(project, config.distribution, distro, VersionProperties.elasticsearch)
Configuration currentDistro = project.configurations.create(distroConfigName)
configureDistributionDependency(project, config.distribution, currentDistro, VersionProperties.elasticsearch)
if (config.bwcVersion != null && config.numBwcNodes > 0) {
// if we have a cluster that has a BWC cluster we also need to configure a dependency on the BWC version
// this version uses the same distribution etc. and only differs in the version we depend on.
@ -85,11 +85,11 @@ class ClusterFormationTasks {
}
configureDistributionDependency(project, config.distribution, project.configurations.elasticsearchBwcDistro, config.bwcVersion)
}
for (int i = 0; i < config.numNodes; ++i) {
for (int i = 0; i < config.numNodes; i++) {
// we start N nodes and out of these N nodes there might be M bwc nodes.
// for each of those nodes we might have a different configuratioon
String elasticsearchVersion = VersionProperties.elasticsearch
Configuration distro = currentDistro
if (i < config.numBwcNodes) {
elasticsearchVersion = config.bwcVersion
distro = project.configurations.elasticsearchBwcDistro
@ -252,7 +252,7 @@ class ClusterFormationTasks {
'path.repo' : "${node.sharedDir}/repo",
'path.shared_data' : "${node.sharedDir}/",
// Define a node attribute so we can test that it exists
'node.attr.testattr' : 'test',
'node.attr.testattr' : 'test',
'repositories.url.allowed_urls': 'http://snapshot.test*'
]
esConfig['node.max_local_storage_nodes'] = node.config.numNodes

13
core/src/main/java/org/elasticsearch/monitor/os/OsStats.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.monitor.os;
import org.elasticsearch.Version;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
@ -31,7 +32,7 @@ import java.util.Arrays;
import java.util.Objects;
public class OsStats implements Writeable, ToXContent {
public static final Version V_5_1_0 = Version.fromId(5010099);
private final long timestamp;
private final Cpu cpu;
private final Mem mem;
@ -51,7 +52,11 @@ public class OsStats implements Writeable, ToXContent {
this.cpu = new Cpu(in);
this.mem = new Mem(in);
this.swap = new Swap(in);
this.cgroup = in.readOptionalWriteable(Cgroup::new);
if (in.getVersion().onOrAfter(V_5_1_0)) {
this.cgroup = in.readOptionalWriteable(Cgroup::new);
} else {
this.cgroup = null;
}
}
@Override
@ -60,7 +65,9 @@ public class OsStats implements Writeable, ToXContent {
cpu.writeTo(out);
mem.writeTo(out);
swap.writeTo(out);
out.writeOptionalWriteable(cgroup);
if (out.getVersion().onOrAfter(V_5_1_0)) {
out.writeOptionalWriteable(cgroup);
}
}
public long getTimestamp() {

2
core/src/test/java/org/elasticsearch/VersionTests.java
View File

@ -23,6 +23,7 @@ import org.elasticsearch.action.ShardValidateQueryRequestTests;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.monitor.os.OsStats;
import org.elasticsearch.search.internal.AliasFilter;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.VersionUtils;
@ -271,6 +272,7 @@ public class VersionTests extends ESTestCase {
assertUnknownVersion(V_20_0_0_UNRELEASED);
expectThrows(AssertionError.class, () -> assertUnknownVersion(Version.CURRENT));
assertUnknownVersion(AliasFilter.V_5_1_0); // once we released 5.1.0 and it's added to Version.java we need to remove this constant
assertUnknownVersion(OsStats.V_5_1_0); // once we released 5.1.0 and it's added to Version.java we need to remove this constant
// once we released 5.0.0 and it's added to Version.java we need to remove this constant
assertUnknownVersion(ShardValidateQueryRequestTests.V_5_0_0);
}

4
rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/10_basic.yaml
View File

@ -219,6 +219,10 @@
---
"Alias sorting":
- skip:
version: " - 5.0.99"
reason: sorting was introduced in 5.1.0
- do:
indices.create:
index: test_index

4
rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml
View File

@ -160,6 +160,10 @@
---
"Test cat indices sort":
- skip:
version: " - 5.0.99"
reason: sorting was introduced in 5.1.0
- do:
indices.create:
index: foo

4
rest-api-spec/src/main/resources/rest-api-spec/test/cat.repositories/10_basic.yaml
View File

@ -46,7 +46,9 @@
---
"Test cat repositories sort":
- skip:
version: " - 5.0.99"
reason: sorting was introduced in 5.1.0
- do:
snapshot.create_repository:
repository: test_cat_repo_1

3
rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml
View File

@ -227,6 +227,9 @@
---
"Test cat shards sort":
- skip:
version: " - 5.0.99"
reason: sorting was introduced in 5.1.0
- do:
indices.create:

28
rest-api-spec/src/main/resources/rest-api-spec/test/cat.templates/10_basic.yaml
View File

@ -1,6 +1,8 @@
---
"Help":
- skip:
version: " - 5.0.99"
reason: templates were introduced in 5.1.0
- do:
cat.templates:
help: true
@ -15,7 +17,9 @@
---
"No templates":
- skip:
version: " - 5.0.99"
reason: templates were introduced in 5.1.0
- do:
cat.templates: {}
@ -26,7 +30,9 @@
---
"Normal templates":
- skip:
version: " - 5.0.99"
reason: templates were introduced in 5.1.0
- do:
indices.put_template:
name: test
@ -72,7 +78,9 @@
---
"Filtered templates":
- skip:
version: " - 5.0.99"
reason: templates were introduced in 5.1.0
- do:
indices.put_template:
name: test
@ -111,7 +119,9 @@
---
"Column headers":
- skip:
version: " - 5.0.99"
reason: templates were introduced in 5.1.0
- do:
indices.put_template:
name: test
@ -145,7 +155,9 @@
---
"Select columns":
- skip:
version: " - 5.0.99"
reason: templates were introduced in 5.1.0
- do:
indices.put_template:
name: test
@ -176,7 +188,9 @@
---
"Sort templates":
- skip:
version: " - 5.0.99"
reason: templates were introduced in 5.1.0
- do:
indices.put_template:
name: test

1
rest-api-spec/src/main/resources/rest-api-spec/test/get/90_versions.yaml
View File

@ -86,3 +86,4 @@
id: 1
version: 1
version_type: external_gte

24
rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml
View File

@ -71,6 +71,22 @@
- match: { hits.total: 1 }
- match: { hits.hits.0._index: "logs-000002"}
---
"Rollover no condition matched":
- skip:
version: " - 5.0.0"
reason: bug fixed in 5.0.1
# create index with alias
- do:
indices.create:
index: logs-1
wait_for_active_shards: 1
body:
aliases:
logs_index: {}
logs_search: {}
# run again and verify results without rolling over
- do:
indices.rollover:
@ -78,11 +94,11 @@
wait_for_active_shards: 1
body:
conditions:
max_docs: 100
max_docs: 1
- match: { old_index: logs-000002 }
- match: { new_index: logs-000003 }
- match: { old_index: logs-1 }
- match: { new_index: logs-000002 }
- match: { rolled_over: false }
- match: { dry_run: false }
- match: { conditions: { "[max_docs: 100]": false } }
- match: { conditions: { "[max_docs: 1]": false } }

6
rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yaml
View File

@ -1,5 +1,11 @@
---
"Shrink index via API":
- skip:
version: " - 5.0.0"
reason: this doesn't work yet with BWC tests since the master is from the old verion
# TODO we need to fix this for BWC tests to make sure we get a node that all shards can allocate on
# today if we run BWC tests and we select the master as a _shrink node but since primaries are allocated
# on the newer version nodes this fails...
# creates an index with one document.
# relocates all it's shards to one node
# shrinks it into a new index with a single shard

9
test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSectionParser.java
View File

@ -36,16 +36,17 @@ public class ClientYamlTestSectionParser implements ClientYamlTestFragmentParser
try {
parser.nextToken();
testSection.setSkipSection(parseContext.parseSkipSection());
while ( parser.currentToken() != XContentParser.Token.END_ARRAY) {
parseContext.advanceToFieldName();
testSection.addExecutableSection(parseContext.parseExecutableSection());
}
parser.nextToken();
assert parser.currentToken() == XContentParser.Token.END_OBJECT;
assert parser.currentToken() == XContentParser.Token.END_OBJECT : "malformed section [" + testSection.getName() + "] expected "
+ XContentParser.Token.END_OBJECT + " but was " + parser.currentToken();
parser.nextToken();
return testSection;
} catch (Exception e) {
throw new ClientYamlTestParseException("Error parsing test named [" + testSection.getName() + "]", e);