Add option to preserve data in test clusters (#65400)

(cherry picked from commit 1ce323e1368cf5231181f1efaba1c4e425066e37)
This commit is contained in:
Mark Vieira 2020-11-24 11:24:26 -08:00
parent 7f7e938a25
commit f8f5d27f6b
8 changed files with 90 additions and 18 deletions

View File

@ -82,12 +82,13 @@ password: `elastic-password`.
==== Other useful arguments ==== Other useful arguments
In order to start a node with a different max heap space add: `-Dtests.heap.size=4G` - In order to start a node with a different max heap space add: `-Dtests.heap.size=4G`
In order to disable assertions add: `-Dtests.asserts=false` - In order to disable assertions add: `-Dtests.asserts=false`
In order to use a custom data directory: `--data-dir=/tmp/foo` - In order to use a custom data directory: `--data-dir=/tmp/foo`
In order to remotely attach a debugger to the process: `--debug-jvm` - In order to preserve data in between executions: `--preserve-data`
In order to set a different keystore password: `--keystore-password` - In order to remotely attach a debugger to the process: `--debug-jvm`
In order to set an Elasticsearch setting, provide a setting with the following prefix: `-Dtests.es.` - In order to set a different keystore password: `--keystore-password`
- In order to set an Elasticsearch setting, provide a setting with the following prefix: `-Dtests.es.`
=== Test case filtering. === Test case filtering.
@ -311,7 +312,7 @@ The REST tests are run automatically when executing the "./gradlew check" comman
YAML REST tests use the following command (modules and plugins may also include YAML REST tests): YAML REST tests use the following command (modules and plugins may also include YAML REST tests):
--------------------------------------------------------------------------- ---------------------------------------------------------------------------
./gradlew :rest-api-spec:yamlRestTest ./gradlew :rest-api-spec:yamlRestTest
--------------------------------------------------------------------------- ---------------------------------------------------------------------------
A specific test case can be run with the following command: A specific test case can be run with the following command:
@ -319,7 +320,7 @@ A specific test case can be run with the following command:
--------------------------------------------------------------------------- ---------------------------------------------------------------------------
./gradlew ':rest-api-spec:yamlRestTest' \ ./gradlew ':rest-api-spec:yamlRestTest' \
--tests "org.elasticsearch.test.rest.ClientYamlTestSuiteIT" \ --tests "org.elasticsearch.test.rest.ClientYamlTestSuiteIT" \
-Dtests.method="test {p0=cat.segments/10_basic/Help}" -Dtests.method="test {p0=cat.segments/10_basic/Help}"
--------------------------------------------------------------------------- ---------------------------------------------------------------------------
The YAML REST tests support all the options provided by the randomized runner, plus the following: The YAML REST tests support all the options provided by the randomized runner, plus the following:
@ -337,14 +338,14 @@ Java REST tests can be run with the "javaRestTest" task.
For example : For example :
--------------------------------------------------------------------------- ---------------------------------------------------------------------------
./gradlew :modules:mapper-extras:javaRestTest ./gradlew :modules:mapper-extras:javaRestTest
--------------------------------------------------------------------------- ---------------------------------------------------------------------------
A specific test case can be run with the following syntax (fqn.test {params}): A specific test case can be run with the following syntax (fqn.test {params}):
--------------------------------------------------------------------------- ---------------------------------------------------------------------------
./gradlew ':modules:mapper-extras:javaRestTest' \ ./gradlew ':modules:mapper-extras:javaRestTest' \
--tests "org.elasticsearch.index.mapper.TokenCountFieldMapperIntegrationIT.testSearchByTokenCount {storeCountedFields=true loadCountedFields=false}" --tests "org.elasticsearch.index.mapper.TokenCountFieldMapperIntegrationIT.testSearchByTokenCount {storeCountedFields=true loadCountedFields=false}"
--------------------------------------------------------------------------- ---------------------------------------------------------------------------
yamlRestTest's and javaRestTest's are easy to identify, since they are found in a yamlRestTest's and javaRestTest's are easy to identify, since they are found in a

View File

@ -282,6 +282,16 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
nodes.all(each -> each.jvmArgs(values)); nodes.all(each -> each.jvmArgs(values));
} }
@Internal
public boolean isPreserveDataDir() {
return nodes.stream().anyMatch(node -> node.isPreserveDataDir());
}
@Override
public void setPreserveDataDir(boolean preserveDataDir) {
nodes.all(each -> each.setPreserveDataDir(preserveDataDir));
}
@Override @Override
public void freeze() { public void freeze() {
nodes.forEach(ElasticsearchNode::freeze); nodes.forEach(ElasticsearchNode::freeze);

View File

@ -167,6 +167,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
private String transportPort = "0"; private String transportPort = "0";
private Path confPathData; private Path confPathData;
private String keystorePassword = ""; private String keystorePassword = "";
private boolean preserveDataDir = false;
ElasticsearchNode( ElasticsearchNode(
String path, String path,
@ -421,6 +422,17 @@ public class ElasticsearchNode implements TestClusterConfiguration {
return configFile.getParent(); return configFile.getParent();
} }
@Override
@Input
public boolean isPreserveDataDir() {
return preserveDataDir;
}
@Override
public void setPreserveDataDir(boolean preserveDataDir) {
this.preserveDataDir = preserveDataDir;
}
@Override @Override
public void freeze() { public void freeze() {
requireNonNull(testDistribution, "null testDistribution passed when configuring test cluster `" + this + "`"); requireNonNull(testDistribution, "null testDistribution passed when configuring test cluster `" + this + "`");
@ -452,7 +464,13 @@ public class ElasticsearchNode implements TestClusterConfiguration {
logToProcessStdout("Configuring working directory: " + workingDir); logToProcessStdout("Configuring working directory: " + workingDir);
// make sure we always start fresh // make sure we always start fresh
if (Files.exists(workingDir)) { if (Files.exists(workingDir)) {
fileSystemOperations.delete(d -> d.delete(workingDir)); if (preserveDataDir) {
Files.list(workingDir)
.filter(path -> path.equals(confPathData) == false)
.forEach(path -> fileSystemOperations.delete(d -> d.delete(path)));
} else {
fileSystemOperations.delete(d -> d.delete(workingDir));
}
} }
isWorkingDirConfigured = true; isWorkingDirConfigured = true;
} }

View File

@ -46,6 +46,8 @@ public class RunTask extends DefaultTestClustersTask {
private Boolean debug = false; private Boolean debug = false;
private Boolean preserveData = false;
private Path dataDir = null; private Path dataDir = null;
private String keystorePassword = ""; private String keystorePassword = "";
@ -65,6 +67,16 @@ public class RunTask extends DefaultTestClustersTask {
dataDir = Paths.get(dataDirStr).toAbsolutePath(); dataDir = Paths.get(dataDirStr).toAbsolutePath();
} }
@Input
public Boolean getPreserveData() {
return preserveData;
}
@Option(option = "preserve-data", description = "Preserves data directory contents (path provided to --data-dir is always preserved)")
public void setPreserveData(Boolean preserveData) {
this.preserveData = preserveData;
}
@Option(option = "keystore-password", description = "Set the elasticsearch keystore password") @Option(option = "keystore-password", description = "Set the elasticsearch keystore password")
public void setKeystorePassword(String password) { public void setKeystorePassword(String password) {
keystorePassword = password; keystorePassword = password;
@ -113,6 +125,7 @@ public class RunTask extends DefaultTestClustersTask {
httpPort++; httpPort++;
cluster.getFirstNode().setTransportPort(String.valueOf(transportPort)); cluster.getFirstNode().setTransportPort(String.valueOf(transportPort));
transportPort++; transportPort++;
cluster.setPreserveDataDir(preserveData);
for (ElasticsearchNode node : cluster.getNodes()) { for (ElasticsearchNode node : cluster.getNodes()) {
additionalSettings.forEach(node::setting); additionalSettings.forEach(node::setting);
if (dataDir != null) { if (dataDir != null) {

View File

@ -66,6 +66,13 @@ public class StandaloneRestIntegTestTask extends Test implements TestClustersAwa
.filter(task -> task != this) .filter(task -> task != this)
.anyMatch(task -> Collections.disjoint(task.getClusters(), getClusters()) == false) .anyMatch(task -> Collections.disjoint(task.getClusters(), getClusters()) == false)
); );
this.getOutputs()
.doNotCacheIf(
"Caching disabled for this task since it is configured to preserve data directory",
// Don't cache the output of this task if it's not running from a clean data directory.
t -> getClusters().stream().anyMatch(cluster -> cluster.isPreserveDataDir())
);
} }
@Override @Override

View File

@ -86,6 +86,10 @@ public interface TestClusterConfiguration {
void jvmArgs(String... values); void jvmArgs(String... values);
boolean isPreserveDataDir();
void setPreserveDataDir(boolean preserveDataDir);
void freeze(); void freeze();
void start(); void start();

View File

@ -11,11 +11,11 @@ the queries and asserts the results that are provided along with the query state
### Running the tests ### Running the tests
To be able to run the tests locally, one should set the environmental variable `eql_test_credentials_file` pointing to To be able to run the tests locally, one should set the environmental variable `eql_test_credentials_file` pointing to
a local file holding the service account credentials which allow access to the gcs bucket where the dataset resides. a local file holding the service account credentials which allow access to the gcs bucket where the dataset resides.
E.g.: E.g.:
```shell script ```shell script
export eql_test_credentials_file=/Users/username/credentials.gcs.json export eql_test_credentials_file=/Users/username/credentials.gcs.json
``` ```
To run the tests you can issue: To run the tests you can issue:
```shell script ```shell script
@ -46,7 +46,7 @@ queries executed, e.g.:*
#### Run a specific query #### Run a specific query
If one wants to run just one query from the set, needs to do it with following command by replacing `<queryNo>` (which If one wants to run just one query from the set, needs to do it with following command by replacing `<queryNo>` (which
can be found in queries.toml file) with the desired number of the query: can be found in queries.toml file) with the desired number of the query:
```shell script ```shell script
@ -66,7 +66,7 @@ or
./gradlew ':x-pack:plugin:eql:qa:correctness:javaRestTest' --tests "org.elasticsearch.xpack.eql.EsEQLCorrectnessIT.test {<queryNo>}" -Dtests.eql_correctness_debug=true ./gradlew ':x-pack:plugin:eql:qa:correctness:javaRestTest' --tests "org.elasticsearch.xpack.eql.EsEQLCorrectnessIT.test {<queryNo>}" -Dtests.eql_correctness_debug=true
``` ```
### Run an ES node manually and run the tests against it ### Run an ES node manually and run the tests against it
If one wants to run an ES node manually (most probably to be able to debug the server), needs to run the following: If one wants to run an ES node manually (most probably to be able to debug the server), needs to run the following:
@ -76,7 +76,7 @@ If one wants to run an ES node manually (most probably to be able to debug the s
**Set the `eql_test_credentials_file` environmental variable correctly in the shell before running the command above,** **Set the `eql_test_credentials_file` environmental variable correctly in the shell before running the command above,**
Once the ES node is up and running, the data can be restored from the snapshot by running the `main` of the Once the ES node is up and running, the data can be restored from the snapshot by running the `main` of the
`EqlDataLoader` class. `EqlDataLoader` class.
Once the data is loaded, a specific query can be run against the running ES node with: Once the data is loaded, a specific query can be run against the running ES node with:
@ -85,3 +85,10 @@ Once the data is loaded, a specific query can be run against the running ES node
``` ```
**Set the `eql_test_credentials_file` environmental variable correctly in the shell before running the command above,** **Set the `eql_test_credentials_file` environmental variable correctly in the shell before running the command above,**
#### Preserve data across node restarts
If you'd like to preserve the restored index and avoid the network download and delay of restoring them on every run of the node,
you can set the `eql.test.preserve.data` system property, e.g.:
```shell script
./gradlew :x-pack:plugin:eql:qa:correctness:javaRestTest -Deql.test.preserve.data=true
```

View File

@ -20,7 +20,16 @@ dependencies {
javaRestTestImplementation 'io.ous:jtoml:2.0.0' javaRestTestImplementation 'io.ous:jtoml:2.0.0'
} }
File serviceAccountFile = (System.getenv("eql_test_credentials_file") ?: System.getProperty("eql.test.credentials.file")) as File File serviceAccountFile = providers.environmentVariable('eql_test_credentials_file')
.forUseAtConfigurationTime()
.orElse(providers.systemProperty('eql.test.credentials.file').forUseAtConfigurationTime())
.map { s -> new File(s)}
.getOrNull()
Boolean preserveData = providers.systemProperty('eql.test.preserve.data')
.forUseAtConfigurationTime()
.map { s -> Boolean.parseBoolean(s) }
.getOrElse(false)
testClusters { testClusters {
all { all {
@ -28,6 +37,9 @@ testClusters {
if (serviceAccountFile) { if (serviceAccountFile) {
keystore 'gcs.client.eql_test.credentials_file', serviceAccountFile keystore 'gcs.client.eql_test.credentials_file', serviceAccountFile
} }
if (preserveData) {
preserveDataDir = true
}
testDistribution = 'DEFAULT' testDistribution = 'DEFAULT'
setting 'xpack.license.self_generated.type', 'basic' setting 'xpack.license.self_generated.type', 'basic'
jvmArgs '-Xms4g', '-Xmx4g' jvmArgs '-Xms4g', '-Xmx4g'
@ -46,6 +58,6 @@ tasks.named('javaRestTest').configure {
} }
tasks.register("runEqlCorrectnessNode", RunTask) { tasks.register("runEqlCorrectnessNode", RunTask) {
useCluster testClusters.runTask; useCluster testClusters.runTask
description = 'Runs elasticsearch in the foreground with gcs plugin and keystore credentials' description = 'Runs elasticsearch in the foreground with gcs plugin and keystore credentials'
} }