Add option to preserve data in test clusters (#65400)
(cherry picked from commit 1ce323e1368cf5231181f1efaba1c4e425066e37)
This commit is contained in:
parent
7f7e938a25
commit
f8f5d27f6b
|
@ -82,12 +82,13 @@ password: `elastic-password`.
|
|||
|
||||
==== Other useful arguments
|
||||
|
||||
In order to start a node with a different max heap space add: `-Dtests.heap.size=4G`
|
||||
In order to disable assertions add: `-Dtests.asserts=false`
|
||||
In order to use a custom data directory: `--data-dir=/tmp/foo`
|
||||
In order to remotely attach a debugger to the process: `--debug-jvm`
|
||||
In order to set a different keystore password: `--keystore-password`
|
||||
In order to set an Elasticsearch setting, provide a setting with the following prefix: `-Dtests.es.`
|
||||
- In order to start a node with a different max heap space add: `-Dtests.heap.size=4G`
|
||||
- In order to disable assertions add: `-Dtests.asserts=false`
|
||||
- In order to use a custom data directory: `--data-dir=/tmp/foo`
|
||||
- In order to preserve data in between executions: `--preserve-data`
|
||||
- In order to remotely attach a debugger to the process: `--debug-jvm`
|
||||
- In order to set a different keystore password: `--keystore-password`
|
||||
- In order to set an Elasticsearch setting, provide a setting with the following prefix: `-Dtests.es.`
|
||||
|
||||
=== Test case filtering.
|
||||
|
||||
|
|
|
@ -282,6 +282,16 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
|
|||
nodes.all(each -> each.jvmArgs(values));
|
||||
}
|
||||
|
||||
@Internal
|
||||
public boolean isPreserveDataDir() {
|
||||
return nodes.stream().anyMatch(node -> node.isPreserveDataDir());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setPreserveDataDir(boolean preserveDataDir) {
|
||||
nodes.all(each -> each.setPreserveDataDir(preserveDataDir));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void freeze() {
|
||||
nodes.forEach(ElasticsearchNode::freeze);
|
||||
|
|
|
@ -167,6 +167,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
|
|||
private String transportPort = "0";
|
||||
private Path confPathData;
|
||||
private String keystorePassword = "";
|
||||
private boolean preserveDataDir = false;
|
||||
|
||||
ElasticsearchNode(
|
||||
String path,
|
||||
|
@ -421,6 +422,17 @@ public class ElasticsearchNode implements TestClusterConfiguration {
|
|||
return configFile.getParent();
|
||||
}
|
||||
|
||||
@Override
|
||||
@Input
|
||||
public boolean isPreserveDataDir() {
|
||||
return preserveDataDir;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setPreserveDataDir(boolean preserveDataDir) {
|
||||
this.preserveDataDir = preserveDataDir;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void freeze() {
|
||||
requireNonNull(testDistribution, "null testDistribution passed when configuring test cluster `" + this + "`");
|
||||
|
@ -452,8 +464,14 @@ public class ElasticsearchNode implements TestClusterConfiguration {
|
|||
logToProcessStdout("Configuring working directory: " + workingDir);
|
||||
// make sure we always start fresh
|
||||
if (Files.exists(workingDir)) {
|
||||
if (preserveDataDir) {
|
||||
Files.list(workingDir)
|
||||
.filter(path -> path.equals(confPathData) == false)
|
||||
.forEach(path -> fileSystemOperations.delete(d -> d.delete(path)));
|
||||
} else {
|
||||
fileSystemOperations.delete(d -> d.delete(workingDir));
|
||||
}
|
||||
}
|
||||
isWorkingDirConfigured = true;
|
||||
}
|
||||
setupNodeDistribution(getExtractedDistributionDir());
|
||||
|
|
|
@ -46,6 +46,8 @@ public class RunTask extends DefaultTestClustersTask {
|
|||
|
||||
private Boolean debug = false;
|
||||
|
||||
private Boolean preserveData = false;
|
||||
|
||||
private Path dataDir = null;
|
||||
|
||||
private String keystorePassword = "";
|
||||
|
@ -65,6 +67,16 @@ public class RunTask extends DefaultTestClustersTask {
|
|||
dataDir = Paths.get(dataDirStr).toAbsolutePath();
|
||||
}
|
||||
|
||||
@Input
|
||||
public Boolean getPreserveData() {
|
||||
return preserveData;
|
||||
}
|
||||
|
||||
@Option(option = "preserve-data", description = "Preserves data directory contents (path provided to --data-dir is always preserved)")
|
||||
public void setPreserveData(Boolean preserveData) {
|
||||
this.preserveData = preserveData;
|
||||
}
|
||||
|
||||
@Option(option = "keystore-password", description = "Set the elasticsearch keystore password")
|
||||
public void setKeystorePassword(String password) {
|
||||
keystorePassword = password;
|
||||
|
@ -113,6 +125,7 @@ public class RunTask extends DefaultTestClustersTask {
|
|||
httpPort++;
|
||||
cluster.getFirstNode().setTransportPort(String.valueOf(transportPort));
|
||||
transportPort++;
|
||||
cluster.setPreserveDataDir(preserveData);
|
||||
for (ElasticsearchNode node : cluster.getNodes()) {
|
||||
additionalSettings.forEach(node::setting);
|
||||
if (dataDir != null) {
|
||||
|
|
|
@ -66,6 +66,13 @@ public class StandaloneRestIntegTestTask extends Test implements TestClustersAwa
|
|||
.filter(task -> task != this)
|
||||
.anyMatch(task -> Collections.disjoint(task.getClusters(), getClusters()) == false)
|
||||
);
|
||||
|
||||
this.getOutputs()
|
||||
.doNotCacheIf(
|
||||
"Caching disabled for this task since it is configured to preserve data directory",
|
||||
// Don't cache the output of this task if it's not running from a clean data directory.
|
||||
t -> getClusters().stream().anyMatch(cluster -> cluster.isPreserveDataDir())
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -86,6 +86,10 @@ public interface TestClusterConfiguration {
|
|||
|
||||
void jvmArgs(String... values);
|
||||
|
||||
boolean isPreserveDataDir();
|
||||
|
||||
void setPreserveDataDir(boolean preserveDataDir);
|
||||
|
||||
void freeze();
|
||||
|
||||
void start();
|
||||
|
|
|
@ -85,3 +85,10 @@ Once the data is loaded, a specific query can be run against the running ES node
|
|||
```
|
||||
|
||||
**Set the `eql_test_credentials_file` environmental variable correctly in the shell before running the command above,**
|
||||
|
||||
#### Preserve data across node restarts
|
||||
If you'd like to preserve the restored index and avoid the network download and delay of restoring them on every run of the node,
|
||||
you can set the `eql.test.preserve.data` system property, e.g.:
|
||||
```shell script
|
||||
./gradlew :x-pack:plugin:eql:qa:correctness:javaRestTest -Deql.test.preserve.data=true
|
||||
```
|
||||
|
|
|
@ -20,7 +20,16 @@ dependencies {
|
|||
javaRestTestImplementation 'io.ous:jtoml:2.0.0'
|
||||
}
|
||||
|
||||
File serviceAccountFile = (System.getenv("eql_test_credentials_file") ?: System.getProperty("eql.test.credentials.file")) as File
|
||||
File serviceAccountFile = providers.environmentVariable('eql_test_credentials_file')
|
||||
.forUseAtConfigurationTime()
|
||||
.orElse(providers.systemProperty('eql.test.credentials.file').forUseAtConfigurationTime())
|
||||
.map { s -> new File(s)}
|
||||
.getOrNull()
|
||||
|
||||
Boolean preserveData = providers.systemProperty('eql.test.preserve.data')
|
||||
.forUseAtConfigurationTime()
|
||||
.map { s -> Boolean.parseBoolean(s) }
|
||||
.getOrElse(false)
|
||||
|
||||
testClusters {
|
||||
all {
|
||||
|
@ -28,6 +37,9 @@ testClusters {
|
|||
if (serviceAccountFile) {
|
||||
keystore 'gcs.client.eql_test.credentials_file', serviceAccountFile
|
||||
}
|
||||
if (preserveData) {
|
||||
preserveDataDir = true
|
||||
}
|
||||
testDistribution = 'DEFAULT'
|
||||
setting 'xpack.license.self_generated.type', 'basic'
|
||||
jvmArgs '-Xms4g', '-Xmx4g'
|
||||
|
@ -46,6 +58,6 @@ tasks.named('javaRestTest').configure {
|
|||
}
|
||||
|
||||
tasks.register("runEqlCorrectnessNode", RunTask) {
|
||||
useCluster testClusters.runTask;
|
||||
useCluster testClusters.runTask
|
||||
description = 'Runs elasticsearch in the foreground with gcs plugin and keystore credentials'
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue