Convert RunTask to use testclusers, remove ClusterFormationTasks (#47572)

* Convert RunTask to use testclusers, remove ClusterFormationTasks

This PR adds a new RunTask and a way for it to start a
testclusters cluster out of band and block on it to replace
the old RunTask that used ClusterFormationTasks.

With this we can now remove ClusterFormationTasks.
This commit is contained in:
Alpar Torok 2019-10-08 14:39:58 +03:00
parent d33dbf82d4
commit 36d018c909
20 changed files with 181 additions and 1732 deletions

View File

@ -17,17 +17,17 @@
* under the License.
*/
import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.BuildPlugin
import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.BwcVersions
import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.plugin.PluginBuildPlugin
import org.elasticsearch.gradle.tool.Boilerplate
import org.gradle.util.GradleVersion
import org.gradle.util.DistributionLocator
import org.gradle.plugins.ide.eclipse.model.SourceFolder
import org.gradle.util.DistributionLocator
import org.gradle.util.GradleVersion
import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
@ -452,7 +452,7 @@ class Run extends DefaultTask {
description = "Enable debugging configuration, to allow attaching a debugger to elasticsearch."
)
public void setDebug(boolean enabled) {
project.project(':distribution').run.clusterConfig.debug = enabled
project.project(':distribution').run.debug = enabled
}
}
task run(type: Run) {

View File

@ -25,7 +25,7 @@ import org.elasticsearch.gradle.NoticeTask
import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.test.RestIntegTestTask
import org.elasticsearch.gradle.test.RunTask
import org.elasticsearch.gradle.testclusters.RunTask
import org.elasticsearch.gradle.testclusters.TestClustersPlugin
import org.elasticsearch.gradle.tool.ClasspathUtils
import org.gradle.api.InvalidUserDataException
@ -68,35 +68,28 @@ class PluginBuildPlugin implements Plugin<Project> {
project.archivesBaseName = name
project.description = extension1.description
configurePublishing(project, extension1)
if (project.plugins.hasPlugin(TestClustersPlugin.class) == false) {
project.integTestCluster.dependsOn(project.tasks.bundlePlugin)
if (isModule) {
project.integTestCluster.module(project)
} else {
project.integTestCluster.plugin(project.path)
}
} else {
project.tasks.integTest.dependsOn(project.tasks.bundlePlugin)
if (isModule) {
project.testClusters.integTest.module(
project.file(project.tasks.bundlePlugin.archiveFile)
)
} else {
project.testClusters.integTest.plugin(
project.file(project.tasks.bundlePlugin.archiveFile)
)
}
project.extensions.getByType(PluginPropertiesExtension).extendedPlugins.each { pluginName ->
// Auto add dependent modules to the test cluster
if (project.findProject(":modules:${pluginName}") != null) {
project.integTest.dependsOn(project.project(":modules:${pluginName}").tasks.bundlePlugin)
project.testClusters.integTest.module(
project.file(project.project(":modules:${pluginName}").tasks.bundlePlugin.archiveFile)
)
}
project.tasks.integTest.dependsOn(project.tasks.bundlePlugin)
if (isModule) {
project.testClusters.integTest.module(
project.file(project.tasks.bundlePlugin.archiveFile)
)
} else {
project.testClusters.integTest.plugin(
project.file(project.tasks.bundlePlugin.archiveFile)
)
}
project.extensions.getByType(PluginPropertiesExtension).extendedPlugins.each { pluginName ->
// Auto add dependent modules to the test cluster
if (project.findProject(":modules:${pluginName}") != null) {
project.integTest.dependsOn(project.project(":modules:${pluginName}").tasks.bundlePlugin)
project.testClusters.integTest.module(
project.file(project.project(":modules:${pluginName}").tasks.bundlePlugin.archiveFile)
)
}
}
if (extension1.name == null) {
throw new InvalidUserDataException('name is a required setting for esplugin')
}
@ -120,14 +113,6 @@ class PluginBuildPlugin implements Plugin<Project> {
]
buildProperties.expand(properties)
buildProperties.inputs.properties(properties)
project.tasks.run.dependsOn(project.tasks.bundlePlugin)
if (isModule) {
project.tasks.run.clusterConfig.distribution = System.getProperty(
'run.distribution', isXPackModule ? 'default' : 'oss'
)
} else {
project.tasks.run.clusterConfig.plugin(project.path)
}
if (isModule == false || isXPackModule) {
addNoticeGeneration(project, extension1)
}
@ -148,7 +133,11 @@ class PluginBuildPlugin implements Plugin<Project> {
createIntegTestTask(project)
createBundleTasks(project, extension)
project.configurations.getByName('default').extendsFrom(project.configurations.getByName('runtime'))
project.tasks.create('run', RunTask) // allow running ES with this plugin in the foreground of a build
// allow running ES with this plugin in the foreground of a build
project.tasks.register('run', RunTask) {
dependsOn(project.tasks.bundlePlugin)
useCluster project.testClusters.integTest
}
}
private void configurePublishing(Project project, PluginPropertiesExtension extension) {
@ -195,10 +184,6 @@ class PluginBuildPlugin implements Plugin<Project> {
private static void createIntegTestTask(Project project) {
RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class)
integTest.mustRunAfter('precommit', 'test')
if (project.plugins.hasPlugin(TestClustersPlugin.class) == false) {
// only if not using test clusters
project.integTestCluster.distribution = System.getProperty('tests.distribution', 'integ-test-zip')
}
project.check.dependsOn(integTest)
}

View File

@ -1,267 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.test
import org.elasticsearch.gradle.Version
import org.gradle.api.GradleException
import org.gradle.api.Project
import org.gradle.api.tasks.Input
/** Configuration for an elasticsearch cluster, used for integration tests. */
class ClusterConfiguration {
private final Project project
@Input
String distribution = 'default'
@Input
int numNodes = 1
@Input
int numBwcNodes = 0
@Input
Version bwcVersion = null
@Input
int httpPort = 0
@Input
int transportPort = 0
/**
* An override of the data directory. Input is the node number and output
* is the override data directory.
*/
@Input
Closure<String> dataDir = null
/** Optional override of the cluster name. */
@Input
String clusterName = null
@Input
boolean daemonize = true
@Input
boolean debug = false
/**
* Configuration of the setting {@code discovery.zen.minimum_master_nodes} on the nodes.
* In case of more than one node, this defaults to the number of nodes
*/
@Input
Closure<Integer> minimumMasterNodes = {
if (bwcVersion != null && bwcVersion.before("6.5.0")) {
return numNodes > 1 ? numNodes : -1
} else {
return numNodes > 1 ? numNodes.intdiv(2) + 1 : -1
}
}
/**
* Whether the initial_master_nodes setting should be automatically derived from the nodes
* in the cluster. Only takes effect if all nodes in the cluster understand this setting
* and the discovery type is not explicitly set.
*/
@Input
boolean autoSetInitialMasterNodes = true
/**
* Whether the file-based discovery provider should be automatically setup based on
* the nodes in the cluster. Only takes effect if no other hosts provider is already
* configured.
*/
@Input
boolean autoSetHostsProvider = true
@Input
String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +
" " + "-Xmx" + System.getProperty('tests.heap.size', '512m') +
" " + System.getProperty('tests.jvm.argline', '')
/**
* Should the shared environment be cleaned on cluster startup? Defaults
* to {@code true} so we run with a clean cluster but some tests wish to
* preserve snapshots between clusters so they set this to true.
*/
@Input
boolean cleanShared = true
/**
* A closure to call which returns the unicast host to connect to for cluster formation.
*
* This allows multi node clusters, or a new cluster to connect to an existing cluster.
* The closure takes three arguments, the NodeInfo for the first node in the cluster,
* the NodeInfo for the node current being configured, an AntBuilder which may be used
* to wait on conditions before returning.
*/
@Input
Closure unicastTransportUri = { NodeInfo seedNode, NodeInfo node, AntBuilder ant ->
if (seedNode == node) {
return null
}
ant.waitfor(maxwait: '40', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond',
timeoutproperty: "failed.${seedNode.transportPortsFile.path}") {
resourceexists {
file(file: seedNode.transportPortsFile.toString())
}
}
if (ant.properties.containsKey("failed.${seedNode.transportPortsFile.path}".toString())) {
throw new GradleException("Failed to locate seed node transport file [${seedNode.transportPortsFile}]: " +
"timed out waiting for it to be created after 40 seconds")
}
return seedNode.transportUri()
}
/**
* A closure to call which returns a manually supplied list of unicast seed hosts.
*/
@Input
Closure<List<String>> otherUnicastHostAddresses = {
Collections.emptyList()
}
/**
* A closure to call before the cluster is considered ready. The closure is passed the node info,
* as well as a groovy AntBuilder, to enable running ant condition checks. The default wait
* condition is for http on the http port.
*/
@Input
Closure waitCondition = { NodeInfo node, AntBuilder ant ->
File tmpFile = new File(node.cwd, 'wait.success')
String waitUrl = "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow"
ant.echo(message: "==> [${new Date()}] checking health: ${waitUrl}",
level: 'info')
// checking here for wait_for_nodes to be >= the number of nodes because its possible
// this cluster is attempting to connect to nodes created by another task (same cluster name),
// so there will be more nodes in that case in the cluster state
ant.get(src: waitUrl,
dest: tmpFile.toString(),
ignoreerrors: true, // do not fail on error, so logging buffers can be flushed by the wait task
retries: 10)
return tmpFile.exists()
}
/**
* The maximum number of seconds to wait for nodes to complete startup, which includes writing
* the ports files for the transports and the pid file. This wait time occurs before the wait
* condition is executed.
*/
@Input
int nodeStartupWaitSeconds = 30
public ClusterConfiguration(Project project) {
this.project = project
}
// **Note** for systemProperties, settings, keystoreFiles etc:
// value could be a GString that is evaluated to just a String
// there are cases when value depends on task that is not executed yet on configuration stage
Map<String, Object> systemProperties = new HashMap<>()
Map<String, Object> environmentVariables = new HashMap<>()
Map<String, Object> settings = new HashMap<>()
Map<String, String> keystoreSettings = new HashMap<>()
Map<String, Object> keystoreFiles = new HashMap<>()
// map from destination path, to source file
Map<String, Object> extraConfigFiles = new HashMap<>()
LinkedHashMap<String, Object> plugins = new LinkedHashMap<>()
List<Project> modules = new ArrayList<>()
LinkedHashMap<String, Object[]> setupCommands = new LinkedHashMap<>()
List<Object> dependencies = new ArrayList<>()
@Input
void systemProperty(String property, Object value) {
systemProperties.put(property, value)
}
@Input
void environment(String variable, Object value) {
environmentVariables.put(variable, value)
}
@Input
void setting(String name, Object value) {
settings.put(name, value)
}
@Input
void keystoreSetting(String name, String value) {
keystoreSettings.put(name, value)
}
/**
* Adds a file to the keystore. The name is the secure setting name, and the sourceFile
* is anything accepted by project.file()
*/
@Input
void keystoreFile(String name, Object sourceFile) {
keystoreFiles.put(name, sourceFile)
}
@Input
void plugin(String path) {
Project pluginProject = project.project(path)
plugins.put(pluginProject.name, pluginProject)
}
@Input
void mavenPlugin(String name, String mavenCoords) {
plugins.put(name, mavenCoords)
}
/** Add a module to the cluster. The project must be an esplugin and have a single zip default artifact. */
@Input
void module(Project moduleProject) {
modules.add(moduleProject)
}
@Input
void setupCommand(String name, Object... args) {
setupCommands.put(name, args)
}
/**
* Add an extra configuration file. The path is relative to the config dir, and the sourceFile
* is anything accepted by project.file()
*/
@Input
void extraConfigFile(String path, Object sourceFile) {
if (path == 'elasticsearch.yml') {
throw new GradleException('Overwriting elasticsearch.yml is not allowed, add additional settings using cluster { setting "foo", "bar" }')
}
extraConfigFiles.put(path, sourceFile)
}
/** Add dependencies that must be run before the first task setting up the cluster. */
@Input
void dependsOn(Object... deps) {
dependencies.addAll(deps)
}
}

View File

@ -1,297 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.test
import com.sun.jna.Native
import com.sun.jna.WString
import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.VersionProperties
import org.gradle.api.Project
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
/**
* A container for the files and configuration associated with a single node in a test cluster.
*/
class NodeInfo {
/** Gradle project this node is part of */
Project project
/** common configuration for all nodes, including this one */
ClusterConfiguration config
/** node number within the cluster, for creating unique names and paths */
int nodeNum
/** name of the cluster this node is part of */
String clusterName
/** root directory all node files and operations happen under */
File baseDir
/** shared data directory all nodes share */
File sharedDir
/** the pid file the node will use */
File pidFile
/** a file written by elasticsearch containing the ports of each bound address for http */
File httpPortsFile
/** a file written by elasticsearch containing the ports of each bound address for transport */
File transportPortsFile
/** elasticsearch home dir */
File homeDir
/** config directory */
File pathConf
/** data directory (as an Object, to allow lazy evaluation) */
Object dataDir
/** THE config file */
File configFile
/** working directory for the node process */
File cwd
/** file that if it exists, indicates the node failed to start */
File failedMarker
/** stdout/stderr log of the elasticsearch process for this node */
File startLog
/** directory to install plugins from */
File pluginsTmpDir
/** Major version of java this node runs with, or {@code null} if using the runtime java version */
Integer javaVersion
/** environment variables to start the node with */
Map<String, String> env
/** arguments to start the node with */
List<String> args
/** Executable to run the bin/elasticsearch with, either cmd or sh */
String executable
/** Path to the elasticsearch start script */
private Object esScript
/** script to run when running in the background */
private File wrapperScript
/** buffer for ant output when starting this node */
ByteArrayOutputStream buffer = new ByteArrayOutputStream()
/** the version of elasticsearch that this node runs */
Version nodeVersion
/** true if the node is not the current version */
boolean isBwcNode
/** Holds node configuration for part of a test cluster. */
NodeInfo(ClusterConfiguration config, int nodeNum, Project project, String prefix, String nodeVersion, File sharedDir) {
this.config = config
this.nodeNum = nodeNum
this.project = project
this.sharedDir = sharedDir
if (config.clusterName != null) {
clusterName = config.clusterName
} else {
clusterName = project.path.replace(':', '_').substring(1) + '_' + prefix
}
baseDir = new File(project.buildDir, "cluster/${prefix} node${nodeNum}")
pidFile = new File(baseDir, 'es.pid')
this.nodeVersion = Version.fromString(nodeVersion)
this.isBwcNode = this.nodeVersion.before(VersionProperties.elasticsearch)
homeDir = new File(baseDir, "elasticsearch-${nodeVersion}")
pathConf = new File(homeDir, 'config')
if (config.dataDir != null) {
dataDir = "${config.dataDir(nodeNum)}"
} else {
dataDir = new File(homeDir, "data")
}
configFile = new File(pathConf, 'elasticsearch.yml')
// even for rpm/deb, the logs are under home because we dont start with real services
File logsDir = new File(homeDir, 'logs')
httpPortsFile = new File(logsDir, 'http.ports')
transportPortsFile = new File(logsDir, 'transport.ports')
cwd = new File(baseDir, "cwd")
failedMarker = new File(cwd, 'run.failed')
startLog = new File(cwd, 'run.log')
pluginsTmpDir = new File(baseDir, "plugins tmp")
args = []
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
executable = 'cmd'
args.add('/C')
args.add('"') // quote the entire command
wrapperScript = new File(cwd, "run.bat")
/*
* We have to delay building the string as the path will not exist during configuration which will fail on Windows due to
* getting the short name requiring the path to already exist.
*/
esScript = "${-> binPath().resolve('elasticsearch.bat').toString()}"
} else {
executable = 'bash'
wrapperScript = new File(cwd, "run")
esScript = binPath().resolve('elasticsearch')
}
if (config.daemonize) {
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
/*
* We have to delay building the string as the path will not exist during configuration which will fail on Windows due to
* getting the short name requiring the path to already exist.
*/
args.add("${-> getShortPathName(wrapperScript.toString())}")
} else {
args.add("${wrapperScript}")
}
} else {
args.add("${esScript}")
}
if (this.nodeVersion.before("6.2.0")) {
javaVersion = 8
} else if (this.nodeVersion.onOrAfter("6.2.0") && this.nodeVersion.before("6.3.0")) {
javaVersion = 9
} else if (this.nodeVersion.onOrAfter("6.3.0") && this.nodeVersion.before("6.5.0")) {
javaVersion = 10
}
args.addAll("-E", "node.portsfile=true")
env = [:]
env.putAll(config.environmentVariables)
for (Map.Entry<String, String> property : System.properties.entrySet()) {
if (property.key.startsWith('tests.es.')) {
args.add("-E")
args.add("${property.key.substring('tests.es.'.size())}=${property.value}")
}
}
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
/*
* We have to delay building the string as the path will not exist during configuration which will fail on Windows due to
* getting the short name requiring the path to already exist.
*/
env.put('ES_PATH_CONF', "${-> getShortPathName(pathConf.toString())}")
}
else {
env.put('ES_PATH_CONF', pathConf)
}
if (!System.properties.containsKey("tests.es.path.data")) {
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
/*
* We have to delay building the string as the path will not exist during configuration which will fail on Windows due to
* getting the short name requiring the path to already exist. This one is extra tricky because usually we rely on the node
* creating its data directory on startup but we simply can not do that here because getting the short path name requires
* the directory to already exist. Therefore, we create this directory immediately before getting the short name.
*/
args.addAll("-E", "path.data=${-> Files.createDirectories(Paths.get(dataDir.toString())); getShortPathName(dataDir.toString())}")
} else {
args.addAll("-E", "path.data=${-> dataDir.toString()}")
}
}
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
args.add('"') // end the entire command, quoted
}
}
Path binPath() {
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
return Paths.get(getShortPathName(new File(homeDir, 'bin').toString()))
} else {
return Paths.get(new File(homeDir, 'bin').toURI())
}
}
static String getShortPathName(String path) {
assert Os.isFamily(Os.FAMILY_WINDOWS)
final WString longPath = new WString("\\\\?\\" + path)
// first we get the length of the buffer needed
final int length = JNAKernel32Library.getInstance().GetShortPathNameW(longPath, null, 0)
if (length == 0) {
throw new IllegalStateException("path [" + path + "] encountered error [" + Native.getLastError() + "]")
}
final char[] shortPath = new char[length]
// knowing the length of the buffer, now we get the short name
if (JNAKernel32Library.getInstance().GetShortPathNameW(longPath, shortPath, length) == 0) {
throw new IllegalStateException("path [" + path + "] encountered error [" + Native.getLastError() + "]")
}
// we have to strip the \\?\ away from the path for cmd.exe
return Native.toString(shortPath).substring(4)
}
/** Returns debug string for the command that started this node. */
String getCommandString() {
String esCommandString = "\nNode ${nodeNum} configuration:\n"
esCommandString += "|-----------------------------------------\n"
esCommandString += "| cwd: ${cwd}\n"
esCommandString += "| command: ${executable} ${args.join(' ')}\n"
esCommandString += '| environment:\n'
env.each { k, v -> esCommandString += "| ${k}: ${v}\n" }
if (config.daemonize) {
esCommandString += "|\n| [${wrapperScript.name}]\n"
wrapperScript.eachLine('UTF-8', { line -> esCommandString += " ${line}\n"})
}
esCommandString += '|\n| [elasticsearch.yml]\n'
configFile.eachLine('UTF-8', { line -> esCommandString += "| ${line}\n" })
esCommandString += "|-----------------------------------------"
return esCommandString
}
void writeWrapperScript() {
String argsPasser = '"$@"'
String exitMarker = "; if [ \$? != 0 ]; then touch run.failed; fi"
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
argsPasser = '%*'
exitMarker = "\r\n if \"%errorlevel%\" neq \"0\" ( type nul >> run.failed )"
}
wrapperScript.setText("\"${esScript}\" ${argsPasser} > run.log 2>&1 ${exitMarker}", 'UTF-8')
}
/** Returns an address and port suitable for a uri to connect to this node over http */
String httpUri() {
return httpPortsFile.readLines("UTF-8").get(0)
}
/** Returns an address and port suitable for a uri to connect to this node over transport protocol */
String transportUri() {
return transportPortsFile.readLines("UTF-8").get(0)
}
/** Returns the file which contains the transport protocol ports for this node */
File getTransportPortsFile() {
return transportPortsFile
}
/** Returns the data directory for this node */
File getDataDir() {
if (!(dataDir instanceof File)) {
return new File(dataDir)
}
return dataDir
}
}

View File

@ -1,41 +0,0 @@
package org.elasticsearch.gradle.test
import org.gradle.api.DefaultTask
import org.gradle.api.Task
import org.gradle.api.tasks.Internal
import org.gradle.api.tasks.options.Option
import org.gradle.util.ConfigureUtil
class RunTask extends DefaultTask {
@Internal
ClusterConfiguration clusterConfig
RunTask() {
description = "Runs elasticsearch with '${project.path}'"
group = 'Verification'
clusterConfig = new ClusterConfiguration(project)
clusterConfig.httpPort = 9200
clusterConfig.transportPort = 9300
clusterConfig.daemonize = false
clusterConfig.distribution = 'default'
project.afterEvaluate {
ClusterFormationTasks.setup(project, name, this, clusterConfig)
}
}
@Option(
option = "debug-jvm",
description = "Enable debugging configuration, to allow attaching a debugger to elasticsearch."
)
void setDebug(boolean enabled) {
clusterConfig.debug = enabled;
}
/** Configure the cluster that will be run. */
@Override
Task configure(Closure closure) {
ConfigureUtil.configure(closure, clusterConfig)
return this
}
}

View File

@ -104,7 +104,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
}
}
private ElasticsearchNode getFirstNode() {
ElasticsearchNode getFirstNode() {
return nodes.getAt(clusterName + "-0");
}

View File

@ -148,6 +148,8 @@ public class ElasticsearchNode implements TestClusterConfiguration {
private volatile Process esProcess;
private Function<String, String> nameCustomization = Function.identity();
private boolean isWorkingDirConfigured = false;
private String httpPort = "0";
private String transportPort = "0";
ElasticsearchNode(String path, String name, Project project, ReaperService reaper, File workingDirBase) {
this.path = path;
@ -358,8 +360,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
@Override
public void freeze() {
requireNonNull(distributions, "null distribution passed when configuring test cluster `" + this + "`");
requireNonNull(javaHome, "null javaHome passed when configuring test cluster `" + this + "`");
requireNonNull(testDistribution, "null testDistribution passed when configuring test cluster `" + this + "`");
LOGGER.info("Locking configuration of `{}`", this);
configurationFrozen.set(true);
}
@ -636,7 +637,9 @@ public class ElasticsearchNode implements TestClusterConfiguration {
private Map<String, String> getESEnvironment() {
Map<String, String> defaultEnv = new HashMap<>();
defaultEnv.put("JAVA_HOME", getJavaHome().getAbsolutePath());
if ( getJavaHome() != null) {
defaultEnv.put("JAVA_HOME", getJavaHome().getAbsolutePath());
}
defaultEnv.put("ES_PATH_CONF", configFile.getParent().toString());
String systemPropertiesString = "";
if (systemProperties.isEmpty() == false) {
@ -695,9 +698,11 @@ public class ElasticsearchNode implements TestClusterConfiguration {
// Don't inherit anything from the environment for as that would lack reproducibility
environment.clear();
environment.putAll(getESEnvironment());
// don't buffer all in memory, make sure we don't block on the default pipes
processBuilder.redirectError(ProcessBuilder.Redirect.appendTo(esStderrFile.toFile()));
processBuilder.redirectOutput(ProcessBuilder.Redirect.appendTo(esStdoutFile.toFile()));
LOGGER.info("Running `{}` in `{}` for {} env: {}", command, workingDir, this, environment);
try {
esProcess = processBuilder.start();
@ -987,11 +992,11 @@ public class ElasticsearchNode implements TestClusterConfiguration {
defaultConfig.put("path.shared_data", workingDir.resolve("sharedData").toString());
defaultConfig.put("node.attr.testattr", "test");
defaultConfig.put("node.portsfile", "true");
defaultConfig.put("http.port", "0");
defaultConfig.put("http.port", httpPort);
if (getVersion().onOrAfter(Version.fromString("6.7.0"))) {
defaultConfig.put("transport.port", "0");
defaultConfig.put("transport.port", transportPort);
} else {
defaultConfig.put("transport.tcp.port", "0");
defaultConfig.put("transport.tcp.port", transportPort);
}
// Default the watermarks to absurdly low to prevent the tests from failing on nodes without enough disk space
defaultConfig.put("cluster.routing.allocation.disk.watermark.low", "1b");
@ -1285,6 +1290,24 @@ public class ElasticsearchNode implements TestClusterConfiguration {
}
}
void setHttpPort(String httpPort) {
this.httpPort = httpPort;
}
void setTransportPort(String transportPort) {
this.transportPort = transportPort;
}
@Internal
Path getEsStdoutFile() {
return esStdoutFile;
}
@Internal
Path getEsStderrFile() {
return esStderrFile;
}
private static class FileEntry implements Named {
private String name;
private File file;

View File

@ -0,0 +1,73 @@
package org.elasticsearch.gradle.testclusters;
import org.gradle.api.logging.Logger;
import org.gradle.api.logging.Logging;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.TaskAction;
import org.gradle.api.tasks.options.Option;
import java.io.BufferedReader;
import java.io.IOException;
import java.nio.file.Files;
import java.util.HashSet;
import java.util.Set;
public class RunTask extends DefaultTestClustersTask {
private static final Logger logger = Logging.getLogger(RunTask.class);
private Boolean debug = false;
@Option(
option = "debug-jvm",
description = "Enable debugging configuration, to allow attaching a debugger to elasticsearch."
)
public void setDebug(boolean enabled) {
this.debug = debug;
}
@Input
public Boolean getDebug() {
return debug;
}
@Override
public void beforeStart() {
int debugPort = 8000;
int httpPort = 9200;
int transportPort = 9300;
for (ElasticsearchCluster cluster : getClusters()) {
cluster.getFirstNode().setHttpPort(String.valueOf(httpPort));
httpPort++;
cluster.getFirstNode().setTransportPort(String.valueOf(transportPort));
transportPort++;
for (ElasticsearchNode node : cluster.getNodes()) {
if (debug) {
logger.lifecycle(
"Running elasticsearch in debug mode, {} suspending until connected on debugPort {}",
node, debugPort
);
node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=" + debugPort);
debugPort += 1;
}
}
}
}
@TaskAction
public void runAndWait() throws IOException {
Set<BufferedReader> toRead = new HashSet<>();
for (ElasticsearchCluster cluster : getClusters()) {
for (ElasticsearchNode node : cluster.getNodes()) {
toRead.add(Files.newBufferedReader(node.getEsStdoutFile()));
}
}
while (Thread.currentThread().isInterrupted() == false) {
for (BufferedReader bufferedReader : toRead) {
if (bufferedReader.ready()) {
logger.lifecycle(bufferedReader.readLine());
}
}
}
}
}

View File

@ -18,9 +18,13 @@ interface TestClustersAware extends Task {
);
}
cluster.getNodes().stream().flatMap(node -> node.getDistributions().stream()).forEach( distro ->
cluster.getNodes().stream().flatMap(node -> node.getDistributions().stream()).forEach(distro ->
dependsOn(distro.getExtracted())
);
getClusters().add(cluster);
}
default void beforeStart() {
}
}

View File

@ -123,7 +123,9 @@ public class TestClustersPlugin implements Plugin<Project> {
return;
}
// we only start the cluster before the actions, so we'll not start it if the task is up-to-date
((TestClustersAware) task).getClusters().forEach(registry::maybeStartCluster);
TestClustersAware awareTask = (TestClustersAware) task;
awareTask.beforeStart();
awareTask.getClusters().forEach(registry::maybeStartCluster);
}
@Override
public void afterActions(Task task) {}

View File

@ -23,11 +23,13 @@ import org.elasticsearch.gradle.ConcatFilesTask
import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.NoticeTask
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.test.RunTask
import org.elasticsearch.gradle.testclusters.RunTask
import java.nio.file.Files
import java.nio.file.Path
apply plugin: 'elasticsearch.testclusters'
/*****************************************************************************
* Third party dependencies report *
*****************************************************************************/
@ -411,28 +413,32 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
}
task run(type: RunTask) {
distribution = System.getProperty('run.distribution', 'default')
if (distribution == 'default') {
String licenseType = System.getProperty("run.license_type", "basic")
if (licenseType == 'trial') {
setting 'xpack.ml.enabled', 'true'
setting 'xpack.graph.enabled', 'true'
setting 'xpack.watcher.enabled', 'true'
setting 'xpack.license.self_generated.type', 'trial'
} else if (licenseType != 'basic') {
throw new IllegalArgumentException("Unsupported self-generated license type: [" + licenseType + "[basic] or [trial].")
testClusters {
runTask {
testDistribution = System.getProperty('run.distribution', 'default')
if (System.getProperty('run.distribution', 'default') == 'default') {
String licenseType = System.getProperty("run.license_type", "basic")
if (licenseType == 'trial') {
setting 'xpack.ml.enabled', 'true'
setting 'xpack.graph.enabled', 'true'
setting 'xpack.watcher.enabled', 'true'
setting 'xpack.license.self_generated.type', 'trial'
} else if (licenseType != 'basic') {
throw new IllegalArgumentException("Unsupported self-generated license type: [" + licenseType + "[basic] or [trial].")
}
setting 'xpack.security.enabled', 'true'
setting 'xpack.monitoring.enabled', 'true'
setting 'xpack.sql.enabled', 'true'
setting 'xpack.rollup.enabled', 'true'
keystore 'bootstrap.password', 'password'
}
setupCommand 'setupTestAdmin',
'bin/elasticsearch-users', 'useradd', 'elastic-admin', '-p', 'elastic-password', '-r', 'superuser'
setting 'xpack.security.enabled', 'true'
setting 'xpack.monitoring.enabled', 'true'
setting 'xpack.sql.enabled', 'true'
setting 'xpack.rollup.enabled', 'true'
keystoreSetting 'bootstrap.password', 'password'
}
}
task run(type: RunTask) {
useCluster testClusters.runTask;
}
/**
* Build some variables that are replaced in the packages. This includes both
* scripts like bin/elasticsearch and bin/elasticsearch-plugin that a user might run and also

View File

@ -27,11 +27,6 @@ configure(subprojects.findAll { it.parent.path == project.path }) {
name project.name
}
run {
// these cannot be run with the normal distribution, since they are included in it!
distribution = 'integ-test-zip'
}
if (project.file('src/main/packaging').exists()) {
throw new InvalidModelException("Modules cannot contain packaging files")
}

View File

@ -16,10 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
import org.elasticsearch.gradle.test.ClusterConfiguration
import org.elasticsearch.gradle.test.ClusterFormationTasks
import org.elasticsearch.gradle.testclusters.DefaultTestClustersTask;
esplugin {
description 'An easy, safe and fast scripting language for Elasticsearch'
classname 'org.elasticsearch.painless.PainlessPlugin'
@ -76,16 +73,21 @@ dependencies {
docCompile project(':modules:lang-painless')
}
ClusterConfiguration clusterConfig = project.extensions.create("generateContextCluster", ClusterConfiguration.class, project)
gradle.projectsEvaluated {
project.ext.generateContextNodes = ClusterFormationTasks.setup(project, "generateContextCluster", generateContextDoc, clusterConfig)
testClusters {
generateContextCluster {
testDistribution = 'DEFAULT'
}
}
clusterConfig.distribution = 'default'
task generateContextDoc(type: JavaExec) {
main = 'org.elasticsearch.painless.ContextDocGenerator'
classpath = sourceSets.doc.runtimeClasspath
systemProperty "cluster.uri", "${-> project.ext.generateContextNodes.collect { it.httpUri() }.join(',') }"
task generateContextDoc(type: DefaultTestClustersTask) {
useCluster testClusters.generateContextCluster
doFirst {
project.javaexec {
main = 'org.elasticsearch.painless.ContextDocGenerator'
classpath = sourceSets.doc.runtimeClasspath
systemProperty "cluster.uri", "${-> testClusters.generateContextCluster.singleNode().getAllHttpSocketURI()}"
}.assertNormalExitValue()
}
}
/**********************************************

View File

@ -27,8 +27,3 @@ testClusters.integTest {
// Modules who's integration is explicitly tested in integration tests
module file(project(':modules:lang-mustache').tasks.bundlePlugin.archiveFile)
}
run {
// Modules who's integration is explicitly tested in integration tests
module project(':modules:lang-mustache')
}

View File

@ -37,14 +37,6 @@ testClusters.integTest {
setting 'reindex.remote.whitelist', '127.0.0.1:*'
}
run {
// Modules who's integration is explicitly tested in integration tests
module project(':modules:parent-join')
module project(':modules:lang-painless')
// Whitelist reindexing from the local node so we can test reindex-from-remote.
setting 'reindex.remote.whitelist', '127.0.0.1:*'
}
test {
/*
* We have to disable setting the number of available processors as tests in the

View File

@ -1,5 +1,3 @@
import org.elasticsearch.gradle.test.NodeInfo
import java.nio.charset.StandardCharsets
apply plugin: 'elasticsearch.docs-test'

View File

@ -1,6 +1,5 @@
import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.plugin.PluginBuildPlugin
import org.elasticsearch.gradle.test.NodeInfo
import java.nio.charset.StandardCharsets

View File

@ -1,6 +1,5 @@
import org.elasticsearch.gradle.BuildPlugin
import org.elasticsearch.gradle.LoggedExec
import org.elasticsearch.gradle.test.NodeInfo
import javax.net.ssl.HttpsURLConnection
import javax.net.ssl.KeyManagerFactory

View File

@ -1,5 +1,5 @@
import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.test.NodeInfo
import org.elasticsearch.gradle.http.WaitForHttpResource
apply plugin: 'elasticsearch.testclusters'