From d44dbd475752a58f76c3354a67339a63b9c3df5c Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 18 Nov 2015 22:22:47 -0800 Subject: [PATCH] Build: Fix integTest output if the elasticsearch script fails If there is a failure in the elasticsearch start script, we currently completely lose the failure. This is due to how spawning works with ant. This change avoids the issue by introducing an intermediate script, built dynamically before running ant exec, which runs elasticsearch and redirects the output to a log file. This essentially causes us to run elasticsearch in the foreground and capture the output, but at the same time keep a running script which ant can pump streams from (which will always be empty). --- build.gradle | 1 + .../gradle/test/ClusterFormationTasks.groovy | 63 +++++++++++++------ 2 files changed, 46 insertions(+), 18 deletions(-) diff --git a/build.gradle b/build.gradle index af0fa12917b..21019ad94db 100644 --- a/build.gradle +++ b/build.gradle @@ -188,5 +188,6 @@ task run(type: Run) { dependsOn ':distribution:run' description = 'Runs elasticsearch in the foreground' group = 'Verification' + impliesSubProjects = true } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 994d47be5b6..d49b1ffe0bf 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -86,6 +86,7 @@ class ClusterFormationTasks { // tasks are chained so their execution order is maintained Task setup = project.tasks.create(name: "${task.name}#clean", type: Delete, dependsOn: task.dependsOn.collect()) { delete home + delete cwd doLast { cwd.mkdirs() } @@ -222,6 +223,8 @@ class ClusterFormationTasks { } String executable + // running with cmd on windows will look for this with the .bat extension + String esScript = new File(home, 'bin/elasticsearch').toString() List esArgs = [] if (Os.isFamily(Os.FAMILY_WINDOWS)) { executable = 'cmd' @@ -230,8 +233,8 @@ class ClusterFormationTasks { } else { executable = 'sh' } - // running with cmd on windows will look for this with the .bat extension - esArgs.add(new File(home, 'bin/elasticsearch').toString()) + + File failedMarker = new File(cwd, 'run.failed') // this closure is converted into ant nodes by groovy's AntBuilder Closure antRunner = { @@ -242,16 +245,42 @@ class ClusterFormationTasks { esEnv['JAVA_OPTS'] += ' -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=8000' } + // Due to how ant exec works with the spawn option, we lose all stdout/stderr from the + // process executed. To work around this, when spawning, we wrap the elasticsearch start + // command inside another shell script, which simply internally redirects the output + // of the real elasticsearch script. This allows ant to keep the streams open with the + // dummy process, but us to have the output available if there is an error in the + // elasticsearch start script + if (config.daemonize) { + String scriptName = 'run' + String argsPasser = '"$@"' + String exitMarker = '; if [ $? != 0 ]; then touch run.failed; fi' + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + scriptName += '.bat' + argsPasser = '%*' + exitMarker = '\r\n if "%errorlevel%" neq "0" ( type nul >> run.failed )' + } + File wrapperScript = new File(cwd, scriptName) + wrapperScript.setText("\"${esScript}\" ${argsPasser} > run.log 2>&1 ${exitMarker}", 'UTF-8') + esScript = wrapperScript.toString() + } + exec(executable: executable, spawn: config.daemonize, dir: cwd, taskname: 'elasticsearch') { esEnv.each { key, value -> env(key: key, value: value) } - (esArgs + esProps).each { arg(value: it) } + arg(value: esScript) + esProps.each { arg(value: it) } } waitfor(maxwait: '30', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: "failed${name}") { - and { + or { resourceexists { - file(file: pidFile.toString()) + file(file: failedMarker.toString()) + } + and { + resourceexists { + file(file: pidFile.toString()) + } + http(url: "http://localhost:${config.httpPort}") } - http(url: "http://localhost:${config.httpPort}") } } } @@ -259,8 +288,8 @@ class ClusterFormationTasks { // this closure is the actual code to run elasticsearch Closure elasticsearchRunner = { // Command as string for logging - String esCommandString = "Elasticsearch command: ${executable} " - esCommandString += (esArgs + esProps).join(' ') + String esCommandString = "Elasticsearch command: ${esScript} " + esCommandString += esProps.join(' ') if (esEnv.isEmpty() == false) { esCommandString += '\nenvironment:' esEnv.each { k, v -> esCommandString += "\n ${k}: ${v}" } @@ -277,20 +306,18 @@ class ClusterFormationTasks { runAntCommand(project, antRunner, captureStream, captureStream) } - if (ant.properties.containsKey("failed${name}".toString())) { - // the waitfor failed, so dump any output we got (may be empty if info logging, but that is ok) - logger.error(buffer.toString('UTF-8')) - // also dump the cluster's log file, it may be useful - File logFile = new File(home, "logs/${clusterName}.log") - if (logFile.exists()) { - logFile.eachLine { line -> logger.error(line) } - } else { - logger.error("Couldn't start elasticsearch and couldn't find ${logFile}") - } + if (ant.properties.containsKey("failed${name}".toString()) || failedMarker.exists()) { if (logger.isInfoEnabled() == false) { // We already log the command at info level. No need to do it twice. logger.error(esCommandString) } + // the waitfor failed, so dump any output we got (may be empty if info logging, but that is ok) + logger.error(buffer.toString('UTF-8')) + // also dump the log file for the startup script (which will include ES logging output to stdout) + File startLog = new File(cwd, 'run.log') + if (startLog.exists()) { + startLog.eachLine { line -> logger.error(line) } + } throw new GradleException('Failed to start elasticsearch') } }