Merge branch 'master' into hdfs2-only

buildSrc/src/main/groovy/org/elasticsearch/gradle/AntTask.groovy

@@ -19,13 +19,13 @@
 
 package org.elasticsearch.gradle
 
-import org.apache.tools.ant.BuildException
 import org.apache.tools.ant.BuildListener
 import org.apache.tools.ant.BuildLogger
 import org.apache.tools.ant.DefaultLogger
 import org.apache.tools.ant.Project
 import org.gradle.api.DefaultTask
 import org.gradle.api.GradleException
+import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.TaskAction
 
 import java.nio.charset.Charset
@@ -58,14 +58,14 @@ public abstract class AntTask extends DefaultTask {
             ant.project.removeBuildListener(listener)
         }
 
-        final int outputLevel = logger.isDebugEnabled() ? Project.MSG_DEBUG : (logger.isInfoEnabled() ? Project.MSG_INFO : Project.MSG_WARN)
+        final int outputLevel = logger.isDebugEnabled() ? Project.MSG_DEBUG : Project.MSG_INFO
         final PrintStream stream = useStdout() ? System.out : new PrintStream(outputBuffer, true, Charset.defaultCharset().name())
         BuildLogger antLogger = makeLogger(stream, outputLevel)
 
         ant.project.addBuildListener(antLogger)
         try {
             runAnt(ant)
-        } catch (BuildException e) {
+        } catch (Exception e) {
             // ant failed, so see if we have buffered output to emit, then rethrow the failure
             String buffer = outputBuffer.toString()
             if (buffer.isEmpty() == false) {
@@ -76,7 +76,7 @@ public abstract class AntTask extends DefaultTask {
     }
 
     /** Runs the doAnt closure. This can be overridden by subclasses instead of having to set a closure. */
-    protected abstract void runAnt(AntBuilder ant);
+    protected abstract void runAnt(AntBuilder ant)
 
     /** Create the logger the ant runner will use, with the given stream for error/output. */
     protected BuildLogger makeLogger(PrintStream stream, int outputLevel) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/Fixture.groovy

@@ -0,0 +1,287 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.test
+
+import org.apache.tools.ant.taskdefs.condition.Os
+import org.elasticsearch.gradle.AntTask
+import org.elasticsearch.gradle.LoggedExec
+import org.gradle.api.GradleException
+import org.gradle.api.Task
+import org.gradle.api.tasks.Exec
+import org.gradle.api.tasks.Input
+
+/**
+ * A fixture for integration tests which runs in a separate process.
+ */
+public class Fixture extends AntTask {
+
+    /** The path to the executable that starts the fixture. */
+    @Input
+    String executable
+
+    private final List<Object> arguments = new ArrayList<>()
+
+    @Input
+    public void args(Object... args) {
+        arguments.addAll(args)
+    }
+
+    /**
+     * Environment variables for the fixture process. The value can be any object, which
+     * will have toString() called at execution time.
+     */
+    private final Map<String, Object> environment = new HashMap<>()
+
+    @Input
+    public void env(String key, Object value) {
+        environment.put(key, value)
+    }
+
+    /** A flag to indicate whether the command should be executed from a shell. */
+    @Input
+    boolean useShell = false
+
+    /**
+     * A flag to indicate whether the fixture should be run in the foreground, or spawned.
+     * It is protected so subclasses can override (eg RunTask).
+     */
+    protected boolean spawn = true
+
+    /**
+     * A closure to call before the fixture is considered ready. The closure is passed the fixture object,
+     * as well as a groovy AntBuilder, to enable running ant condition checks. The default wait
+     * condition is for http on the http port.
+     */
+    @Input
+    Closure waitCondition = { Fixture fixture, AntBuilder ant ->
+        File tmpFile = new File(fixture.cwd, 'wait.success')
+        ant.get(src: "http://${fixture.addressAndPort}",
+                dest: tmpFile.toString(),
+                ignoreerrors: true, // do not fail on error, so logging information can be flushed
+                retries: 10)
+        return tmpFile.exists()
+    }
+
+    /** A task which will stop this fixture. This should be used as a finalizedBy for any tasks that use the fixture. */
+    public final Task stopTask
+
+    public Fixture() {
+        stopTask = createStopTask()
+        finalizedBy(stopTask)
+    }
+
+    @Override
+    protected void runAnt(AntBuilder ant) {
+        project.delete(baseDir) // reset everything
+        cwd.mkdirs()
+        final String realExecutable
+        final List<Object> realArgs = new ArrayList<>()
+        final Map<String, Object> realEnv = environment
+        // We need to choose which executable we are using. In shell mode, or when we
+        // are spawning and thus using the wrapper script, the executable is the shell.
+        if (useShell || spawn) {
+            if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+                realExecutable = 'cmd'
+                realArgs.add('/C')
+                realArgs.add('"') // quote the entire command
+            } else {
+                realExecutable = 'sh'
+            }
+        } else {
+            realExecutable = executable
+            realArgs.addAll(arguments)
+        }
+        if (spawn) {
+            writeWrapperScript(executable)
+            realArgs.add(wrapperScript)
+            realArgs.addAll(arguments)
+        }
+        if (Os.isFamily(Os.FAMILY_WINDOWS) && (useShell || spawn)) {
+            realArgs.add('"')
+        }
+        commandString.eachLine { line -> logger.info(line) }
+
+        ant.exec(executable: realExecutable, spawn: spawn, dir: cwd, taskname: name) {
+            realEnv.each { key, value -> env(key: key, value: value) }
+            realArgs.each { arg(value: it) }
+        }
+
+        String failedProp = "failed${name}"
+        // first wait for resources, or the failure marker from the wrapper script
+        ant.waitfor(maxwait: '30', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: failedProp) {
+            or {
+                resourceexists {
+                    file(file: failureMarker.toString())
+                }
+                and {
+                    resourceexists {
+                        file(file: pidFile.toString())
+                    }
+                    resourceexists {
+                        file(file: portsFile.toString())
+                    }
+                }
+            }
+        }
+
+        if (ant.project.getProperty(failedProp) || failureMarker.exists()) {
+            fail("Failed to start ${name}")
+        }
+
+        // the process is started (has a pid) and is bound to a network interface
+        // so now wait undil the waitCondition has been met
+        // TODO: change this to a loop?
+        boolean success
+        try {
+            success = waitCondition(this, ant) == false
+        } catch (Exception e) {
+            String msg = "Wait condition caught exception for ${name}"
+            logger.error(msg, e)
+            fail(msg, e)
+        }
+        if (success == false) {
+            fail("Wait condition failed for ${name}")
+        }
+    }
+
+    /** Returns a debug string used to log information about how the fixture was run. */
+    protected String getCommandString() {
+        String commandString = "\n${name} configuration:\n"
+        commandString += "-----------------------------------------\n"
+        commandString += "  cwd: ${cwd}\n"
+        commandString += "  command: ${executable} ${arguments.join(' ')}\n"
+        commandString += '  environment:\n'
+        environment.each { k, v -> commandString += "    ${k}: ${v}\n" }
+        if (spawn) {
+            commandString += "\n  [${wrapperScript.name}]\n"
+            wrapperScript.eachLine('UTF-8', { line -> commandString += "    ${line}\n"})
+        }
+        return commandString
+    }
+
+    /**
+     * Writes a script to run the real executable, so that stdout/stderr can be captured.
+     * TODO: this could be removed if we do use our own ProcessBuilder and pump output from the process
+     */
+    private void writeWrapperScript(String executable) {
+        wrapperScript.parentFile.mkdirs()
+        String argsPasser = '"$@"'
+        String exitMarker = "; if [ \$? != 0 ]; then touch run.failed; fi"
+        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+            argsPasser = '%*'
+            exitMarker = "\r\n if \"%errorlevel%\" neq \"0\" ( type nul >> run.failed )"
+        }
+        wrapperScript.setText("\"${executable}\" ${argsPasser} > run.log 2>&1 ${exitMarker}", 'UTF-8')
+    }
+
+    /** Fail the build with the given message, and logging relevant info*/
+    private void fail(String msg, Exception... suppressed) {
+        if (logger.isInfoEnabled() == false) {
+            // We already log the command at info level. No need to do it twice.
+            commandString.eachLine { line -> logger.error(line) }
+        }
+        logger.error("${name} output:")
+        logger.error("-----------------------------------------")
+        logger.error("  failure marker exists: ${failureMarker.exists()}")
+        logger.error("  pid file exists: ${pidFile.exists()}")
+        logger.error("  ports file exists: ${portsFile.exists()}")
+        // also dump the log file for the startup script (which will include ES logging output to stdout)
+        if (runLog.exists()) {
+            logger.error("\n  [log]")
+            runLog.eachLine { line -> logger.error("    ${line}") }
+        }
+        logger.error("-----------------------------------------")
+        GradleException toThrow = new GradleException(msg)
+        for (Exception e : suppressed) {
+            toThrow.addSuppressed(e)
+        }
+        throw toThrow
+    }
+
+    /** Adds a task to kill an elasticsearch node with the given pidfile */
+    private Task createStopTask() {
+        final Fixture fixture = this
+        final Object pid = "${ -> fixture.pid }"
+        Exec stop = project.tasks.create(name: "${name}#stop", type: LoggedExec)
+        stop.onlyIf { fixture.pidFile.exists() }
+        stop.doFirst {
+            logger.info("Shutting down ${fixture.name} with pid ${pid}")
+        }
+        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+            stop.executable = 'Taskkill'
+            stop.args('/PID', pid, '/F')
+        } else {
+            stop.executable = 'kill'
+            stop.args('-9', pid)
+        }
+        stop.doLast {
+            project.delete(fixture.pidFile)
+        }
+        return stop
+    }
+
+    /**
+     * A path relative to the build dir that all configuration and runtime files
+     * will live in for this fixture
+     */
+    protected File getBaseDir() {
+        return new File(project.buildDir, "fixtures/${name}")
+    }
+
+    /** Returns the working directory for the process. Defaults to "cwd" inside baseDir. */
+    protected File getCwd() {
+        return new File(baseDir, 'cwd')
+    }
+
+    /** Returns the file the process writes its pid to. Defaults to "pid" inside baseDir. */
+    protected File getPidFile() {
+        return new File(baseDir, 'pid')
+    }
+
+    /** Reads the pid file and returns the process' pid */
+    public int getPid() {
+        return Integer.parseInt(pidFile.getText('UTF-8').trim())
+    }
+
+    /** Returns the file the process writes its bound ports to. Defaults to "ports" inside baseDir. */
+    protected File getPortsFile() {
+        return new File(baseDir, 'ports')
+    }
+
+    /** Returns an address and port suitable for a uri to connect to this node over http */
+    public String getAddressAndPort() {
+        return portsFile.readLines("UTF-8").get(0)
+    }
+
+    /** Returns a file that wraps around the actual command when {@code spawn == true}. */
+    protected File getWrapperScript() {
+        return new File(cwd, Os.isFamily(Os.FAMILY_WINDOWS) ? 'run.bat' : 'run')
+    }
+
+    /** Returns a file that the wrapper script writes when the command failed. */
+    protected File getFailureMarker() {
+        return new File(cwd, 'run.failed')
+    }
+
+    /** Returns a file that the wrapper script writes when the command failed. */
+    protected File getRunLog() {
+        return new File(cwd, 'run.log')
+    }
+}

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -20,6 +20,8 @@ package org.elasticsearch.gradle.test
 
 import com.carrotsearch.gradle.junit4.RandomizedTestingTask
 import org.elasticsearch.gradle.BuildPlugin
+import org.gradle.api.GradleException
+import org.gradle.api.Task
 import org.gradle.api.internal.tasks.options.Option
 import org.gradle.api.plugins.JavaBasePlugin
 import org.gradle.api.tasks.Input
@@ -80,4 +82,25 @@ public class RestIntegTestTask extends RandomizedTestingTask {
     public ClusterConfiguration getCluster() {
         return clusterConfig
     }
+
+    @Override
+    public Task dependsOn(Object... dependencies) {
+        super.dependsOn(dependencies)
+        for (Object dependency : dependencies) {
+            if (dependency instanceof Fixture) {
+                finalizedBy(((Fixture)dependency).stopTask)
+            }
+        }
+        return this
+    }
+
+    @Override
+    public void setDependsOn(Iterable<?> dependencies) {
+        super.setDependsOn(dependencies)
+        for (Object dependency : dependencies) {
+            if (dependency instanceof Fixture) {
+                finalizedBy(((Fixture)dependency).stopTask)
+            }
+        }
+    }
 }

plugins/build.gradle

@@ -17,7 +17,8 @@
  * under the License.
  */
 
-subprojects {
+// only configure immediate children of plugins dir
+configure(subprojects.findAll { it.parent.path == project.path }) {
   group = 'org.elasticsearch.plugin'
 
   apply plugin: 'elasticsearch.esplugin'

plugins/jvm-example/build.gradle

@@ -27,3 +27,24 @@ test.enabled = false
 
 compileJava.options.compilerArgs << "-Xlint:-rawtypes"
 
+configurations {
+  exampleFixture
+}
+
+dependencies {
+  exampleFixture project(':test:fixtures:example-fixture')
+}
+
+task exampleFixture(type: org.elasticsearch.gradle.test.Fixture) {
+  dependsOn project.configurations.exampleFixture
+  executable = new File(project.javaHome, 'bin/java')
+  args '-cp', "${ -> project.configurations.exampleFixture.asPath }",
+       'example.ExampleTestFixture',
+       baseDir
+}
+
+integTest {
+  dependsOn exampleFixture
+  systemProperty 'external.address', "${ -> exampleFixture.addressAndPort }"
+}
+

plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/ExampleExternalIT.java

@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugin.example;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.InetAddress;
+import java.net.Socket;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
+
+public class ExampleExternalIT extends ESTestCase {
+    public void testExample() throws Exception {
+        String stringAddress = Objects.requireNonNull(System.getProperty("external.address"));
+        URL url = new URL("http://" + stringAddress);
+        InetAddress address = InetAddress.getByName(url.getHost());
+        try (Socket socket = new Socket(address, url.getPort());
+             BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))) {
+           assertEquals("TEST", reader.readLine());
+        }
+    }
+}

qa/smoke-test-plugins/build.gradle

@@ -22,7 +22,7 @@ import org.elasticsearch.gradle.MavenFilteringHack
 apply plugin: 'elasticsearch.rest-test'
 
 ext.pluginsCount = 0
-project.rootProject.subprojects.findAll { it.path.startsWith(':plugins:') }.each { subproj ->
+project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj ->
   integTest {
     cluster {
       // need to get a non-decorated project object, so must re-lookup the project by path

settings.gradle

@@ -9,6 +9,7 @@ List projects = [
   'distribution:deb',
   'distribution:rpm',
   'test:framework',
+  'test:fixtures:example-fixture',
   'modules:lang-expression',
   'modules:lang-groovy',
   'modules:lang-mustache',

test/build.gradle

@@ -17,7 +17,27 @@
  * under the License.
  */
 
+import org.elasticsearch.gradle.precommit.PrecommitTasks
+
 subprojects {
+  // fixtures is just an intermediate parent project
+  if (name == 'fixtures') return
+
   group = 'org.elasticsearch.test'
-  apply plugin: 'com.bmuschko.nexus'
+  apply plugin: 'elasticsearch.build'
+
+  
+  // the main files are actually test files, so use the appopriate forbidden api sigs
+  forbiddenApisMain {
+    bundledSignatures = ['jdk-unsafe', 'jdk-deprecated']
+    signaturesURLs = [PrecommitTasks.getResource('/forbidden/all-signatures.txt'),
+                      PrecommitTasks.getResource('/forbidden/test-signatures.txt')]
+  }
+
+  // TODO: should we have licenses for our test deps?
+  dependencyLicenses.enabled = false
+
+  // TODO: why is the test framework pulled in...
+  forbiddenApisMain.enabled = false
+  jarHell.enabled = false
 }

test/fixtures/example-fixture/build.gradle

@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+apply plugin: 'elasticsearch.build'

test/fixtures/example-fixture/src/main/java/example/ExampleTestFixture.java

@@ -0,0 +1,83 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package example;
+
+import java.lang.management.ManagementFactory;
+import java.net.Inet6Address;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.AsynchronousServerSocketChannel;
+import java.nio.channels.AsynchronousSocketChannel;
+import java.nio.channels.CompletionHandler;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.Collections;
+
+/** Crappy example test fixture that responds with TEST and closes the connection */
+public class ExampleTestFixture {
+    public static void main(String args[]) throws Exception {
+        if (args.length != 1) {
+            throw new IllegalArgumentException("ExampleTestFixture <logDirectory>");
+        }
+        Path dir = Paths.get(args[0]);
+        AsynchronousServerSocketChannel server = AsynchronousServerSocketChannel
+                .open()
+                .bind(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0));
+
+        // write pid file
+        Path tmp = Files.createTempFile(dir, null, null);
+        String pid = ManagementFactory.getRuntimeMXBean().getName().split("@")[0];
+        Files.write(tmp, Collections.singleton(pid));
+        Files.move(tmp, dir.resolve("pid"), StandardCopyOption.ATOMIC_MOVE);
+
+        // write port file
+        tmp = Files.createTempFile(dir, null, null);
+        InetSocketAddress bound = (InetSocketAddress) server.getLocalAddress();
+        if (bound.getAddress() instanceof Inet6Address) {
+            Files.write(tmp, Collections.singleton("[" + bound.getHostString() + "]:" + bound.getPort()));
+        } else {
+            Files.write(tmp, Collections.singleton(bound.getHostString() + ":" + bound.getPort()));
+        }
+        Files.move(tmp, dir.resolve("ports"), StandardCopyOption.ATOMIC_MOVE);
+
+        // go time
+        server.accept(null, new CompletionHandler<AsynchronousSocketChannel,Void>() {
+            @Override
+            public void completed(AsynchronousSocketChannel socket, Void attachment) {
+                server.accept(null, this);
+                try (AsynchronousSocketChannel ch = socket) {
+                    ch.write(ByteBuffer.wrap("TEST\n".getBytes(StandardCharsets.UTF_8))).get();
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+            }
+
+            @Override
+            public void failed(Throwable exc, Void attachment) {}
+        });
+
+        // wait forever, until you kill me
+        Thread.sleep(Long.MAX_VALUE);
+    }
+}

test/framework/build.gradle

@@ -16,9 +16,6 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-import org.elasticsearch.gradle.precommit.PrecommitTasks
-
-apply plugin: 'elasticsearch.build'
 
 dependencies {
   compile "org.elasticsearch:elasticsearch:${version}"
@@ -36,15 +33,5 @@ dependencies {
 compileJava.options.compilerArgs << '-Xlint:-cast,-deprecation,-fallthrough,-overrides,-rawtypes,-serial,-try,-unchecked'
 compileTestJava.options.compilerArgs << '-Xlint:-rawtypes'
 
-// the main files are actually test files, so use the appopriate forbidden api sigs
-forbiddenApisMain {
-  bundledSignatures = ['jdk-unsafe', 'jdk-deprecated']
-  signaturesURLs = [PrecommitTasks.getResource('/forbidden/all-signatures.txt'),
-                    PrecommitTasks.getResource('/forbidden/test-signatures.txt')]
-}
-
-// TODO: should we have licenses for our test deps?
-dependencyLicenses.enabled = false
-
 // we intentionally exclude the ant tasks because people were depending on them from their tests!!!!!!!
 thirdPartyAudit.missingClasses = true