honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into hdfs2-only

This commit is contained in:
Robert Muir 2015-12-19 21:52:52 -05:00
parent 323111b715 403644cb48
commit ae89c6e51c
13 changed files with 505 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
buildSrc/src/main/groovy/org/elasticsearch/gradle/AntTask.groovy
View File

@ -19,13 +19,13 @@
package org.elasticsearch.gradle
import org.apache.tools.ant.BuildException
import org.apache.tools.ant.BuildListener
import org.apache.tools.ant.BuildLogger
import org.apache.tools.ant.DefaultLogger
import org.apache.tools.ant.Project
import org.gradle.api.DefaultTask
import org.gradle.api.GradleException
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.TaskAction
import java.nio.charset.Charset
@ -58,14 +58,14 @@ public abstract class AntTask extends DefaultTask {
ant.project.removeBuildListener(listener)
}
final int outputLevel = logger.isDebugEnabled() ? Project.MSG_DEBUG : (logger.isInfoEnabled() ? Project.MSG_INFO : Project.MSG_WARN)
final int outputLevel = logger.isDebugEnabled() ? Project.MSG_DEBUG : Project.MSG_INFO
final PrintStream stream = useStdout() ? System.out : new PrintStream(outputBuffer, true, Charset.defaultCharset().name())
BuildLogger antLogger = makeLogger(stream, outputLevel)
ant.project.addBuildListener(antLogger)
try {
runAnt(ant)
} catch (BuildException e) {
} catch (Exception e) {
// ant failed, so see if we have buffered output to emit, then rethrow the failure
String buffer = outputBuffer.toString()
if (buffer.isEmpty() == false) {
@ -76,7 +76,7 @@ public abstract class AntTask extends DefaultTask {
}
/** Runs the doAnt closure. This can be overridden by subclasses instead of having to set a closure. */
protected abstract void runAnt(AntBuilder ant);
protected abstract void runAnt(AntBuilder ant)
/** Create the logger the ant runner will use, with the given stream for error/output. */
protected BuildLogger makeLogger(PrintStream stream, int outputLevel) {

287
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/Fixture.groovy Normal file
View File

@ -0,0 +1,287 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.gradle.test
import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.AntTask
import org.elasticsearch.gradle.LoggedExec
import org.gradle.api.GradleException
import org.gradle.api.Task
import org.gradle.api.tasks.Exec
import org.gradle.api.tasks.Input
/**
* A fixture for integration tests which runs in a separate process.
*/
public class Fixture extends AntTask {
/** The path to the executable that starts the fixture. */
@Input
String executable
private final List<Object> arguments = new ArrayList<>()
@Input
public void args(Object... args) {
arguments.addAll(args)
}
/**
* Environment variables for the fixture process. The value can be any object, which
* will have toString() called at execution time.
*/
private final Map<String, Object> environment = new HashMap<>()
@Input
public void env(String key, Object value) {
environment.put(key, value)
}
/** A flag to indicate whether the command should be executed from a shell. */
@Input
boolean useShell = false
/**
* A flag to indicate whether the fixture should be run in the foreground, or spawned.
* It is protected so subclasses can override (eg RunTask).
*/
protected boolean spawn = true
/**
* A closure to call before the fixture is considered ready. The closure is passed the fixture object,
* as well as a groovy AntBuilder, to enable running ant condition checks. The default wait
* condition is for http on the http port.
*/
@Input
Closure waitCondition = { Fixture fixture, AntBuilder ant ->
File tmpFile = new File(fixture.cwd, 'wait.success')
ant.get(src: "http://${fixture.addressAndPort}",
dest: tmpFile.toString(),
ignoreerrors: true, // do not fail on error, so logging information can be flushed
retries: 10)
return tmpFile.exists()
}
/** A task which will stop this fixture. This should be used as a finalizedBy for any tasks that use the fixture. */
public final Task stopTask
public Fixture() {
stopTask = createStopTask()
finalizedBy(stopTask)
}
@Override
protected void runAnt(AntBuilder ant) {
project.delete(baseDir) // reset everything
cwd.mkdirs()
final String realExecutable
final List<Object> realArgs = new ArrayList<>()
final Map<String, Object> realEnv = environment
// We need to choose which executable we are using. In shell mode, or when we
// are spawning and thus using the wrapper script, the executable is the shell.
if (useShell || spawn) {
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
realExecutable = 'cmd'
realArgs.add('/C')
realArgs.add('"') // quote the entire command
} else {
realExecutable = 'sh'
}
} else {
realExecutable = executable
realArgs.addAll(arguments)
}
if (spawn) {
writeWrapperScript(executable)
realArgs.add(wrapperScript)
realArgs.addAll(arguments)
}
if (Os.isFamily(Os.FAMILY_WINDOWS) && (useShell || spawn)) {
realArgs.add('"')
}
commandString.eachLine { line -> logger.info(line) }
ant.exec(executable: realExecutable, spawn: spawn, dir: cwd, taskname: name) {
realEnv.each { key, value -> env(key: key, value: value) }
realArgs.each { arg(value: it) }
}
String failedProp = "failed${name}"
// first wait for resources, or the failure marker from the wrapper script
ant.waitfor(maxwait: '30', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: failedProp) {
or {
resourceexists {
file(file: failureMarker.toString())
}
and {
resourceexists {
file(file: pidFile.toString())
}
resourceexists {
file(file: portsFile.toString())
}
}
}
}
if (ant.project.getProperty(failedProp) || failureMarker.exists()) {
fail("Failed to start ${name}")
}
// the process is started (has a pid) and is bound to a network interface
// so now wait undil the waitCondition has been met
// TODO: change this to a loop?
boolean success
try {
success = waitCondition(this, ant) == false
} catch (Exception e) {
String msg = "Wait condition caught exception for ${name}"
logger.error(msg, e)
fail(msg, e)
}
if (success == false) {
fail("Wait condition failed for ${name}")
}
}
/** Returns a debug string used to log information about how the fixture was run. */
protected String getCommandString() {
String commandString = "\n${name} configuration:\n"
commandString += "-----------------------------------------\n"
commandString += " cwd: ${cwd}\n"
commandString += " command: ${executable} ${arguments.join(' ')}\n"
commandString += ' environment:\n'
environment.each { k, v -> commandString += " ${k}: ${v}\n" }
if (spawn) {
commandString += "\n [${wrapperScript.name}]\n"
wrapperScript.eachLine('UTF-8', { line -> commandString += " ${line}\n"})
}
return commandString
}
/**
* Writes a script to run the real executable, so that stdout/stderr can be captured.
* TODO: this could be removed if we do use our own ProcessBuilder and pump output from the process
*/
private void writeWrapperScript(String executable) {
wrapperScript.parentFile.mkdirs()
String argsPasser = '"$@"'
String exitMarker = "; if [ \$? != 0 ]; then touch run.failed; fi"
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
argsPasser = '%*'
exitMarker = "\r\n if \"%errorlevel%\" neq \"0\" ( type nul >> run.failed )"
}
wrapperScript.setText("\"${executable}\" ${argsPasser} > run.log 2>&1 ${exitMarker}", 'UTF-8')
}
/** Fail the build with the given message, and logging relevant info*/
private void fail(String msg, Exception... suppressed) {
if (logger.isInfoEnabled() == false) {
// We already log the command at info level. No need to do it twice.
commandString.eachLine { line -> logger.error(line) }
}
logger.error("${name} output:")
logger.error("-----------------------------------------")
logger.error(" failure marker exists: ${failureMarker.exists()}")
logger.error(" pid file exists: ${pidFile.exists()}")
logger.error(" ports file exists: ${portsFile.exists()}")
// also dump the log file for the startup script (which will include ES logging output to stdout)
if (runLog.exists()) {
logger.error("\n [log]")
runLog.eachLine { line -> logger.error(" ${line}") }
}
logger.error("-----------------------------------------")
GradleException toThrow = new GradleException(msg)
for (Exception e : suppressed) {
toThrow.addSuppressed(e)
}
throw toThrow
}
/** Adds a task to kill an elasticsearch node with the given pidfile */
private Task createStopTask() {
final Fixture fixture = this
final Object pid = "${ -> fixture.pid }"
Exec stop = project.tasks.create(name: "${name}#stop", type: LoggedExec)
stop.onlyIf { fixture.pidFile.exists() }
stop.doFirst {
logger.info("Shutting down ${fixture.name} with pid ${pid}")
}
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
stop.executable = 'Taskkill'
stop.args('/PID', pid, '/F')
} else {
stop.executable = 'kill'
stop.args('-9', pid)
}
stop.doLast {
project.delete(fixture.pidFile)
}
return stop
}
/**
* A path relative to the build dir that all configuration and runtime files
* will live in for this fixture
*/
protected File getBaseDir() {
return new File(project.buildDir, "fixtures/${name}")
}
/** Returns the working directory for the process. Defaults to "cwd" inside baseDir. */
protected File getCwd() {
return new File(baseDir, 'cwd')
}
/** Returns the file the process writes its pid to. Defaults to "pid" inside baseDir. */
protected File getPidFile() {
return new File(baseDir, 'pid')
}
/** Reads the pid file and returns the process' pid */
public int getPid() {
return Integer.parseInt(pidFile.getText('UTF-8').trim())
}
/** Returns the file the process writes its bound ports to. Defaults to "ports" inside baseDir. */
protected File getPortsFile() {
return new File(baseDir, 'ports')
}
/** Returns an address and port suitable for a uri to connect to this node over http */
public String getAddressAndPort() {
return portsFile.readLines("UTF-8").get(0)
}
/** Returns a file that wraps around the actual command when {@code spawn == true}. */
protected File getWrapperScript() {
return new File(cwd, Os.isFamily(Os.FAMILY_WINDOWS) ? 'run.bat' : 'run')
}
/** Returns a file that the wrapper script writes when the command failed. */
protected File getFailureMarker() {
return new File(cwd, 'run.failed')
}
/** Returns a file that the wrapper script writes when the command failed. */
protected File getRunLog() {
return new File(cwd, 'run.log')
}
}

23
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
View File

@ -20,6 +20,8 @@ package org.elasticsearch.gradle.test
import com.carrotsearch.gradle.junit4.RandomizedTestingTask
import org.elasticsearch.gradle.BuildPlugin
import org.gradle.api.GradleException
import org.gradle.api.Task
import org.gradle.api.internal.tasks.options.Option
import org.gradle.api.plugins.JavaBasePlugin
import org.gradle.api.tasks.Input
@ -80,4 +82,25 @@ public class RestIntegTestTask extends RandomizedTestingTask {
public ClusterConfiguration getCluster() {
return clusterConfig
}
@Override
public Task dependsOn(Object... dependencies) {
super.dependsOn(dependencies)
for (Object dependency : dependencies) {
if (dependency instanceof Fixture) {
finalizedBy(((Fixture)dependency).stopTask)
}
}
return this
}
@Override
public void setDependsOn(Iterable<?> dependencies) {
super.setDependsOn(dependencies)
for (Object dependency : dependencies) {
if (dependency instanceof Fixture) {
finalizedBy(((Fixture)dependency).stopTask)
}
}
}
}

3
plugins/build.gradle
View File

@ -17,7 +17,8 @@
* under the License.
*/
subprojects {
// only configure immediate children of plugins dir
configure(subprojects.findAll { it.parent.path == project.path }) {
group = 'org.elasticsearch.plugin'
apply plugin: 'elasticsearch.esplugin'

21
plugins/jvm-example/build.gradle
View File

@ -27,3 +27,24 @@ test.enabled = false
compileJava.options.compilerArgs << "-Xlint:-rawtypes"
configurations {
exampleFixture
}
dependencies {
exampleFixture project(':test:fixtures:example-fixture')
}
task exampleFixture(type: org.elasticsearch.gradle.test.Fixture) {
dependsOn project.configurations.exampleFixture
executable = new File(project.javaHome, 'bin/java')
args '-cp', "${ -> project.configurations.exampleFixture.asPath }",
'example.ExampleTestFixture',
baseDir
}
integTest {
dependsOn exampleFixture
systemProperty 'external.address', "${ -> exampleFixture.addressAndPort }"
}

42
plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/ExampleExternalIT.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.plugin.example;
import org.elasticsearch.test.ESTestCase;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.InetAddress;
import java.net.Socket;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Objects;
public class ExampleExternalIT extends ESTestCase {
public void testExample() throws Exception {
String stringAddress = Objects.requireNonNull(System.getProperty("external.address"));
URL url = new URL("http://" + stringAddress);
InetAddress address = InetAddress.getByName(url.getHost());
try (Socket socket = new Socket(address, url.getPort());
BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8))) {
assertEquals("TEST", reader.readLine());
}
}
}

2
qa/smoke-test-plugins/build.gradle
View File

@ -22,7 +22,7 @@ import org.elasticsearch.gradle.MavenFilteringHack
apply plugin: 'elasticsearch.rest-test'
ext.pluginsCount = 0
project.rootProject.subprojects.findAll { it.path.startsWith(':plugins:') }.each { subproj ->
project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj ->
integTest {
cluster {
// need to get a non-decorated project object, so must re-lookup the project by path

1
settings.gradle
View File

@ -9,6 +9,7 @@ List projects = [
'distribution:deb',
'distribution:rpm',
'test:framework',
'test:fixtures:example-fixture',
'modules:lang-expression',
'modules:lang-groovy',
'modules:lang-mustache',

22
test/build.gradle
View File

@ -17,7 +17,27 @@
* under the License.
*/
import org.elasticsearch.gradle.precommit.PrecommitTasks
subprojects {
// fixtures is just an intermediate parent project
if (name == 'fixtures') return
group = 'org.elasticsearch.test'
apply plugin: 'com.bmuschko.nexus'
apply plugin: 'elasticsearch.build'
// the main files are actually test files, so use the appopriate forbidden api sigs
forbiddenApisMain {
bundledSignatures = ['jdk-unsafe', 'jdk-deprecated']
signaturesURLs = [PrecommitTasks.getResource('/forbidden/all-signatures.txt'),
PrecommitTasks.getResource('/forbidden/test-signatures.txt')]
}
// TODO: should we have licenses for our test deps?
dependencyLicenses.enabled = false
// TODO: why is the test framework pulled in...
forbiddenApisMain.enabled = false
jarHell.enabled = false
}

0
test/fixtures/build.gradle vendored Normal file
View File

20
test/fixtures/example-fixture/build.gradle vendored Normal file
View File

@ -0,0 +1,20 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
apply plugin: 'elasticsearch.build'

83
test/fixtures/example-fixture/src/main/java/example/ExampleTestFixture.java vendored Normal file
View File

@ -0,0 +1,83 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package example;
import java.lang.management.ManagementFactory;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.nio.channels.AsynchronousServerSocketChannel;
import java.nio.channels.AsynchronousSocketChannel;
import java.nio.channels.CompletionHandler;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.Collections;
/** Crappy example test fixture that responds with TEST and closes the connection */
public class ExampleTestFixture {
public static void main(String args[]) throws Exception {
if (args.length != 1) {
throw new IllegalArgumentException("ExampleTestFixture <logDirectory>");
}
Path dir = Paths.get(args[0]);
AsynchronousServerSocketChannel server = AsynchronousServerSocketChannel
.open()
.bind(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0));
// write pid file
Path tmp = Files.createTempFile(dir, null, null);
String pid = ManagementFactory.getRuntimeMXBean().getName().split("@")[0];
Files.write(tmp, Collections.singleton(pid));
Files.move(tmp, dir.resolve("pid"), StandardCopyOption.ATOMIC_MOVE);
// write port file
tmp = Files.createTempFile(dir, null, null);
InetSocketAddress bound = (InetSocketAddress) server.getLocalAddress();
if (bound.getAddress() instanceof Inet6Address) {
Files.write(tmp, Collections.singleton("[" + bound.getHostString() + "]:" + bound.getPort()));
} else {
Files.write(tmp, Collections.singleton(bound.getHostString() + ":" + bound.getPort()));
}
Files.move(tmp, dir.resolve("ports"), StandardCopyOption.ATOMIC_MOVE);
// go time
server.accept(null, new CompletionHandler<AsynchronousSocketChannel,Void>() {
@Override
public void completed(AsynchronousSocketChannel socket, Void attachment) {
server.accept(null, this);
try (AsynchronousSocketChannel ch = socket) {
ch.write(ByteBuffer.wrap("TEST\n".getBytes(StandardCharsets.UTF_8))).get();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void failed(Throwable exc, Void attachment) {}
});
// wait forever, until you kill me
Thread.sleep(Long.MAX_VALUE);
}
}

13
test/framework/build.gradle
View File

@ -16,9 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
import org.elasticsearch.gradle.precommit.PrecommitTasks
apply plugin: 'elasticsearch.build'
dependencies {
compile "org.elasticsearch:elasticsearch:${version}"
@ -36,15 +33,5 @@ dependencies {
compileJava.options.compilerArgs << '-Xlint:-cast,-deprecation,-fallthrough,-overrides,-rawtypes,-serial,-try,-unchecked'
compileTestJava.options.compilerArgs << '-Xlint:-rawtypes'
// the main files are actually test files, so use the appopriate forbidden api sigs
forbiddenApisMain {
bundledSignatures = ['jdk-unsafe', 'jdk-deprecated']
signaturesURLs = [PrecommitTasks.getResource('/forbidden/all-signatures.txt'),
PrecommitTasks.getResource('/forbidden/test-signatures.txt')]
}
// TODO: should we have licenses for our test deps?
dependencyLicenses.enabled = false
// we intentionally exclude the ant tasks because people were depending on them from their tests!!!!!!!
thirdPartyAudit.missingClasses = true