HADOOP-14696. parallel tests don't work for Windows. Contributed by Allen Wittenauer
This commit is contained in:
parent
19292bc264
commit
45d1b0fdcc
|
@ -979,30 +979,13 @@
|
|||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-maven-plugins</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>create-parallel-tests-dirs</id>
|
||||
<phase>test-compile</phase>
|
||||
<configuration>
|
||||
<target>
|
||||
<script language="javascript"><![CDATA[
|
||||
var baseDirs = [
|
||||
"${test.build.data}",
|
||||
"${test.build.dir}",
|
||||
"${hadoop.tmp.dir}" ];
|
||||
for (var i in baseDirs) {
|
||||
for (var j = 1; j <= ${testsThreadCount}; ++j) {
|
||||
var mkdir = project.createTask("mkdir");
|
||||
mkdir.setDir(new java.io.File(baseDirs[i], j));
|
||||
mkdir.perform();
|
||||
}
|
||||
}
|
||||
]]></script>
|
||||
</target>
|
||||
</configuration>
|
||||
<id>parallel-tests-createdir</id>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
<goal>parallel-tests-createdir</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
|
@ -1015,6 +998,7 @@
|
|||
<reuseForks>false</reuseForks>
|
||||
<argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
|
||||
<systemPropertyVariables>
|
||||
<testsThreadCount>${testsThreadCount}</testsThreadCount>
|
||||
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
|
||||
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
|
||||
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
|
||||
|
|
|
@ -286,7 +286,7 @@ public abstract class GenericTestUtils {
|
|||
public static void assertExists(File f) {
|
||||
Assert.assertTrue("File " + f + " should exist", f.exists());
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* List all of the files in 'dir' that match the regex 'pattern'.
|
||||
* Then check that this list is identical to 'expectedMatches'.
|
||||
|
@ -294,7 +294,7 @@ public abstract class GenericTestUtils {
|
|||
*/
|
||||
public static void assertGlobEquals(File dir, String pattern,
|
||||
String ... expectedMatches) throws IOException {
|
||||
|
||||
|
||||
Set<String> found = Sets.newTreeSet();
|
||||
for (File f : FileUtil.listFiles(dir)) {
|
||||
if (f.getName().matches(pattern)) {
|
||||
|
@ -349,7 +349,7 @@ public abstract class GenericTestUtils {
|
|||
StringUtils.stringifyException(t)),
|
||||
t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for the specified test to return true. The test will be performed
|
||||
|
@ -499,18 +499,18 @@ public abstract class GenericTestUtils {
|
|||
*/
|
||||
public static class DelayAnswer implements Answer<Object> {
|
||||
private final Log LOG;
|
||||
|
||||
|
||||
private final CountDownLatch fireLatch = new CountDownLatch(1);
|
||||
private final CountDownLatch waitLatch = new CountDownLatch(1);
|
||||
private final CountDownLatch resultLatch = new CountDownLatch(1);
|
||||
|
||||
|
||||
private final AtomicInteger fireCounter = new AtomicInteger(0);
|
||||
private final AtomicInteger resultCounter = new AtomicInteger(0);
|
||||
|
||||
|
||||
// Result fields set after proceed() is called.
|
||||
private volatile Throwable thrown;
|
||||
private volatile Object returnValue;
|
||||
|
||||
|
||||
public DelayAnswer(Log log) {
|
||||
this.LOG = log;
|
||||
}
|
||||
|
@ -521,7 +521,7 @@ public abstract class GenericTestUtils {
|
|||
public void waitForCall() throws InterruptedException {
|
||||
fireLatch.await();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Tell the method to proceed.
|
||||
* This should only be called after waitForCall()
|
||||
|
@ -529,7 +529,7 @@ public abstract class GenericTestUtils {
|
|||
public void proceed() {
|
||||
waitLatch.countDown();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Object answer(InvocationOnMock invocation) throws Throwable {
|
||||
LOG.info("DelayAnswer firing fireLatch");
|
||||
|
@ -558,7 +558,7 @@ public abstract class GenericTestUtils {
|
|||
resultLatch.countDown();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* After calling proceed(), this will wait until the call has
|
||||
* completed and a result has been returned to the caller.
|
||||
|
@ -566,7 +566,7 @@ public abstract class GenericTestUtils {
|
|||
public void waitForResult() throws InterruptedException {
|
||||
resultLatch.await();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* After the call has gone through, return any exception that
|
||||
* was thrown, or null if no exception was thrown.
|
||||
|
@ -574,7 +574,7 @@ public abstract class GenericTestUtils {
|
|||
public Throwable getThrown() {
|
||||
return thrown;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* After the call has gone through, return the call's return value,
|
||||
* or null in case it was void or an exception was thrown.
|
||||
|
@ -582,20 +582,20 @@ public abstract class GenericTestUtils {
|
|||
public Object getReturnValue() {
|
||||
return returnValue;
|
||||
}
|
||||
|
||||
|
||||
public int getFireCount() {
|
||||
return fireCounter.get();
|
||||
}
|
||||
|
||||
|
||||
public int getResultCount() {
|
||||
return resultCounter.get();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* An Answer implementation that simply forwards all calls through
|
||||
* to a delegate.
|
||||
*
|
||||
*
|
||||
* This is useful as the default Answer for a mock object, to create
|
||||
* something like a spy on an RPC proxy. For example:
|
||||
* <code>
|
||||
|
@ -606,14 +606,14 @@ public abstract class GenericTestUtils {
|
|||
* ...
|
||||
* </code>
|
||||
*/
|
||||
public static class DelegateAnswer implements Answer<Object> {
|
||||
public static class DelegateAnswer implements Answer<Object> {
|
||||
private final Object delegate;
|
||||
private final Log log;
|
||||
|
||||
|
||||
public DelegateAnswer(Object delegate) {
|
||||
this(null, delegate);
|
||||
}
|
||||
|
||||
|
||||
public DelegateAnswer(Log log, Object delegate) {
|
||||
this.log = log;
|
||||
this.delegate = delegate;
|
||||
|
@ -653,7 +653,7 @@ public abstract class GenericTestUtils {
|
|||
this.minSleepTime = minSleepTime;
|
||||
this.maxSleepTime = maxSleepTime;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Object answer(InvocationOnMock invocation) throws Throwable {
|
||||
boolean interrupted = false;
|
||||
|
@ -683,11 +683,11 @@ public abstract class GenericTestUtils {
|
|||
" but got:\n" + output,
|
||||
Pattern.compile(pattern).matcher(output).find());
|
||||
}
|
||||
|
||||
|
||||
public static void assertValueNear(long expected, long actual, long allowedError) {
|
||||
assertValueWithinRange(expected - allowedError, expected + allowedError, actual);
|
||||
}
|
||||
|
||||
|
||||
public static void assertValueWithinRange(long expectedMin, long expectedMax,
|
||||
long actual) {
|
||||
Assert.assertTrue("Expected " + actual + " to be in range (" + expectedMin + ","
|
||||
|
@ -842,4 +842,28 @@ public abstract class GenericTestUtils {
|
|||
failf(format, args);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retreive the max number of parallel test threads when running under maven.
|
||||
* @return int number of threads
|
||||
*/
|
||||
public static int getTestsThreadCount() {
|
||||
String propString = System.getProperty("testsThreadCount", "1");
|
||||
int threadCount = 1;
|
||||
if (propString != null) {
|
||||
String trimProp = propString.trim();
|
||||
if (trimProp.endsWith("C")) {
|
||||
double multiplier = Double.parseDouble(
|
||||
trimProp.substring(0, trimProp.length()-1));
|
||||
double calculated = multiplier * ((double) Runtime
|
||||
.getRuntime()
|
||||
.availableProcessors());
|
||||
threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0;
|
||||
} else {
|
||||
threadCount = Integer.parseInt(trimProp);
|
||||
}
|
||||
}
|
||||
return threadCount;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -515,30 +515,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-maven-plugins</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>create-parallel-tests-dirs</id>
|
||||
<phase>test-compile</phase>
|
||||
<configuration>
|
||||
<target>
|
||||
<script language="javascript"><![CDATA[
|
||||
var baseDirs = [
|
||||
"${test.build.data}",
|
||||
"${test.build.dir}",
|
||||
"${hadoop.tmp.dir}" ];
|
||||
for (var i in baseDirs) {
|
||||
for (var j = 1; j <= ${testsThreadCount}; ++j) {
|
||||
var mkdir = project.createTask("mkdir");
|
||||
mkdir.setDir(new java.io.File(baseDirs[i], j));
|
||||
mkdir.perform();
|
||||
}
|
||||
}
|
||||
]]></script>
|
||||
</target>
|
||||
</configuration>
|
||||
<id>parallel-tests-createdir</id>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
<goal>parallel-tests-createdir</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
|
@ -551,6 +534,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|||
<reuseForks>false</reuseForks>
|
||||
<argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
|
||||
<systemPropertyVariables>
|
||||
<testsThreadCount>${testsThreadCount}</testsThreadCount>
|
||||
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
|
||||
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
|
||||
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
|
||||
|
|
|
@ -0,0 +1,100 @@
|
|||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.maven.plugin.paralleltests;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
import org.apache.maven.plugin.AbstractMojo;
|
||||
import org.apache.maven.plugin.MojoExecutionException;
|
||||
import org.apache.maven.plugins.annotations.LifecyclePhase;
|
||||
import org.apache.maven.plugins.annotations.Mojo;
|
||||
import org.apache.maven.plugins.annotations.Parameter;
|
||||
|
||||
|
||||
/**
|
||||
* Goal which creates the parallel-test directories.
|
||||
*/
|
||||
@Mojo(name="parallel-tests-createdir",
|
||||
defaultPhase = LifecyclePhase.GENERATE_TEST_RESOURCES)
|
||||
public class CreateDirsMojo extends AbstractMojo {
|
||||
|
||||
/**
|
||||
* Location of the test.build.dir.
|
||||
*/
|
||||
@Parameter(defaultValue="${project.build.directory}/test-dir")
|
||||
private File testBuildDir;
|
||||
|
||||
/**
|
||||
* Location of the test.build.data.
|
||||
*/
|
||||
@Parameter(defaultValue="${project.build.directory}/test-dir")
|
||||
private File testBuildData;
|
||||
|
||||
/**
|
||||
* Location of the test.build.data.
|
||||
*/
|
||||
@Parameter(defaultValue="${project.build.directory}/tmp")
|
||||
private File hadoopTmpDir;
|
||||
|
||||
/**
|
||||
* Thread count.
|
||||
*/
|
||||
@Parameter(defaultValue="${testsThreadCount}")
|
||||
private String testsThreadCount;
|
||||
|
||||
public void execute() throws MojoExecutionException {
|
||||
int numDirs=getTestsThreadCount();
|
||||
|
||||
mkParallelDirs(testBuildDir, numDirs);
|
||||
mkParallelDirs(testBuildData, numDirs);
|
||||
mkParallelDirs(hadoopTmpDir, numDirs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the real number of parallel threads.
|
||||
* @return int number of threads
|
||||
*/
|
||||
|
||||
public int getTestsThreadCount() {
|
||||
int threadCount = 1;
|
||||
if (testsThreadCount != null) {
|
||||
String trimProp = testsThreadCount.trim();
|
||||
if (trimProp.endsWith("C")) {
|
||||
double multiplier = Double.parseDouble(
|
||||
trimProp.substring(0, trimProp.length()-1));
|
||||
double calculated = multiplier * ((double) Runtime
|
||||
.getRuntime()
|
||||
.availableProcessors());
|
||||
threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0;
|
||||
} else {
|
||||
threadCount = Integer.parseInt(testsThreadCount);
|
||||
}
|
||||
}
|
||||
return threadCount;
|
||||
}
|
||||
|
||||
private void mkParallelDirs(File testDir, int numDirs)
|
||||
throws MojoExecutionException {
|
||||
for (int i=1; i<=numDirs; i++) {
|
||||
File newDir = new File(testDir, String.valueOf(i));
|
||||
if (!newDir.exists()) {
|
||||
getLog().info("Creating " + newDir.toString());
|
||||
if (!newDir.mkdirs()) {
|
||||
throw new MojoExecutionException("Unable to create "
|
||||
+ newDir.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -85,30 +85,13 @@
|
|||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-maven-plugins</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>create-parallel-tests-dirs</id>
|
||||
<phase>test-compile</phase>
|
||||
<configuration>
|
||||
<target>
|
||||
<script language="javascript"><![CDATA[
|
||||
var baseDirs = [
|
||||
"${test.build.data}",
|
||||
"${test.build.dir}",
|
||||
"${hadoop.tmp.dir}" ];
|
||||
for (var i in baseDirs) {
|
||||
for (var j = 1; j <= ${testsThreadCount}; ++j) {
|
||||
var mkdir = project.createTask("mkdir");
|
||||
mkdir.setDir(new java.io.File(baseDirs[i], j));
|
||||
mkdir.perform();
|
||||
}
|
||||
}
|
||||
]]></script>
|
||||
</target>
|
||||
</configuration>
|
||||
<id>parallel-tests-createdir</id>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
<goal>parallel-tests-createdir</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
|
@ -121,6 +104,7 @@
|
|||
<reuseForks>false</reuseForks>
|
||||
<argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
|
||||
<systemPropertyVariables>
|
||||
<testsThreadCount>${testsThreadCount}</testsThreadCount>
|
||||
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
|
||||
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
|
||||
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
|
||||
|
|
Loading…
Reference in New Issue