HADOOP-14696. parallel tests don't work for Windows. Contributed by Allen Wittenauer
This commit is contained in:
Chris Douglas
parent
19292bc264
commit
45d1b0fdcc
|
|
@ -979,30 +979,13 @@
|
|||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-maven-plugins</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>create-parallel-tests-dirs</id>
|
||||
<phase>test-compile</phase>
|
||||
<configuration>
|
||||
<target>
|
||||
<script language="javascript"><![CDATA[
|
||||
var baseDirs = [
|
||||
"${test.build.data}",
|
||||
"${test.build.dir}",
|
||||
"${hadoop.tmp.dir}" ];
|
||||
for (var i in baseDirs) {
|
||||
for (var j = 1; j <= ${testsThreadCount}; ++j) {
|
||||
var mkdir = project.createTask("mkdir");
|
||||
mkdir.setDir(new java.io.File(baseDirs[i], j));
|
||||
mkdir.perform();
|
||||
}
|
||||
}
|
||||
]]></script>
|
||||
</target>
|
||||
</configuration>
|
||||
<id>parallel-tests-createdir</id>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
<goal>parallel-tests-createdir</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
|
|
@ -1015,6 +998,7 @@
|
|||
<reuseForks>false</reuseForks>
|
||||
<argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
|
||||
<systemPropertyVariables>
|
||||
<testsThreadCount>${testsThreadCount}</testsThreadCount>
|
||||
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
|
||||
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
|
||||
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
|
||||
|
|
|
|||
|
|
@ -842,4 +842,28 @@ public abstract class GenericTestUtils {
|
|||
failf(format, args);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retreive the max number of parallel test threads when running under maven.
|
||||
* @return int number of threads
|
||||
*/
|
||||
public static int getTestsThreadCount() {
|
||||
String propString = System.getProperty("testsThreadCount", "1");
|
||||
int threadCount = 1;
|
||||
if (propString != null) {
|
||||
String trimProp = propString.trim();
|
||||
if (trimProp.endsWith("C")) {
|
||||
double multiplier = Double.parseDouble(
|
||||
trimProp.substring(0, trimProp.length()-1));
|
||||
double calculated = multiplier * ((double) Runtime
|
||||
.getRuntime()
|
||||
.availableProcessors());
|
||||
threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0;
|
||||
} else {
|
||||
threadCount = Integer.parseInt(trimProp);
|
||||
}
|
||||
}
|
||||
return threadCount;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -515,30 +515,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-maven-plugins</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>create-parallel-tests-dirs</id>
|
||||
<phase>test-compile</phase>
|
||||
<configuration>
|
||||
<target>
|
||||
<script language="javascript"><![CDATA[
|
||||
var baseDirs = [
|
||||
"${test.build.data}",
|
||||
"${test.build.dir}",
|
||||
"${hadoop.tmp.dir}" ];
|
||||
for (var i in baseDirs) {
|
||||
for (var j = 1; j <= ${testsThreadCount}; ++j) {
|
||||
var mkdir = project.createTask("mkdir");
|
||||
mkdir.setDir(new java.io.File(baseDirs[i], j));
|
||||
mkdir.perform();
|
||||
}
|
||||
}
|
||||
]]></script>
|
||||
</target>
|
||||
</configuration>
|
||||
<id>parallel-tests-createdir</id>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
<goal>parallel-tests-createdir</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
|
|
@ -551,6 +534,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|||
<reuseForks>false</reuseForks>
|
||||
<argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
|
||||
<systemPropertyVariables>
|
||||
<testsThreadCount>${testsThreadCount}</testsThreadCount>
|
||||
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
|
||||
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
|
||||
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,100 @@
|
|||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.maven.plugin.paralleltests;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
import org.apache.maven.plugin.AbstractMojo;
|
||||
import org.apache.maven.plugin.MojoExecutionException;
|
||||
import org.apache.maven.plugins.annotations.LifecyclePhase;
|
||||
import org.apache.maven.plugins.annotations.Mojo;
|
||||
import org.apache.maven.plugins.annotations.Parameter;
|
||||
|
||||
|
||||
/**
|
||||
* Goal which creates the parallel-test directories.
|
||||
*/
|
||||
@Mojo(name="parallel-tests-createdir",
|
||||
defaultPhase = LifecyclePhase.GENERATE_TEST_RESOURCES)
|
||||
public class CreateDirsMojo extends AbstractMojo {
|
||||
|
||||
/**
|
||||
* Location of the test.build.dir.
|
||||
*/
|
||||
@Parameter(defaultValue="${project.build.directory}/test-dir")
|
||||
private File testBuildDir;
|
||||
|
||||
/**
|
||||
* Location of the test.build.data.
|
||||
*/
|
||||
@Parameter(defaultValue="${project.build.directory}/test-dir")
|
||||
private File testBuildData;
|
||||
|
||||
/**
|
||||
* Location of the test.build.data.
|
||||
*/
|
||||
@Parameter(defaultValue="${project.build.directory}/tmp")
|
||||
private File hadoopTmpDir;
|
||||
|
||||
/**
|
||||
* Thread count.
|
||||
*/
|
||||
@Parameter(defaultValue="${testsThreadCount}")
|
||||
private String testsThreadCount;
|
||||
|
||||
public void execute() throws MojoExecutionException {
|
||||
int numDirs=getTestsThreadCount();
|
||||
|
||||
mkParallelDirs(testBuildDir, numDirs);
|
||||
mkParallelDirs(testBuildData, numDirs);
|
||||
mkParallelDirs(hadoopTmpDir, numDirs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the real number of parallel threads.
|
||||
* @return int number of threads
|
||||
*/
|
||||
|
||||
public int getTestsThreadCount() {
|
||||
int threadCount = 1;
|
||||
if (testsThreadCount != null) {
|
||||
String trimProp = testsThreadCount.trim();
|
||||
if (trimProp.endsWith("C")) {
|
||||
double multiplier = Double.parseDouble(
|
||||
trimProp.substring(0, trimProp.length()-1));
|
||||
double calculated = multiplier * ((double) Runtime
|
||||
.getRuntime()
|
||||
.availableProcessors());
|
||||
threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0;
|
||||
} else {
|
||||
threadCount = Integer.parseInt(testsThreadCount);
|
||||
}
|
||||
}
|
||||
return threadCount;
|
||||
}
|
||||
|
||||
private void mkParallelDirs(File testDir, int numDirs)
|
||||
throws MojoExecutionException {
|
||||
for (int i=1; i<=numDirs; i++) {
|
||||
File newDir = new File(testDir, String.valueOf(i));
|
||||
if (!newDir.exists()) {
|
||||
getLog().info("Creating " + newDir.toString());
|
||||
if (!newDir.mkdirs()) {
|
||||
throw new MojoExecutionException("Unable to create "
|
||||
+ newDir.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -85,30 +85,13 @@
|
|||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-maven-plugins</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>create-parallel-tests-dirs</id>
|
||||
<phase>test-compile</phase>
|
||||
<configuration>
|
||||
<target>
|
||||
<script language="javascript"><![CDATA[
|
||||
var baseDirs = [
|
||||
"${test.build.data}",
|
||||
"${test.build.dir}",
|
||||
"${hadoop.tmp.dir}" ];
|
||||
for (var i in baseDirs) {
|
||||
for (var j = 1; j <= ${testsThreadCount}; ++j) {
|
||||
var mkdir = project.createTask("mkdir");
|
||||
mkdir.setDir(new java.io.File(baseDirs[i], j));
|
||||
mkdir.perform();
|
||||
}
|
||||
}
|
||||
]]></script>
|
||||
</target>
|
||||
</configuration>
|
||||
<id>parallel-tests-createdir</id>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
<goal>parallel-tests-createdir</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
|
|
@ -121,6 +104,7 @@
|
|||
<reuseForks>false</reuseForks>
|
||||
<argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
|
||||
<systemPropertyVariables>
|
||||
<testsThreadCount>${testsThreadCount}</testsThreadCount>
|
||||
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
|
||||
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
|
||||
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
|
||||
|
|
|
|||
Loading…
Reference in New Issue