HADOOP-14696. parallel tests don't work for Windows. Contributed by Allen Wittenauer
(cherry picked from commit 45d1b0fdcc04a86be91a9b72073cdc30bec04d3b) (cherry picked from commit 09940b1eb3b7ed764149f4a993c1857e9c6ad938) (cherry picked from commit a03c8ea61f6e30a9d462571ace23858b6e0fd1c9) (cherry picked from commit 340cd5f1b137dccc033023ed92cc87b7e47f45c2)
This commit is contained in:
Chris Douglas
parent
da270052a2
commit
f12f540c62
@ -795,30 +795,13 @@
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-maven-plugins</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>create-parallel-tests-dirs</id>
|
||||
<phase>test-compile</phase>
|
||||
<configuration>
|
||||
<target>
|
||||
<script language="javascript"><![CDATA[
|
||||
var baseDirs = [
|
||||
"${test.build.data}",
|
||||
"${test.build.dir}",
|
||||
"${hadoop.tmp.dir}" ];
|
||||
for (var i in baseDirs) {
|
||||
for (var j = 1; j <= ${testsThreadCount}; ++j) {
|
||||
var mkdir = project.createTask("mkdir");
|
||||
mkdir.setDir(new java.io.File(baseDirs[i], j));
|
||||
mkdir.perform();
|
||||
}
|
||||
}
|
||||
]]></script>
|
||||
</target>
|
||||
</configuration>
|
||||
<id>parallel-tests-createdir</id>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
<goal>parallel-tests-createdir</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
@ -831,6 +814,7 @@
|
||||
<reuseForks>false</reuseForks>
|
||||
<argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
|
||||
<systemPropertyVariables>
|
||||
<testsThreadCount>${testsThreadCount}</testsThreadCount>
|
||||
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
|
||||
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
|
||||
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
|
||||
|
||||
@ -826,4 +826,28 @@ public static void failif(boolean condition,
|
||||
failf(format, args);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retreive the max number of parallel test threads when running under maven.
|
||||
* @return int number of threads
|
||||
*/
|
||||
public static int getTestsThreadCount() {
|
||||
String propString = System.getProperty("testsThreadCount", "1");
|
||||
int threadCount = 1;
|
||||
if (propString != null) {
|
||||
String trimProp = propString.trim();
|
||||
if (trimProp.endsWith("C")) {
|
||||
double multiplier = Double.parseDouble(
|
||||
trimProp.substring(0, trimProp.length()-1));
|
||||
double calculated = multiplier * ((double) Runtime
|
||||
.getRuntime()
|
||||
.availableProcessors());
|
||||
threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0;
|
||||
} else {
|
||||
threadCount = Integer.parseInt(trimProp);
|
||||
}
|
||||
}
|
||||
return threadCount;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -434,30 +434,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-maven-plugins</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>create-parallel-tests-dirs</id>
|
||||
<phase>test-compile</phase>
|
||||
<configuration>
|
||||
<target>
|
||||
<script language="javascript"><![CDATA[
|
||||
var baseDirs = [
|
||||
"${test.build.data}",
|
||||
"${test.build.dir}",
|
||||
"${hadoop.tmp.dir}" ];
|
||||
for (var i in baseDirs) {
|
||||
for (var j = 1; j <= ${testsThreadCount}; ++j) {
|
||||
var mkdir = project.createTask("mkdir");
|
||||
mkdir.setDir(new java.io.File(baseDirs[i], j));
|
||||
mkdir.perform();
|
||||
}
|
||||
}
|
||||
]]></script>
|
||||
</target>
|
||||
</configuration>
|
||||
<id>parallel-tests-createdir</id>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
<goal>parallel-tests-createdir</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
@ -470,6 +453,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<reuseForks>false</reuseForks>
|
||||
<argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
|
||||
<systemPropertyVariables>
|
||||
<testsThreadCount>${testsThreadCount}</testsThreadCount>
|
||||
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
|
||||
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
|
||||
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
|
||||
|
||||
@ -0,0 +1,100 @@
|
||||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.maven.plugin.paralleltests;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
import org.apache.maven.plugin.AbstractMojo;
|
||||
import org.apache.maven.plugin.MojoExecutionException;
|
||||
import org.apache.maven.plugins.annotations.LifecyclePhase;
|
||||
import org.apache.maven.plugins.annotations.Mojo;
|
||||
import org.apache.maven.plugins.annotations.Parameter;
|
||||
|
||||
|
||||
/**
|
||||
* Goal which creates the parallel-test directories.
|
||||
*/
|
||||
@Mojo(name="parallel-tests-createdir",
|
||||
defaultPhase = LifecyclePhase.GENERATE_TEST_RESOURCES)
|
||||
public class CreateDirsMojo extends AbstractMojo {
|
||||
|
||||
/**
|
||||
* Location of the test.build.dir.
|
||||
*/
|
||||
@Parameter(defaultValue="${project.build.directory}/test-dir")
|
||||
private File testBuildDir;
|
||||
|
||||
/**
|
||||
* Location of the test.build.data.
|
||||
*/
|
||||
@Parameter(defaultValue="${project.build.directory}/test-dir")
|
||||
private File testBuildData;
|
||||
|
||||
/**
|
||||
* Location of the test.build.data.
|
||||
*/
|
||||
@Parameter(defaultValue="${project.build.directory}/tmp")
|
||||
private File hadoopTmpDir;
|
||||
|
||||
/**
|
||||
* Thread count.
|
||||
*/
|
||||
@Parameter(defaultValue="${testsThreadCount}")
|
||||
private String testsThreadCount;
|
||||
|
||||
public void execute() throws MojoExecutionException {
|
||||
int numDirs=getTestsThreadCount();
|
||||
|
||||
mkParallelDirs(testBuildDir, numDirs);
|
||||
mkParallelDirs(testBuildData, numDirs);
|
||||
mkParallelDirs(hadoopTmpDir, numDirs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the real number of parallel threads.
|
||||
* @return int number of threads
|
||||
*/
|
||||
|
||||
public int getTestsThreadCount() {
|
||||
int threadCount = 1;
|
||||
if (testsThreadCount != null) {
|
||||
String trimProp = testsThreadCount.trim();
|
||||
if (trimProp.endsWith("C")) {
|
||||
double multiplier = Double.parseDouble(
|
||||
trimProp.substring(0, trimProp.length()-1));
|
||||
double calculated = multiplier * ((double) Runtime
|
||||
.getRuntime()
|
||||
.availableProcessors());
|
||||
threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0;
|
||||
} else {
|
||||
threadCount = Integer.parseInt(testsThreadCount);
|
||||
}
|
||||
}
|
||||
return threadCount;
|
||||
}
|
||||
|
||||
private void mkParallelDirs(File testDir, int numDirs)
|
||||
throws MojoExecutionException {
|
||||
for (int i=1; i<=numDirs; i++) {
|
||||
File newDir = new File(testDir, String.valueOf(i));
|
||||
if (!newDir.exists()) {
|
||||
getLog().info("Creating " + newDir.toString());
|
||||
if (!newDir.mkdirs()) {
|
||||
throw new MojoExecutionException("Unable to create "
|
||||
+ newDir.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -85,30 +85,13 @@
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<artifactId>maven-antrun-plugin</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-maven-plugins</artifactId>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>create-parallel-tests-dirs</id>
|
||||
<phase>test-compile</phase>
|
||||
<configuration>
|
||||
<target>
|
||||
<script language="javascript"><![CDATA[
|
||||
var baseDirs = [
|
||||
"${test.build.data}",
|
||||
"${test.build.dir}",
|
||||
"${hadoop.tmp.dir}" ];
|
||||
for (var i in baseDirs) {
|
||||
for (var j = 1; j <= ${testsThreadCount}; ++j) {
|
||||
var mkdir = project.createTask("mkdir");
|
||||
mkdir.setDir(new java.io.File(baseDirs[i], j));
|
||||
mkdir.perform();
|
||||
}
|
||||
}
|
||||
]]></script>
|
||||
</target>
|
||||
</configuration>
|
||||
<id>parallel-tests-createdir</id>
|
||||
<goals>
|
||||
<goal>run</goal>
|
||||
<goal>parallel-tests-createdir</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
@ -121,6 +104,7 @@
|
||||
<reuseForks>false</reuseForks>
|
||||
<argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
|
||||
<systemPropertyVariables>
|
||||
<testsThreadCount>${testsThreadCount}</testsThreadCount>
|
||||
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
|
||||
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
|
||||
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user