From f12f540c62b91552d25e9ca1e9a387992823f350 Mon Sep 17 00:00:00 2001 From: Chris Douglas Date: Mon, 12 Mar 2018 19:47:42 -0700 Subject: [PATCH] HADOOP-14696. parallel tests don't work for Windows. Contributed by Allen Wittenauer (cherry picked from commit 45d1b0fdcc04a86be91a9b72073cdc30bec04d3b) (cherry picked from commit 09940b1eb3b7ed764149f4a993c1857e9c6ad938) (cherry picked from commit a03c8ea61f6e30a9d462571ace23858b6e0fd1c9) (cherry picked from commit 340cd5f1b137dccc033023ed92cc87b7e47f45c2) --- hadoop-common-project/hadoop-common/pom.xml | 26 +---- .../apache/hadoop/test/GenericTestUtils.java | 68 ++++++++---- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 26 +---- .../plugin/paralleltests/CreateDirsMojo.java | 100 ++++++++++++++++++ hadoop-tools/hadoop-aws/pom.xml | 26 +---- 5 files changed, 161 insertions(+), 85 deletions(-) create mode 100644 hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 06a9252c381..88c3b10878f 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -795,30 +795,13 @@ - maven-antrun-plugin + org.apache.hadoop + hadoop-maven-plugins - create-parallel-tests-dirs - test-compile - - - - - + parallel-tests-createdir - run + parallel-tests-createdir @@ -831,6 +814,7 @@ false ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true + ${testsThreadCount} ${test.build.data}/${surefire.forkNumber} ${test.build.dir}/${surefire.forkNumber} ${hadoop.tmp.dir}/${surefire.forkNumber} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java index af47d2981ac..2810213e9f7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java @@ -287,7 +287,7 @@ public abstract class GenericTestUtils { public static void assertExists(File f) { Assert.assertTrue("File " + f + " should exist", f.exists()); } - + /** * List all of the files in 'dir' that match the regex 'pattern'. * Then check that this list is identical to 'expectedMatches'. @@ -295,7 +295,7 @@ public abstract class GenericTestUtils { */ public static void assertGlobEquals(File dir, String pattern, String ... expectedMatches) throws IOException { - + Set found = Sets.newTreeSet(); for (File f : FileUtil.listFiles(dir)) { if (f.getName().matches(pattern)) { @@ -333,7 +333,7 @@ public abstract class GenericTestUtils { + StringUtils.stringifyException(t), t); } - } + } /** * Wait for the specified test to return true. The test will be performed @@ -483,18 +483,18 @@ public abstract class GenericTestUtils { */ public static class DelayAnswer implements Answer { private final Log LOG; - + private final CountDownLatch fireLatch = new CountDownLatch(1); private final CountDownLatch waitLatch = new CountDownLatch(1); private final CountDownLatch resultLatch = new CountDownLatch(1); - + private final AtomicInteger fireCounter = new AtomicInteger(0); private final AtomicInteger resultCounter = new AtomicInteger(0); - + // Result fields set after proceed() is called. private volatile Throwable thrown; private volatile Object returnValue; - + public DelayAnswer(Log log) { this.LOG = log; } @@ -505,7 +505,7 @@ public abstract class GenericTestUtils { public void waitForCall() throws InterruptedException { fireLatch.await(); } - + /** * Tell the method to proceed. * This should only be called after waitForCall() @@ -513,7 +513,7 @@ public abstract class GenericTestUtils { public void proceed() { waitLatch.countDown(); } - + @Override public Object answer(InvocationOnMock invocation) throws Throwable { LOG.info("DelayAnswer firing fireLatch"); @@ -542,7 +542,7 @@ public abstract class GenericTestUtils { resultLatch.countDown(); } } - + /** * After calling proceed(), this will wait until the call has * completed and a result has been returned to the caller. @@ -550,7 +550,7 @@ public abstract class GenericTestUtils { public void waitForResult() throws InterruptedException { resultLatch.await(); } - + /** * After the call has gone through, return any exception that * was thrown, or null if no exception was thrown. @@ -558,7 +558,7 @@ public abstract class GenericTestUtils { public Throwable getThrown() { return thrown; } - + /** * After the call has gone through, return the call's return value, * or null in case it was void or an exception was thrown. @@ -566,20 +566,20 @@ public abstract class GenericTestUtils { public Object getReturnValue() { return returnValue; } - + public int getFireCount() { return fireCounter.get(); } - + public int getResultCount() { return resultCounter.get(); } } - + /** * An Answer implementation that simply forwards all calls through * to a delegate. - * + * * This is useful as the default Answer for a mock object, to create * something like a spy on an RPC proxy. For example: * @@ -590,14 +590,14 @@ public abstract class GenericTestUtils { * ... * */ - public static class DelegateAnswer implements Answer { + public static class DelegateAnswer implements Answer { private final Object delegate; private final Log log; - + public DelegateAnswer(Object delegate) { this(null, delegate); } - + public DelegateAnswer(Log log, Object delegate) { this.log = log; this.delegate = delegate; @@ -637,7 +637,7 @@ public abstract class GenericTestUtils { this.minSleepTime = minSleepTime; this.maxSleepTime = maxSleepTime; } - + @Override public Object answer(InvocationOnMock invocation) throws Throwable { boolean interrupted = false; @@ -667,11 +667,11 @@ public abstract class GenericTestUtils { " but got:\n" + output, Pattern.compile(pattern).matcher(output).find()); } - + public static void assertValueNear(long expected, long actual, long allowedError) { assertValueWithinRange(expected - allowedError, expected + allowedError, actual); } - + public static void assertValueWithinRange(long expectedMin, long expectedMax, long actual) { Assert.assertTrue("Expected " + actual + " to be in range (" + expectedMin + "," @@ -826,4 +826,28 @@ public abstract class GenericTestUtils { failf(format, args); } } + + /** + * Retreive the max number of parallel test threads when running under maven. + * @return int number of threads + */ + public static int getTestsThreadCount() { + String propString = System.getProperty("testsThreadCount", "1"); + int threadCount = 1; + if (propString != null) { + String trimProp = propString.trim(); + if (trimProp.endsWith("C")) { + double multiplier = Double.parseDouble( + trimProp.substring(0, trimProp.length()-1)); + double calculated = multiplier * ((double) Runtime + .getRuntime() + .availableProcessors()); + threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0; + } else { + threadCount = Integer.parseInt(trimProp); + } + } + return threadCount; + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 87650eceb43..5e74159cd65 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -434,30 +434,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> - maven-antrun-plugin + org.apache.hadoop + hadoop-maven-plugins - create-parallel-tests-dirs - test-compile - - - - - + parallel-tests-createdir - run + parallel-tests-createdir @@ -470,6 +453,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> false ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true + ${testsThreadCount} ${test.build.data}/${surefire.forkNumber} ${test.build.dir}/${surefire.forkNumber} ${hadoop.tmp.dir}/${surefire.forkNumber} diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java new file mode 100644 index 00000000000..4c9b9d73177 --- /dev/null +++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java @@ -0,0 +1,100 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.maven.plugin.paralleltests; + +import java.io.File; + +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.maven.plugins.annotations.LifecyclePhase; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; + + +/** + * Goal which creates the parallel-test directories. + */ +@Mojo(name="parallel-tests-createdir", + defaultPhase = LifecyclePhase.GENERATE_TEST_RESOURCES) +public class CreateDirsMojo extends AbstractMojo { + + /** + * Location of the test.build.dir. + */ + @Parameter(defaultValue="${project.build.directory}/test-dir") + private File testBuildDir; + + /** + * Location of the test.build.data. + */ + @Parameter(defaultValue="${project.build.directory}/test-dir") + private File testBuildData; + + /** + * Location of the test.build.data. + */ + @Parameter(defaultValue="${project.build.directory}/tmp") + private File hadoopTmpDir; + + /** + * Thread count. + */ + @Parameter(defaultValue="${testsThreadCount}") + private String testsThreadCount; + + public void execute() throws MojoExecutionException { + int numDirs=getTestsThreadCount(); + + mkParallelDirs(testBuildDir, numDirs); + mkParallelDirs(testBuildData, numDirs); + mkParallelDirs(hadoopTmpDir, numDirs); + } + + /** + * Get the real number of parallel threads. + * @return int number of threads + */ + + public int getTestsThreadCount() { + int threadCount = 1; + if (testsThreadCount != null) { + String trimProp = testsThreadCount.trim(); + if (trimProp.endsWith("C")) { + double multiplier = Double.parseDouble( + trimProp.substring(0, trimProp.length()-1)); + double calculated = multiplier * ((double) Runtime + .getRuntime() + .availableProcessors()); + threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0; + } else { + threadCount = Integer.parseInt(testsThreadCount); + } + } + return threadCount; + } + + private void mkParallelDirs(File testDir, int numDirs) + throws MojoExecutionException { + for (int i=1; i<=numDirs; i++) { + File newDir = new File(testDir, String.valueOf(i)); + if (!newDir.exists()) { + getLog().info("Creating " + newDir.toString()); + if (!newDir.mkdirs()) { + throw new MojoExecutionException("Unable to create " + + newDir.toString()); + } + } + } + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 920f985d550..90e144c461b 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -85,30 +85,13 @@ - maven-antrun-plugin + org.apache.hadoop + hadoop-maven-plugins - create-parallel-tests-dirs - test-compile - - - - - + parallel-tests-createdir - run + parallel-tests-createdir @@ -121,6 +104,7 @@ false ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true + ${testsThreadCount} ${test.build.data}/${surefire.forkNumber} ${test.build.dir}/${surefire.forkNumber} ${hadoop.tmp.dir}/${surefire.forkNumber}