HADOOP-14696. parallel tests don't work for Windows. Contributed by Allen Wittenauer

(cherry picked from commit 45d1b0fdcc) (cherry picked from commit 09940b1eb3) (cherry picked from commit a03c8ea61f) (cherry picked from commit 340cd5f1b1)
2018-03-12 19:47:42 -07:00 · 2018-03-12 19:47:42 -07:00 · f12f540c62
parent da270052a2
commit f12f540c62
5 changed files with 161 additions and 85 deletions
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@ -795,30 +795,13 @@
      <build>
        <plugins>
          <plugin>
-            <artifactId>maven-antrun-plugin</artifactId>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-maven-plugins</artifactId>
            <executions>
              <execution>
-                <id>create-parallel-tests-dirs</id>
-                <phase>test-compile</phase>
-                <configuration>
-                  <target>
-                    <script language="javascript"><![CDATA[
-                      var baseDirs = [
-                          "${test.build.data}",
-                          "${test.build.dir}",
-                          "${hadoop.tmp.dir}" ];
-                      for (var i in baseDirs) {
-                        for (var j = 1; j <= ${testsThreadCount}; ++j) {
-                          var mkdir = project.createTask("mkdir");
-                          mkdir.setDir(new java.io.File(baseDirs[i], j));
-                          mkdir.perform();
-                        }
-                      }
-                    ]]></script>
-                  </target>
-                </configuration>
+                <id>parallel-tests-createdir</id>
                <goals>
-                  <goal>run</goal>
+                  <goal>parallel-tests-createdir</goal>
                </goals>
              </execution>
            </executions>
@ -831,6 +814,7 @@
              <reuseForks>false</reuseForks>
              <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
              <systemPropertyVariables>
+                <testsThreadCount>${testsThreadCount}</testsThreadCount>
                <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
                <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
                <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
@ -287,7 +287,7 @@ public abstract class GenericTestUtils {
  public static void assertExists(File f) {
    Assert.assertTrue("File " + f + " should exist", f.exists());
  }
-    
+
  /**
   * List all of the files in 'dir' that match the regex 'pattern'.
   * Then check that this list is identical to 'expectedMatches'.
@ -295,7 +295,7 @@ public abstract class GenericTestUtils {
   */
  public static void assertGlobEquals(File dir, String pattern,
      String ... expectedMatches) throws IOException {
-    
+
    Set<String> found = Sets.newTreeSet();
    for (File f : FileUtil.listFiles(dir)) {
      if (f.getName().matches(pattern)) {
@ -333,7 +333,7 @@ public abstract class GenericTestUtils {
          + StringUtils.stringifyException(t),
          t);
    }
-  }  
+  }

  /**
   * Wait for the specified test to return true. The test will be performed
@ -483,18 +483,18 @@ public abstract class GenericTestUtils {
   */
  public static class DelayAnswer implements Answer<Object> {
    private final Log LOG;
-    
+
    private final CountDownLatch fireLatch = new CountDownLatch(1);
    private final CountDownLatch waitLatch = new CountDownLatch(1);
    private final CountDownLatch resultLatch = new CountDownLatch(1);
-    
+
    private final AtomicInteger fireCounter = new AtomicInteger(0);
    private final AtomicInteger resultCounter = new AtomicInteger(0);
-    
+
    // Result fields set after proceed() is called.
    private volatile Throwable thrown;
    private volatile Object returnValue;
-    
+
    public DelayAnswer(Log log) {
      this.LOG = log;
    }
@ -505,7 +505,7 @@ public abstract class GenericTestUtils {
    public void waitForCall() throws InterruptedException {
      fireLatch.await();
    }
-  
+
    /**
     * Tell the method to proceed.
     * This should only be called after waitForCall()
@ -513,7 +513,7 @@ public abstract class GenericTestUtils {
    public void proceed() {
      waitLatch.countDown();
    }
-  
+
    @Override
    public Object answer(InvocationOnMock invocation) throws Throwable {
      LOG.info("DelayAnswer firing fireLatch");
@ -542,7 +542,7 @@ public abstract class GenericTestUtils {
        resultLatch.countDown();
      }
    }
-    
+
    /**
     * After calling proceed(), this will wait until the call has
     * completed and a result has been returned to the caller.
@ -550,7 +550,7 @@ public abstract class GenericTestUtils {
    public void waitForResult() throws InterruptedException {
      resultLatch.await();
    }
-    
+
    /**
     * After the call has gone through, return any exception that
     * was thrown, or null if no exception was thrown.
@ -558,7 +558,7 @@ public abstract class GenericTestUtils {
    public Throwable getThrown() {
      return thrown;
    }
-    
+
    /**
     * After the call has gone through, return the call's return value,
     * or null in case it was void or an exception was thrown.
@ -566,20 +566,20 @@ public abstract class GenericTestUtils {
    public Object getReturnValue() {
      return returnValue;
    }
-    
+
    public int getFireCount() {
      return fireCounter.get();
    }
-    
+
    public int getResultCount() {
      return resultCounter.get();
    }
  }
-  
+
  /**
   * An Answer implementation that simply forwards all calls through
   * to a delegate.
-   * 
+   *
   * This is useful as the default Answer for a mock object, to create
   * something like a spy on an RPC proxy. For example:
   * <code>
@ -590,14 +590,14 @@ public abstract class GenericTestUtils {
   *    ...
   * </code>
   */
-  public static class DelegateAnswer implements Answer<Object> { 
+  public static class DelegateAnswer implements Answer<Object> {
    private final Object delegate;
    private final Log log;
-    
+
    public DelegateAnswer(Object delegate) {
      this(null, delegate);
    }
-    
+
    public DelegateAnswer(Log log, Object delegate) {
      this.log = log;
      this.delegate = delegate;
@ -637,7 +637,7 @@ public abstract class GenericTestUtils {
      this.minSleepTime = minSleepTime;
      this.maxSleepTime = maxSleepTime;
    }
-    
+
    @Override
    public Object answer(InvocationOnMock invocation) throws Throwable {
      boolean interrupted = false;
@ -667,11 +667,11 @@ public abstract class GenericTestUtils {
        " but got:\n" + output,
        Pattern.compile(pattern).matcher(output).find());
  }
-  
+
  public static void assertValueNear(long expected, long actual, long allowedError) {
    assertValueWithinRange(expected - allowedError, expected + allowedError, actual);
  }
-  
+
  public static void assertValueWithinRange(long expectedMin, long expectedMax,
      long actual) {
    Assert.assertTrue("Expected " + actual + " to be in range (" + expectedMin + ","
@ -826,4 +826,28 @@ public abstract class GenericTestUtils {
      failf(format, args);
    }
  }
+
+  /**
+   * Retreive the max number of parallel test threads when running under maven.
+   * @return int number of threads
+   */
+  public static int getTestsThreadCount() {
+    String propString = System.getProperty("testsThreadCount", "1");
+    int threadCount = 1;
+    if (propString != null) {
+      String trimProp = propString.trim();
+      if (trimProp.endsWith("C")) {
+        double multiplier = Double.parseDouble(
+            trimProp.substring(0, trimProp.length()-1));
+        double calculated = multiplier * ((double) Runtime
+            .getRuntime()
+            .availableProcessors());
+        threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0;
+      } else {
+        threadCount = Integer.parseInt(trimProp);
+      }
+    }
+    return threadCount;
+  }
+
 }
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@ -434,30 +434,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
      <build>
        <plugins>
          <plugin>
-            <artifactId>maven-antrun-plugin</artifactId>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-maven-plugins</artifactId>
            <executions>
              <execution>
-                <id>create-parallel-tests-dirs</id>
-                <phase>test-compile</phase>
-                <configuration>
-                  <target>
-                    <script language="javascript"><![CDATA[
-                      var baseDirs = [
-                          "${test.build.data}",
-                          "${test.build.dir}",
-                          "${hadoop.tmp.dir}" ];
-                      for (var i in baseDirs) {
-                        for (var j = 1; j <= ${testsThreadCount}; ++j) {
-                          var mkdir = project.createTask("mkdir");
-                          mkdir.setDir(new java.io.File(baseDirs[i], j));
-                          mkdir.perform();
-                        }
-                      }
-                    ]]></script>
-                  </target>
-                </configuration>
+                <id>parallel-tests-createdir</id>
                <goals>
-                  <goal>run</goal>
+                  <goal>parallel-tests-createdir</goal>
                </goals>
              </execution>
            </executions>
@ -470,6 +453,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
              <reuseForks>false</reuseForks>
              <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
              <systemPropertyVariables>
+                <testsThreadCount>${testsThreadCount}</testsThreadCount>
                <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
                <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
                <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
--- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java
+++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/paralleltests/CreateDirsMojo.java
@ -0,0 +1,100 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.maven.plugin.paralleltests;
+
+import java.io.File;
+
+import org.apache.maven.plugin.AbstractMojo;
+import org.apache.maven.plugin.MojoExecutionException;
+import org.apache.maven.plugins.annotations.LifecyclePhase;
+import org.apache.maven.plugins.annotations.Mojo;
+import org.apache.maven.plugins.annotations.Parameter;
+
+
+/**
+ * Goal which creates the parallel-test directories.
+ */
+@Mojo(name="parallel-tests-createdir",
+      defaultPhase = LifecyclePhase.GENERATE_TEST_RESOURCES)
+public class CreateDirsMojo extends AbstractMojo {
+
+  /**
+   * Location of the test.build.dir.
+   */
+  @Parameter(defaultValue="${project.build.directory}/test-dir")
+  private File testBuildDir;
+
+  /**
+   * Location of the test.build.data.
+   */
+  @Parameter(defaultValue="${project.build.directory}/test-dir")
+  private File testBuildData;
+
+  /**
+   * Location of the test.build.data.
+   */
+  @Parameter(defaultValue="${project.build.directory}/tmp")
+  private File hadoopTmpDir;
+
+  /**
+   * Thread count.
+   */
+  @Parameter(defaultValue="${testsThreadCount}")
+  private String testsThreadCount;
+
+  public void execute() throws MojoExecutionException {
+    int numDirs=getTestsThreadCount();
+
+    mkParallelDirs(testBuildDir, numDirs);
+    mkParallelDirs(testBuildData, numDirs);
+    mkParallelDirs(hadoopTmpDir, numDirs);
+  }
+
+  /**
+   * Get the real number of parallel threads.
+   * @return int number of threads
+   */
+
+  public int getTestsThreadCount() {
+    int threadCount = 1;
+    if (testsThreadCount != null) {
+      String trimProp = testsThreadCount.trim();
+      if (trimProp.endsWith("C")) {
+        double multiplier = Double.parseDouble(
+            trimProp.substring(0, trimProp.length()-1));
+        double calculated = multiplier * ((double) Runtime
+            .getRuntime()
+            .availableProcessors());
+        threadCount = calculated > 0d ? Math.max((int) calculated, 1) : 0;
+      } else {
+        threadCount = Integer.parseInt(testsThreadCount);
+      }
+    }
+    return threadCount;
+  }
+
+  private void mkParallelDirs(File testDir, int numDirs)
+      throws MojoExecutionException {
+    for (int i=1; i<=numDirs; i++) {
+      File newDir = new File(testDir, String.valueOf(i));
+      if (!newDir.exists()) {
+        getLog().info("Creating " + newDir.toString());
+        if (!newDir.mkdirs()) {
+          throw new MojoExecutionException("Unable to create "
+              + newDir.toString());
+        }
+      }
+    }
+  }
+}
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@ -85,30 +85,13 @@
      <build>
        <plugins>
          <plugin>
-            <artifactId>maven-antrun-plugin</artifactId>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-maven-plugins</artifactId>
            <executions>
              <execution>
-                <id>create-parallel-tests-dirs</id>
-                <phase>test-compile</phase>
-                <configuration>
-                  <target>
-                    <script language="javascript"><![CDATA[
-                      var baseDirs = [
-                          "${test.build.data}",
-                          "${test.build.dir}",
-                          "${hadoop.tmp.dir}" ];
-                      for (var i in baseDirs) {
-                        for (var j = 1; j <= ${testsThreadCount}; ++j) {
-                          var mkdir = project.createTask("mkdir");
-                          mkdir.setDir(new java.io.File(baseDirs[i], j));
-                          mkdir.perform();
-                        }
-                      }
-                    ]]></script>
-                  </target>
-                </configuration>
+                <id>parallel-tests-createdir</id>
                <goals>
-                  <goal>run</goal>
+                  <goal>parallel-tests-createdir</goal>
                </goals>
              </execution>
            </executions>
@ -121,6 +104,7 @@
              <reuseForks>false</reuseForks>
              <argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
              <systemPropertyVariables>
+                <testsThreadCount>${testsThreadCount}</testsThreadCount>
                <test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
                <test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
                <hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>