Implemented a retry policy for tests that may randomly fail.

Tests classes that may fail should be annotated with @RunWith(RetryRunner.class). This custom test runner attempts to re-run tests that fail, up to a maximum number of attempts defined as a constant in the test runner (currently set to 3 attempts max).

Jira: MATH-423

git-svn-id: https://svn.apache.org/repos/asf/commons/proper/math/trunk@1085954 13f79535-47bb-0310-9956-ffa450edef68

src/test/java/org/apache/commons/math/RetryRunner.java

@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.commons.math;
+
+import org.junit.runners.BlockJUnit4ClassRunner;
+import org.junit.runners.model.FrameworkMethod;
+import org.junit.runners.model.InitializationError;
+import org.junit.runners.model.Statement;
+
+
+/**
+ * A test runner that retries tests when assertions fail.
+ * @version $Revision$ $Date$
+ */
+public class RetryRunner extends BlockJUnit4ClassRunner {
+
+    /** Maximal number of test run attempts. */
+    private static final int MAX_ATTEMPTS = 3;
+
+    /** Simple constructor.
+     * @param testClass class to test
+     * @throws InitializationError if default runner cannot be built
+     */
+    public RetryRunner(final Class<?> testClass)
+        throws InitializationError {
+        super(testClass);
+    }
+
+    @Override
+    public Statement methodInvoker(FrameworkMethod method, Object test) {
+        final Statement singleTryStatement = super.methodInvoker(method, test);
+        return new Statement() {
+
+            /** Evaluate the statement.
+             * We attempt several runs for the test, at most MAX_ATTEMPTS.
+             * if one attempt succeeds, we succeed, if all attempts fail, we
+             * fail with the reason corresponding to the last attempt
+             */
+            public void evaluate() throws Throwable {
+                Throwable failureReason = null;
+                for (int i = 0; i < MAX_ATTEMPTS; ++i) {
+                    try {
+
+                        // do one test run attempt
+                        singleTryStatement.evaluate();
+
+                        // attempt succeeded, stop evaluation here
+                        return;
+
+                    } catch (Throwable t) {
+                        // attempt failed, store the reason why
+                        failureReason = t;
+                    }
+                }
+
+                // all attempts failed
+                throw failureReason;
+
+            }
+        };
+    }
+
+}

src/test/java/org/apache/commons/math/RetryTestCase.java

@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.commons.math;
-
-
-/**
- * A Test case that retries tests when assertions fail.
- * @version $Revision$ $Date$
- */
-public abstract class RetryTestCase {
-
-    // TODO implement retry policy using Junit 4  API
-
-    //    /**
-//     *  Override runTest() to catch AssertionFailedError and retry
-//     */
-//    @Override
-//    protected void runTest() throws Throwable {
-//        try {
-//            super.runTest();
-//        } catch (AssertionFailedError err) {
-//            // System.out.println("Retrying " + this.getName());
-//            super.runTest();
-//        }
-//    }
-
-}

src/test/java/org/apache/commons/math/random/EmpiricalDistributionTest.java

@@ -23,12 +23,13 @@ import java.io.InputStreamReader;
 import java.net.URL;
 import java.util.ArrayList;
 
-import org.apache.commons.math.RetryTestCase;
+import org.apache.commons.math.RetryRunner;
 import org.apache.commons.math.TestUtils;
 import org.apache.commons.math.stat.descriptive.SummaryStatistics;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
 
 /**
  * Test cases for the EmpiricalDistribution class
@@ -36,7 +37,8 @@ import org.junit.Test;
  * @version $Revision$ $Date$
  */
 
-public final class EmpiricalDistributionTest extends RetryTestCase {
+@RunWith(RetryRunner.class)
+public final class EmpiricalDistributionTest {
 
     protected EmpiricalDistribution empiricalDistribution = null;
     protected EmpiricalDistribution empiricalDistribution2 = null;

src/test/java/org/apache/commons/math/random/RandomDataTest.java

@@ -22,7 +22,7 @@ import java.util.HashSet;
 import java.util.List;
 
 
-import org.apache.commons.math.RetryTestCase;
+import org.apache.commons.math.RetryRunner;
 import org.apache.commons.math.TestUtils;
 import org.apache.commons.math.distribution.BetaDistributionImpl;
 import org.apache.commons.math.distribution.BinomialDistributionImpl;
@@ -49,6 +49,7 @@ import org.apache.commons.math.util.FastMath;
 import org.apache.commons.math.exception.MathIllegalArgumentException;
 import org.junit.Assert;
 import org.junit.Test;
+import org.junit.runner.RunWith;
 
 /**
  * Test cases for the RandomData class.
@@ -57,7 +58,8 @@ import org.junit.Test;
  *          2009) $
  */
 
-public class RandomDataTest extends RetryTestCase {
+@RunWith(RetryRunner.class)
+public class RandomDataTest {
 
     public RandomDataTest() {
         randomData = new RandomDataImpl();

src/test/java/org/apache/commons/math/random/ValueServerTest.java

@@ -19,11 +19,12 @@ package org.apache.commons.math.random;
 import java.io.EOFException;
 import java.net.URL;
 
-import org.apache.commons.math.RetryTestCase;
+import org.apache.commons.math.RetryRunner;
 import org.apache.commons.math.stat.descriptive.SummaryStatistics;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
 
 /**
  * Test cases for the ValueServer class.
@@ -31,7 +32,8 @@ import org.junit.Test;
  * @version $Revision$ $Date$
  */
 
-public final class ValueServerTest extends RetryTestCase {
+@RunWith(RetryRunner.class)
+public final class ValueServerTest {
 
     private ValueServer vs = new ValueServer();