From 3c4a3dbd7c34ee09b414400fb57477563359962a Mon Sep 17 00:00:00 2001 From: Luc Maisonobe Date: Sun, 27 Mar 2011 15:02:47 +0000 Subject: [PATCH] Implemented a retry policy for tests that may randomly fail. Tests classes that may fail should be annotated with @RunWith(RetryRunner.class). This custom test runner attempts to re-run tests that fail, up to a maximum number of attempts defined as a constant in the test runner (currently set to 3 attempts max). Jira: MATH-423 git-svn-id: https://svn.apache.org/repos/asf/commons/proper/math/trunk@1085954 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/commons/math/RetryRunner.java | 78 +++++++++++++++++++ .../apache/commons/math/RetryTestCase.java | 42 ---------- .../random/EmpiricalDistributionTest.java | 6 +- .../commons/math/random/RandomDataTest.java | 6 +- .../commons/math/random/ValueServerTest.java | 6 +- 5 files changed, 90 insertions(+), 48 deletions(-) create mode 100644 src/test/java/org/apache/commons/math/RetryRunner.java delete mode 100644 src/test/java/org/apache/commons/math/RetryTestCase.java diff --git a/src/test/java/org/apache/commons/math/RetryRunner.java b/src/test/java/org/apache/commons/math/RetryRunner.java new file mode 100644 index 000000000..489bef662 --- /dev/null +++ b/src/test/java/org/apache/commons/math/RetryRunner.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.commons.math; + +import org.junit.runners.BlockJUnit4ClassRunner; +import org.junit.runners.model.FrameworkMethod; +import org.junit.runners.model.InitializationError; +import org.junit.runners.model.Statement; + + +/** + * A test runner that retries tests when assertions fail. + * @version $Revision$ $Date$ + */ +public class RetryRunner extends BlockJUnit4ClassRunner { + + /** Maximal number of test run attempts. */ + private static final int MAX_ATTEMPTS = 3; + + /** Simple constructor. + * @param testClass class to test + * @throws InitializationError if default runner cannot be built + */ + public RetryRunner(final Class testClass) + throws InitializationError { + super(testClass); + } + + @Override + public Statement methodInvoker(FrameworkMethod method, Object test) { + final Statement singleTryStatement = super.methodInvoker(method, test); + return new Statement() { + + /** Evaluate the statement. + * We attempt several runs for the test, at most MAX_ATTEMPTS. + * if one attempt succeeds, we succeed, if all attempts fail, we + * fail with the reason corresponding to the last attempt + */ + public void evaluate() throws Throwable { + Throwable failureReason = null; + for (int i = 0; i < MAX_ATTEMPTS; ++i) { + try { + + // do one test run attempt + singleTryStatement.evaluate(); + + // attempt succeeded, stop evaluation here + return; + + } catch (Throwable t) { + // attempt failed, store the reason why + failureReason = t; + } + } + + // all attempts failed + throw failureReason; + + } + }; + } + +} diff --git a/src/test/java/org/apache/commons/math/RetryTestCase.java b/src/test/java/org/apache/commons/math/RetryTestCase.java deleted file mode 100644 index 1692faed4..000000000 --- a/src/test/java/org/apache/commons/math/RetryTestCase.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.commons.math; - - -/** - * A Test case that retries tests when assertions fail. - * @version $Revision$ $Date$ - */ -public abstract class RetryTestCase { - - // TODO implement retry policy using Junit 4 API - - // /** -// * Override runTest() to catch AssertionFailedError and retry -// */ -// @Override -// protected void runTest() throws Throwable { -// try { -// super.runTest(); -// } catch (AssertionFailedError err) { -// // System.out.println("Retrying " + this.getName()); -// super.runTest(); -// } -// } - -} diff --git a/src/test/java/org/apache/commons/math/random/EmpiricalDistributionTest.java b/src/test/java/org/apache/commons/math/random/EmpiricalDistributionTest.java index 8f5473206..5c7ac9b3a 100644 --- a/src/test/java/org/apache/commons/math/random/EmpiricalDistributionTest.java +++ b/src/test/java/org/apache/commons/math/random/EmpiricalDistributionTest.java @@ -23,12 +23,13 @@ import java.io.InputStreamReader; import java.net.URL; import java.util.ArrayList; -import org.apache.commons.math.RetryTestCase; +import org.apache.commons.math.RetryRunner; import org.apache.commons.math.TestUtils; import org.apache.commons.math.stat.descriptive.SummaryStatistics; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; /** * Test cases for the EmpiricalDistribution class @@ -36,7 +37,8 @@ import org.junit.Test; * @version $Revision$ $Date$ */ -public final class EmpiricalDistributionTest extends RetryTestCase { +@RunWith(RetryRunner.class) +public final class EmpiricalDistributionTest { protected EmpiricalDistribution empiricalDistribution = null; protected EmpiricalDistribution empiricalDistribution2 = null; diff --git a/src/test/java/org/apache/commons/math/random/RandomDataTest.java b/src/test/java/org/apache/commons/math/random/RandomDataTest.java index 15dda7df1..e7d3626bc 100644 --- a/src/test/java/org/apache/commons/math/random/RandomDataTest.java +++ b/src/test/java/org/apache/commons/math/random/RandomDataTest.java @@ -22,7 +22,7 @@ import java.util.HashSet; import java.util.List; -import org.apache.commons.math.RetryTestCase; +import org.apache.commons.math.RetryRunner; import org.apache.commons.math.TestUtils; import org.apache.commons.math.distribution.BetaDistributionImpl; import org.apache.commons.math.distribution.BinomialDistributionImpl; @@ -49,6 +49,7 @@ import org.apache.commons.math.util.FastMath; import org.apache.commons.math.exception.MathIllegalArgumentException; import org.junit.Assert; import org.junit.Test; +import org.junit.runner.RunWith; /** * Test cases for the RandomData class. @@ -57,7 +58,8 @@ import org.junit.Test; * 2009) $ */ -public class RandomDataTest extends RetryTestCase { +@RunWith(RetryRunner.class) +public class RandomDataTest { public RandomDataTest() { randomData = new RandomDataImpl(); diff --git a/src/test/java/org/apache/commons/math/random/ValueServerTest.java b/src/test/java/org/apache/commons/math/random/ValueServerTest.java index 495a81893..d520e200f 100644 --- a/src/test/java/org/apache/commons/math/random/ValueServerTest.java +++ b/src/test/java/org/apache/commons/math/random/ValueServerTest.java @@ -19,11 +19,12 @@ package org.apache.commons.math.random; import java.io.EOFException; import java.net.URL; -import org.apache.commons.math.RetryTestCase; +import org.apache.commons.math.RetryRunner; import org.apache.commons.math.stat.descriptive.SummaryStatistics; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; /** * Test cases for the ValueServer class. @@ -31,7 +32,8 @@ import org.junit.Test; * @version $Revision$ $Date$ */ -public final class ValueServerTest extends RetryTestCase { +@RunWith(RetryRunner.class) +public final class ValueServerTest { private ValueServer vs = new ValueServer();