honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added support for `tests.failfast` and `tests.maxiters` 

This commit adds support for failing fast when running a test
case with `-Dtests.iters=N` and uses some goodness from LuceneTestCase
in a new base `AbstractRandomizedTest`. This class checks among other
things if a tests doesn't call `super.setup` / `super.tearDown` when it
should do and checks if a large static resources are not cleaned up
after the tests ie. a running node.
This commit is contained in:
Simon Willnauer 2013-08-12 13:16:21 +02:00
parent 2ed3dbbf67
commit dbed36a13f
15 changed files with 368 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
pom.xml
View File

@ -33,7 +33,8 @@
<lucene.version>4.4.0</lucene.version>
<tests.jvms>auto</tests.jvms>
<tests.shuffle>true</tests.shuffle>
<tests.failfast>true</tests.failfast>
<tests.failfast>yes</tests.failfast>
<tests.maxfailures>1</tests.maxfailures>
</properties>
<repositories>
@ -372,6 +373,8 @@
<systemProperties>
<!-- RandomizedTesting library system properties -->
<tests.iters>${tests.iters}</tests.iters>
<tests.maxfailures>${tests.maxfailures}</tests.maxfailures>
<tests.failfast>${tests.failfast}</tests.failfast>
<tests.class>${tests.class}</tests.class>
<tests.method>${tests.method}</tests.method>
<tests.nightly>${tests.nightly}</tests.nightly>
@ -380,6 +383,8 @@
<tests.slow>${tests.slow}</tests.slow>
<tests.awaitsfix>${tests.awaitsfix}</tests.awaitsfix>
<tests.slow>${tests.slow}</tests.slow>
<tests.timeoutSuite>${tests.timeoutSuite}</tests.timeoutSuite>
<tests.showSuccess>${tests.showSuccess}</tests.showSuccess>
<es.node.local>${env.ES_TEST_LOCAL}</es.node.local>
<es.transport.tcp.compress>${env.ES_TEST_COMPRESS}</es.transport.tcp.compress>
<es.action.wait_on_mapping_change>${env.ES_WAIT_ON_MAPPING_CHANGE}

4
src/test/java/org/apache/lucene/analysis/miscellaneous/UniqueTokenFilterTests.java
View File

@ -25,17 +25,17 @@ import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.test.integration.ElasticsearchTestCase;
import org.junit.Test;
import java.io.IOException;
import java.io.Reader;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
/**
*/
public class UniqueTokenFilterTests {
public class UniqueTokenFilterTests extends ElasticsearchTestCase {
@Test
public void simpleTest() throws IOException {

323
src/test/java/org/apache/lucene/util/AbstractRandomizedTest.java Normal file
View File

@ -0,0 +1,323 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.lucene.util;
import com.carrotsearch.randomizedtesting.JUnit4MethodProvider;
import com.carrotsearch.randomizedtesting.LifecycleScope;
import com.carrotsearch.randomizedtesting.RandomizedContext;
import com.carrotsearch.randomizedtesting.RandomizedTest;
import com.carrotsearch.randomizedtesting.annotations.Listeners;
import com.carrotsearch.randomizedtesting.annotations.TestMethodProviders;
import com.carrotsearch.randomizedtesting.rules.NoClassHooksShadowingRule;
import com.carrotsearch.randomizedtesting.rules.NoInstanceHooksOverridesRule;
import com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule;
import com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.junit.listerners.ReproduceInfoPrinter;
import org.junit.After;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.rules.RuleChain;
import org.junit.rules.TestRule;
import org.junit.runner.RunWith;
import java.io.Closeable;
import java.io.File;
import java.lang.reflect.Method;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Logger;
@TestMethodProviders({
LuceneJUnit3MethodProvider.class,
JUnit4MethodProvider.class
})
@Listeners({
ReproduceInfoPrinter.class
})
@RunWith(value = com.carrotsearch.randomizedtesting.RandomizedRunner.class)
@SuppressCodecs(value="Lucene3x")
// NOTE: this class is in o.a.lucene.util since it uses some classes that are related
// to the test framework that didn't make sense to copy but are package private access
public class AbstractRandomizedTest extends RandomizedTest {
// --------------------------------------------------------------------
// Test groups, system properties and other annotations modifying tests
// --------------------------------------------------------------------
/** @see #ignoreAfterMaxFailures*/
public static final String SYSPROP_MAXFAILURES = "tests.maxfailures";
/** @see #ignoreAfterMaxFailures*/
public static final String SYSPROP_FAILFAST = "tests.failfast";
// -----------------------------------------------------------------
// Truly immutable fields and constants, initialized once and valid
// for all suites ever since.
// -----------------------------------------------------------------
/**
* Use this constant when creating Analyzers and any other version-dependent stuff.
* <p><b>NOTE:</b> Change this when development starts for new Lucene version:
*/
public static final Version TEST_VERSION_CURRENT = Lucene.VERSION;
/**
* True if and only if tests are run in verbose mode. If this flag is false
* tests are not expected to print any messages.
*/
public static final boolean VERBOSE = systemPropertyAsBoolean("tests.verbose", false);
/**
* A random multiplier which you should use when writing random tests:
* multiply it by the number of iterations to scale your tests (for nightly builds).
*/
public static final int RANDOM_MULTIPLIER = systemPropertyAsInt("tests.multiplier", 1);
/** TODO: javadoc? */
public static final String DEFAULT_LINE_DOCS_FILE = "europarl.lines.txt.gz";
/** the line file used by LineFileDocs */
public static final String TEST_LINE_DOCS_FILE = System.getProperty("tests.linedocsfile", DEFAULT_LINE_DOCS_FILE);
/** Create indexes in this directory, optimally use a subdir, named after the test */
public static final File TEMP_DIR;
static {
String s = System.getProperty("tempDir", System.getProperty("java.io.tmpdir"));
if (s == null)
throw new RuntimeException("To run tests, you need to define system property 'tempDir' or 'java.io.tmpdir'.");
TEMP_DIR = new File(s);
TEMP_DIR.mkdirs();
}
/**
* These property keys will be ignored in verification of altered properties.
* @see SystemPropertiesInvariantRule
* @see #ruleChain
* @see #classRules
*/
private static final String [] IGNORED_INVARIANT_PROPERTIES = {
"user.timezone", "java.rmi.server.randomIDs", "sun.nio.ch.bugLevel"
};
// -----------------------------------------------------------------
// Fields initialized in class or instance rules.
// -----------------------------------------------------------------
// -----------------------------------------------------------------
// Class level (suite) rules.
// -----------------------------------------------------------------
/**
* Stores the currently class under test.
*/
private static final TestRuleStoreClassName classNameRule;
/**
* Class environment setup rule.
*/
static final TestRuleSetupAndRestoreClassEnv classEnvRule;
/**
* Suite failure marker (any error in the test or suite scope).
*/
public final static TestRuleMarkFailure suiteFailureMarker =
new TestRuleMarkFailure();
/**
* Ignore tests after hitting a designated number of initial failures. This
* is truly a "static" global singleton since it needs to span the lifetime of all
* test classes running inside this JVM (it cannot be part of a class rule).
*
* <p>This poses some problems for the test framework's tests because these sometimes
* trigger intentional failures which add up to the global count. This field contains
* a (possibly) changing reference to {@link TestRuleIgnoreAfterMaxFailures} and we
* dispatch to its current value from the {@link #classRules} chain using {@link TestRuleDelegate}.
*/
private static final AtomicReference<TestRuleIgnoreAfterMaxFailures> ignoreAfterMaxFailuresDelegate;
private static final TestRule ignoreAfterMaxFailures;
static {
int maxFailures = systemPropertyAsInt(SYSPROP_MAXFAILURES, Integer.MAX_VALUE);
boolean failFast = systemPropertyAsBoolean(SYSPROP_FAILFAST, false);
if (failFast) {
if (maxFailures == Integer.MAX_VALUE) {
maxFailures = 1;
} else {
Logger.getLogger(LuceneTestCase.class.getSimpleName()).warning(
"Property '" + SYSPROP_MAXFAILURES + "'=" + maxFailures + ", 'failfast' is" +
" ignored.");
}
}
ignoreAfterMaxFailuresDelegate =
new AtomicReference<TestRuleIgnoreAfterMaxFailures>(
new TestRuleIgnoreAfterMaxFailures(maxFailures));
ignoreAfterMaxFailures = TestRuleDelegate.of(ignoreAfterMaxFailuresDelegate);
}
/**
* Temporarily substitute the global {@link TestRuleIgnoreAfterMaxFailures}. See
* {@link #ignoreAfterMaxFailuresDelegate} for some explanation why this method
* is needed.
*/
public static TestRuleIgnoreAfterMaxFailures replaceMaxFailureRule(TestRuleIgnoreAfterMaxFailures newValue) {
return ignoreAfterMaxFailuresDelegate.getAndSet(newValue);
}
/**
* Max 10mb of static data stored in a test suite class after the suite is complete.
* Prevents static data structures leaking and causing OOMs in subsequent tests.
*/
private final static long STATIC_LEAK_THRESHOLD = 10 * 1024 * 1024;
/** By-name list of ignored types like loggers etc. */
private final static Set<String> STATIC_LEAK_IGNORED_TYPES =
Collections.unmodifiableSet(new HashSet<String>(Arrays.asList(
EnumSet.class.getName())));
/**
* This controls how suite-level rules are nested. It is important that _all_ rules declared
* in {@link LuceneTestCase} are executed in proper order if they depend on each
* other.
*/
@ClassRule
public static TestRule classRules = RuleChain
.outerRule(new TestRuleIgnoreTestSuites())
.around(ignoreAfterMaxFailures)
.around(suiteFailureMarker)
.around(new TestRuleAssertionsRequired())
.around(new StaticFieldsInvariantRule(STATIC_LEAK_THRESHOLD, true) {
@Override
protected boolean accept(java.lang.reflect.Field field) {
// Don't count known classes that consume memory once.
if (STATIC_LEAK_IGNORED_TYPES.contains(field.getType().getName())) {
return false;
}
// Don't count references from ourselves, we're top-level.
if (field.getDeclaringClass() == LuceneTestCase.class) {
return false;
}
return super.accept(field);
}
})
.around(new NoClassHooksShadowingRule())
.around(new NoInstanceHooksOverridesRule() {
@Override
protected boolean verify(Method key) {
String name = key.getName();
return !(name.equals("setUp") || name.equals("tearDown"));
}
})
.around(new SystemPropertiesInvariantRule(IGNORED_INVARIANT_PROPERTIES))
.around(classNameRule = new TestRuleStoreClassName())
.around(classEnvRule = new TestRuleSetupAndRestoreClassEnv());
// -----------------------------------------------------------------
// Test level rules.
// -----------------------------------------------------------------
/** Enforces {@link #setUp()} and {@link #tearDown()} calls are chained. */
private TestRuleSetupTeardownChained parentChainCallRule = new TestRuleSetupTeardownChained();
/** Save test thread and name. */
private TestRuleThreadAndTestName threadAndTestNameRule = new TestRuleThreadAndTestName();
/** Taint suite result with individual test failures. */
private TestRuleMarkFailure testFailureMarker = new TestRuleMarkFailure(suiteFailureMarker);
/**
* This controls how individual test rules are nested. It is important that
* _all_ rules declared in {@link LuceneTestCase} are executed in proper order
* if they depend on each other.
*/
@Rule
public final TestRule ruleChain = RuleChain
.outerRule(testFailureMarker)
.around(ignoreAfterMaxFailures)
.around(threadAndTestNameRule)
.around(new SystemPropertiesInvariantRule(IGNORED_INVARIANT_PROPERTIES))
.around(new TestRuleSetupAndRestoreInstanceEnv())
.around(new TestRuleFieldCacheSanity())
.around(parentChainCallRule);
// -----------------------------------------------------------------
// Suite and test case setup/ cleanup.
// -----------------------------------------------------------------
/**
* For subclasses to override. Overrides must call {@code super.setUp()}.
*/
@Before
public void setUp() throws Exception {
parentChainCallRule.setupCalled = true;
}
/**
* For subclasses to override. Overrides must call {@code super.tearDown()}.
*/
@After
public void tearDown() throws Exception {
parentChainCallRule.teardownCalled = true;
}
// -----------------------------------------------------------------
// Test facilities and facades for subclasses.
// -----------------------------------------------------------------
/**
* Registers a {@link Closeable} resource that should be closed after the test
* completes.
*
* @return <code>resource</code> (for call chaining).
*/
public <T extends Closeable> T closeAfterTest(T resource) {
return RandomizedContext.current().closeAtEnd(resource, LifecycleScope.TEST);
}
/**
* Registers a {@link Closeable} resource that should be closed after the suite
* completes.
*
* @return <code>resource</code> (for call chaining).
*/
public static <T extends Closeable> T closeAfterSuite(T resource) {
return RandomizedContext.current().closeAtEnd(resource, LifecycleScope.SUITE);
}
/**
* Return the current class being tested.
*/
public static Class<?> getTestClass() {
return classNameRule.getTestClass();
}
/**
* Return the name of the currently executing test case.
*/
public String getTestName() {
return threadAndTestNameRule.testMethodName;
}
}

16
src/test/java/org/elasticsearch/test/integration/AbstractNodesTests.java
View File

@ -29,6 +29,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.indices.IndexMissingException;
import org.elasticsearch.node.Node;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
@ -192,7 +193,7 @@ public abstract class AbstractNodesTests extends ElasticsearchTestCase {
private static volatile AbstractNodesTests testInstance; // this test class only works once per JVM
@BeforeClass
@AfterClass
public static void tearDownOnce() throws Exception {
synchronized (AbstractNodesTests.class) {
if (testInstance != null) {
@ -203,8 +204,21 @@ public abstract class AbstractNodesTests extends ElasticsearchTestCase {
}
}
@BeforeClass
public static void setUpOnce() throws Exception {
synchronized (AbstractNodesTests.class) {
if (testInstance != null) {
testInstance.afterClass();
testInstance.closeAllNodes();
testInstance = null;
}
}
}
@Before
public final void setUp() throws Exception {
super.setUp();
synchronized (AbstractNodesTests.class) {
if (testInstance == null) {
testInstance = this;

29
src/test/java/org/elasticsearch/test/integration/ElasticsearchTestCase.java
View File

@ -18,44 +18,28 @@
*/
package org.elasticsearch.test.integration;
import com.carrotsearch.randomizedtesting.JUnit4MethodProvider;
import com.carrotsearch.randomizedtesting.RandomizedTest;
import com.carrotsearch.randomizedtesting.ThreadFilter;
import com.carrotsearch.randomizedtesting.annotations.*;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope;
import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
import com.google.common.base.Predicate;
import org.apache.lucene.util.LuceneJUnit3MethodProvider;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.AbstractRandomizedTest;
import org.apache.lucene.util.TimeUnits;
import org.apache.lucene.util.Version;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.junit.listerners.ReproduceInfoPrinter;
import org.junit.runner.RunWith;
import java.util.concurrent.TimeUnit;
@TestMethodProviders({
LuceneJUnit3MethodProvider.class,
JUnit4MethodProvider.class
})
@Listeners({
ReproduceInfoPrinter.class
})
@ThreadLeakFilters(defaultFilters = true, filters = {ElasticsearchTestCase.ElasticSearchThreadFilter.class})
@ThreadLeakScope(Scope.NONE)
@TimeoutSuite(millis = TimeUnits.HOUR) // timeout the suite after 1h and fail the test.
@RunWith(value = com.carrotsearch.randomizedtesting.RandomizedRunner.class)
public abstract class ElasticsearchTestCase extends RandomizedTest {
public static final Version TEST_VERSION_CURRENT = LuceneTestCase.TEST_VERSION_CURRENT;
public abstract class ElasticsearchTestCase extends AbstractRandomizedTest {
protected final ESLogger logger = Loggers.getLogger(getClass());
public static final String CHILD_VM_ID = System.getProperty("junit4.childvm.id", "" + System.currentTimeMillis());
public static final String SYSPROP_BADAPPLES = "tests.badapples";
public static class ElasticSearchThreadFilter implements ThreadFilter {
@Override
public boolean reject(Thread t) {
@ -68,7 +52,6 @@ public abstract class ElasticsearchTestCase extends RandomizedTest {
awaitBusy(breakPredicate, 10, TimeUnit.SECONDS);
}
public void awaitBusy(Predicate<?> breakPredicate, long maxWaitTime, TimeUnit unit) throws InterruptedException {
long maxTimeInMillis = TimeUnit.MILLISECONDS.convert(maxWaitTime, unit);
long iterations = Math.max(Math.round(Math.log10(maxTimeInMillis) / Math.log10(2)), 1);
@ -87,4 +70,4 @@ public abstract class ElasticsearchTestCase extends RandomizedTest {
}
}
}

4
src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientDocumentActionsTests.java
View File

@ -30,8 +30,8 @@ public class TransportClientDocumentActionsTests extends DocumentActionsTests {
@BeforeClass
public static void beforeClass() throws Exception {
public static void beforeTransportClientDocumentActionsTests() throws Exception {
cluster().setClientFactory(TestCluster.TransportClientFactory.NO_SNIFF_CLIENT_FACTORY);
DocumentActionsTests.beforeClass();
DocumentActionsTests.beforeDocumentActionsTests();
}
}

5
src/test/java/org/elasticsearch/test/integration/client/transport/TransportClientSniffDocumentActionsTests.java
View File

@ -29,9 +29,10 @@ import org.junit.BeforeClass;
public class TransportClientSniffDocumentActionsTests extends DocumentActionsTests {
@BeforeClass
public static void beforeClass() throws Exception {
public static void beforTransportClientSniffDocumentActionsTests() throws Exception {
DocumentActionsTests.beforeDocumentActionsTests();
cluster().setClientFactory(TestCluster.TransportClientFactory.SNIFF_CLIENT_FACTORY);
DocumentActionsTests.beforeClass();
DocumentActionsTests.beforeDocumentActionsTests();
}
}

1
src/test/java/org/elasticsearch/test/integration/cluster/shards/ClusterSearchShardsTests.java
View File

@ -27,7 +27,6 @@ import org.elasticsearch.test.integration.AbstractNodesTests;
import org.junit.Test;
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
/**

2
src/test/java/org/elasticsearch/test/integration/document/DocumentActionsTests.java
View File

@ -55,7 +55,7 @@ import static org.hamcrest.Matchers.nullValue;
public class DocumentActionsTests extends AbstractSharedClusterTest {
@BeforeClass
public static void beforeClass() throws Exception {
public static void beforeDocumentActionsTests() throws Exception {
AbstractSharedClusterTest.beforeClass();
wipeIndices();
// no indices, check that simple operations fail

1
src/test/java/org/elasticsearch/test/integration/gateway/fs/IndexGatewayTests.java
View File

@ -58,6 +58,7 @@ public class IndexGatewayTests extends AbstractNodesTests {
@After
public void closeNodes() throws Exception {
tearDown();
node("server1").stop();
// since we store (by default) the index snapshot under the gateway, resetting it will reset the index data as well
((InternalNode) node("server1")).injector().getInstance(Gateway.class).reset();

4
src/test/java/org/elasticsearch/test/integration/gateway/none/RecoverAfterNodesTests.java
View File

@ -27,7 +27,6 @@ import org.junit.After;
import org.junit.Test;
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
@ -39,7 +38,8 @@ public class RecoverAfterNodesTests extends AbstractNodesTests {
private final static TimeValue BLOCK_WAIT_TIMEOUT = TimeValue.timeValueSeconds(1);
@After
public void closeNodes() {
public void closeNodes() throws Exception {
tearDown();
closeAllNodes();
}

1
src/test/java/org/elasticsearch/test/integration/indices/wamer/LocalGatewayIndicesWarmerTests.java
View File

@ -46,6 +46,7 @@ public class LocalGatewayIndicesWarmerTests extends AbstractNodesTests {
@After
public void cleanAndCloseNodes() throws Exception {
super.tearDown();
for (int i = 0; i < 10; i++) {
if (node("node" + i) != null) {
node("node" + i).stop();

2
src/test/java/org/elasticsearch/test/unit/index/engine/robin/RobinEngineTests.java
View File

@ -106,6 +106,7 @@ public class RobinEngineTests extends ElasticsearchTestCase {
@Before
public void setUp() throws Exception {
super.setUp();
defaultSettings = ImmutableSettings.builder()
.put(RobinEngine.INDEX_COMPOUND_ON_FLUSH, getRandom().nextBoolean())
.build(); // TODO randomize more settings
@ -124,6 +125,7 @@ public class RobinEngineTests extends ElasticsearchTestCase {
@After
public void tearDown() throws Exception {
super.tearDown();
replicaEngine.close();
storeReplica.close();

1
src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractFieldDataTests.java
View File

@ -63,6 +63,7 @@ public abstract class AbstractFieldDataTests extends ElasticsearchTestCase {
@After
public void tearDown() throws Exception {
super.tearDown();
if (readerContext != null) {
readerContext.reader().close();
}

4
src/test/java/org/elasticsearch/test/unit/transport/AbstractSimpleTransportTests.java
View File

@ -60,6 +60,7 @@ public abstract class AbstractSimpleTransportTests extends ElasticsearchTestCase
@Before
public void setUp() throws Exception {
super.setUp();
threadPool = new ThreadPool();
serviceA = build(ImmutableSettings.builder().put("name", "TS_A").build(), version0);
nodeA = new DiscoveryNode("TS_A", "TS_A", serviceA.boundAddress().publishAddress(), ImmutableMap.<String, String>of(), version0);
@ -94,7 +95,8 @@ public abstract class AbstractSimpleTransportTests extends ElasticsearchTestCase
}
@After
public void tearDown() {
public void tearDown() throws Exception {
super.tearDown();
serviceA.close();
serviceB.close();
threadPool.shutdown();