From 9ae47eaa359ff21e81a06bd289c3381ec5eca5b3 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Wed, 5 Oct 2011 13:16:22 +0000 Subject: [PATCH] LUCENE-3463: Add @UseNoMemoryExpensiveCodec annotation git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1179210 13f79535-47bb-0310-9956-ffa450edef68 --- .../lucene/index/RandomCodecProvider.java | 8 +++--- .../apache/lucene/util/LuceneTestCase.java | 27 ++++++++++++++++++- .../lucene/util/LuceneTestCaseRunner.java | 14 ++++++++++ .../apache/lucene/index/TestLongPostings.java | 7 ++--- .../apache/lucene/index/TestTermsEnum.java | 2 ++ .../util/automaton/TestCompiledAutomaton.java | 13 ++++----- .../org/apache/lucene/util/fst/TestFSTs.java | 2 ++ 7 files changed, 58 insertions(+), 15 deletions(-) diff --git a/lucene/src/test-framework/org/apache/lucene/index/RandomCodecProvider.java b/lucene/src/test-framework/org/apache/lucene/index/RandomCodecProvider.java index 833d6e979af..b92ce7a004a 100644 --- a/lucene/src/test-framework/org/apache/lucene/index/RandomCodecProvider.java +++ b/lucene/src/test-framework/org/apache/lucene/index/RandomCodecProvider.java @@ -48,7 +48,7 @@ public class RandomCodecProvider extends CodecProvider { private Map previousMappings = new HashMap(); private final int perFieldSeed; - public RandomCodecProvider(Random random) { + public RandomCodecProvider(Random random, boolean useNoMemoryExpensiveCodec) { this.perFieldSeed = random.nextInt(); // TODO: make it possible to specify min/max iterms per // block via CL: @@ -61,8 +61,10 @@ public class RandomCodecProvider extends CodecProvider { minItemsPerBlock = _TestUtil.nextInt(random, 2, 100); maxItemsPerBlock = 2*(Math.max(1, minItemsPerBlock-1)) + random.nextInt(100); register(new PulsingCodec( 1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock)); - register(new SimpleTextCodec()); - register(new MemoryCodec()); + if (!useNoMemoryExpensiveCodec) { + register(new SimpleTextCodec()); + register(new MemoryCodec()); + } Collections.shuffle(knownCodecs, random); } diff --git a/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java b/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java index 688fe110b42..f9df8e5f15b 100644 --- a/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java +++ b/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java @@ -21,9 +21,11 @@ import java.io.File; import java.io.IOException; import java.io.PrintStream; import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; import java.lang.annotation.Inherited; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; import java.lang.reflect.Constructor; import java.util.*; import java.util.Map.Entry; @@ -346,7 +348,7 @@ public abstract class LuceneTestCase extends Assert { if (random.nextInt(4) == 0) { // preflex-only setup codec = installTestCodecs("PreFlex", CodecProvider.getDefault()); } else { // per-field setup - CodecProvider.setDefault(new RandomCodecProvider(random)); + CodecProvider.setDefault(new RandomCodecProvider(random, useNoMemoryExpensiveCodec)); codec = installTestCodecs(TEST_CODEC, CodecProvider.getDefault()); } } else { // ordinary setup @@ -636,6 +638,17 @@ public abstract class LuceneTestCase extends Assert { if (!testsFailed) { assertTrue("ensure your tearDown() calls super.tearDown()!!!", (s == State.INITIAL || s == State.TEARDOWN)); } + + if (useNoMemoryExpensiveCodec) { + final String defCodec = CodecProvider.getDefault().getDefaultFieldCodec(); + // Stupid: assumeFalse in setUp() does not print any information, because + // TestWatchman does not watch test during setUp() - getName() is also not defined... + // => print info directly and use assume without message: + if ("SimpleText".equals(defCodec) || "Memory".equals(defCodec)) { + System.err.println("NOTE: A test method in " + getClass().getSimpleName() + " was ignored, as it uses too much memory with " + defCodec + "."); + Assume.assumeTrue(false); + } + } } /** @@ -1397,6 +1410,9 @@ public abstract class LuceneTestCase extends Assert { return context; } + // initialized by the TestRunner + static boolean useNoMemoryExpensiveCodec; + // recorded seed: for beforeClass private static long staticSeed; // seed for individual test methods, changed in @before @@ -1415,6 +1431,15 @@ public abstract class LuceneTestCase extends Assert { @Retention(RetentionPolicy.RUNTIME) public @interface Nightly {} + /** + * Annotation for test classes that should only use codecs that are not memory expensive (avoid SimpleText, MemoryCodec). + */ + @Documented + @Inherited + @Retention(RetentionPolicy.RUNTIME) + @Target(ElementType.TYPE) + public @interface UseNoMemoryExpensiveCodec {} + @Ignore("just a hack") public final void alwaysIgnoredTestMethod() {} } diff --git a/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCaseRunner.java b/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCaseRunner.java index c970252c296..cbdd5d77151 100644 --- a/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCaseRunner.java +++ b/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCaseRunner.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Random; import org.apache.lucene.util.LuceneTestCase.Nightly; +import org.apache.lucene.util.LuceneTestCase.UseNoMemoryExpensiveCodec; import org.junit.Ignore; import org.junit.Test; import org.junit.runner.Description; @@ -156,6 +157,19 @@ public class LuceneTestCaseRunner extends BlockJUnit4ClassRunner { public LuceneTestCaseRunner(Class clazz) throws InitializationError { super(clazz); + + // This TestRunner can handle only LuceneTestCase subclasses + if (!LuceneTestCase.class.isAssignableFrom(clazz)) { + throw new UnsupportedOperationException("LuceneTestCaseRunner can only be used with LuceneTestCase."); + } + + final boolean useNoMemoryExpensiveCodec = LuceneTestCase.useNoMemoryExpensiveCodec = + clazz.isAnnotationPresent(UseNoMemoryExpensiveCodec.class); + if (useNoMemoryExpensiveCodec) { + System.err.println("NOTE: Using no memory expensive codecs (Memory, SimpleText) for " + + clazz.getSimpleName() + "."); + } + // evil we cannot init our random here, because super() calls computeTestMethods!!!!; Filter f = new Filter() { diff --git a/lucene/src/test/org/apache/lucene/index/TestLongPostings.java b/lucene/src/test/org/apache/lucene/index/TestLongPostings.java index 6bec750b5db..034c6f12f9d 100644 --- a/lucene/src/test/org/apache/lucene/index/TestLongPostings.java +++ b/lucene/src/test/org/apache/lucene/index/TestLongPostings.java @@ -33,9 +33,11 @@ import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.LuceneTestCase.UseNoMemoryExpensiveCodec; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util._TestUtil; +@UseNoMemoryExpensiveCodec public class TestLongPostings extends LuceneTestCase { // Produces a realistic unicode random string that @@ -76,9 +78,6 @@ public class TestLongPostings extends LuceneTestCase { } public void testLongPostings() throws Exception { - assumeFalse("Too slow with SimpleText codec at night", TEST_NIGHTLY && CodecProvider.getDefault().getFieldCodec("field").equals("SimpleText")); - assumeFalse("Too slow with Memory codec at night", TEST_NIGHTLY && CodecProvider.getDefault().getFieldCodec("field").equals("Memory")); - // Don't use _TestUtil.getTempDir so that we own the // randomness (ie same seed will point to same dir): Directory dir = newFSDirectory(_TestUtil.getTempDir("longpostings" + "." + random.nextLong())); @@ -271,8 +270,6 @@ public class TestLongPostings extends LuceneTestCase { } public void doTestLongPostingsNoPositions(IndexOptions options) throws Exception { - assumeFalse("Too slow with SimpleText codec at night", TEST_NIGHTLY && CodecProvider.getDefault().getFieldCodec("field").equals("SimpleText")); - assumeFalse("Too slow with Memory codec at night", TEST_NIGHTLY && CodecProvider.getDefault().getFieldCodec("field").equals("Memory")); // Don't use _TestUtil.getTempDir so that we own the // randomness (ie same seed will point to same dir): Directory dir = newFSDirectory(_TestUtil.getTempDir("longpostings" + "." + random.nextLong())); diff --git a/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java b/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java index 4fbcf5229b9..40159eb1b8d 100644 --- a/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java +++ b/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java @@ -40,12 +40,14 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.LuceneTestCase.UseNoMemoryExpensiveCodec; import org.apache.lucene.util._TestUtil; import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.BasicAutomata; import org.apache.lucene.util.automaton.CompiledAutomaton; import org.apache.lucene.util.automaton.DaciukMihovAutomatonBuilder; +@UseNoMemoryExpensiveCodec public class TestTermsEnum extends LuceneTestCase { public void test() throws Exception { diff --git a/lucene/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java b/lucene/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java index f346d4cf5f9..31d0a573585 100644 --- a/lucene/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java +++ b/lucene/src/test/org/apache/lucene/util/automaton/TestCompiledAutomaton.java @@ -19,6 +19,7 @@ package org.apache.lucene.util.automaton; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -30,12 +31,12 @@ import org.apache.lucene.util._TestUtil; public class TestCompiledAutomaton extends LuceneTestCase { private CompiledAutomaton build(String... strings) { - final List as = new ArrayList(); + final List terms = new ArrayList(); for(String s : strings) { - as.add(BasicAutomata.makeString(s)); + terms.add(new BytesRef(s)); } - Automaton a = BasicOperations.union(as); - a.determinize(); + Collections.sort(terms); + final Automaton a = DaciukMihovAutomatonBuilder.build(terms); return new CompiledAutomaton(a, true, false); } @@ -93,7 +94,7 @@ public class TestCompiledAutomaton extends LuceneTestCase { } public void testRandom() throws Exception { - final int numTerms = atLeast(1000); + final int numTerms = atLeast(400); final Set terms = new HashSet(); while(terms.size() != numTerms) { terms.add(randomString()); @@ -107,7 +108,7 @@ public class TestCompiledAutomaton extends LuceneTestCase { } public void testBasic() throws Exception { - CompiledAutomaton c = build("foo", "fob", "goo"); + CompiledAutomaton c = build("fob", "foo", "goo"); testFloor(c, "goo", "goo"); testFloor(c, "ga", "foo"); testFloor(c, "g", "foo"); diff --git a/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java b/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java index 3276397fc0c..33496b6fa5c 100644 --- a/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java +++ b/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java @@ -53,10 +53,12 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IntsRef; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.LuceneTestCase.UseNoMemoryExpensiveCodec; import org.apache.lucene.util.UnicodeUtil; import org.apache.lucene.util._TestUtil; import org.apache.lucene.util.fst.FST.Arc; +@UseNoMemoryExpensiveCodec public class TestFSTs extends LuceneTestCase { private MockDirectoryWrapper dir;