Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-2764: Allow tests to use random codec per field 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1036543 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Simon Willnauer 2010-11-18 17:28:27 +00:00
parent e60764241b
commit a5d73e7dec
9 changed files with 80 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lucene/common-build.xml
View File

@ -65,6 +65,7 @@
</condition>
<property name="tests.multiplier" value="1" />
<property name="tests.codec" value="random" />
<property name="tests.randomCodecPerField" value="true"/>
<property name="tests.locale" value="random" />
<property name="tests.timezone" value="random" />
<property name="tests.directory" value="random" />

1
lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
View File

@ -133,7 +133,6 @@ public class CreateIndexTask extends PerfTask {
final String defaultCodec = config.get("default.codec", null);
if (defaultCodec != null) {
CodecProvider.setDefaultCodec(defaultCodec);
CodecProvider.getDefault().setDefaultFieldCodec(defaultCodec);
}

13
lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java
View File

@ -39,7 +39,7 @@ import org.apache.lucene.index.codecs.standard.StandardCodec;
public class CodecProvider {
private SegmentInfosWriter infosWriter = new DefaultSegmentInfosWriter();
private SegmentInfosReader infosReader = new DefaultSegmentInfosReader();
private String defaultFieldCodec = defaultCodec;
private String defaultFieldCodec = "Standard";
private final Map<String, String> perFieldMap = new HashMap<String, String>();
@ -47,7 +47,6 @@ public class CodecProvider {
private final Set<String> knownExtensions = new HashSet<String>();
private static String defaultCodec = "Standard";
public final static String[] CORE_CODECS = new String[] {"Standard", "Pulsing", "PreFlex", "SimpleText"};
@ -103,15 +102,6 @@ public class CodecProvider {
return defaultCodecs;
}
/** Used for testing. @lucene.internal */
public synchronized static void setDefaultCodec(String s) {
defaultCodec = s;
}
/** Used for testing. @lucene.internal */
public synchronized static String getDefaultCodec() {
return defaultCodec;
}
/**
* Sets the {@link Codec} for a given field. Not that setting a fields code is
* write-once. If the fields codec is already set this method will throw an
@ -175,6 +165,5 @@ class DefaultCodecProvider extends CodecProvider {
register(new PreFlexCodec());
register(new PulsingCodec(1));
register(new SimpleTextCodec());
setDefaultFieldCodec(CodecProvider.getDefaultCodec());
}
}

3
lucene/src/test/org/apache/lucene/index/RandomIndexWriter.java
View File

@ -24,7 +24,6 @@ import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.Version;
@ -84,7 +83,7 @@ public class RandomIndexWriter implements Closeable {
flushAt = _TestUtil.nextInt(r, 10, 1000);
if (LuceneTestCase.VERBOSE) {
System.out.println("RIW config=" + w.getConfig());
System.out.println("codec default=" + CodecProvider.getDefaultCodec());
System.out.println("codec default=" + w.getConfig().getCodecProvider().getDefaultFieldCodec());
}
}

2
lucene/src/test/org/apache/lucene/index/Test2BTerms.java
View File

@ -123,7 +123,7 @@ public class Test2BTerms extends LuceneTestCase {
@Ignore("Takes ~4 hours to run on a fast machine!! And requires that you don't use PreFlex codec.")
public void test2BTerms() throws IOException {
if ("PreFlex".equals(CodecProvider.getDefaultCodec())) {
if ("PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())) {
throw new RuntimeException("thist test cannot run with PreFlex codec");
}

4
lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
View File

@ -69,7 +69,7 @@ public class TestLazyProxSkipping extends LuceneTestCase {
int numDocs = 500;
Directory directory = new SeekCountingDirectory(new RAMDirectory());
IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, true, false)).setMaxBufferedDocs(10));
IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, true, false)).setMaxBufferedDocs(10));
((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false);
((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false);
for (int i = 0; i < numDocs; i++) {
@ -120,7 +120,7 @@ public class TestLazyProxSkipping extends LuceneTestCase {
}
public void testLazySkipping() throws IOException {
assumeFalse("This test cannot run with SimpleText codec", CodecProvider.getDefaultCodec().equals("SimpleText"));
assumeFalse("This test cannot run with SimpleText codec", getRandomFieldCodec(this.field).equals("SimpleText"));
// test whether only the minimum amount of seeks()
// are performed
performTest(5);

3
lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java
View File

@ -27,7 +27,6 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.BytesRef;
@ -57,7 +56,7 @@ public class TestPrefixRandom extends LuceneTestCase {
// we generate aweful prefixes: good for testing.
// but for preflex codec, the test can be very slow, so use less iterations.
String codec = CodecProvider.getDefaultCodec();
final String codec = getRandomFieldCodec("field");
int num = codec.equals("PreFlex") ? 200 * RANDOM_MULTIPLIER : 2000 * RANDOM_MULTIPLIER;
for (int i = 0; i < num; i++) {
field.setValue(_TestUtil.randomUnicodeString(random, 10));

4
lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java
View File

@ -58,7 +58,6 @@ public class TestRegexpRandom2 extends LuceneTestCase {
RandomIndexWriter writer = new RandomIndexWriter(random, dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.KEYWORD, false))
.setMaxBufferedDocs(_TestUtil.nextInt(random, 50, 1000)));
Document doc = new Document();
Field field = newField("field", "", Field.Store.NO, Field.Index.NOT_ANALYZED);
doc.add(field);
@ -135,8 +134,7 @@ public class TestRegexpRandom2 extends LuceneTestCase {
public void testRegexps() throws Exception {
// we generate aweful regexps: good for testing.
// but for preflex codec, the test can be very slow, so use less iterations.
String codec = CodecProvider.getDefaultCodec();
int num = codec.equals("PreFlex") ? 100 * RANDOM_MULTIPLIER : 1000 * RANDOM_MULTIPLIER;
int num = getRandomFieldCodec("field").equals("PreFlex") ? 100 * RANDOM_MULTIPLIER : 1000 * RANDOM_MULTIPLIER;
for (int i = 0; i < num; i++) {
String reg = AutomatonTestUtil.randomRegexp(random);
assertSame(reg);

80
lucene/src/test/org/apache/lucene/util/LuceneTestCase.java
View File

@ -35,6 +35,8 @@ import org.apache.lucene.index.codecs.mocksep.MockSepCodec;
import org.apache.lucene.index.codecs.preflex.PreFlexCodec;
import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec;
import org.apache.lucene.index.codecs.pulsing.PulsingCodec;
import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec;
import org.apache.lucene.index.codecs.standard.StandardCodec;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.FieldCache.CacheEntry;
@ -73,6 +75,7 @@ import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.List;
@ -146,6 +149,8 @@ public abstract class LuceneTestCase extends Assert {
// tests)
/** Gets the codec to run tests with. */
static final String TEST_CODEC = System.getProperty("tests.codec", "random");
/** Get if a random codec per field should be used */ // only use random per field if no explicit codec is set
static final boolean TEST_RANDOM_CODEC_PER_FIELD = "random".equals(TEST_CODEC) && Boolean.parseBoolean(System.getProperty("tests.randomCodecPerField", "true"));
/** Gets the locale to run tests with */
static final String TEST_LOCALE = System.getProperty("tests.locale", "random");
/** Gets the timezone to run tests with */
@ -215,7 +220,7 @@ public abstract class LuceneTestCase extends Assert {
static Codec installTestCodecs() {
final CodecProvider cp = CodecProvider.getDefault();
savedDefaultCodec = CodecProvider.getDefaultCodec();
savedDefaultCodec = cp.getDefaultFieldCodec();
String codec = TEST_CODEC;
final boolean codecHasParam;
@ -235,7 +240,6 @@ public abstract class LuceneTestCase extends Assert {
}
}
CodecProvider.setDefaultCodec(codec);
cp.setDefaultFieldCodec(codec);
if (codec.equals("PreFlex")) {
@ -268,7 +272,6 @@ public abstract class LuceneTestCase extends Assert {
cp.unregister(cp.lookup("MockFixedIntBlock"));
cp.unregister(cp.lookup("MockVariableIntBlock"));
swapCodec(new PulsingCodec(1));
CodecProvider.setDefaultCodec(savedDefaultCodec);
cp.setDefaultFieldCodec(savedDefaultCodec);
}
@ -322,6 +325,7 @@ public abstract class LuceneTestCase extends Assert {
timeZone = TEST_TIMEZONE.equals("random") ? randomTimeZone(random) : TimeZone.getTimeZone(TEST_TIMEZONE);
TimeZone.setDefault(timeZone);
testsFailed = false;
randomCodecProvider = new RandomCodecProvider(random);
}
@AfterClass
@ -342,11 +346,11 @@ public abstract class LuceneTestCase extends Assert {
}
stores = null;
// if tests failed, report some information back
if (testsFailed)
if (testsFailed) {
System.out.println("NOTE: test params are: codec=" + codec +
", locale=" + locale +
", timezone=" + (timeZone == null ? "(null)" : timeZone.getID()));
if (testsFailed) {
", timezone=" + (timeZone == null ? "(null)" : timeZone.getID()) +
(TEST_RANDOM_CODEC_PER_FIELD?", "+randomCodecProvider.toString():""));
System.err.println("NOTE: all tests run in this JVM:");
System.err.println(Arrays.toString(testClassesRun.toArray()));
}
@ -617,8 +621,9 @@ public abstract class LuceneTestCase extends Assert {
return newIndexWriterConfig(random, v, a);
}
/** create a new index writer config with random defaults */
public static IndexWriterConfig newIndexWriterConfig(Random r, Version v, Analyzer a) {
IndexWriterConfig c = new IndexWriterConfig(v, a);
final IndexWriterConfig c = new IndexWriterConfig(v, a);
if (r.nextBoolean()) {
c.setMergePolicy(new LogDocMergePolicy());
}
@ -642,7 +647,9 @@ public abstract class LuceneTestCase extends Assert {
logmp.setCalibrateSizeByDeletes(r.nextBoolean());
logmp.setMergeFactor(_TestUtil.nextInt(r, 2, 20));
}
if (TEST_RANDOM_CODEC_PER_FIELD) {
c.setCodecProvider(randomCodecProvider);
}
c.setReaderPooling(r.nextBoolean());
c.setReaderTermsIndexDivisor(_TestUtil.nextInt(r, 1, 4));
return c;
@ -844,6 +851,8 @@ public abstract class LuceneTestCase extends Assert {
// seed for individual test methods, changed in @before
private long seed;
protected static CodecProvider randomCodecProvider;
private static final Random seedRand = new Random();
protected static final Random random = new Random();
@ -944,6 +953,61 @@ public abstract class LuceneTestCase extends Assert {
}
}
/** Returns the codec for the given field uses in this testcase */
public static String getRandomFieldCodec(String field) {
final CodecProvider provider = TEST_RANDOM_CODEC_PER_FIELD ? randomCodecProvider : CodecProvider.getDefault();
return provider.getFieldCodec(field);
}
static class RandomCodecProvider extends CodecProvider {
private final Codec[] codecs;
private final Map<String, String> perFieldMap = new HashMap<String, String>();
private final Random random;
RandomCodecProvider(Random random) {
if (random.nextInt(5) == 0) {
/*
* We randomly swap in a exclusive PreFlexCodec to provide random test coverage
* for preFlex indexes. In realistic a PreFlex-Environment newer codecs don't occur.
*/
this.codecs = new Codec[] { new PreFlexRWCodec() };
} else {
this.codecs = new Codec[] { new StandardCodec(), new SimpleTextCodec(),
new MockSepCodec(), new PulsingCodec(1 + random.nextInt(10)),
new MockVariableIntBlockCodec(1 + random.nextInt(10)),
new MockFixedIntBlockCodec(1 + random.nextInt(10)), };
register(new PreFlexCodec()); // register this for read support
}
for (int i = 0; i < codecs.length; i++) {
register(codecs[i]);
}
this.random = random;
}
@Override
public synchronized String getFieldCodec(String name) {
if (!perFieldMap.containsKey(name)) { // select a codec at random
setFieldCodec(name, codecs[random.nextInt(codecs.length)].name);
}
return super.getFieldCodec(name);
}
@Override
public synchronized void setFieldCodec(String field, String codec) {
if (!perFieldMap.containsKey(field)) {
perFieldMap.put(field, codec);
}
super.setFieldCodec(field, codec);
}
@Override
public String toString() {
return "RandomCodecProvider [perFieldMap=" + perFieldMap + "]";
}
}
@Ignore("just a hack")
public final void alwaysIgnoredTestMethod() {}
}