Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

make MockRandomCodec more evil: now it randomly swaps different int codecs per-file 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1070106 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2011-02-12 16:21:03 +00:00
parent a01e9cbb86
commit 449f739498
3 changed files with 78 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java
View File

@ -18,15 +18,17 @@ package org.apache.lucene.index.codecs.mockrandom;
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
import java.util.Set;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.codecs.BlockTermsReader;
import org.apache.lucene.index.codecs.BlockTermsWriter;
import org.apache.lucene.index.codecs.Codec;
@ -46,6 +48,9 @@ import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockCodec;
import org.apache.lucene.index.codecs.mocksep.MockSingleIntFactory;
import org.apache.lucene.index.codecs.pulsing.PulsingPostingsReaderImpl;
import org.apache.lucene.index.codecs.pulsing.PulsingPostingsWriterImpl;
import org.apache.lucene.index.codecs.sep.IntIndexInput;
import org.apache.lucene.index.codecs.sep.IntIndexOutput;
import org.apache.lucene.index.codecs.sep.IntStreamFactory;
import org.apache.lucene.index.codecs.sep.SepPostingsReaderImpl;
import org.apache.lucene.index.codecs.sep.SepPostingsWriterImpl;
import org.apache.lucene.index.codecs.standard.StandardPostingsReader;
@ -71,11 +76,57 @@ public class MockRandomCodec extends Codec {
this.seedRandom = new Random(random.nextLong());
}
// Chooses random IntStreamFactory depending on file's extension
private static class MockIntStreamFactory extends IntStreamFactory {
private final int salt;
private final List<IntStreamFactory> delegates = new ArrayList<IntStreamFactory>();
public MockIntStreamFactory(Random random) {
salt = random.nextInt();
delegates.add(new MockSingleIntFactory());
final int blockSize = _TestUtil.nextInt(random, 1, 2000);
delegates.add(new MockFixedIntBlockCodec.MockIntFactory(blockSize));
final int baseBlockSize = _TestUtil.nextInt(random, 1, 127);
delegates.add(new MockVariableIntBlockCodec.MockIntFactory(baseBlockSize));
// TODO: others
}
private static String getExtension(String fileName) {
final int idx = fileName.indexOf('.');
assert idx != -1;
return fileName.substring(idx);
}
@Override
public IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException {
// Must only use extension, because IW.addIndexes can
// rename segment!
final IntStreamFactory f = delegates.get((Math.abs(salt ^ getExtension(fileName).hashCode())) % delegates.size());
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: read using int factory " + f + " from fileName=" + fileName);
}
return f.openInput(dir, fileName, readBufferSize);
}
@Override
public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException {
final IntStreamFactory f = delegates.get((Math.abs(salt ^ getExtension(fileName).hashCode())) % delegates.size());
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: write using int factory " + f + " to fileName=" + fileName);
}
return f.createOutput(dir, fileName);
}
}
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
final long seed = seedRandom.nextLong();
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: writing to seg=" + state.segmentName + " seed=" + seed);
}
final String seedFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, SEED_EXT);
final IndexOutput out = state.directory.createOutput(seedFileName);
out.writeLong(seed);
@ -83,25 +134,9 @@ public class MockRandomCodec extends Codec {
final Random random = new Random(seed);
PostingsWriterBase postingsWriter;
final int n = random.nextInt(4);
if (n == 0) {
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: writing MockSep postings");
}
postingsWriter = new SepPostingsWriterImpl(state, new MockSingleIntFactory());
} else if (n == 1) {
final int blockSize = _TestUtil.nextInt(random, 1, 2000);
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: writing MockFixedIntBlock(" + blockSize + ") postings");
}
postingsWriter = new SepPostingsWriterImpl(state, new MockFixedIntBlockCodec.MockIntFactory(blockSize));
} else if (n == 2) {
final int baseBlockSize = _TestUtil.nextInt(random, 1, 127);
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: writing MockVariableIntBlock(" + baseBlockSize + ") postings");
}
postingsWriter = new SepPostingsWriterImpl(state, new MockVariableIntBlockCodec.MockIntFactory(baseBlockSize));
if (random.nextBoolean()) {
postingsWriter = new SepPostingsWriterImpl(state, new MockIntStreamFactory(random));
} else {
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: writing Standard postings");
@ -190,32 +225,17 @@ public class MockRandomCodec extends Codec {
final String seedFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.codecId, SEED_EXT);
final IndexInput in = state.dir.openInput(seedFileName);
final long seed = in.readLong();
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: reading from seg=" + state.segmentInfo.name + " seed=" + seed);
}
in.close();
final Random random = new Random(seed);
PostingsReaderBase postingsReader;
final int n = random.nextInt(4);
if (n == 0) {
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: reading MockSep postings");
}
if (random.nextBoolean()) {
postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo,
state.readBufferSize, new MockSingleIntFactory(), state.codecId);
} else if (n == 1) {
final int blockSize = _TestUtil.nextInt(random, 1, 2000);
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: reading MockFixedIntBlock(" + blockSize + ") postings");
}
postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo,
state.readBufferSize, new MockFixedIntBlockCodec.MockIntFactory(blockSize), state.codecId);
} else if (n == 2) {
final int baseBlockSize = _TestUtil.nextInt(random, 1, 127);
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: reading MockVariableIntBlock(" + baseBlockSize + ") postings");
}
postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo,
state.readBufferSize, new MockVariableIntBlockCodec.MockIntFactory(baseBlockSize), state.codecId);
state.readBufferSize, new MockIntStreamFactory(random), state.codecId);
} else {
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: reading Standard postings");

3
lucene/src/test/org/apache/lucene/index/TestDoc.java
View File

@ -57,6 +57,9 @@ public class TestDoc extends LuceneTestCase {
@Override
public void setUp() throws Exception {
super.setUp();
if (VERBOSE) {
System.out.println("TEST: setUp");
}
workDir = new File(TEMP_DIR,"TestDoc");
workDir.mkdirs();

17
lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
View File

@ -106,13 +106,17 @@ public class TestIndexReaderCloneNorms extends LuceneTestCase {
Directory dir3 = newDirectory();
createIndex(random, dir3);
if (VERBOSE) {
System.out.println("TEST: now addIndexes/optimize");
}
IndexWriter iw = new IndexWriter(
dir3,
newIndexWriterConfig(TEST_VERSION_CURRENT, anlzr).
setOpenMode(OpenMode.APPEND).
setMaxBufferedDocs(5).
setMergePolicy(newLogMergePolicy(3))
setMergePolicy(newLogMergePolicy(3))
);
iw.setInfoStream(VERBOSE ? System.out : null);
iw.addIndexes(dir1, dir2);
iw.optimize();
iw.close();
@ -146,6 +150,9 @@ public class TestIndexReaderCloneNorms extends LuceneTestCase {
// try cloning and reopening the norms
private void doTestNorms(Random random, Directory dir) throws IOException {
if (VERBOSE) {
System.out.println("TEST: now doTestNorms");
}
addDocs(random, dir, 12, true);
IndexReader ir = IndexReader.open(dir, false);
verifyIndex(ir);
@ -237,13 +244,20 @@ public class TestIndexReaderCloneNorms extends LuceneTestCase {
}
private void createIndex(Random random, Directory dir) throws IOException {
if (VERBOSE) {
System.out.println("TEST: createIndex");
}
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.CREATE)
.setMaxBufferedDocs(5).setSimilarityProvider(similarityOne).setMergePolicy(newLogMergePolicy()));
LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy();
lmp.setMergeFactor(3);
lmp.setUseCompoundFile(true);
iw.close();
if (VERBOSE) {
System.out.println("TEST: done createIndex");
}
}
private void modifyNormsForF1(IndexReader ir) throws IOException {
@ -298,6 +312,7 @@ public class TestIndexReaderCloneNorms extends LuceneTestCase {
lmp.setMergeFactor(3);
lmp.setUseCompoundFile(compound);
IndexWriter iw = new IndexWriter(dir, conf);
iw.setInfoStream(VERBOSE ? System.out : null);
for (int i = 0; i < ndocs; i++) {
iw.addDocument(newDoc());
}