Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-3892: turn on all options testing in TestPostingsFormat; fix it to skip offsets if the PF can't index them 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/pforcodec_3892@1364085 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2012-07-21 13:31:16 +00:00
parent 5d36bf73e1
commit cd2bf59fdf
3 changed files with 54 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java
View File

@ -330,15 +330,25 @@ public class TestPostingsFormat extends LuceneTestCase {
System.out.println("\nTEST: now build index");
}
int maxIndexOptionNoOffsets = Arrays.asList(IndexOptions.values()).indexOf(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
// nocommit use allowPayloads
FieldInfo[] newFieldInfoArray = new FieldInfo[fields.size()];
for(int fieldUpto=0;fieldUpto<fields.size();fieldUpto++) {
FieldInfo oldFieldInfo = fieldInfos.fieldInfo(fieldUpto);
String pf = _TestUtil.getPostingsFormat(oldFieldInfo.name);
int fieldMaxIndexOption;
if (doesntSupportOffsets.contains(pf)) {
fieldMaxIndexOption = Math.min(maxIndexOptionNoOffsets, maxIndexOption);
} else {
fieldMaxIndexOption = maxIndexOption;
}
// Randomly picked the IndexOptions to index this
// field with:
IndexOptions indexOptions = IndexOptions.values()[random().nextInt(1+maxIndexOption)];
IndexOptions indexOptions = IndexOptions.values()[random().nextInt(1+fieldMaxIndexOption)];
boolean doPayloads = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 && allowPayloads;
newFieldInfoArray[fieldUpto] = new FieldInfo(oldFieldInfo.name,
@ -366,12 +376,13 @@ public class TestPostingsFormat extends LuceneTestCase {
Map<BytesRef,List<Posting>> terms = fieldEnt.getValue();
FieldInfo fieldInfo = newFieldInfos.fieldInfo(field);
if (VERBOSE) {
System.out.println("field=" + field);
}
IndexOptions indexOptions = fieldInfo.getIndexOptions();
if (VERBOSE) {
System.out.println("field=" + field + " indexOtions=" + indexOptions);
}
boolean doFreq = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
boolean doPos = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
boolean doPayloads = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 && allowPayloads;
@ -413,6 +424,7 @@ public class TestPostingsFormat extends LuceneTestCase {
} else {
totalTF++;
}
postingsConsumer.finishDoc();
docCount++;
}
termsConsumer.finishTerm(term, new TermStats(postings.size(), totalTF));
@ -539,9 +551,9 @@ public class TestPostingsFormat extends LuceneTestCase {
docsAndPositionsEnum = threadState.reuseDocsAndPositionsEnum;
}
assertNotNull(docsEnum);
assertNotNull("null DocsEnum", docsEnum);
int initialDocID = docsEnum.docID();
assertTrue(initialDocID == -1 || initialDocID == DocsEnum.NO_MORE_DOCS);
assertTrue("inital docID should be -1 or NO_MORE_DOCS", initialDocID == -1 || initialDocID == DocsEnum.NO_MORE_DOCS);
if (VERBOSE) {
if (prevDocsEnum == null) {
@ -600,10 +612,10 @@ public class TestPostingsFormat extends LuceneTestCase {
while (nextPosting <= stopAt) {
if (nextPosting == stopAt) {
if (stopAt == expected.size()) {
assertEquals(DocsEnum.NO_MORE_DOCS, docsEnum.nextDoc());
assertEquals("DocsEnum should have ended but didn't", DocsEnum.NO_MORE_DOCS, docsEnum.nextDoc());
// Common bug is to forget to set this.doc=NO_MORE_DOCS in the enum!:
assertEquals(DocsEnum.NO_MORE_DOCS, docsEnum.docID());
assertEquals("DocsEnum should have ended but didn't", DocsEnum.NO_MORE_DOCS, docsEnum.docID());
}
break;
}
@ -631,7 +643,7 @@ public class TestPostingsFormat extends LuceneTestCase {
if (VERBOSE) {
System.out.println(" now advance to end (target=" + target + ")");
}
assertEquals(DocsEnum.NO_MORE_DOCS, docsEnum.advance(target));
assertEquals("DocsEnum should have ended but didn't", DocsEnum.NO_MORE_DOCS, docsEnum.advance(target));
break;
} else {
posting = expected.get(nextPosting++);
@ -643,7 +655,7 @@ public class TestPostingsFormat extends LuceneTestCase {
}
}
int docID = docsEnum.advance(targetDocID != -1 ? targetDocID : posting.docID);
assertEquals(posting.docID, docID);
assertEquals("docID is wrong", posting.docID, docID);
}
} else {
posting = expected.get(nextPosting++);
@ -651,7 +663,7 @@ public class TestPostingsFormat extends LuceneTestCase {
System.out.println(" now nextDoc to " + posting.docID + " (" + nextPosting + " of " + stopAt + ")");
}
int docID = docsEnum.nextDoc();
assertEquals(posting.docID, docID);
assertEquals("docID is wrong", posting.docID, docID);
}
if (doCheckFreqs && random().nextDouble() <= freqAskChance) {
@ -659,7 +671,7 @@ public class TestPostingsFormat extends LuceneTestCase {
System.out.println(" now freq()=" + posting.positions.size());
}
int freq = docsEnum.freq();
assertEquals(posting.positions.size(), freq);
assertEquals("freq is wrong", posting.positions.size(), freq);
}
if (doCheckPositions) {
@ -676,7 +688,7 @@ public class TestPostingsFormat extends LuceneTestCase {
if (VERBOSE) {
System.out.println(" now nextPosition to " + position.position);
}
assertEquals(position.position, docsAndPositionsEnum.nextPosition());
assertEquals("position is wrong", position.position, docsAndPositionsEnum.nextPosition());
// nocommit sometimes don't pull the payload even
// though we pulled the position
@ -687,17 +699,18 @@ public class TestPostingsFormat extends LuceneTestCase {
System.out.println(" now check payload length=" + (position.payload == null ? 0 : position.payload.length));
}
if (position.payload == null || position.payload.length == 0) {
assertFalse(docsAndPositionsEnum.hasPayload());
assertFalse("should not have payload", docsAndPositionsEnum.hasPayload());
} else {
assertTrue(docsAndPositionsEnum.hasPayload());
assertTrue("should have payload but doesn't", docsAndPositionsEnum.hasPayload());
BytesRef payload = docsAndPositionsEnum.getPayload();
assertFalse(docsAndPositionsEnum.hasPayload());
assertFalse("2nd call to hasPayload should be false", docsAndPositionsEnum.hasPayload());
assertNotNull(payload);
assertEquals(position.payload.length, payload.length);
assertNotNull("payload should not be null", payload);
assertEquals("payload length is wrong", position.payload.length, payload.length);
for(int byteUpto=0;byteUpto<position.payload.length;byteUpto++) {
assertEquals(position.payload[byteUpto],
assertEquals("payload bytes are wrong",
position.payload[byteUpto],
payload.bytes[payload.offset+byteUpto]);
}
}
@ -713,16 +726,19 @@ public class TestPostingsFormat extends LuceneTestCase {
if (VERBOSE) {
System.out.println(" now check offsets: startOff=" + position.startOffset + " endOffset=" + position.endOffset);
}
assertEquals(position.startOffset, docsAndPositionsEnum.startOffset());
assertEquals(position.endOffset, docsAndPositionsEnum.endOffset());
assertEquals("startOffset is wrong", position.startOffset, docsAndPositionsEnum.startOffset());
assertEquals("endOffset is wrong", position.endOffset, docsAndPositionsEnum.endOffset());
} else {
if (VERBOSE) {
System.out.println(" skip check offsets");
}
}
} else {
assertEquals(-1, docsAndPositionsEnum.startOffset());
assertEquals(-1, docsAndPositionsEnum.endOffset());
if (VERBOSE) {
System.out.println(" now check offsets are -1");
}
assertEquals("startOffset isn't -1", -1, docsAndPositionsEnum.startOffset());
assertEquals("endOffset isn't -1", -1, docsAndPositionsEnum.endOffset());
}
}
}
@ -846,6 +862,7 @@ public class TestPostingsFormat extends LuceneTestCase {
boolean indexPayloads = random().nextBoolean();
// nocommit test thread safety of buildIndex too
FieldsProducer fieldsProducer = buildIndex(dir, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, indexPayloads);
//testTerms(fieldsProducer, EnumSet.noneOf(Option.class), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
@ -855,7 +872,11 @@ public class TestPostingsFormat extends LuceneTestCase {
//testTerms(fieldsProducer, EnumSet.of(Option.SKIPPING), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
//testTerms(fieldsProducer, EnumSet.of(Option.THREADS, Option.TERM_STATE, Option.SKIPPING, Option.PARTIAL_DOC_CONSUME, Option.PARTIAL_POS_CONSUME), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
//testTerms(fieldsProducer, EnumSet.of(Option.TERM_STATE, Option.SKIPPING, Option.PARTIAL_DOC_CONSUME, Option.PARTIAL_POS_CONSUME), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
testTerms(fieldsProducer, EnumSet.of(Option.TERM_STATE, Option.PAYLOADS, Option.PARTIAL_DOC_CONSUME, Option.PARTIAL_POS_CONSUME, Option.SKIPPING), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
//testTerms(fieldsProducer, EnumSet.of(Option.TERM_STATE, Option.PAYLOADS, Option.PARTIAL_DOC_CONSUME, Option.PARTIAL_POS_CONSUME, Option.SKIPPING), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
// NOTE: you can also test "weaker" index options than
// you indexed with:
testTerms(fieldsProducer, EnumSet.allOf(Option.class), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
fieldsProducer.close();
dir.close();

10
lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
View File

@ -473,16 +473,6 @@ public abstract class BaseTokenStreamTestCase extends LuceneTestCase {
}
}
static final Set<String> doesntSupportOffsets = new HashSet<String>(Arrays.asList(
"Lucene3x",
"MockFixedIntBlock",
"MockVariableIntBlock",
"MockSep",
"MockRandom",
"For",
"PFor"
));
private static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength, boolean useCharFilter, boolean simple, boolean offsetsAreCorrect, RandomIndexWriter iw) throws IOException {
final LineFileDocs docs = new LineFileDocs(random);

9
lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
View File

@ -277,6 +277,15 @@ public abstract class LuceneTestCase extends Assert {
CORE_DIRECTORIES.add("RAMDirectory");
};
protected static final Set<String> doesntSupportOffsets = new HashSet<String>(Arrays.asList(
"Lucene3x",
"MockFixedIntBlock",
"MockVariableIntBlock",
"MockSep",
"MockRandom",
"For",
"PFor"
));
// -----------------------------------------------------------------
// Fields initialized in class or instance rules.