mirror of https://github.com/apache/lucene.git
LUCENE-9186: remove linefiledocs usage from BaseTokenStreamTestCase
This commit is contained in:
parent
4350efa932
commit
3bcc97c8eb
|
@ -45,7 +45,6 @@ import org.apache.lucene.util.BytesRef;
|
||||||
import org.apache.lucene.util.BytesRefBuilder;
|
import org.apache.lucene.util.BytesRefBuilder;
|
||||||
import org.apache.lucene.util.IOUtils;
|
import org.apache.lucene.util.IOUtils;
|
||||||
import org.apache.lucene.util.IntsRef;
|
import org.apache.lucene.util.IntsRef;
|
||||||
import org.apache.lucene.util.LineFileDocs;
|
|
||||||
import org.apache.lucene.util.LuceneTestCase;
|
import org.apache.lucene.util.LuceneTestCase;
|
||||||
import org.apache.lucene.util.Rethrow;
|
import org.apache.lucene.util.Rethrow;
|
||||||
import org.apache.lucene.util.TestUtil;
|
import org.apache.lucene.util.TestUtil;
|
||||||
|
@ -596,7 +595,6 @@ public abstract class BaseTokenStreamTestCase extends LuceneTestCase {
|
||||||
|
|
||||||
private static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength, boolean useCharFilter, boolean simple, boolean graphOffsetsAreCorrect, RandomIndexWriter iw) throws IOException {
|
private static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength, boolean useCharFilter, boolean simple, boolean graphOffsetsAreCorrect, RandomIndexWriter iw) throws IOException {
|
||||||
|
|
||||||
final LineFileDocs docs = new LineFileDocs(random);
|
|
||||||
Document doc = null;
|
Document doc = null;
|
||||||
Field field = null, currentField = null;
|
Field field = null, currentField = null;
|
||||||
StringReader bogus = new StringReader("");
|
StringReader bogus = new StringReader("");
|
||||||
|
@ -625,61 +623,33 @@ public abstract class BaseTokenStreamTestCase extends LuceneTestCase {
|
||||||
doc.add(currentField);
|
doc.add(currentField);
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
for (int i = 0; i < iterations; i++) {
|
||||||
for (int i = 0; i < iterations; i++) {
|
String text = TestUtil.randomAnalysisString(random, maxWordLength, simple);
|
||||||
String text;
|
|
||||||
|
|
||||||
if (random.nextInt(10) == 7) {
|
try {
|
||||||
// real data from linedocs
|
checkAnalysisConsistency(random, a, useCharFilter, text, graphOffsetsAreCorrect, currentField);
|
||||||
text = docs.nextDoc().get("body");
|
if (iw != null) {
|
||||||
if (text.length() > maxWordLength) {
|
if (random.nextInt(7) == 0) {
|
||||||
|
// pile up a multivalued field
|
||||||
// Take a random slice from the text...:
|
IndexableFieldType ft = field.fieldType();
|
||||||
int startPos = random.nextInt(text.length() - maxWordLength);
|
currentField = new Field("dummy", bogus, ft);
|
||||||
if (startPos > 0 && Character.isLowSurrogate(text.charAt(startPos))) {
|
doc.add(currentField);
|
||||||
// Take care not to split up a surrogate pair:
|
} else {
|
||||||
startPos--;
|
iw.addDocument(doc);
|
||||||
assert Character.isHighSurrogate(text.charAt(startPos));
|
if (doc.getFields().size() > 1) {
|
||||||
}
|
// back to 1 field
|
||||||
int endPos = startPos + maxWordLength - 1;
|
currentField = field;
|
||||||
if (Character.isHighSurrogate(text.charAt(endPos))) {
|
doc.removeFields("dummy");
|
||||||
// Take care not to split up a surrogate pair:
|
|
||||||
endPos--;
|
|
||||||
}
|
|
||||||
text = text.substring(startPos, 1+endPos);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// synthetic
|
|
||||||
text = TestUtil.randomAnalysisString(random, maxWordLength, simple);
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
checkAnalysisConsistency(random, a, useCharFilter, text, graphOffsetsAreCorrect, currentField);
|
|
||||||
if (iw != null) {
|
|
||||||
if (random.nextInt(7) == 0) {
|
|
||||||
// pile up a multivalued field
|
|
||||||
IndexableFieldType ft = field.fieldType();
|
|
||||||
currentField = new Field("dummy", bogus, ft);
|
|
||||||
doc.add(currentField);
|
doc.add(currentField);
|
||||||
} else {
|
|
||||||
iw.addDocument(doc);
|
|
||||||
if (doc.getFields().size() > 1) {
|
|
||||||
// back to 1 field
|
|
||||||
currentField = field;
|
|
||||||
doc.removeFields("dummy");
|
|
||||||
doc.add(currentField);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (Throwable t) {
|
|
||||||
// TODO: really we should pass a random seed to
|
|
||||||
// checkAnalysisConsistency then print it here too:
|
|
||||||
System.err.println("TEST FAIL: useCharFilter=" + useCharFilter + " text='" + escape(text) + "'");
|
|
||||||
Rethrow.rethrow(t);
|
|
||||||
}
|
}
|
||||||
|
} catch (Throwable t) {
|
||||||
|
// TODO: really we should pass a random seed to
|
||||||
|
// checkAnalysisConsistency then print it here too:
|
||||||
|
System.err.println("TEST FAIL: useCharFilter=" + useCharFilter + " text='" + escape(text) + "'");
|
||||||
|
Rethrow.rethrow(t);
|
||||||
}
|
}
|
||||||
} finally {
|
|
||||||
IOUtils.closeWhileHandlingException(docs);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue