Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-6127: fix mockanalyzer to not use asserts 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1646878 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2014-12-19 21:29:56 +00:00
parent 2bafbb726b
commit 8390ae9b11
4 changed files with 86 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java
View File

@ -90,7 +90,7 @@ public abstract class Tokenizer extends TokenStream {
throw new IllegalStateException("TokenStream contract violation: close() call missing"); throw new IllegalStateException("TokenStream contract violation: close() call missing");
} }
this.inputPending = input; this.inputPending = input;
assert setReaderTestPoint(); setReaderTestPoint();
} }
@Override @Override
@ -100,10 +100,8 @@ public abstract class Tokenizer extends TokenStream {
inputPending = ILLEGAL_STATE_READER; inputPending = ILLEGAL_STATE_READER;
} }
// only used by assert, for testing // only used for testing
boolean setReaderTestPoint() { void setReaderTestPoint() {}
return true;
}
private static final Reader ILLEGAL_STATE_READER = new Reader() { private static final Reader ILLEGAL_STATE_READER = new Reader() {
@Override @Override

9
lucene/core/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java
View File

@ -120,19 +120,16 @@ public class TestCachingTokenFilter extends BaseTokenStreamTestCase {
} }
public void testDoubleResetFails() throws IOException { public void testDoubleResetFails() throws IOException {
assumeTrue("We want MockAnalyzer to detect double-reset", TEST_ASSERTS_ENABLED);
Analyzer analyzer = new MockAnalyzer(random()); Analyzer analyzer = new MockAnalyzer(random());
final TokenStream input = analyzer.tokenStream("field", "abc"); final TokenStream input = analyzer.tokenStream("field", "abc");
CachingTokenFilter buffer = new CachingTokenFilter(input); CachingTokenFilter buffer = new CachingTokenFilter(input);
buffer.reset();//ok buffer.reset();//ok
boolean madeIt = false;
try { try {
buffer.reset();//bad (this used to work which we don't want) buffer.reset();//bad (this used to work which we don't want)
madeIt = true; fail("didn't get expected exception");
} catch (Throwable e) { } catch (IllegalStateException e) {
//ignore assertEquals("double reset()", e.getMessage());
} }
assertFalse(madeIt);
} }
private void checkTokens(TokenStream stream) throws IOException { private void checkTokens(TokenStream stream) throws IOException {

15
lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
View File

@ -391,9 +391,6 @@ public abstract class BaseTokenStreamTestCase extends LuceneTestCase {
} }
} catch (IllegalStateException expected) { } catch (IllegalStateException expected) {
// ok // ok
} catch (AssertionError expected) {
// ok: MockTokenizer
assertTrue(expected.getMessage(), expected.getMessage() != null && expected.getMessage().contains("wrong state"));
} catch (Exception unexpected) { } catch (Exception unexpected) {
unexpected.printStackTrace(System.err); unexpected.printStackTrace(System.err);
fail("got wrong exception when reset() not called: " + unexpected); fail("got wrong exception when reset() not called: " + unexpected);
@ -752,13 +749,13 @@ public abstract class BaseTokenStreamTestCase extends LuceneTestCase {
} }
try { try {
ts.end(); ts.end();
} catch (AssertionError ae) { } catch (IllegalStateException ise) {
// Catch & ignore MockTokenizer's // Catch & ignore MockTokenizer's
// anger... // anger...
if ("end() called before incrementToken() returned false!".equals(ae.getMessage())) { if ("end() called before incrementToken() returned false!".equals(ise.getMessage())) {
// OK // OK
} else { } else {
throw ae; throw ise;
} }
} }
ts.close(); ts.close();
@ -777,13 +774,13 @@ public abstract class BaseTokenStreamTestCase extends LuceneTestCase {
} }
try { try {
ts.end(); ts.end();
} catch (AssertionError ae) { } catch (IllegalStateException ise) {
// Catch & ignore MockTokenizer's // Catch & ignore MockTokenizer's
// anger... // anger...
if ("end() called before incrementToken() returned false!".equals(ae.getMessage())) { if ("end() called before incrementToken() returned false!".equals(ise.getMessage())) {
// OK // OK
} else { } else {
throw ae; throw ise;
} }
} }
ts.close(); ts.close();

106
lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java
View File

@ -83,7 +83,7 @@ public class MockTokenizer extends Tokenizer {
}; };
private State streamState = State.CLOSE; private State streamState = State.CLOSE;
private int lastOffset = 0; // only for asserting private int lastOffset = 0; // only for checks
private boolean enableChecks = true; private boolean enableChecks = true;
// evil: but we don't change the behavior with this random, we only switch up how we read // evil: but we don't change the behavior with this random, we only switch up how we read
@ -119,10 +119,24 @@ public class MockTokenizer extends Tokenizer {
this(factory, WHITESPACE, true); this(factory, WHITESPACE, true);
} }
// we allow some checks (e.g. state machine) to be turned off.
// turning off checks just means we suppress exceptions from them
private void fail(String message) {
if (enableChecks) {
throw new IllegalStateException(message);
}
}
private void failAlways(String message) {
throw new IllegalStateException(message);
}
@Override @Override
public final boolean incrementToken() throws IOException { public final boolean incrementToken() throws IOException {
assert !enableChecks || (streamState == State.RESET || streamState == State.INCREMENT) if (streamState != State.RESET && streamState != State.INCREMENT) {
: "incrementToken() called while in wrong state: " + streamState; fail("incrementToken() called while in wrong state: " + streamState);
}
clearAttributes(); clearAttributes();
for (;;) { for (;;) {
int startOffset; int startOffset;
@ -160,11 +174,19 @@ public class MockTokenizer extends Tokenizer {
} }
int correctedStartOffset = correctOffset(startOffset); int correctedStartOffset = correctOffset(startOffset);
int correctedEndOffset = correctOffset(endOffset); int correctedEndOffset = correctOffset(endOffset);
assert correctedStartOffset >= 0; if (correctedStartOffset < 0) {
assert correctedEndOffset >= 0; failAlways("invalid start offset: " + correctedStartOffset + ", before correction: " + startOffset);
assert correctedStartOffset >= lastOffset; }
if (correctedEndOffset < 0) {
failAlways("invalid end offset: " + correctedEndOffset + ", before correction: " + endOffset);
}
if (correctedStartOffset < lastOffset) {
failAlways("start offset went backwards: " + correctedStartOffset + ", before correction: " + startOffset + ", lastOffset: " + lastOffset);
}
lastOffset = correctedStartOffset; lastOffset = correctedStartOffset;
assert correctedEndOffset >= correctedStartOffset; if (correctedEndOffset < correctedStartOffset) {
failAlways("end offset: " + correctedEndOffset + " is before start offset: " + correctedStartOffset);
}
offsetAtt.setOffset(correctedStartOffset, correctedEndOffset); offsetAtt.setOffset(correctedStartOffset, correctedEndOffset);
if (state == -1 || runAutomaton.isAccept(state)) { if (state == -1 || runAutomaton.isAccept(state)) {
// either we hit a reject state (longest match), or end-of-text, but in an accept state // either we hit a reject state (longest match), or end-of-text, but in an accept state
@ -182,16 +204,20 @@ public class MockTokenizer extends Tokenizer {
if (ch < 0) { if (ch < 0) {
return ch; return ch;
} else { } else {
assert !Character.isLowSurrogate((char) ch) : "unpaired low surrogate: " + Integer.toHexString(ch); if (Character.isLowSurrogate((char) ch)) {
failAlways("unpaired low surrogate: " + Integer.toHexString(ch));
}
off++; off++;
if (Character.isHighSurrogate((char) ch)) { if (Character.isHighSurrogate((char) ch)) {
int ch2 = readChar(); int ch2 = readChar();
if (ch2 >= 0) { if (ch2 >= 0) {
off++; off++;
assert Character.isLowSurrogate((char) ch2) : "unpaired high surrogate: " + Integer.toHexString(ch) + ", followed by: " + Integer.toHexString(ch2); if (!Character.isLowSurrogate((char) ch2)) {
failAlways("unpaired high surrogate: " + Integer.toHexString(ch) + ", followed by: " + Integer.toHexString(ch2));
}
return Character.toCodePoint((char) ch, (char) ch2); return Character.toCodePoint((char) ch, (char) ch2);
} else { } else {
assert false : "stream ends with unpaired high surrogate: " + Integer.toHexString(ch); failAlways("stream ends with unpaired high surrogate: " + Integer.toHexString(ch));
} }
} }
return ch; return ch;
@ -243,40 +269,56 @@ public class MockTokenizer extends Tokenizer {
@Override @Override
public void reset() throws IOException { public void reset() throws IOException {
super.reset(); try {
state = runAutomaton.getInitialState(); super.reset();
lastOffset = off = 0; state = runAutomaton.getInitialState();
bufferedCodePoint = -1; lastOffset = off = 0;
assert !enableChecks || streamState != State.RESET : "double reset()"; bufferedCodePoint = -1;
streamState = State.RESET; if (streamState == State.RESET) {
fail("double reset()");
}
} finally {
streamState = State.RESET;
}
} }
@Override @Override
public void close() throws IOException { public void close() throws IOException {
super.close(); try {
// in some exceptional cases (e.g. TestIndexWriterExceptions) a test can prematurely close() super.close();
// these tests should disable this check, by default we check the normal workflow. // in some exceptional cases (e.g. TestIndexWriterExceptions) a test can prematurely close()
// TODO: investigate the CachingTokenFilter "double-close"... for now we ignore this // these tests should disable this check, by default we check the normal workflow.
assert !enableChecks || streamState == State.END || streamState == State.CLOSE : "close() called in wrong state: " + streamState; // TODO: investigate the CachingTokenFilter "double-close"... for now we ignore this
streamState = State.CLOSE; if (!(streamState == State.END || streamState == State.CLOSE)) {
fail("close() called in wrong state: " + streamState);
}
} finally {
streamState = State.CLOSE;
}
} }
@Override @Override
boolean setReaderTestPoint() { void setReaderTestPoint() {
assert !enableChecks || streamState == State.CLOSE : "setReader() called in wrong state: " + streamState; try {
streamState = State.SETREADER; if (streamState != State.CLOSE) {
return true; fail("setReader() called in wrong state: " + streamState);
}
} finally {
streamState = State.SETREADER;
}
} }
@Override @Override
public void end() throws IOException { public void end() throws IOException {
super.end();
int finalOffset = correctOffset(off);
offsetAtt.setOffset(finalOffset, finalOffset);
// some tokenizers, such as limiting tokenizers, call end() before incrementToken() returns false.
// these tests should disable this check (in general you should consume the entire stream)
try { try {
assert !enableChecks || streamState == State.INCREMENT_FALSE : "end() called before incrementToken() returned false!"; super.end();
int finalOffset = correctOffset(off);
offsetAtt.setOffset(finalOffset, finalOffset);
// some tokenizers, such as limiting tokenizers, call end() before incrementToken() returns false.
// these tests should disable this check (in general you should consume the entire stream)
if (streamState != State.INCREMENT_FALSE) {
fail("end() called before incrementToken() returned false!");
}
} finally { } finally {
streamState = State.END; streamState = State.END;
} }