From fe3982c746606e595073675063dd42d0698b644d Mon Sep 17 00:00:00 2001 From: Christopher John Male Date: Tue, 27 Sep 2011 04:07:38 +0000 Subject: [PATCH] LUCENE-3455: Moved remaining Analysis consumers over to using reusableTokenStream git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1176191 13f79535-47bb-0310-9956-ffa450edef68 --- .../queryparser/classic/ParseException.java | 12 +++++++ .../queryparser/classic/QueryParserBase.java | 27 ++++++++------- .../apache/solr/schema/ICUCollationField.java | 7 ++-- .../handler/AnalysisRequestHandlerBase.java | 15 ++++++--- .../apache/solr/schema/CollationField.java | 7 ++-- .../org/apache/solr/schema/TextField.java | 19 +++++------ .../solr/spelling/SimpleQueryConverter.java | 33 +++++++++++-------- 7 files changed, 72 insertions(+), 48 deletions(-) diff --git a/modules/queryparser/src/java/org/apache/lucene/queryparser/classic/ParseException.java b/modules/queryparser/src/java/org/apache/lucene/queryparser/classic/ParseException.java index f638ae1554b..a452ac39b6a 100644 --- a/modules/queryparser/src/java/org/apache/lucene/queryparser/classic/ParseException.java +++ b/modules/queryparser/src/java/org/apache/lucene/queryparser/classic/ParseException.java @@ -58,6 +58,18 @@ public class ParseException extends Exception { specialConstructor = false; } + /** + * Creates a new ParseException which is wrapping another Throwable with an + * additional message + * + * @param message Message for the Exception + * @param throwable Wrapped Throwable + */ + public ParseException(String message, Throwable throwable) { + super(message, throwable); + specialConstructor = false; + } + /** * This variable determines which constructor was used to create * this object and thereby affects the semantics of the diff --git a/modules/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java b/modules/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java index 9abddc031b7..6d3d8d01db3 100644 --- a/modules/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java +++ b/modules/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java @@ -477,27 +477,24 @@ public abstract class QueryParserBase { source = analyzer.reusableTokenStream(field, new StringReader(queryText)); source.reset(); } catch (IOException e) { - source = analyzer.tokenStream(field, new StringReader(queryText)); + throw new ParseException("Unable to initialize TokenStream to analyze query text", e); } CachingTokenFilter buffer = new CachingTokenFilter(source); TermToBytesRefAttribute termAtt = null; PositionIncrementAttribute posIncrAtt = null; int numTokens = 0; - boolean success = false; try { buffer.reset(); - success = true; } catch (IOException e) { - // success==false if we hit an exception + throw new ParseException("Unable to initialize TokenStream to analyze query text", e); } - if (success) { - if (buffer.hasAttribute(TermToBytesRefAttribute.class)) { - termAtt = buffer.getAttribute(TermToBytesRefAttribute.class); - } - if (buffer.hasAttribute(PositionIncrementAttribute.class)) { - posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class); - } + + if (buffer.hasAttribute(TermToBytesRefAttribute.class)) { + termAtt = buffer.getAttribute(TermToBytesRefAttribute.class); + } + if (buffer.hasAttribute(PositionIncrementAttribute.class)) { + posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class); } int positionCount = 0; @@ -529,7 +526,7 @@ public abstract class QueryParserBase { source.close(); } catch (IOException e) { - // ignore + throw new ParseException("Cannot close TokenStream analyzing query text", e); } BytesRef bytes = termAtt == null ? null : termAtt.getBytesRef(); @@ -789,7 +786,7 @@ public abstract class QueryParserBase { source = analyzer.reusableTokenStream(field, new StringReader(part)); source.reset(); } catch (IOException e) { - source = analyzer.tokenStream(field, new StringReader(part)); + throw new RuntimeException("Unable to initialize TokenStream to analyze range part: " + part, e); } TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class); @@ -808,7 +805,9 @@ public abstract class QueryParserBase { try { source.end(); source.close(); - } catch (IOException ignored) {} + } catch (IOException e) { + throw new RuntimeException("Unable to end & close TokenStream after analyzing range part: " + part, e); + } return new BytesRef(bytes); } diff --git a/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java b/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java index daf6dff4cc1..5eb59de3d58 100644 --- a/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java +++ b/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java @@ -195,7 +195,7 @@ public class ICUCollationField extends FieldType { source = analyzer.reusableTokenStream(field, new StringReader(part)); source.reset(); } catch (IOException e) { - source = analyzer.tokenStream(field, new StringReader(part)); + throw new RuntimeException("Unable to initialize TokenStream to analyze range part: " + part, e); } TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class); @@ -212,8 +212,11 @@ public class ICUCollationField extends FieldType { } try { + source.end(); source.close(); - } catch (IOException ignored) {} + } catch (IOException e) { + throw new RuntimeException("Unable to end & close TokenStream after analyzing range part: " + part, e); + } return new BytesRef(bytes); } diff --git a/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java index 16c598fe1a1..0e614e77143 100644 --- a/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java +++ b/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java @@ -140,20 +140,25 @@ public abstract class AnalysisRequestHandlerBase extends RequestHandlerBase { * @param analyzer The analyzer to use. */ protected Set getQueryTokenSet(String query, Analyzer analyzer) { - final Set tokens = new HashSet(); - final TokenStream tokenStream = analyzer.tokenStream("", new StringReader(query)); - final TermToBytesRefAttribute bytesAtt = tokenStream.getAttribute(TermToBytesRefAttribute.class); - final BytesRef bytes = bytesAtt.getBytesRef(); try { + final Set tokens = new HashSet(); + final TokenStream tokenStream = analyzer.reusableTokenStream("", new StringReader(query)); + final TermToBytesRefAttribute bytesAtt = tokenStream.getAttribute(TermToBytesRefAttribute.class); + final BytesRef bytes = bytesAtt.getBytesRef(); + tokenStream.reset(); + while (tokenStream.incrementToken()) { bytesAtt.fillBytesRef(); tokens.add(new BytesRef(bytes)); } + + tokenStream.end(); + tokenStream.close(); + return tokens; } catch (IOException ioe) { throw new RuntimeException("Error occured while iterating over tokenstream", ioe); } - return tokens; } /** diff --git a/solr/core/src/java/org/apache/solr/schema/CollationField.java b/solr/core/src/java/org/apache/solr/schema/CollationField.java index 143ae6fb971..afdfcff64b6 100644 --- a/solr/core/src/java/org/apache/solr/schema/CollationField.java +++ b/solr/core/src/java/org/apache/solr/schema/CollationField.java @@ -217,7 +217,7 @@ public class CollationField extends FieldType { source = analyzer.reusableTokenStream(field, new StringReader(part)); source.reset(); } catch (IOException e) { - source = analyzer.tokenStream(field, new StringReader(part)); + throw new RuntimeException("Unable to initialize TokenStream to analyze range part: " + part, e); } TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class); @@ -234,8 +234,11 @@ public class CollationField extends FieldType { } try { + source.end(); source.close(); - } catch (IOException ignored) {} + } catch (IOException e) { + throw new RuntimeException("Unable to end & close TokenStream after analyzing range part: " + part, e); + } return new BytesRef(bytes); } diff --git a/solr/core/src/java/org/apache/solr/schema/TextField.java b/solr/core/src/java/org/apache/solr/schema/TextField.java index 2e6645c75a8..fb4bf596bd2 100644 --- a/solr/core/src/java/org/apache/solr/schema/TextField.java +++ b/solr/core/src/java/org/apache/solr/schema/TextField.java @@ -112,27 +112,24 @@ public class TextField extends FieldType { source = analyzer.reusableTokenStream(field, new StringReader(queryText)); source.reset(); } catch (IOException e) { - source = analyzer.tokenStream(field, new StringReader(queryText)); + throw new RuntimeException("Unable to initialize TokenStream to analyze query text", e); } CachingTokenFilter buffer = new CachingTokenFilter(source); CharTermAttribute termAtt = null; PositionIncrementAttribute posIncrAtt = null; int numTokens = 0; - boolean success = false; try { buffer.reset(); - success = true; } catch (IOException e) { - // success==false if we hit an exception + throw new RuntimeException("Unable to initialize TokenStream to analyze query text", e); } - if (success) { - if (buffer.hasAttribute(CharTermAttribute.class)) { - termAtt = buffer.getAttribute(CharTermAttribute.class); - } - if (buffer.hasAttribute(PositionIncrementAttribute.class)) { - posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class); - } + + if (buffer.hasAttribute(CharTermAttribute.class)) { + termAtt = buffer.getAttribute(CharTermAttribute.class); + } + if (buffer.hasAttribute(PositionIncrementAttribute.class)) { + posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class); } int positionCount = 0; diff --git a/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java b/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java index 0dcfee46b16..df9eb75c6eb 100644 --- a/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java +++ b/solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java @@ -37,23 +37,25 @@ import java.io.IOException; * * @since solr 1.3 **/ -class SimpleQueryConverter extends SpellingQueryConverter{ +class SimpleQueryConverter extends SpellingQueryConverter { + @Override public Collection convert(String origQuery) { - Collection result = new HashSet(); - WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_40); - TokenStream ts = analyzer.tokenStream("", new StringReader(origQuery)); - // TODO: support custom attributes - CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); - OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class); - TypeAttribute typeAtt = ts.addAttribute(TypeAttribute.class); - FlagsAttribute flagsAtt = ts.addAttribute(FlagsAttribute.class); - PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class); - PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class); - try { + Collection result = new HashSet(); + WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_40); + TokenStream ts = analyzer.reusableTokenStream("", new StringReader(origQuery)); + // TODO: support custom attributes + CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); + OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class); + TypeAttribute typeAtt = ts.addAttribute(TypeAttribute.class); + FlagsAttribute flagsAtt = ts.addAttribute(FlagsAttribute.class); + PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class); + PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class); + ts.reset(); - while (ts.incrementToken()){ + + while (ts.incrementToken()) { Token tok = new Token(); tok.copyBuffer(termAtt.buffer(), 0, termAtt.length()); tok.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset()); @@ -63,9 +65,12 @@ class SimpleQueryConverter extends SpellingQueryConverter{ tok.setType(typeAtt.type()); result.add(tok); } + ts.end(); + ts.close(); + + return result; } catch (IOException e) { throw new RuntimeException(e); } - return result; } }