LUCENE-3455: Moved remaining Analysis consumers over to using reusableTokenStream

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1176191 13f79535-47bb-0310-9956-ffa450edef68

modules/queryparser/src/java/org/apache/lucene/queryparser/classic/ParseException.java

@@ -58,6 +58,18 @@ public class ParseException extends Exception {
     specialConstructor = false;
   }
 
+  /**
+   * Creates a new ParseException which is wrapping another Throwable with an
+   * additional message
+   *
+   * @param message Message for the Exception
+   * @param throwable Wrapped Throwable
+   */
+  public ParseException(String message, Throwable throwable) {
+    super(message, throwable);
+    specialConstructor = false;
+  }
+
   /**
    * This variable determines which constructor was used to create
    * this object and thereby affects the semantics of the

modules/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java

@@ -477,27 +477,24 @@ public abstract class QueryParserBase {
       source = analyzer.reusableTokenStream(field, new StringReader(queryText));
       source.reset();
     } catch (IOException e) {
-      source = analyzer.tokenStream(field, new StringReader(queryText));
+      throw new ParseException("Unable to initialize TokenStream to analyze query text", e);
     }
     CachingTokenFilter buffer = new CachingTokenFilter(source);
     TermToBytesRefAttribute termAtt = null;
     PositionIncrementAttribute posIncrAtt = null;
     int numTokens = 0;
 
-    boolean success = false;
     try {
       buffer.reset();
-      success = true;
     } catch (IOException e) {
-      // success==false if we hit an exception
+      throw new ParseException("Unable to initialize TokenStream to analyze query text", e);
     }
-    if (success) {
+
-      if (buffer.hasAttribute(TermToBytesRefAttribute.class)) {
+    if (buffer.hasAttribute(TermToBytesRefAttribute.class)) {
-        termAtt = buffer.getAttribute(TermToBytesRefAttribute.class);
+      termAtt = buffer.getAttribute(TermToBytesRefAttribute.class);
-      }
+    }
-      if (buffer.hasAttribute(PositionIncrementAttribute.class)) {
+    if (buffer.hasAttribute(PositionIncrementAttribute.class)) {
-        posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class);
+      posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class);
-      }
     }
 
     int positionCount = 0;
@@ -529,7 +526,7 @@ public abstract class QueryParserBase {
       source.close();
     }
     catch (IOException e) {
-      // ignore
+      throw new ParseException("Cannot close TokenStream analyzing query text", e);
     }
 
     BytesRef bytes = termAtt == null ? null : termAtt.getBytesRef();
@@ -789,7 +786,7 @@ public abstract class QueryParserBase {
       source = analyzer.reusableTokenStream(field, new StringReader(part));
       source.reset();
     } catch (IOException e) {
-      source = analyzer.tokenStream(field, new StringReader(part));
+      throw new RuntimeException("Unable to initialize TokenStream to analyze range part: " + part, e);
     }
       
     TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class);
@@ -808,7 +805,9 @@ public abstract class QueryParserBase {
     try {
       source.end();
       source.close();
-    } catch (IOException ignored) {}
+    } catch (IOException e) {
+      throw new RuntimeException("Unable to end & close TokenStream after analyzing range part: " + part, e);
+    }
     
     return new BytesRef(bytes);
   }

solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java

@@ -195,7 +195,7 @@ public class ICUCollationField extends FieldType {
       source = analyzer.reusableTokenStream(field, new StringReader(part));
       source.reset();
     } catch (IOException e) {
-      source = analyzer.tokenStream(field, new StringReader(part));
+      throw new RuntimeException("Unable to initialize TokenStream to analyze range part: " + part, e);
     }
       
     TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class);
@@ -212,8 +212,11 @@ public class ICUCollationField extends FieldType {
     }
       
     try {
+      source.end();
       source.close();
-    } catch (IOException ignored) {}
+    } catch (IOException e) {
+      throw new RuntimeException("Unable to end & close TokenStream after analyzing range part: " + part, e);
+    }
       
     return new BytesRef(bytes);
   }

solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java

@@ -140,20 +140,25 @@ public abstract class AnalysisRequestHandlerBase extends RequestHandlerBase {
    * @param analyzer The analyzer to use.
    */
   protected Set<BytesRef> getQueryTokenSet(String query, Analyzer analyzer) {
-    final Set<BytesRef> tokens = new HashSet<BytesRef>();
-    final TokenStream tokenStream = analyzer.tokenStream("", new StringReader(query));
-    final TermToBytesRefAttribute bytesAtt = tokenStream.getAttribute(TermToBytesRefAttribute.class);
-    final BytesRef bytes = bytesAtt.getBytesRef();
     try {
+      final Set<BytesRef> tokens = new HashSet<BytesRef>();
+      final TokenStream tokenStream = analyzer.reusableTokenStream("", new StringReader(query));
+      final TermToBytesRefAttribute bytesAtt = tokenStream.getAttribute(TermToBytesRefAttribute.class);
+      final BytesRef bytes = bytesAtt.getBytesRef();
+
       tokenStream.reset();
+
       while (tokenStream.incrementToken()) {
         bytesAtt.fillBytesRef();
         tokens.add(new BytesRef(bytes));
       }
+
+      tokenStream.end();
+      tokenStream.close();
+      return tokens;
     } catch (IOException ioe) {
       throw new RuntimeException("Error occured while iterating over tokenstream", ioe);
     }
-    return tokens;
   }
 
   /**

solr/core/src/java/org/apache/solr/schema/CollationField.java

@@ -217,7 +217,7 @@ public class CollationField extends FieldType {
       source = analyzer.reusableTokenStream(field, new StringReader(part));
       source.reset();
     } catch (IOException e) {
-      source = analyzer.tokenStream(field, new StringReader(part));
+      throw new RuntimeException("Unable to initialize TokenStream to analyze range part: " + part, e);
     }
       
     TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class);
@@ -234,8 +234,11 @@ public class CollationField extends FieldType {
     }
       
     try {
+      source.end();
       source.close();
-    } catch (IOException ignored) {}
+    } catch (IOException e) {
+      throw new RuntimeException("Unable to end & close TokenStream after analyzing range part: " + part, e);
+    }
       
     return new BytesRef(bytes);
   }

solr/core/src/java/org/apache/solr/schema/TextField.java

@@ -112,27 +112,24 @@ public class TextField extends FieldType {
       source = analyzer.reusableTokenStream(field, new StringReader(queryText));
       source.reset();
     } catch (IOException e) {
-      source = analyzer.tokenStream(field, new StringReader(queryText));
+      throw new RuntimeException("Unable to initialize TokenStream to analyze query text", e);
     }
     CachingTokenFilter buffer = new CachingTokenFilter(source);
     CharTermAttribute termAtt = null;
     PositionIncrementAttribute posIncrAtt = null;
     int numTokens = 0;
 
-    boolean success = false;
     try {
       buffer.reset();
-      success = true;
     } catch (IOException e) {
-      // success==false if we hit an exception
+      throw new RuntimeException("Unable to initialize TokenStream to analyze query text", e);
     }
-    if (success) {
+
-      if (buffer.hasAttribute(CharTermAttribute.class)) {
+    if (buffer.hasAttribute(CharTermAttribute.class)) {
-        termAtt = buffer.getAttribute(CharTermAttribute.class);
+      termAtt = buffer.getAttribute(CharTermAttribute.class);
-      }
+    }
-      if (buffer.hasAttribute(PositionIncrementAttribute.class)) {
+    if (buffer.hasAttribute(PositionIncrementAttribute.class)) {
-        posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class);
+      posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class);
-      }
     }
 
     int positionCount = 0;

solr/core/src/test/org/apache/solr/spelling/SimpleQueryConverter.java

@@ -37,23 +37,25 @@ import java.io.IOException;
  *
  * @since solr 1.3
  **/
-class SimpleQueryConverter extends SpellingQueryConverter{
+class SimpleQueryConverter extends SpellingQueryConverter {
+
   @Override
   public Collection<Token> convert(String origQuery) {
-    Collection<Token> result = new HashSet<Token>();
-    WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_40);
-    TokenStream ts = analyzer.tokenStream("", new StringReader(origQuery));
-    // TODO: support custom attributes
-    CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
-    OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
-    TypeAttribute typeAtt = ts.addAttribute(TypeAttribute.class);
-    FlagsAttribute flagsAtt = ts.addAttribute(FlagsAttribute.class);
-    PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class);
-    PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class);
-    
     try {
+      Collection<Token> result = new HashSet<Token>();
+      WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_40);
+      TokenStream ts = analyzer.reusableTokenStream("", new StringReader(origQuery));
+      // TODO: support custom attributes
+      CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+      OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
+      TypeAttribute typeAtt = ts.addAttribute(TypeAttribute.class);
+      FlagsAttribute flagsAtt = ts.addAttribute(FlagsAttribute.class);
+      PayloadAttribute payloadAtt = ts.addAttribute(PayloadAttribute.class);
+      PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class);
+
       ts.reset();
-      while (ts.incrementToken()){
+
+      while (ts.incrementToken()) {
         Token tok = new Token();
         tok.copyBuffer(termAtt.buffer(), 0, termAtt.length());
         tok.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
@@ -63,9 +65,12 @@ class SimpleQueryConverter extends SpellingQueryConverter{
         tok.setType(typeAtt.type());
         result.add(tok);
       }
+      ts.end();
+      ts.close();
+      
+      return result;
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
-    return result;
   }
 }