From 433774fa6a6a6a722fb616f1eb332e65cf832469 Mon Sep 17 00:00:00 2001 From: mikemccand Date: Wed, 30 Sep 2015 12:59:00 +0200 Subject: [PATCH 1/4] close TokenStream in finally --- .../elasticsearch/search/suggest/SuggestUtils.java | 12 ++++++------ .../suggest/phrase/NoisyChannelSpellChecker.java | 7 ++++--- .../search/suggest/phrase/PhraseSuggester.java | 12 +++++++----- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java index bcf8cee64c2..6d2767c1f4e 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java @@ -116,12 +116,13 @@ public final class SuggestUtils { } public static int analyze(Analyzer analyzer, CharsRef toAnalyze, String field, TokenConsumer consumer) throws IOException { - TokenStream ts = analyzer.tokenStream( - field, new FastCharArrayReader(toAnalyze.chars, toAnalyze.offset, toAnalyze.length) - ); - return analyze(ts, consumer); + try (TokenStream ts = analyzer.tokenStream( + field, new FastCharArrayReader(toAnalyze.chars, toAnalyze.offset, toAnalyze.length))) { + return analyze(ts, consumer); + } } - + + /** NOTE: caller must close the TokenStream */ public static int analyze(TokenStream stream, TokenConsumer consumer) throws IOException { stream.reset(); consumer.reset(stream); @@ -131,7 +132,6 @@ public final class SuggestUtils { numTokens++; } consumer.end(); - stream.close(); return numTokens; } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellChecker.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellChecker.java index ec9ca6e1da2..ac03a736526 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellChecker.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellChecker.java @@ -133,11 +133,12 @@ public final class NoisyChannelSpellChecker { public Result getCorrections(Analyzer analyzer, BytesRef query, CandidateGenerator generator, float maxErrors, int numCorrections, IndexReader reader, String analysisField, WordScorer scorer, float confidence, int gramSize) throws IOException { - - return getCorrections(tokenStream(analyzer, query, new CharsRefBuilder(), analysisField), generator, maxErrors, numCorrections, scorer, confidence, gramSize); - + try (TokenStream ts = tokenStream(analyzer, query, new CharsRefBuilder(), analysisField)) { + return getCorrections(ts, generator, maxErrors, numCorrections, scorer, confidence, gramSize); + } } + /** NOTE: caller must close returned TokenStream */ public TokenStream tokenStream(Analyzer analyzer, BytesRef query, CharsRefBuilder spare, String field) throws IOException { spare.copyUTF8Bytes(query); return analyzer.tokenStream(field, new FastCharArrayReader(spare.chars(), 0, spare.length())); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java index e7d0eb378c3..232b9e6d21c 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java @@ -92,12 +92,14 @@ public final class PhraseSuggester extends Suggester { if (gens.size() > 0 && suggestTerms != null) { final NoisyChannelSpellChecker checker = new NoisyChannelSpellChecker(realWordErrorLikelihood, suggestion.getRequireUnigram(), suggestion.getTokenLimit()); final BytesRef separator = suggestion.separator(); - TokenStream stream = checker.tokenStream(suggestion.getAnalyzer(), suggestion.getText(), spare, suggestion.getField()); + Result checkerResult; + try (TokenStream stream = checker.tokenStream(suggestion.getAnalyzer(), suggestion.getText(), spare, suggestion.getField())) { - WordScorer wordScorer = suggestion.model().newScorer(indexReader, suggestTerms, suggestField, realWordErrorLikelihood, separator); - Result checkerResult = checker.getCorrections(stream, new MultiCandidateGeneratorWrapper(suggestion.getShardSize(), - gens.toArray(new CandidateGenerator[gens.size()])), suggestion.maxErrors(), - suggestion.getShardSize(), wordScorer, suggestion.confidence(), suggestion.gramSize()); + WordScorer wordScorer = suggestion.model().newScorer(indexReader, suggestTerms, suggestField, realWordErrorLikelihood, separator); + checkerResult = checker.getCorrections(stream, new MultiCandidateGeneratorWrapper(suggestion.getShardSize(), + gens.toArray(new CandidateGenerator[gens.size()])), suggestion.maxErrors(), + suggestion.getShardSize(), wordScorer, suggestion.confidence(), suggestion.gramSize()); + } PhraseSuggestion.Entry resultEntry = buildResultEntry(suggestion, spare, checkerResult.cutoffScore); response.addTerm(resultEntry); From bb613bcacdb5d643ef5fc8820500599db1e083b0 Mon Sep 17 00:00:00 2001 From: mikemccand Date: Wed, 30 Sep 2015 15:44:35 +0200 Subject: [PATCH 2/4] move close responsibility back down to SuggestUtils.analyze --- .../search/suggest/SuggestUtils.java | 37 +++++++++++++------ .../phrase/NoisyChannelSpellChecker.java | 7 ++-- .../suggest/phrase/PhraseSuggester.java | 12 +++--- 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java index 6d2767c1f4e..ce994d0993c 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java @@ -28,6 +28,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.CharsRefBuilder; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.automaton.LevenshteinAutomata; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseFieldMatcher; @@ -116,22 +117,34 @@ public final class SuggestUtils { } public static int analyze(Analyzer analyzer, CharsRef toAnalyze, String field, TokenConsumer consumer) throws IOException { - try (TokenStream ts = analyzer.tokenStream( - field, new FastCharArrayReader(toAnalyze.chars, toAnalyze.offset, toAnalyze.length))) { - return analyze(ts, consumer); - } + TokenStream ts = analyzer.tokenStream( + field, new FastCharArrayReader(toAnalyze.chars, toAnalyze.offset, toAnalyze.length) + ); + return analyze(ts, consumer); } - - /** NOTE: caller must close the TokenStream */ + + /** NOTE: this method closes the TokenStream, even on exception, which is awkward + * because really the caller who called {@link Analyzer#tokenStream} should close it, + * but when trying that there are recursion issues when we try to use the same + * TokenStrem twice in the same recursion... */ public static int analyze(TokenStream stream, TokenConsumer consumer) throws IOException { - stream.reset(); - consumer.reset(stream); int numTokens = 0; - while (stream.incrementToken()) { - consumer.nextToken(); - numTokens++; + boolean success = false; + try { + stream.reset(); + consumer.reset(stream); + while (stream.incrementToken()) { + consumer.nextToken(); + numTokens++; + } + consumer.end(); + } finally { + if (success) { + stream.close(); + } else { + IOUtils.closeWhileHandlingException(stream); + } } - consumer.end(); return numTokens; } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellChecker.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellChecker.java index ac03a736526..ec9ca6e1da2 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellChecker.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellChecker.java @@ -133,12 +133,11 @@ public final class NoisyChannelSpellChecker { public Result getCorrections(Analyzer analyzer, BytesRef query, CandidateGenerator generator, float maxErrors, int numCorrections, IndexReader reader, String analysisField, WordScorer scorer, float confidence, int gramSize) throws IOException { - try (TokenStream ts = tokenStream(analyzer, query, new CharsRefBuilder(), analysisField)) { - return getCorrections(ts, generator, maxErrors, numCorrections, scorer, confidence, gramSize); - } + + return getCorrections(tokenStream(analyzer, query, new CharsRefBuilder(), analysisField), generator, maxErrors, numCorrections, scorer, confidence, gramSize); + } - /** NOTE: caller must close returned TokenStream */ public TokenStream tokenStream(Analyzer analyzer, BytesRef query, CharsRefBuilder spare, String field) throws IOException { spare.copyUTF8Bytes(query); return analyzer.tokenStream(field, new FastCharArrayReader(spare.chars(), 0, spare.length())); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java index 232b9e6d21c..e7d0eb378c3 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java @@ -92,14 +92,12 @@ public final class PhraseSuggester extends Suggester { if (gens.size() > 0 && suggestTerms != null) { final NoisyChannelSpellChecker checker = new NoisyChannelSpellChecker(realWordErrorLikelihood, suggestion.getRequireUnigram(), suggestion.getTokenLimit()); final BytesRef separator = suggestion.separator(); - Result checkerResult; - try (TokenStream stream = checker.tokenStream(suggestion.getAnalyzer(), suggestion.getText(), spare, suggestion.getField())) { + TokenStream stream = checker.tokenStream(suggestion.getAnalyzer(), suggestion.getText(), spare, suggestion.getField()); - WordScorer wordScorer = suggestion.model().newScorer(indexReader, suggestTerms, suggestField, realWordErrorLikelihood, separator); - checkerResult = checker.getCorrections(stream, new MultiCandidateGeneratorWrapper(suggestion.getShardSize(), - gens.toArray(new CandidateGenerator[gens.size()])), suggestion.maxErrors(), - suggestion.getShardSize(), wordScorer, suggestion.confidence(), suggestion.gramSize()); - } + WordScorer wordScorer = suggestion.model().newScorer(indexReader, suggestTerms, suggestField, realWordErrorLikelihood, separator); + Result checkerResult = checker.getCorrections(stream, new MultiCandidateGeneratorWrapper(suggestion.getShardSize(), + gens.toArray(new CandidateGenerator[gens.size()])), suggestion.maxErrors(), + suggestion.getShardSize(), wordScorer, suggestion.confidence(), suggestion.gramSize()); PhraseSuggestion.Entry resultEntry = buildResultEntry(suggestion, spare, checkerResult.cutoffScore); response.addTerm(resultEntry); From dc01450d955fe7cdba5228ed27fbc7aa9795010f Mon Sep 17 00:00:00 2001 From: mikemccand Date: Wed, 30 Sep 2015 17:37:26 +0200 Subject: [PATCH 3/4] cutover more Analyzer.tokenStream to try-with-resources --- .../classic/MapperQueryParser.java | 93 +++++++++++-------- .../analyzing/XAnalyzingSuggester.java | 6 +- .../analyze/TransportAnalyzeAction.java | 7 +- .../index/analysis/Analysis.java | 4 +- .../mapper/core/TokenCountFieldMapper.java | 13 +-- .../MultiDocumentPercolatorIndex.java | 9 +- .../SingleDocumentPercolatorIndex.java | 9 +- .../search/highlight/PlainHighlighter.java | 30 +++--- .../search/suggest/SuggestUtils.java | 8 +- .../completion/CompletionTokenStream.java | 4 +- .../suggest/phrase/PhraseSuggester.java | 11 ++- .../core/TokenCountFieldMapperTests.java | 13 ++- 12 files changed, 112 insertions(+), 95 deletions(-) diff --git a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java index 3a61daeca12..916d049312c 100644 --- a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java +++ b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java @@ -26,6 +26,7 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.index.Term; import org.apache.lucene.search.*; import org.apache.lucene.util.automaton.RegExp; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.index.mapper.MappedFieldType; @@ -484,30 +485,31 @@ public class MapperQueryParser extends QueryParser { if (!settings.analyzeWildcard()) { return super.getPrefixQuery(field, termStr); } + List tlist; // get Analyzer from superclass and tokenize the term - TokenStream source; + TokenStream source = null; try { - source = getAnalyzer().tokenStream(field, termStr); - source.reset(); - } catch (IOException e) { - return super.getPrefixQuery(field, termStr); - } - List tlist = new ArrayList<>(); - CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); - - while (true) { try { - if (!source.incrementToken()) break; + source = getAnalyzer().tokenStream(field, termStr); + source.reset(); } catch (IOException e) { - break; + return super.getPrefixQuery(field, termStr); } - tlist.add(termAtt.toString()); - } + tlist = new ArrayList<>(); + CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); - try { - source.close(); - } catch (IOException e) { - // ignore + while (true) { + try { + if (!source.incrementToken()) break; + } catch (IOException e) { + break; + } + tlist.add(termAtt.toString()); + } + } finally { + if (source != null) { + IOUtils.closeWhileHandlingException(source); + } } if (tlist.size() == 1) { @@ -619,21 +621,30 @@ public class MapperQueryParser extends QueryParser { if (isWithinToken) { try { TokenStream source = getAnalyzer().tokenStream(field, tmp.toString()); - source.reset(); - CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); - if (source.incrementToken()) { - String term = termAtt.toString(); - if (term.length() == 0) { + boolean success = false; + try { + source.reset(); + CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); + if (source.incrementToken()) { + String term = termAtt.toString(); + if (term.length() == 0) { + // no tokens, just use what we have now + aggStr.append(tmp); + } else { + aggStr.append(term); + } + } else { // no tokens, just use what we have now aggStr.append(tmp); - } else { - aggStr.append(term); } - } else { - // no tokens, just use what we have now - aggStr.append(tmp); + success = true; + } finally { + if (success) { + source.close(); + } else { + IOUtils.close(source); + } } - source.close(); } catch (IOException e) { aggStr.append(tmp); } @@ -648,22 +659,22 @@ public class MapperQueryParser extends QueryParser { } if (isWithinToken) { try { - TokenStream source = getAnalyzer().tokenStream(field, tmp.toString()); - source.reset(); - CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); - if (source.incrementToken()) { - String term = termAtt.toString(); - if (term.length() == 0) { + try (TokenStream source = getAnalyzer().tokenStream(field, tmp.toString())) { + source.reset(); + CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); + if (source.incrementToken()) { + String term = termAtt.toString(); + if (term.length() == 0) { + // no tokens, just use what we have now + aggStr.append(tmp); + } else { + aggStr.append(term); + } + } else { // no tokens, just use what we have now aggStr.append(tmp); - } else { - aggStr.append(term); } - } else { - // no tokens, just use what we have now - aggStr.append(tmp); } - source.close(); } catch (IOException e) { aggStr.append(tmp); } diff --git a/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java b/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java index 5db4f932c67..b2b23a29981 100644 --- a/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java +++ b/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java @@ -959,11 +959,9 @@ public long ramBytesUsed() { // TODO: is there a Reader from a CharSequence? // Turn tokenstream into automaton: Automaton automaton = null; - TokenStream ts = queryAnalyzer.tokenStream("", key.toString()); - try { + + try (TokenStream ts = queryAnalyzer.tokenStream("", key.toString())) { automaton = getTokenStreamToAutomaton().toAutomaton(ts); - } finally { - IOUtils.closeWhileHandlingException(ts); } automaton = replaceSep(automaton); diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java index 42d05ea4637..4f7a605341e 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java @@ -217,12 +217,10 @@ public class TransportAnalyzeAction extends TransportSingleShardAction tokens = new ArrayList<>(); - TokenStream stream = null; int lastPosition = -1; int lastOffset = 0; for (String text : request.text()) { - try { - stream = analyzer.tokenStream(field, text); + try (TokenStream stream = analyzer.tokenStream(field, text)) { stream.reset(); CharTermAttribute term = stream.addAttribute(CharTermAttribute.class); PositionIncrementAttribute posIncr = stream.addAttribute(PositionIncrementAttribute.class); @@ -243,11 +241,8 @@ public class TransportAnalyzeAction extends TransportSingleShardAction> and pass previous, // like the indexer does - TokenStream tokenStream = field.tokenStream(analyzer, null); - if (tokenStream != null) { - memoryIndex.addField(field.name(), tokenStream, field.boost()); - } + try (TokenStream tokenStream = field.tokenStream(analyzer, null)) { + if (tokenStream != null) { + memoryIndex.addField(field.name(), tokenStream, field.boost()); + } + } } catch (IOException e) { throw new ElasticsearchException("Failed to create token stream", e); } diff --git a/core/src/main/java/org/elasticsearch/percolator/SingleDocumentPercolatorIndex.java b/core/src/main/java/org/elasticsearch/percolator/SingleDocumentPercolatorIndex.java index 3233cdcd756..1271872cab6 100644 --- a/core/src/main/java/org/elasticsearch/percolator/SingleDocumentPercolatorIndex.java +++ b/core/src/main/java/org/elasticsearch/percolator/SingleDocumentPercolatorIndex.java @@ -56,10 +56,11 @@ class SingleDocumentPercolatorIndex implements PercolatorIndex { Analyzer analyzer = context.mapperService().documentMapper(parsedDocument.type()).mappers().indexAnalyzer(); // TODO: instead of passing null here, we can have a CTL> and pass previous, // like the indexer does - TokenStream tokenStream = field.tokenStream(analyzer, null); - if (tokenStream != null) { - memoryIndex.addField(field.name(), tokenStream, field.boost()); - } + try (TokenStream tokenStream = field.tokenStream(analyzer, null)) { + if (tokenStream != null) { + memoryIndex.addField(field.name(), tokenStream, field.boost()); + } + } } catch (Exception e) { throw new ElasticsearchException("Failed to create token stream for [" + field.name() + "]", e); } diff --git a/core/src/main/java/org/elasticsearch/search/highlight/PlainHighlighter.java b/core/src/main/java/org/elasticsearch/search/highlight/PlainHighlighter.java index d50c53a1380..041ed754d76 100644 --- a/core/src/main/java/org/elasticsearch/search/highlight/PlainHighlighter.java +++ b/core/src/main/java/org/elasticsearch/search/highlight/PlainHighlighter.java @@ -33,6 +33,7 @@ import org.apache.lucene.search.highlight.SimpleSpanFragmenter; import org.apache.lucene.search.highlight.TextFragment; import org.apache.lucene.util.BytesRefHash; import org.apache.lucene.util.CollectionUtil; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.text.StringText; import org.elasticsearch.common.text.Text; @@ -109,15 +110,16 @@ public class PlainHighlighter implements Highlighter { for (Object textToHighlight : textsToHighlight) { String text = textToHighlight.toString(); - TokenStream tokenStream = analyzer.tokenStream(mapper.fieldType().names().indexName(), text); - if (!tokenStream.hasAttribute(CharTermAttribute.class) || !tokenStream.hasAttribute(OffsetAttribute.class)) { - // can't perform highlighting if the stream has no terms (binary token stream) or no offsets - continue; - } - TextFragment[] bestTextFragments = entry.getBestTextFragments(tokenStream, text, false, numberOfFragments); - for (TextFragment bestTextFragment : bestTextFragments) { - if (bestTextFragment != null && bestTextFragment.getScore() > 0) { - fragsList.add(bestTextFragment); + try (TokenStream tokenStream = analyzer.tokenStream(mapper.fieldType().names().indexName(), text)) { + if (!tokenStream.hasAttribute(CharTermAttribute.class) || !tokenStream.hasAttribute(OffsetAttribute.class)) { + // can't perform highlighting if the stream has no terms (binary token stream) or no offsets + continue; + } + TextFragment[] bestTextFragments = entry.getBestTextFragments(tokenStream, text, false, numberOfFragments); + for (TextFragment bestTextFragment : bestTextFragments) { + if (bestTextFragment != null && bestTextFragment.getScore() > 0) { + fragsList.add(bestTextFragment); + } } } } @@ -165,7 +167,7 @@ public class PlainHighlighter implements Highlighter { String fieldContents = textsToHighlight.get(0).toString(); int end; try { - end = findGoodEndForNoHighlightExcerpt(noMatchSize, analyzer.tokenStream(mapper.fieldType().names().indexName(), fieldContents)); + end = findGoodEndForNoHighlightExcerpt(noMatchSize, analyzer, mapper.fieldType().names().indexName(), fieldContents); } catch (Exception e) { throw new FetchPhaseExecutionException(context, "Failed to highlight field [" + highlighterContext.fieldName + "]", e); } @@ -181,8 +183,8 @@ public class PlainHighlighter implements Highlighter { return true; } - private static int findGoodEndForNoHighlightExcerpt(int noMatchSize, TokenStream tokenStream) throws IOException { - try { + private static int findGoodEndForNoHighlightExcerpt(int noMatchSize, Analyzer analyzer, String fieldName, String contents) throws IOException { + try (TokenStream tokenStream = analyzer.tokenStream(fieldName, contents)) { if (!tokenStream.hasAttribute(OffsetAttribute.class)) { // Can't split on term boundaries without offsets return -1; @@ -200,11 +202,9 @@ public class PlainHighlighter implements Highlighter { } end = attr.endOffset(); } + tokenStream.end(); // We've exhausted the token stream so we should just highlight everything. return end; - } finally { - tokenStream.end(); - tokenStream.close(); } } } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java index ce994d0993c..8dd193f6c24 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java @@ -117,10 +117,10 @@ public final class SuggestUtils { } public static int analyze(Analyzer analyzer, CharsRef toAnalyze, String field, TokenConsumer consumer) throws IOException { - TokenStream ts = analyzer.tokenStream( - field, new FastCharArrayReader(toAnalyze.chars, toAnalyze.offset, toAnalyze.length) - ); - return analyze(ts, consumer); + try (TokenStream ts = analyzer.tokenStream( + field, new FastCharArrayReader(toAnalyze.chars, toAnalyze.offset, toAnalyze.length))) { + return analyze(ts, consumer); + } } /** NOTE: this method closes the TokenStream, even on exception, which is awkward diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionTokenStream.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionTokenStream.java index ebcf0456f87..5edf848dda3 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionTokenStream.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionTokenStream.java @@ -100,9 +100,7 @@ public final class CompletionTokenStream extends TokenStream { @Override public void close() throws IOException { - if (posInc == -1) { - input.close(); - } + input.close(); } public static interface ToFiniteStrings { diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java index e7d0eb378c3..724e3d40e25 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java @@ -92,12 +92,13 @@ public final class PhraseSuggester extends Suggester { if (gens.size() > 0 && suggestTerms != null) { final NoisyChannelSpellChecker checker = new NoisyChannelSpellChecker(realWordErrorLikelihood, suggestion.getRequireUnigram(), suggestion.getTokenLimit()); final BytesRef separator = suggestion.separator(); - TokenStream stream = checker.tokenStream(suggestion.getAnalyzer(), suggestion.getText(), spare, suggestion.getField()); - WordScorer wordScorer = suggestion.model().newScorer(indexReader, suggestTerms, suggestField, realWordErrorLikelihood, separator); - Result checkerResult = checker.getCorrections(stream, new MultiCandidateGeneratorWrapper(suggestion.getShardSize(), - gens.toArray(new CandidateGenerator[gens.size()])), suggestion.maxErrors(), - suggestion.getShardSize(), wordScorer, suggestion.confidence(), suggestion.gramSize()); + Result checkerResult; + try (TokenStream stream = checker.tokenStream(suggestion.getAnalyzer(), suggestion.getText(), spare, suggestion.getField())) { + checkerResult = checker.getCorrections(stream, new MultiCandidateGeneratorWrapper(suggestion.getShardSize(), + gens.toArray(new CandidateGenerator[gens.size()])), suggestion.maxErrors(), + suggestion.getShardSize(), wordScorer, suggestion.confidence(), suggestion.gramSize()); + } PhraseSuggestion.Entry resultEntry = buildResultEntry(suggestion, spare, checkerResult.cutoffScore); response.addTerm(resultEntry); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperTests.java index 818366647d1..5a644e56f48 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperTests.java @@ -19,7 +19,9 @@ package org.elasticsearch.index.mapper.core; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CannedTokenStream; +import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; import org.elasticsearch.common.xcontent.XContentFactory; @@ -87,7 +89,14 @@ public class TokenCountFieldMapperTests extends ESSingleNodeTestCase { int finalTokenIncrement = 4; // Count the final token increment on the rare token streams that have them Token[] tokens = new Token[] {t1, t2, t3}; Collections.shuffle(Arrays.asList(tokens), getRandom()); - TokenStream tokenStream = new CannedTokenStream(finalTokenIncrement, 0, tokens); - assertThat(TokenCountFieldMapper.countPositions(tokenStream), equalTo(7)); + final TokenStream tokenStream = new CannedTokenStream(finalTokenIncrement, 0, tokens); + // TODO: we have no CannedAnalyzer? + Analyzer analyzer = new Analyzer() { + @Override + public TokenStreamComponents createComponents(String fieldName) { + return new TokenStreamComponents(new MockTokenizer(), tokenStream); + } + }; + assertThat(TokenCountFieldMapper.countPositions(analyzer, "", ""), equalTo(7)); } } From a321300e9c1faeb67fa0fe0a906c968352fae873 Mon Sep 17 00:00:00 2001 From: mikemccand Date: Wed, 30 Sep 2015 17:46:22 +0200 Subject: [PATCH 4/4] another try-with-resources --- .../classic/MapperQueryParser.java | 31 ++++++------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java index 916d049312c..ca1524f1214 100644 --- a/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java +++ b/core/src/main/java/org/apache/lucene/queryparser/classic/MapperQueryParser.java @@ -619,31 +619,20 @@ public class MapperQueryParser extends QueryParser { char c = termStr.charAt(i); if (c == '?' || c == '*') { if (isWithinToken) { - try { - TokenStream source = getAnalyzer().tokenStream(field, tmp.toString()); - boolean success = false; - try { - source.reset(); - CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); - if (source.incrementToken()) { - String term = termAtt.toString(); - if (term.length() == 0) { - // no tokens, just use what we have now - aggStr.append(tmp); - } else { - aggStr.append(term); - } - } else { + try (TokenStream source = getAnalyzer().tokenStream(field, tmp.toString())) { + source.reset(); + CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); + if (source.incrementToken()) { + String term = termAtt.toString(); + if (term.length() == 0) { // no tokens, just use what we have now aggStr.append(tmp); - } - success = true; - } finally { - if (success) { - source.close(); } else { - IOUtils.close(source); + aggStr.append(term); } + } else { + // no tokens, just use what we have now + aggStr.append(tmp); } } catch (IOException e) { aggStr.append(tmp);