From bb613bcacdb5d643ef5fc8820500599db1e083b0 Mon Sep 17 00:00:00 2001 From: mikemccand Date: Wed, 30 Sep 2015 15:44:35 +0200 Subject: [PATCH] move close responsibility back down to SuggestUtils.analyze --- .../search/suggest/SuggestUtils.java | 37 +++++++++++++------ .../phrase/NoisyChannelSpellChecker.java | 7 ++-- .../suggest/phrase/PhraseSuggester.java | 12 +++--- 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java index 6d2767c1f4e..ce994d0993c 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java @@ -28,6 +28,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.CharsRefBuilder; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.automaton.LevenshteinAutomata; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseFieldMatcher; @@ -116,22 +117,34 @@ public final class SuggestUtils { } public static int analyze(Analyzer analyzer, CharsRef toAnalyze, String field, TokenConsumer consumer) throws IOException { - try (TokenStream ts = analyzer.tokenStream( - field, new FastCharArrayReader(toAnalyze.chars, toAnalyze.offset, toAnalyze.length))) { - return analyze(ts, consumer); - } + TokenStream ts = analyzer.tokenStream( + field, new FastCharArrayReader(toAnalyze.chars, toAnalyze.offset, toAnalyze.length) + ); + return analyze(ts, consumer); } - - /** NOTE: caller must close the TokenStream */ + + /** NOTE: this method closes the TokenStream, even on exception, which is awkward + * because really the caller who called {@link Analyzer#tokenStream} should close it, + * but when trying that there are recursion issues when we try to use the same + * TokenStrem twice in the same recursion... */ public static int analyze(TokenStream stream, TokenConsumer consumer) throws IOException { - stream.reset(); - consumer.reset(stream); int numTokens = 0; - while (stream.incrementToken()) { - consumer.nextToken(); - numTokens++; + boolean success = false; + try { + stream.reset(); + consumer.reset(stream); + while (stream.incrementToken()) { + consumer.nextToken(); + numTokens++; + } + consumer.end(); + } finally { + if (success) { + stream.close(); + } else { + IOUtils.closeWhileHandlingException(stream); + } } - consumer.end(); return numTokens; } diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellChecker.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellChecker.java index ac03a736526..ec9ca6e1da2 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellChecker.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellChecker.java @@ -133,12 +133,11 @@ public final class NoisyChannelSpellChecker { public Result getCorrections(Analyzer analyzer, BytesRef query, CandidateGenerator generator, float maxErrors, int numCorrections, IndexReader reader, String analysisField, WordScorer scorer, float confidence, int gramSize) throws IOException { - try (TokenStream ts = tokenStream(analyzer, query, new CharsRefBuilder(), analysisField)) { - return getCorrections(ts, generator, maxErrors, numCorrections, scorer, confidence, gramSize); - } + + return getCorrections(tokenStream(analyzer, query, new CharsRefBuilder(), analysisField), generator, maxErrors, numCorrections, scorer, confidence, gramSize); + } - /** NOTE: caller must close returned TokenStream */ public TokenStream tokenStream(Analyzer analyzer, BytesRef query, CharsRefBuilder spare, String field) throws IOException { spare.copyUTF8Bytes(query); return analyzer.tokenStream(field, new FastCharArrayReader(spare.chars(), 0, spare.length())); diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java index 232b9e6d21c..e7d0eb378c3 100644 --- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java +++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java @@ -92,14 +92,12 @@ public final class PhraseSuggester extends Suggester { if (gens.size() > 0 && suggestTerms != null) { final NoisyChannelSpellChecker checker = new NoisyChannelSpellChecker(realWordErrorLikelihood, suggestion.getRequireUnigram(), suggestion.getTokenLimit()); final BytesRef separator = suggestion.separator(); - Result checkerResult; - try (TokenStream stream = checker.tokenStream(suggestion.getAnalyzer(), suggestion.getText(), spare, suggestion.getField())) { + TokenStream stream = checker.tokenStream(suggestion.getAnalyzer(), suggestion.getText(), spare, suggestion.getField()); - WordScorer wordScorer = suggestion.model().newScorer(indexReader, suggestTerms, suggestField, realWordErrorLikelihood, separator); - checkerResult = checker.getCorrections(stream, new MultiCandidateGeneratorWrapper(suggestion.getShardSize(), - gens.toArray(new CandidateGenerator[gens.size()])), suggestion.maxErrors(), - suggestion.getShardSize(), wordScorer, suggestion.confidence(), suggestion.gramSize()); - } + WordScorer wordScorer = suggestion.model().newScorer(indexReader, suggestTerms, suggestField, realWordErrorLikelihood, separator); + Result checkerResult = checker.getCorrections(stream, new MultiCandidateGeneratorWrapper(suggestion.getShardSize(), + gens.toArray(new CandidateGenerator[gens.size()])), suggestion.maxErrors(), + suggestion.getShardSize(), wordScorer, suggestion.confidence(), suggestion.gramSize()); PhraseSuggestion.Entry resultEntry = buildResultEntry(suggestion, spare, checkerResult.cutoffScore); response.addTerm(resultEntry);