Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-4845: close tmp directory; fix test to catch un-closed files; add missing suggester.close() 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1503458 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2013-07-15 20:29:32 +00:00
parent 50372a0200
commit ce962bd8be
2 changed files with 105 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
View File

@ -100,10 +100,10 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
private final Analyzer queryAnalyzer;
private final Analyzer indexAnalyzer;
private final Directory dir;
private final Version matchVersion;
private final File indexPath;
private final int minPrefixChars;
private Directory dir;
/** {@link IndexSearcher} used for lookups. */
protected IndexSearcher searcher;
@ -143,7 +143,7 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
this.matchVersion = matchVersion;
this.indexPath = indexPath;
this.minPrefixChars = minPrefixChars;
dir = FSDirectory.open(indexPath);
dir = getDirectory(indexPath);
if (DirectoryReader.indexExists(dir)) {
// Already built; open it:
@ -166,43 +166,54 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
return iwc;
}
/** Subclass can override to choose a specific {@link
* Directory} implementation. */
protected Directory getDirectory(File path) throws IOException {
return FSDirectory.open(path);
}
@Override
public void build(TermFreqIterator iter) throws IOException {
if (searcher != null) {
searcher.getIndexReader().close();
searcher = null;
}
TermFreqPayloadIterator payloads;
if (iter instanceof TermFreqPayloadIterator) {
payloads = (TermFreqPayloadIterator) iter;
} else {
payloads = null;
}
Directory dirTmp = FSDirectory.open(new File(indexPath.toString() + ".tmp"));
Directory dirTmp = getDirectory(new File(indexPath.toString() + ".tmp"));
Analyzer gramAnalyzer = new AnalyzerWrapper() {
@Override
protected Analyzer getWrappedAnalyzer(String fieldName) {
return indexAnalyzer;
}
@Override
protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
if (fieldName.equals("textgrams") && minPrefixChars > 0) {
return new TokenStreamComponents(components.getTokenizer(),
new EdgeNGramTokenFilter(matchVersion,
components.getTokenStream(),
1, minPrefixChars));
} else {
return components;
}
}
};
IndexWriter w = new IndexWriter(dirTmp,
getIndexWriterConfig(matchVersion, gramAnalyzer));
IndexWriter w = null;
IndexWriter w2 = null;
AtomicReader r = null;
boolean success = false;
try {
Analyzer gramAnalyzer = new AnalyzerWrapper() {
@Override
protected Analyzer getWrappedAnalyzer(String fieldName) {
return indexAnalyzer;
}
@Override
protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
if (fieldName.equals("textgrams") && minPrefixChars > 0) {
return new TokenStreamComponents(components.getTokenizer(),
new EdgeNGramTokenFilter(matchVersion,
components.getTokenStream(),
1, minPrefixChars));
} else {
return components;
}
}
};
w = new IndexWriter(dirTmp,
getIndexWriterConfig(matchVersion, gramAnalyzer));
BytesRef text;
Document doc = new Document();
FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
@ -298,9 +309,9 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
success = true;
} finally {
if (success) {
IOUtils.close(w, w2, r);
IOUtils.close(w, w2, r, dirTmp);
} else {
IOUtils.closeWhileHandlingException(w, w2, r);
IOUtils.closeWhileHandlingException(w, w2, r, dirTmp);
}
}
}
@ -327,6 +338,10 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
* should be highlighted ({@code doHighlight}). */
public List<LookupResult> lookup(CharSequence key, int num, boolean allTermsRequired, boolean doHighlight) {
if (searcher == null) {
throw new IllegalStateException("suggester was not built");
}
final BooleanClause.Occur occur;
if (allTermsRequired) {
occur = BooleanClause.Occur.MUST;
@ -563,7 +578,10 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
if (searcher != null) {
searcher.getIndexReader().close();
searcher = null;
}
if (dir != null) {
dir.close();
dir = null;
}
}
};

68
lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java
View File

@ -33,6 +33,7 @@ import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
import org.apache.lucene.search.suggest.TermFreqPayload;
import org.apache.lucene.search.suggest.TermFreqPayloadArrayIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.LuceneTestCase;
@ -51,7 +52,12 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase {
File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3);
AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3) {
@Override
protected Directory getDirectory(File path) {
return newDirectory();
}
};
suggester.build(new TermFreqPayloadArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("ear", random()), 10, true, true);
@ -94,11 +100,21 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase {
File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3);
AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3) {
@Override
protected Directory getDirectory(File path) {
return newFSDirectory(path);
}
};
suggester.build(new TermFreqPayloadArrayIterator(keys));
suggester.close();
suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3);
suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3) {
@Override
protected Directory getDirectory(File path) {
return newFSDirectory(path);
}
};
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("ear", random()), 10, true, true);
assertEquals(2, results.size());
assertEquals("a penny saved is a penny <b>ear</b>ned", results.get(0).key);
@ -117,7 +133,12 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase {
Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
int minPrefixLength = random().nextInt(10);
AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, minPrefixLength);
AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, minPrefixLength) {
@Override
protected Directory getDirectory(File path) {
return newFSDirectory(path);
}
};
suggester.build(new TermFreqPayloadArrayIterator(keys));
for(int i=0;i<2;i++) {
@ -174,8 +195,14 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase {
// Make sure things still work after close and reopen:
suggester.close();
suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, minPrefixLength);
suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, minPrefixLength) {
@Override
protected Directory getDirectory(File path) {
return newFSDirectory(path);
}
};
}
suggester.close();
}
public void testHighlight() throws Exception {
@ -186,7 +213,12 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase {
File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3);
AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3) {
@Override
protected Directory getDirectory(File path) {
return newDirectory();
}
};
suggester.build(new TermFreqPayloadArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("penn", random()), 10, true, true);
assertEquals(1, results.size());
@ -202,7 +234,12 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase {
File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true);
AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3);
AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3) {
@Override
protected Directory getDirectory(File path) {
return newDirectory();
}
};
suggester.build(new TermFreqPayloadArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("penn", random()), 10, true, true);
assertEquals(1, results.size());
@ -225,6 +262,11 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase {
sb.append("</b>");
}
}
@Override
protected Directory getDirectory(File path) {
return newDirectory();
}
};
suggester.build(new TermFreqPayloadArrayIterator(keys));
results = suggester.lookup(_TestUtil.stringToCharSequence("penn", random()), 10, true, true);
@ -241,7 +283,12 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase {
File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3);
AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3) {
@Override
protected Directory getDirectory(File path) {
return newDirectory();
}
};
suggester.build(new TermFreqPayloadArrayIterator(keys));
suggester.close();
suggester.close();
@ -298,6 +345,11 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase {
return null;
}
}
@Override
protected Directory getDirectory(File path) {
return newDirectory();
}
};
suggester.build(new TermFreqPayloadArrayIterator(keys));