mirror of https://github.com/apache/lucene.git
Fixed deprecation issues, adjusted test cases to use assertEquals better, reformatted style
git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@160987 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Erik Hatcher
parent
0c99b57cc1
commit
ec522fc1c8
|
|
@ -1,7 +1,7 @@
|
|||
package org.apache.lucene.search.spell;
|
||||
|
||||
/**
|
||||
* Copyright 2002-2004 The Apache Software Foundation
|
||||
* Copyright 2002-2005 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -17,78 +17,79 @@ package org.apache.lucene.search.spell;
|
|||
*/
|
||||
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
|
||||
import java.util.Iterator;
|
||||
|
||||
import org.apache.lucene.index.TermEnum;
|
||||
import org.apache.lucene.index.Term;
|
||||
|
||||
import java.io.*;
|
||||
|
||||
/**
|
||||
* Lucene Dictionnary
|
||||
* Lucene Dictionnary
|
||||
*
|
||||
* @author Nicolas Maisonneuve
|
||||
*/
|
||||
public class LuceneDictionary
|
||||
implements Dictionary {
|
||||
IndexReader reader;
|
||||
String field;
|
||||
public class LuceneDictionary implements Dictionary {
|
||||
IndexReader reader;
|
||||
String field;
|
||||
|
||||
public LuceneDictionary (IndexReader reader, String field) {
|
||||
this.reader=reader;
|
||||
this.field=field;
|
||||
public LuceneDictionary(IndexReader reader, String field) {
|
||||
this.reader = reader;
|
||||
this.field = field;
|
||||
}
|
||||
|
||||
public final Iterator getWordsIterator() {
|
||||
return new LuceneIterator();
|
||||
}
|
||||
|
||||
|
||||
final class LuceneIterator implements Iterator {
|
||||
private TermEnum enum;
|
||||
private Term actualTerm;
|
||||
private boolean has_next_called;
|
||||
|
||||
public LuceneIterator() {
|
||||
try {
|
||||
enum = reader.terms(new Term(field, ""));
|
||||
} catch (IOException ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public final Iterator getWordsIterator () {
|
||||
return new LuceneIterator();
|
||||
public Object next() {
|
||||
if (!has_next_called) {
|
||||
hasNext();
|
||||
}
|
||||
has_next_called = false;
|
||||
return (actualTerm != null) ? actualTerm.text() : null;
|
||||
}
|
||||
|
||||
|
||||
final class LuceneIterator implements Iterator {
|
||||
private TermEnum enum;
|
||||
private Term actualTerm;
|
||||
private boolean has_next_called;
|
||||
|
||||
public LuceneIterator () {
|
||||
try {
|
||||
enum=reader.terms(new Term(field, ""));
|
||||
}
|
||||
catch (IOException ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
public boolean hasNext() {
|
||||
has_next_called = true;
|
||||
try {
|
||||
// if there is still words
|
||||
if (!enum.next()) {
|
||||
actualTerm = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
public Object next () {
|
||||
if (!has_next_called) {hasNext();}
|
||||
has_next_called=false;
|
||||
return (actualTerm!=null) ? actualTerm.text(): null;
|
||||
// if the next word are in the field
|
||||
actualTerm = enum.term();
|
||||
String fieldt = actualTerm.field();
|
||||
if (fieldt != field) {
|
||||
actualTerm = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
public boolean hasNext () {
|
||||
has_next_called=true;
|
||||
try {
|
||||
// if there is still words
|
||||
if (!enum.next()) {
|
||||
actualTerm=null;
|
||||
return false;
|
||||
}
|
||||
// if the next word are in the field
|
||||
actualTerm=enum.term();
|
||||
String fieldt=actualTerm.field();
|
||||
if (fieldt!=field) {
|
||||
actualTerm=null;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
catch (IOException ex) {
|
||||
ex.printStackTrace();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void remove () {};
|
||||
return true;
|
||||
} catch (IOException ex) {
|
||||
ex.printStackTrace();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public void remove() {
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
package org.apache.lucene.search.spell;
|
||||
|
||||
/**
|
||||
* Copyright 2002-2004 The Apache Software Foundation
|
||||
* Copyright 2002-2005 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -35,52 +35,48 @@ import java.io.*;
|
|||
*/
|
||||
public class PlainTextDictionary implements Dictionary {
|
||||
|
||||
private BufferedReader in;
|
||||
private String line;
|
||||
private boolean has_next_called;
|
||||
private BufferedReader in;
|
||||
private String line;
|
||||
private boolean has_next_called;
|
||||
|
||||
public PlainTextDictionary (File file) throws FileNotFoundException {
|
||||
in=new BufferedReader(new FileReader(file));
|
||||
public PlainTextDictionary(File file) throws FileNotFoundException {
|
||||
in = new BufferedReader(new FileReader(file));
|
||||
}
|
||||
|
||||
public PlainTextDictionary(InputStream dictFile) {
|
||||
in = new BufferedReader(new InputStreamReader(dictFile));
|
||||
}
|
||||
|
||||
public Iterator getWordsIterator() {
|
||||
return new fileIterator();
|
||||
}
|
||||
|
||||
|
||||
final class fileIterator implements Iterator {
|
||||
public Object next() {
|
||||
if (!has_next_called) {
|
||||
hasNext();
|
||||
}
|
||||
has_next_called = false;
|
||||
return line;
|
||||
}
|
||||
|
||||
|
||||
public PlainTextDictionary (InputStream dictFile) {
|
||||
in=new BufferedReader(new InputStreamReader(dictFile));
|
||||
public boolean hasNext() {
|
||||
has_next_called = true;
|
||||
try {
|
||||
line = in.readLine();
|
||||
} catch (IOException ex) {
|
||||
ex.printStackTrace();
|
||||
line = null;
|
||||
return false;
|
||||
}
|
||||
return (line != null) ? true : false;
|
||||
}
|
||||
|
||||
|
||||
public Iterator getWordsIterator () {
|
||||
|
||||
return new fileIterator();
|
||||
}
|
||||
|
||||
|
||||
final class fileIterator
|
||||
implements Iterator {
|
||||
public Object next () {
|
||||
if (!has_next_called) {
|
||||
hasNext();
|
||||
}
|
||||
has_next_called=false;
|
||||
return line;
|
||||
}
|
||||
|
||||
|
||||
public boolean hasNext () {
|
||||
has_next_called=true;
|
||||
try {
|
||||
line=in.readLine();
|
||||
}
|
||||
catch (IOException ex) {
|
||||
ex.printStackTrace();
|
||||
line=null;
|
||||
return false;
|
||||
}
|
||||
return (line!=null)?true:false;
|
||||
}
|
||||
|
||||
|
||||
public void remove () {};
|
||||
}
|
||||
public void remove() {
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,7 +26,6 @@ import org.apache.lucene.document.Field;
|
|||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.index.IndexWriter;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.index.TermEnum;
|
||||
import org.apache.lucene.search.BooleanClause;
|
||||
import org.apache.lucene.search.BooleanQuery;
|
||||
import org.apache.lucene.search.Hits;
|
||||
|
|
@ -113,7 +112,7 @@ public class SpellChecker {
|
|||
* Suggest similar words (restricted or not of a field of a user index)
|
||||
* @param word String the word you want a spell check done on
|
||||
* @param num_sug int the number of suggest words
|
||||
* @param IndexReader the indexReader of the user index (can be null see field param)
|
||||
* @param ir the indexReader of the user index (can be null see field param)
|
||||
* @param field String the field of the user index: if field is not null ,the suggest
|
||||
* words are restricted to the words present in this field.
|
||||
* @param morePopular boolean return only the suggest words that are more frequent than the searched word
|
||||
|
|
@ -214,7 +213,7 @@ public class SpellChecker {
|
|||
private static void add (BooleanQuery q, String k, String v, float boost) {
|
||||
Query tq=new TermQuery(new Term(k, v));
|
||||
tq.setBoost(boost);
|
||||
q.add(new BooleanClause(tq, false, false));
|
||||
q.add(new BooleanClause(tq, BooleanClause.Occur.SHOULD));
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -222,7 +221,7 @@ public class SpellChecker {
|
|||
* Add a clause to a boolean query.
|
||||
*/
|
||||
private static void add (BooleanQuery q, String k, String v) {
|
||||
q.add(new BooleanClause(new TermQuery(new Term(k, v)), false, false));
|
||||
q.add(new BooleanClause(new TermQuery(new Term(k, v)), BooleanClause.Occur.SHOULD));
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -269,12 +268,10 @@ public class SpellChecker {
|
|||
* @throws IOException
|
||||
*/
|
||||
public void indexDictionnary (Dictionary dict) throws IOException {
|
||||
|
||||
int ng1, ng2;
|
||||
IndexReader.unlock(spellindex);
|
||||
IndexWriter writer=new IndexWriter(spellindex, new WhitespaceAnalyzer(), !IndexReader.indexExists(spellindex));
|
||||
writer.mergeFactor=300;
|
||||
writer.minMergeDocs=150;
|
||||
writer.setMergeFactor(300);
|
||||
writer.setMaxBufferedDocs(150);
|
||||
|
||||
Iterator iter=dict.getWordsIterator();
|
||||
while (iter.hasNext()) {
|
||||
|
|
@ -328,7 +325,7 @@ public class SpellChecker {
|
|||
|
||||
private static Document createDocument (String text, int ng1, int ng2) {
|
||||
Document doc=new Document();
|
||||
doc.add(Field.Keyword(F_WORD, text)); // orig term
|
||||
doc.add(new Field(F_WORD, text, Field.Store.YES, Field.Index.UN_TOKENIZED)); // orig term
|
||||
addGram(text, doc, ng1, ng2);
|
||||
return doc;
|
||||
}
|
||||
|
|
@ -341,14 +338,14 @@ public class SpellChecker {
|
|||
String end=null;
|
||||
for (int i=0; i<len-ng+1; i++) {
|
||||
String gram=text.substring(i, i+ng);
|
||||
doc.add(Field.Keyword(key, gram));
|
||||
doc.add(new Field(key, gram, Field.Store.YES, Field.Index.UN_TOKENIZED));
|
||||
if (i==0) {
|
||||
doc.add(Field.Keyword("start"+ng, gram));
|
||||
doc.add(new Field("start"+ng, gram, Field.Store.YES, Field.Index.UN_TOKENIZED));
|
||||
}
|
||||
end=gram;
|
||||
}
|
||||
if (end!=null) { // may not be present if len==ng1
|
||||
doc.add(Field.Keyword("end"+ng, end));
|
||||
doc.add(new Field("end"+ng, end, Field.Store.YES, Field.Index.UN_TOKENIZED));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ package org.apache.lucene.search.spell;
|
|||
|
||||
|
||||
import junit.framework.*;
|
||||
import org.apache.lucene.search.spell.*;
|
||||
import org.apache.lucene.store.RAMDirectory;
|
||||
import org.apache.lucene.index.IndexWriter;
|
||||
import org.apache.lucene.analysis.SimpleAnalyzer;
|
||||
|
|
@ -10,113 +9,108 @@ import org.apache.lucene.document.Document;
|
|||
import org.apache.lucene.util.English;
|
||||
import org.apache.lucene.document.Field;
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
|
||||
import java.io.IOException;
|
||||
import org.apache.lucene.store.FSDirectory;
|
||||
|
||||
import org.apache.lucene.store.Directory;
|
||||
import java.io.File;
|
||||
|
||||
|
||||
/**
|
||||
* Test case
|
||||
*
|
||||
* @author Nicolas Maisonneuve
|
||||
*/
|
||||
public class TestSpellChecker extends TestCase {
|
||||
private SpellChecker spellChecker;
|
||||
Directory userindex, spellindex;
|
||||
|
||||
public class TestSpellChecker
|
||||
extends TestCase {
|
||||
private SpellChecker spellChecker;
|
||||
Directory userindex, spellindex;
|
||||
protected void setUp() throws Exception {
|
||||
super.setUp();
|
||||
|
||||
protected void setUp () throws Exception {
|
||||
super.setUp();
|
||||
//create a user index
|
||||
userindex = new RAMDirectory();
|
||||
IndexWriter writer = new IndexWriter(userindex, new SimpleAnalyzer(), true);
|
||||
|
||||
//create a user index
|
||||
userindex=new RAMDirectory();
|
||||
IndexWriter writer=new IndexWriter(userindex, new SimpleAnalyzer(), true);
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
Document doc = new Document();
|
||||
doc.add(new Field("field1", English.intToEnglish(i), Field.Store.YES, Field.Index.TOKENIZED));
|
||||
doc.add(new Field("field2", English.intToEnglish(i + 1), Field.Store.YES, Field.Index.TOKENIZED)); // + word thousand
|
||||
writer.addDocument(doc);
|
||||
}
|
||||
writer.close();
|
||||
|
||||
for (int i=0; i<1000; i++) {
|
||||
Document doc=new Document();
|
||||
doc.add(Field.Text("field1", English.intToEnglish(i)));
|
||||
doc.add(Field.Text("field2", English.intToEnglish(i+1))); // + word thousand
|
||||
writer.addDocument(doc);
|
||||
}
|
||||
writer.close();
|
||||
// create the spellChecker
|
||||
spellindex = new RAMDirectory();
|
||||
spellChecker = new SpellChecker(spellindex);
|
||||
}
|
||||
|
||||
// create the spellChecker
|
||||
File file=new File("d://test");
|
||||
spellindex=FSDirectory.getDirectory(file, true);
|
||||
spellChecker=new SpellChecker(spellindex);
|
||||
|
||||
public void testBuild() {
|
||||
try {
|
||||
IndexReader r = IndexReader.open(userindex);
|
||||
|
||||
spellChecker.clearIndex();
|
||||
|
||||
addwords(r, "field1");
|
||||
int num_field1 = this.numdoc();
|
||||
|
||||
addwords(r, "field2");
|
||||
int num_field2 = this.numdoc();
|
||||
|
||||
assertEquals(num_field2, num_field1 + 1);
|
||||
|
||||
// test small word
|
||||
String[] similar = spellChecker.suggestSimilar("fvie", 2);
|
||||
assertEquals(similar[0], "five");
|
||||
|
||||
similar = spellChecker.suggestSimilar("fiv", 2);
|
||||
assertEquals(similar[0], "five");
|
||||
|
||||
similar = spellChecker.suggestSimilar("ive", 2);
|
||||
assertEquals(similar[0], "five");
|
||||
|
||||
similar = spellChecker.suggestSimilar("fives", 2);
|
||||
assertEquals(similar[0], "five");
|
||||
|
||||
similar = spellChecker.suggestSimilar("fie", 2);
|
||||
assertEquals(similar[0], "five");
|
||||
|
||||
similar = spellChecker.suggestSimilar("fi", 2);
|
||||
assertEquals(0, similar.length);
|
||||
|
||||
// test restraint to a field
|
||||
similar = spellChecker.suggestSimilar("tousand", 10, r, "field1", false);
|
||||
assertEquals(0, similar.length); // there isn't the term thousand in the field field1
|
||||
|
||||
similar = spellChecker.suggestSimilar("tousand", 10, r, "field2", false);
|
||||
assertEquals(1, similar.length); // there is the term thousand in the field field2
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
fail();
|
||||
}
|
||||
|
||||
|
||||
public void testBuild () {
|
||||
try {
|
||||
IndexReader r=IndexReader.open(userindex);
|
||||
|
||||
spellChecker.clearIndex();
|
||||
|
||||
addwords(r, "field1");
|
||||
int num_field1=this.numdoc();
|
||||
|
||||
addwords(r, "field2");
|
||||
int num_field2=this.numdoc();
|
||||
|
||||
this.assertTrue(num_field2==num_field1+1);
|
||||
|
||||
// test small word
|
||||
String[] l=spellChecker.suggestSimilar("fvie", 2);
|
||||
this.assertTrue(l[0].equals("five"));
|
||||
|
||||
l=spellChecker.suggestSimilar("fiv", 2);
|
||||
this.assertTrue(l[0].equals("five"));
|
||||
|
||||
l=spellChecker.suggestSimilar("ive", 2);
|
||||
this.assertTrue(l[0].equals("five"));
|
||||
|
||||
l=spellChecker.suggestSimilar("fives", 2);
|
||||
this.assertTrue(l[0].equals("five"));
|
||||
|
||||
l=spellChecker.suggestSimilar("fie", 2);
|
||||
this.assertTrue(l[0].equals("five"));
|
||||
|
||||
l=spellChecker.suggestSimilar("fi", 2);
|
||||
this.assertEquals(0,l.length);
|
||||
|
||||
// test restreint to a field
|
||||
l=spellChecker.suggestSimilar("tousand", 10, r, "field1", false);
|
||||
this.assertEquals(0,l.length); // there isn't the term thousand in the field field1
|
||||
|
||||
l=spellChecker.suggestSimilar("tousand", 10, r, "field2", false);
|
||||
this.assertEquals(1,l.length); // there is the term thousand in the field field2
|
||||
}
|
||||
catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
this.assertTrue(false);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void addwords (IndexReader r, String field) throws IOException {
|
||||
long time=System.currentTimeMillis();
|
||||
spellChecker.indexDictionnary(new LuceneDictionary(r, field));
|
||||
time=System.currentTimeMillis()-time;
|
||||
System.out.println("time to build "+field+": "+time);
|
||||
}
|
||||
private void addwords(IndexReader r, String field) throws IOException {
|
||||
long time = System.currentTimeMillis();
|
||||
spellChecker.indexDictionnary(new LuceneDictionary(r, field));
|
||||
time = System.currentTimeMillis() - time;
|
||||
System.out.println("time to build " + field + ": " + time);
|
||||
}
|
||||
|
||||
private int numdoc() throws IOException {
|
||||
IndexReader rs = IndexReader.open(spellindex);
|
||||
int num = rs.numDocs();
|
||||
assertTrue(num != 0);
|
||||
System.out.println("num docs: " + num);
|
||||
rs.close();
|
||||
return num;
|
||||
}
|
||||
|
||||
private int numdoc () throws IOException {
|
||||
IndexReader rs=IndexReader.open(spellindex);
|
||||
int num=rs.numDocs();
|
||||
this.assertTrue(num!=0);
|
||||
System.out.println("num docs: "+num);
|
||||
rs.close();
|
||||
return num;
|
||||
}
|
||||
|
||||
|
||||
protected void tearDown () throws Exception {
|
||||
spellChecker=null;
|
||||
super.tearDown();
|
||||
}
|
||||
protected void tearDown() throws Exception {
|
||||
spellChecker = null;
|
||||
super.tearDown();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue