Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed deprecation issues, adjusted test cases to use assertEquals better, reformatted style 

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@160987 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Erik Hatcher 2005-04-11 23:48:02 +00:00
parent 0c99b57cc1
commit ec522fc1c8
4 changed files with 187 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
contrib/spellchecker/src/java/org/apache/lucene/search/spell/LuceneDictionary.java
View File

@ -1,7 +1,7 @@
package org.apache.lucene.search.spell; package org.apache.lucene.search.spell;
/** /**
* Copyright 2002-2004 The Apache Software Foundation * Copyright 2002-2005 The Apache Software Foundation
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -17,78 +17,79 @@ package org.apache.lucene.search.spell;
*/ */
import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader;
import java.util.Iterator; import java.util.Iterator;
import org.apache.lucene.index.TermEnum; import org.apache.lucene.index.TermEnum;
import org.apache.lucene.index.Term; import org.apache.lucene.index.Term;
import java.io.*; import java.io.*;
/** /**
* Lucene Dictionnary * Lucene Dictionnary
*
* @author Nicolas Maisonneuve * @author Nicolas Maisonneuve
*/ */
public class LuceneDictionary public class LuceneDictionary implements Dictionary {
implements Dictionary {
IndexReader reader; IndexReader reader;
String field; String field;
public LuceneDictionary (IndexReader reader, String field) { public LuceneDictionary(IndexReader reader, String field) {
this.reader=reader; this.reader = reader;
this.field=field; this.field = field;
} }
public final Iterator getWordsIterator() {
public final Iterator getWordsIterator () {
return new LuceneIterator(); return new LuceneIterator();
} }
final class LuceneIterator implements Iterator { final class LuceneIterator implements Iterator {
private TermEnum enum; private TermEnum enum;
private Term actualTerm; private Term actualTerm;
private boolean has_next_called; private boolean has_next_called;
public LuceneIterator () { public LuceneIterator() {
try { try {
enum=reader.terms(new Term(field, "")); enum = reader.terms(new Term(field, ""));
} } catch (IOException ex) {
catch (IOException ex) {
ex.printStackTrace(); ex.printStackTrace();
} }
} }
public Object next () { public Object next() {
if (!has_next_called) {hasNext();} if (!has_next_called) {
has_next_called=false; hasNext();
return (actualTerm!=null) ? actualTerm.text(): null; }
has_next_called = false;
return (actualTerm != null) ? actualTerm.text() : null;
} }
public boolean hasNext () { public boolean hasNext() {
has_next_called=true; has_next_called = true;
try { try {
// if there is still words // if there is still words
if (!enum.next()) { if (!enum.next()) {
actualTerm=null; actualTerm = null;
return false; return false;
} }
// if the next word are in the field // if the next word are in the field
actualTerm=enum.term(); actualTerm = enum.term();
String fieldt=actualTerm.field(); String fieldt = actualTerm.field();
if (fieldt!=field) { if (fieldt != field) {
actualTerm=null; actualTerm = null;
return false; return false;
} }
return true; return true;
} } catch (IOException ex) {
catch (IOException ex) {
ex.printStackTrace(); ex.printStackTrace();
return false; return false;
} }
} }
public void remove() {
public void remove () {}; };
} }
} }

38
contrib/spellchecker/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java
View File

@ -1,7 +1,7 @@
package org.apache.lucene.search.spell; package org.apache.lucene.search.spell;
/** /**
* Copyright 2002-2004 The Apache Software Foundation * Copyright 2002-2005 The Apache Software Foundation
* *
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
@ -39,48 +39,44 @@ public class PlainTextDictionary implements Dictionary {
private String line; private String line;
private boolean has_next_called; private boolean has_next_called;
public PlainTextDictionary (File file) throws FileNotFoundException { public PlainTextDictionary(File file) throws FileNotFoundException {
in=new BufferedReader(new FileReader(file)); in = new BufferedReader(new FileReader(file));
} }
public PlainTextDictionary(InputStream dictFile) {
public PlainTextDictionary (InputStream dictFile) { in = new BufferedReader(new InputStreamReader(dictFile));
in=new BufferedReader(new InputStreamReader(dictFile));
} }
public Iterator getWordsIterator() {
public Iterator getWordsIterator () {
return new fileIterator(); return new fileIterator();
} }
final class fileIterator final class fileIterator implements Iterator {
implements Iterator { public Object next() {
public Object next () {
if (!has_next_called) { if (!has_next_called) {
hasNext(); hasNext();
} }
has_next_called=false; has_next_called = false;
return line; return line;
} }
public boolean hasNext () { public boolean hasNext() {
has_next_called=true; has_next_called = true;
try { try {
line=in.readLine(); line = in.readLine();
} } catch (IOException ex) {
catch (IOException ex) {
ex.printStackTrace(); ex.printStackTrace();
line=null; line = null;
return false; return false;
} }
return (line!=null)?true:false; return (line != null) ? true : false;
} }
public void remove () {}; public void remove() {
};
} }
} }

21
contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java
View File

@ -26,7 +26,6 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term; import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermEnum;
import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Hits; import org.apache.lucene.search.Hits;
@ -113,7 +112,7 @@ public class SpellChecker {
* Suggest similar words (restricted or not of a field of a user index) * Suggest similar words (restricted or not of a field of a user index)
* @param word String the word you want a spell check done on * @param word String the word you want a spell check done on
* @param num_sug int the number of suggest words * @param num_sug int the number of suggest words
* @param IndexReader the indexReader of the user index (can be null see field param) * @param ir the indexReader of the user index (can be null see field param)
* @param field String the field of the user index: if field is not null ,the suggest * @param field String the field of the user index: if field is not null ,the suggest
* words are restricted to the words present in this field. * words are restricted to the words present in this field.
* @param morePopular boolean return only the suggest words that are more frequent than the searched word * @param morePopular boolean return only the suggest words that are more frequent than the searched word
@ -214,7 +213,7 @@ public class SpellChecker {
private static void add (BooleanQuery q, String k, String v, float boost) { private static void add (BooleanQuery q, String k, String v, float boost) {
Query tq=new TermQuery(new Term(k, v)); Query tq=new TermQuery(new Term(k, v));
tq.setBoost(boost); tq.setBoost(boost);
q.add(new BooleanClause(tq, false, false)); q.add(new BooleanClause(tq, BooleanClause.Occur.SHOULD));
} }
@ -222,7 +221,7 @@ public class SpellChecker {
* Add a clause to a boolean query. * Add a clause to a boolean query.
*/ */
private static void add (BooleanQuery q, String k, String v) { private static void add (BooleanQuery q, String k, String v) {
q.add(new BooleanClause(new TermQuery(new Term(k, v)), false, false)); q.add(new BooleanClause(new TermQuery(new Term(k, v)), BooleanClause.Occur.SHOULD));
} }
@ -269,12 +268,10 @@ public class SpellChecker {
* @throws IOException * @throws IOException
*/ */
public void indexDictionnary (Dictionary dict) throws IOException { public void indexDictionnary (Dictionary dict) throws IOException {
int ng1, ng2;
IndexReader.unlock(spellindex); IndexReader.unlock(spellindex);
IndexWriter writer=new IndexWriter(spellindex, new WhitespaceAnalyzer(), !IndexReader.indexExists(spellindex)); IndexWriter writer=new IndexWriter(spellindex, new WhitespaceAnalyzer(), !IndexReader.indexExists(spellindex));
writer.mergeFactor=300; writer.setMergeFactor(300);
writer.minMergeDocs=150; writer.setMaxBufferedDocs(150);
Iterator iter=dict.getWordsIterator(); Iterator iter=dict.getWordsIterator();
while (iter.hasNext()) { while (iter.hasNext()) {
@ -328,7 +325,7 @@ public class SpellChecker {
private static Document createDocument (String text, int ng1, int ng2) { private static Document createDocument (String text, int ng1, int ng2) {
Document doc=new Document(); Document doc=new Document();
doc.add(Field.Keyword(F_WORD, text)); // orig term doc.add(new Field(F_WORD, text, Field.Store.YES, Field.Index.UN_TOKENIZED)); // orig term
addGram(text, doc, ng1, ng2); addGram(text, doc, ng1, ng2);
return doc; return doc;
} }
@ -341,14 +338,14 @@ public class SpellChecker {
String end=null; String end=null;
for (int i=0; i<len-ng+1; i++) { for (int i=0; i<len-ng+1; i++) {
String gram=text.substring(i, i+ng); String gram=text.substring(i, i+ng);
doc.add(Field.Keyword(key, gram)); doc.add(new Field(key, gram, Field.Store.YES, Field.Index.UN_TOKENIZED));
if (i==0) { if (i==0) {
doc.add(Field.Keyword("start"+ng, gram)); doc.add(new Field("start"+ng, gram, Field.Store.YES, Field.Index.UN_TOKENIZED));
} }
end=gram; end=gram;
} }
if (end!=null) { // may not be present if len==ng1 if (end!=null) { // may not be present if len==ng1
doc.add(Field.Keyword("end"+ng, end)); doc.add(new Field("end"+ng, end, Field.Store.YES, Field.Index.UN_TOKENIZED));
} }
} }
} }

102
contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestSpellChecker.java
View File

@ -2,7 +2,6 @@ package org.apache.lucene.search.spell;
import junit.framework.*; import junit.framework.*;
import org.apache.lucene.search.spell.*;
import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.analysis.SimpleAnalyzer; import org.apache.lucene.analysis.SimpleAnalyzer;
@ -10,112 +9,107 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.util.English; import org.apache.lucene.util.English;
import org.apache.lucene.document.Field; import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader;
import java.io.IOException; import java.io.IOException;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import java.io.File;
/** /**
* Test case * Test case
*
* @author Nicolas Maisonneuve * @author Nicolas Maisonneuve
*/ */
public class TestSpellChecker extends TestCase {
public class TestSpellChecker
extends TestCase {
private SpellChecker spellChecker; private SpellChecker spellChecker;
Directory userindex, spellindex; Directory userindex, spellindex;
protected void setUp () throws Exception { protected void setUp() throws Exception {
super.setUp(); super.setUp();
//create a user index //create a user index
userindex=new RAMDirectory(); userindex = new RAMDirectory();
IndexWriter writer=new IndexWriter(userindex, new SimpleAnalyzer(), true); IndexWriter writer = new IndexWriter(userindex, new SimpleAnalyzer(), true);
for (int i=0; i<1000; i++) { for (int i = 0; i < 1000; i++) {
Document doc=new Document(); Document doc = new Document();
doc.add(Field.Text("field1", English.intToEnglish(i))); doc.add(new Field("field1", English.intToEnglish(i), Field.Store.YES, Field.Index.TOKENIZED));
doc.add(Field.Text("field2", English.intToEnglish(i+1))); // + word thousand doc.add(new Field("field2", English.intToEnglish(i + 1), Field.Store.YES, Field.Index.TOKENIZED)); // + word thousand
writer.addDocument(doc); writer.addDocument(doc);
} }
writer.close(); writer.close();
// create the spellChecker // create the spellChecker
File file=new File("d://test"); spellindex = new RAMDirectory();
spellindex=FSDirectory.getDirectory(file, true); spellChecker = new SpellChecker(spellindex);
spellChecker=new SpellChecker(spellindex);
} }
public void testBuild () { public void testBuild() {
try { try {
IndexReader r=IndexReader.open(userindex); IndexReader r = IndexReader.open(userindex);
spellChecker.clearIndex(); spellChecker.clearIndex();
addwords(r, "field1"); addwords(r, "field1");
int num_field1=this.numdoc(); int num_field1 = this.numdoc();
addwords(r, "field2"); addwords(r, "field2");
int num_field2=this.numdoc(); int num_field2 = this.numdoc();
this.assertTrue(num_field2==num_field1+1); assertEquals(num_field2, num_field1 + 1);
// test small word // test small word
String[] l=spellChecker.suggestSimilar("fvie", 2); String[] similar = spellChecker.suggestSimilar("fvie", 2);
this.assertTrue(l[0].equals("five")); assertEquals(similar[0], "five");
l=spellChecker.suggestSimilar("fiv", 2); similar = spellChecker.suggestSimilar("fiv", 2);
this.assertTrue(l[0].equals("five")); assertEquals(similar[0], "five");
l=spellChecker.suggestSimilar("ive", 2); similar = spellChecker.suggestSimilar("ive", 2);
this.assertTrue(l[0].equals("five")); assertEquals(similar[0], "five");
l=spellChecker.suggestSimilar("fives", 2); similar = spellChecker.suggestSimilar("fives", 2);
this.assertTrue(l[0].equals("five")); assertEquals(similar[0], "five");
l=spellChecker.suggestSimilar("fie", 2); similar = spellChecker.suggestSimilar("fie", 2);
this.assertTrue(l[0].equals("five")); assertEquals(similar[0], "five");
l=spellChecker.suggestSimilar("fi", 2); similar = spellChecker.suggestSimilar("fi", 2);
this.assertEquals(0,l.length); assertEquals(0, similar.length);
// test restreint to a field // test restraint to a field
l=spellChecker.suggestSimilar("tousand", 10, r, "field1", false); similar = spellChecker.suggestSimilar("tousand", 10, r, "field1", false);
this.assertEquals(0,l.length); // there isn't the term thousand in the field field1 assertEquals(0, similar.length); // there isn't the term thousand in the field field1
l=spellChecker.suggestSimilar("tousand", 10, r, "field2", false); similar = spellChecker.suggestSimilar("tousand", 10, r, "field2", false);
this.assertEquals(1,l.length); // there is the term thousand in the field field2 assertEquals(1, similar.length); // there is the term thousand in the field field2
} } catch (IOException e) {
catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
this.assertTrue(false); fail();
} }
} }
private void addwords (IndexReader r, String field) throws IOException { private void addwords(IndexReader r, String field) throws IOException {
long time=System.currentTimeMillis(); long time = System.currentTimeMillis();
spellChecker.indexDictionnary(new LuceneDictionary(r, field)); spellChecker.indexDictionnary(new LuceneDictionary(r, field));
time=System.currentTimeMillis()-time; time = System.currentTimeMillis() - time;
System.out.println("time to build "+field+": "+time); System.out.println("time to build " + field + ": " + time);
} }
private int numdoc() throws IOException {
private int numdoc () throws IOException { IndexReader rs = IndexReader.open(spellindex);
IndexReader rs=IndexReader.open(spellindex); int num = rs.numDocs();
int num=rs.numDocs(); assertTrue(num != 0);
this.assertTrue(num!=0); System.out.println("num docs: " + num);
System.out.println("num docs: "+num);
rs.close(); rs.close();
return num; return num;
} }
protected void tearDown() throws Exception {
protected void tearDown () throws Exception { spellChecker = null;
spellChecker=null;
super.tearDown(); super.tearDown();
} }