mirror of https://github.com/apache/lucene.git
prelim checking of spellchecker, v1.1
git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@151014 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
David Spencer
parent
a8d98638a0
commit
57cd076565
|
|
@ -0,0 +1,156 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
<project basedir="." default="rebuild" name="Spelling checker">
|
||||
|
||||
<property name="lucene.lib" value="d:/dev/lib/lucene.jar"/>
|
||||
<property name="lucenetest.lib" value="D:/dev/jakarta-lucene/build/classes/test"/>
|
||||
|
||||
|
||||
<property name="name" value="spellchecker"/>
|
||||
<property name="Name" value="spellchecker"/>
|
||||
<property name="version" value="1.1"/>
|
||||
<property name="year" value="2004"/>
|
||||
<property name="final.name" value="${name}-${version}"/>
|
||||
<property name="java" location="src/java"/>
|
||||
<property name="test" location="src/test"/>
|
||||
<property name="build.dir" location="build"/>
|
||||
<property name="build.java" location="${build.dir}/classes/java"/>
|
||||
<property name="build.test" location="${build.dir}/classes/test"/>
|
||||
<property name="build.javadocs" location="doc"/>
|
||||
<property name="javadoc.link" value="http://java.sun.com/j2se/1.4/docs/api/"/>
|
||||
<property name="javac.debug" value="off"/>
|
||||
<property name="junit.output.dir" location="${build.dir}/test"/>
|
||||
<property name="junit.reports" location="${build.dir}/test/reports"/>
|
||||
|
||||
|
||||
|
||||
<!-- Build classpath -->
|
||||
<path id="classpath">
|
||||
<pathelement location="${lucene.lib}"/>
|
||||
|
||||
<pathelement location="${build.java}"/>
|
||||
</path>
|
||||
|
||||
<path id="test.classpath">
|
||||
<path refid="classpath"/>
|
||||
<pathelement location="${lucenetest.lib}"/>
|
||||
<pathelement location="${build.dir}/classes/test"/>
|
||||
</path>
|
||||
<!--Patternset to exclude files from the output directory:-->
|
||||
|
||||
<!-- ================================================================== -->
|
||||
<!-- C O M P I L E -->
|
||||
<!-- ================================================================== -->
|
||||
<!-- -->
|
||||
<!-- ================================================================== -->
|
||||
<target name="javacompile"
|
||||
description="Compiles core classes">
|
||||
<mkdir dir="${build.java}"/>
|
||||
<javac
|
||||
srcdir="${java}"
|
||||
includes="**/*.java"
|
||||
destdir="${build.java}"
|
||||
debug="${javac.debug}"
|
||||
optimize="on">
|
||||
<classpath refid="classpath"/>
|
||||
</javac>
|
||||
</target>
|
||||
|
||||
<!-- ================================================================== -->
|
||||
<!-- J A R -->
|
||||
<!-- ================================================================== -->
|
||||
<!-- -->
|
||||
<!-- ================================================================== -->
|
||||
<target name="jar" depends="javacompile" description="Generates the Jar file">
|
||||
<jar
|
||||
destfile="${build.dir}/${final.name}.jar"
|
||||
basedir="${build.java}" />
|
||||
</target>
|
||||
|
||||
<!-- ================================================================== -->
|
||||
<!-- J A V A D O C -->
|
||||
<!-- ================================================================== -->
|
||||
<!-- -->
|
||||
<!-- ================================================================== -->
|
||||
<target name="javadoc">
|
||||
<mkdir dir="${build.javadocs}"/>
|
||||
<javadoc
|
||||
sourcepath="${java}"
|
||||
overview="src/java/overview.html"
|
||||
packagenames="org.apache.lucene.*"
|
||||
destdir="${build.javadocs}"
|
||||
author="true"
|
||||
version="true"
|
||||
use="true"
|
||||
link="${javadoc.link}"
|
||||
windowtitle="${Name} ${version} API"
|
||||
doctitle="${Name} ${version} API"
|
||||
bottom="Author: Nicolas Maisonneuve (${year})" >
|
||||
</javadoc>
|
||||
</target>
|
||||
|
||||
<!-- ================================================================== -->
|
||||
<!-- C L E A N -->
|
||||
<!-- ================================================================== -->
|
||||
<!-- -->
|
||||
<!-- ================================================================== -->
|
||||
<target name="clean">
|
||||
<delete failonerror="false" includeemptydirs="true">
|
||||
<fileset dir="${build.dir}"/>
|
||||
</delete>
|
||||
</target>
|
||||
|
||||
|
||||
<!-- ================================================================== -->
|
||||
<!-- B U I L D T E S T -->
|
||||
<!-- ================================================================== -->
|
||||
<!-- -->
|
||||
<!-- ================================================================== -->
|
||||
<target name="compile-test" depends="javacompile">
|
||||
<mkdir dir="${build.test}"/>
|
||||
<javac
|
||||
srcdir="${test}"
|
||||
includes="**/*.java"
|
||||
destdir="${build.test}"
|
||||
debug="true">
|
||||
<classpath refid="test.classpath"/>
|
||||
</javac>
|
||||
</target>
|
||||
|
||||
<!-- ================================================================== -->
|
||||
<!-- R U N T E S T S -->
|
||||
<!-- ================================================================== -->
|
||||
<!-- -->
|
||||
<!-- ================================================================== -->
|
||||
<target name="test" depends="compile-test" description="Runs unit tests">
|
||||
<fail unless="junit.present">
|
||||
##################################################################
|
||||
JUnit not found.
|
||||
Please make sure junit.jar is in ANT_HOME/lib, or made available
|
||||
to Ant using other mechanisms like -lib or CLASSPATH.
|
||||
##################################################################
|
||||
</fail>
|
||||
<mkdir dir="${junit.output.dir}"/>
|
||||
<junit printsummary="off" haltonfailure="no"
|
||||
errorProperty="tests.failed" failureProperty="tests.failed">
|
||||
<classpath refid="junit.classpath"/>
|
||||
<sysproperty key="dataDir" file="src/test"/>
|
||||
<sysproperty key="tempDir" file="${build.dir}/test"/>
|
||||
<formatter type="xml"/>
|
||||
<formatter type="brief" usefile="false"/>
|
||||
<batchtest fork="yes" todir="${junit.output.dir}" unless="testcase">
|
||||
<fileset dir="src/test" includes="**/Test*.java"/>
|
||||
</batchtest>
|
||||
<batchtest fork="yes" todir="${junit.output.dir}" if="testcase">
|
||||
<fileset dir="src/test" includes="**/${testcase}.java"/>
|
||||
</batchtest>
|
||||
</junit>
|
||||
|
||||
<fail if="tests.failed">Tests failed!</fail>
|
||||
</target>
|
||||
|
||||
<target depends="javacompile" name="make"/>
|
||||
|
||||
<target depends="clean,make" name="rebuild"/>
|
||||
|
||||
</project>
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
package org.apache.lucene.search.spell;
|
||||
/**
|
||||
* Copyright 2002-2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.util.Iterator;
|
||||
|
||||
/**
|
||||
* A simple interface representing a Dictionary
|
||||
* @author Nicolas Maisonneuve
|
||||
* @version 1.0
|
||||
*/
|
||||
public interface Dictionary {
|
||||
|
||||
/**
|
||||
* return all the words present in the dictionnary
|
||||
* @return Iterator
|
||||
*/
|
||||
public Iterator getWordsIterator();
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
package org.apache.lucene.search.spell;
|
||||
|
||||
/**
|
||||
* Copyright 2002-2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import java.util.Iterator;
|
||||
import org.apache.lucene.index.TermEnum;
|
||||
import org.apache.lucene.index.Term;
|
||||
import java.io.*;
|
||||
|
||||
/**
|
||||
* Lucene Dictionnary
|
||||
* @author Nicolas Maisonneuve
|
||||
*/
|
||||
public class LuceneDictionary
|
||||
implements Dictionary {
|
||||
IndexReader reader;
|
||||
String field;
|
||||
|
||||
public LuceneDictionary (IndexReader reader, String field) {
|
||||
this.reader=reader;
|
||||
this.field=field;
|
||||
|
||||
}
|
||||
|
||||
|
||||
public final Iterator getWordsIterator () {
|
||||
return new LuceneIterator();
|
||||
}
|
||||
|
||||
|
||||
final class LuceneIterator implements Iterator {
|
||||
private TermEnum enum;
|
||||
private Term actualTerm;
|
||||
private boolean has_next_called;
|
||||
|
||||
public LuceneIterator () {
|
||||
try {
|
||||
enum=reader.terms(new Term(field, ""));
|
||||
}
|
||||
catch (IOException ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public Object next () {
|
||||
if (!has_next_called) {hasNext();}
|
||||
has_next_called=false;
|
||||
return (actualTerm!=null) ? actualTerm.text(): null;
|
||||
}
|
||||
|
||||
|
||||
public boolean hasNext () {
|
||||
has_next_called=true;
|
||||
try {
|
||||
// if there is still words
|
||||
if (!enum.next()) {
|
||||
actualTerm=null;
|
||||
return false;
|
||||
}
|
||||
// if the next word are in the field
|
||||
actualTerm=enum.term();
|
||||
String fieldt=actualTerm.field();
|
||||
if (fieldt!=field) {
|
||||
actualTerm=null;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
catch (IOException ex) {
|
||||
ex.printStackTrace();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void remove () {};
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
package org.apache.lucene.search.spell;
|
||||
|
||||
/**
|
||||
* Copyright 2002-2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.io.InputStream;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.*;
|
||||
|
||||
|
||||
/**
|
||||
* dictionary represented by a file text
|
||||
* Format allowed: 1 word per line:
|
||||
* word1
|
||||
* word2
|
||||
* word3
|
||||
*
|
||||
* @author Nicolas Maisonneuve
|
||||
*/
|
||||
public class PlainTextDictionary implements Dictionary {
|
||||
|
||||
private BufferedReader in;
|
||||
private String line;
|
||||
private boolean has_next_called;
|
||||
|
||||
public PlainTextDictionary (File file) throws FileNotFoundException {
|
||||
in=new BufferedReader(new FileReader(file));
|
||||
}
|
||||
|
||||
|
||||
public PlainTextDictionary (InputStream dictFile) {
|
||||
in=new BufferedReader(new InputStreamReader(System.in));
|
||||
}
|
||||
|
||||
|
||||
public Iterator getWordsIterator () {
|
||||
|
||||
return new fileIterator();
|
||||
}
|
||||
|
||||
|
||||
final class fileIterator
|
||||
implements Iterator {
|
||||
public Object next () {
|
||||
if (!has_next_called) {
|
||||
hasNext();
|
||||
}
|
||||
has_next_called=false;
|
||||
return line;
|
||||
}
|
||||
|
||||
|
||||
public boolean hasNext () {
|
||||
has_next_called=true;
|
||||
try {
|
||||
line=in.readLine();
|
||||
}
|
||||
catch (IOException ex) {
|
||||
ex.printStackTrace();
|
||||
line=null;
|
||||
return false;
|
||||
}
|
||||
return (line!=null)?true:false;
|
||||
}
|
||||
|
||||
|
||||
public void remove () {};
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,363 @@
|
|||
package org.apache.lucene.search.spell;
|
||||
|
||||
|
||||
/**
|
||||
* Copyright 2002-2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.analysis.WhitespaceAnalyzer;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.document.Field;
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.index.IndexWriter;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.index.TermEnum;
|
||||
import org.apache.lucene.search.BooleanClause;
|
||||
import org.apache.lucene.search.BooleanQuery;
|
||||
import org.apache.lucene.search.Hits;
|
||||
import org.apache.lucene.search.IndexSearcher;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.TermQuery;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import java.util.*;
|
||||
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* Spell Checker class (Main class) <br/>
|
||||
* (initially inspired by the David Spencer code)
|
||||
* </p>
|
||||
*
|
||||
* <p>
|
||||
* Spell Checker spellchecker= new SpellChecker (spellDirectory);<br/>
|
||||
* <br/>
|
||||
* //To index a field of a user index <br/>
|
||||
* spellchecker.indexDictionary(new LuceneDictionary(my_lucene_reader, a_field));<br/>
|
||||
*<br/>
|
||||
* //To index a file containing words <br/>
|
||||
* spellchecker.indexDictionary(new PlainTextDictionary(new File("myfile.txt")));<br/>
|
||||
*</p>
|
||||
*
|
||||
* @author Nicolas Maisonneuve
|
||||
* @version 1.0
|
||||
*/
|
||||
public class SpellChecker {
|
||||
|
||||
/**
|
||||
* Field name for each word in the ngram index.
|
||||
*/
|
||||
public static final String F_WORD="word";
|
||||
|
||||
|
||||
/**
|
||||
* the spell index
|
||||
*/
|
||||
Directory spellindex;
|
||||
|
||||
/**
|
||||
* Boost value for start and end grams
|
||||
*/private float bStart=2.0f;
|
||||
private float bEnd=1.0f;
|
||||
|
||||
|
||||
private IndexReader reader;
|
||||
float min=0.5f;
|
||||
|
||||
public void setSpellIndex (Directory spellindex) {
|
||||
this.spellindex=spellindex;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Set the accuraty 0<min<1 default 0.5
|
||||
* @param min float
|
||||
*/
|
||||
public void setAccuraty (float min) {
|
||||
this.min=min;
|
||||
}
|
||||
|
||||
|
||||
public SpellChecker (Directory gramIndex) {
|
||||
this.setSpellIndex(gramIndex);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Suggest similar words
|
||||
* @param word String the word you want a spell check done on
|
||||
* @param num_sug int the number of suggest words
|
||||
* @throws IOException
|
||||
* @return String[]
|
||||
*/
|
||||
public String[] suggestSimilar (String word, int num_sug) throws IOException {
|
||||
return this.suggestSimilar(word, num_sug, null, null, false);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Suggest similar words (restricted or not of a field of a user index)
|
||||
* @param word String the word you want a spell check done on
|
||||
* @param num_sug int the number of suggest words
|
||||
* @param IndexReader the indexReader of the user index (can be null see field param)
|
||||
* @param field String the field of the user index: if field is not null ,the suggest
|
||||
* words are restricted to the words present in this field.
|
||||
* @param morePopular boolean return only the suggest words that are more frequent than the searched word
|
||||
* (only if restricted mode = (indexReader!=null and field!=null)
|
||||
* @throws IOException
|
||||
* @return String[] the sorted list of the suggest words with this 2 criteri
|
||||
* first criteria : the edit distance, second criteria (only if restricted mode): the popularity
|
||||
* of the suggest words in the field of the user index
|
||||
*/
|
||||
public String[] suggestSimilar (String word, int num_sug, IndexReader ir, String field
|
||||
, boolean morePopular) throws IOException {
|
||||
|
||||
final TRStringDistance sd=new TRStringDistance(word);
|
||||
final int lengthWord=word.length();
|
||||
|
||||
final int goalFreq=(morePopular&&ir!=null)?ir.docFreq(new Term(field, word)):0;
|
||||
if (!morePopular&&goalFreq>0) {
|
||||
return new String[] {
|
||||
word}; // return the word if it exist in the index and i don't want a more popular word
|
||||
}
|
||||
|
||||
BooleanQuery query=new BooleanQuery();
|
||||
String[] grams;
|
||||
String key;
|
||||
|
||||
for (int ng=getMin(lengthWord); ng<=getMax(lengthWord); ng++) {
|
||||
|
||||
key="gram"+ng; // form key
|
||||
|
||||
grams=formGrams(word, ng); // form word into ngrams (allow dups too)
|
||||
|
||||
if (grams.length==0) {
|
||||
continue; // hmm
|
||||
}
|
||||
|
||||
if (bStart>0) { // should we boost prefixes?
|
||||
add(query, "start"+ng, grams[0], bStart); // matches start of word
|
||||
|
||||
}
|
||||
if (bEnd>0) { // should we boost suffixes
|
||||
add(query, "end"+ng, grams[grams.length-1], bEnd); // matches end of word
|
||||
|
||||
}
|
||||
for (int i=0; i<grams.length; i++) {
|
||||
add(query, key, grams[i]);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
IndexSearcher searcher=new IndexSearcher(this.spellindex);
|
||||
Hits hits=searcher.search(query);
|
||||
SuggestWordQueue sugqueue=new SuggestWordQueue(num_sug);
|
||||
|
||||
int stop=Math.min(hits.length(), 10*num_sug); // go thru more than 'maxr' matches in case the distance filter triggers
|
||||
SuggestWord sugword=new SuggestWord();
|
||||
for (int i=0; i<stop; i++) {
|
||||
|
||||
sugword.string=hits.doc(i).get(F_WORD); // get orig word)
|
||||
|
||||
if (sugword.string==word) {
|
||||
continue; // don't suggest a word for itself, that would be silly
|
||||
}
|
||||
|
||||
//edit distance/normalize with the min word length
|
||||
sugword.score=1.0f-((float) sd.getDistance(sugword.string)/Math.min(sugword.string.length(), lengthWord));
|
||||
if (sugword.score<min) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ir!=null) { // use the user index
|
||||
sugword.freq=ir.docFreq(new Term(field, sugword.string)); // freq in the index
|
||||
if ((morePopular&&goalFreq>sugword.freq)||sugword.freq<1) { // don't suggest a word that is not present in the field
|
||||
continue;
|
||||
}
|
||||
}
|
||||
sugqueue.insert(sugword);
|
||||
if (sugqueue.size()==num_sug) {
|
||||
//if queue full , maintain the min score
|
||||
min=((SuggestWord) sugqueue.top()).score;
|
||||
}
|
||||
sugword=new SuggestWord();
|
||||
}
|
||||
|
||||
// convert to array string
|
||||
String[] list=new String[sugqueue.size()];
|
||||
for (int i=sugqueue.size()-1; i>=0; i--) {
|
||||
list[i]=((SuggestWord) sugqueue.pop()).string;
|
||||
}
|
||||
|
||||
searcher.close();
|
||||
return list;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Add a clause to a boolean query.
|
||||
*/
|
||||
private static void add (BooleanQuery q, String k, String v, float boost) {
|
||||
Query tq=new TermQuery(new Term(k, v));
|
||||
tq.setBoost(boost);
|
||||
q.add(new BooleanClause(tq, false, false));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Add a clause to a boolean query.
|
||||
*/
|
||||
private static void add (BooleanQuery q, String k, String v) {
|
||||
q.add(new BooleanClause(new TermQuery(new Term(k, v)), false, false));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Form all ngrams for a given word.
|
||||
* @param text the word to parse
|
||||
* @param ng the ngram length e.g. 3
|
||||
* @return an array of all ngrams in the word and note that duplicates are not removed
|
||||
*/
|
||||
private static String[] formGrams (String text, int ng) {
|
||||
int len=text.length();
|
||||
String[] res=new String[len-ng+1];
|
||||
for (int i=0; i<len-ng+1; i++) {
|
||||
res[i]=text.substring(i, i+ng);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
public void clearIndex () throws IOException {
|
||||
IndexReader.unlock(spellindex);
|
||||
IndexWriter writer=new IndexWriter(spellindex, null, true);
|
||||
writer.close();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* if the word exist in the index
|
||||
* @param word String
|
||||
* @throws IOException
|
||||
* @return boolean
|
||||
*/
|
||||
public boolean exist (String word) throws IOException {
|
||||
if (reader==null) {
|
||||
reader=IndexReader.open(spellindex);
|
||||
}
|
||||
return reader.docFreq(new Term(F_WORD, word))>0;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Index a Dictionnary
|
||||
* @param dict the dictionnary to index
|
||||
* @throws IOException
|
||||
*/
|
||||
public void indexDictionnary (Dictionary dict) throws IOException {
|
||||
|
||||
int ng1, ng2;
|
||||
IndexReader.unlock(spellindex);
|
||||
IndexWriter writer=new IndexWriter(spellindex, new WhitespaceAnalyzer(), !IndexReader.indexExists(spellindex));
|
||||
writer.mergeFactor=300;
|
||||
writer.minMergeDocs=150;
|
||||
|
||||
Iterator iter=dict.getWordsIterator();
|
||||
while (iter.hasNext()) {
|
||||
String word=(String) iter.next();
|
||||
|
||||
int len=word.length();
|
||||
if (len<3) {
|
||||
continue; // too short we bail but "too long" is fine...
|
||||
}
|
||||
|
||||
if (this.exist(word)) { // if the word already exist in the gramindex
|
||||
continue;
|
||||
}
|
||||
|
||||
// ok index the word
|
||||
Document doc=createDocument(word, getMin(len), getMax(len));
|
||||
writer.addDocument(doc);
|
||||
}
|
||||
// close writer
|
||||
writer.optimize();
|
||||
writer.close();
|
||||
|
||||
// close reader
|
||||
reader.close();
|
||||
reader=null;
|
||||
}
|
||||
|
||||
|
||||
private int getMin (int l) {
|
||||
if (l>5) {
|
||||
return 3;
|
||||
}
|
||||
if (l==5) {
|
||||
return 2;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
private int getMax (int l) {
|
||||
if (l>5) {
|
||||
return 4;
|
||||
}
|
||||
if (l==5) {
|
||||
return 3;
|
||||
}
|
||||
return 2;
|
||||
|
||||
}
|
||||
|
||||
|
||||
private static Document createDocument (String text, int ng1, int ng2) {
|
||||
Document doc=new Document();
|
||||
doc.add(Field.Keyword(F_WORD, text)); // orig term
|
||||
addGram(text, doc, ng1, ng2);
|
||||
return doc;
|
||||
}
|
||||
|
||||
|
||||
private static void addGram (String text, Document doc, int ng1, int ng2) {
|
||||
int len=text.length();
|
||||
for (int ng=ng1; ng<=ng2; ng++) {
|
||||
String key="gram"+ng;
|
||||
String end=null;
|
||||
for (int i=0; i<len-ng+1; i++) {
|
||||
String gram=text.substring(i, i+ng);
|
||||
doc.add(Field.Keyword(key, gram));
|
||||
if (i==0) {
|
||||
doc.add(Field.Keyword("start"+ng, gram));
|
||||
}
|
||||
end=gram;
|
||||
}
|
||||
if (end!=null) { // may not be present if len==ng1
|
||||
doc.add(Field.Keyword("end"+ng, end));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected void finalize () throws Throwable {
|
||||
if (reader!=null) {
|
||||
reader.close();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
package org.apache.lucene.search.spell;
|
||||
|
||||
|
||||
/**
|
||||
* Copyright 2002-2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* SuggestWord Class
|
||||
* used in suggestSimilat method in SpellChecker class
|
||||
* @author Nicolas Maisonneuve
|
||||
*/
|
||||
final class SuggestWord {
|
||||
/**
|
||||
* the score of the word
|
||||
*/
|
||||
public float score;
|
||||
|
||||
|
||||
/**
|
||||
* The freq of the word
|
||||
*/
|
||||
public int freq;
|
||||
|
||||
|
||||
/**
|
||||
* the suggested word
|
||||
*/
|
||||
public String string;
|
||||
|
||||
|
||||
public final int compareTo (SuggestWord a) {
|
||||
//first criteria: the edit distance
|
||||
if (score>a.score) {
|
||||
return 1;
|
||||
}
|
||||
if (score<a.score) {
|
||||
return-1;
|
||||
}
|
||||
|
||||
//second criteria (if first criteria is equal): the popularity
|
||||
if (freq>a.freq) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (freq<a.freq) {
|
||||
return-1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
package org.apache.lucene.search.spell;
|
||||
|
||||
|
||||
/**
|
||||
* Copyright 2002-2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* to sort SuggestWord
|
||||
* @author Nicolas Maisonneuve
|
||||
*/
|
||||
import org.apache.lucene.util.PriorityQueue;
|
||||
|
||||
|
||||
final class SuggestWordQueue
|
||||
extends PriorityQueue {
|
||||
|
||||
SuggestWordQueue (int size) {
|
||||
initialize(size);
|
||||
}
|
||||
|
||||
protected final boolean lessThan (Object a, Object b) {
|
||||
SuggestWord wa=(SuggestWord) a;
|
||||
SuggestWord wb=(SuggestWord) b;
|
||||
int val=wa.compareTo(wb);
|
||||
return val<0;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,132 @@
|
|||
package org.apache.lucene.search.spell;
|
||||
|
||||
|
||||
/**
|
||||
* Copyright 2002-2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Edit distance class
|
||||
*/
|
||||
public final class TRStringDistance {
|
||||
|
||||
final char[] sa;
|
||||
final int n;
|
||||
final int[][][] cache=new int[30][][];
|
||||
|
||||
|
||||
/**
|
||||
* Optimized to run a bit faster than the static getDistance().
|
||||
* In one benchmark times were 5.3sec using ctr vs 8.5sec w/ static method, thus 37% faster.
|
||||
*/
|
||||
public TRStringDistance (String target) {
|
||||
sa=target.toCharArray();
|
||||
n=sa.length;
|
||||
}
|
||||
|
||||
|
||||
//*****************************
|
||||
// Compute Levenshtein distance
|
||||
//*****************************
|
||||
public final int getDistance (String other) {
|
||||
int d[][]; // matrix
|
||||
int cost; // cost
|
||||
|
||||
// Step 1
|
||||
final char[] ta=other.toCharArray();
|
||||
final int m=ta.length;
|
||||
if (n==0) {
|
||||
return m;
|
||||
}
|
||||
if (m==0) {
|
||||
return n;
|
||||
}
|
||||
|
||||
if (m>=cache.length) {
|
||||
d=form(n, m);
|
||||
}
|
||||
else if (cache[m]!=null) {
|
||||
d=cache[m];
|
||||
}
|
||||
else {
|
||||
d=cache[m]=form(n, m);
|
||||
|
||||
// Step 3
|
||||
|
||||
}
|
||||
for (int i=1; i<=n; i++) {
|
||||
final char s_i=sa[i-1];
|
||||
|
||||
// Step 4
|
||||
|
||||
for (int j=1; j<=m; j++) {
|
||||
final char t_j=ta[j-1];
|
||||
|
||||
// Step 5
|
||||
|
||||
if (s_i==t_j) { // same
|
||||
cost=0;
|
||||
}
|
||||
else { // not a match
|
||||
cost=1;
|
||||
|
||||
// Step 6
|
||||
|
||||
}
|
||||
d[i][j]=min3(d[i-1][j]+1, d[i][j-1]+1, d[i-1][j-1]+cost);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Step 7
|
||||
return d[n][m];
|
||||
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
private static int[][] form (int n, int m) {
|
||||
int[][] d=new int[n+1][m+1];
|
||||
// Step 2
|
||||
|
||||
for (int i=0; i<=n; i++) {
|
||||
d[i][0]=i;
|
||||
|
||||
}
|
||||
for (int j=0; j<=m; j++) {
|
||||
d[0][j]=j;
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
|
||||
//****************************
|
||||
// Get minimum of three values
|
||||
//****************************
|
||||
private static int min3 (int a, int b, int c) {
|
||||
int mi=a;
|
||||
if (b<mi) {
|
||||
mi=b;
|
||||
}
|
||||
if (c<mi) {
|
||||
mi=c;
|
||||
}
|
||||
return mi;
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
package org.apache.lucene.search.spell;
|
||||
|
||||
|
||||
import junit.framework.*;
|
||||
import org.apache.lucene.search.spell.*;
|
||||
import org.apache.lucene.store.RAMDirectory;
|
||||
import org.apache.lucene.index.IndexWriter;
|
||||
import org.apache.lucene.analysis.SimpleAnalyzer;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.util.English;
|
||||
import org.apache.lucene.document.Field;
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import java.io.IOException;
|
||||
import org.apache.lucene.store.FSDirectory;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import java.io.File;
|
||||
|
||||
|
||||
/**
|
||||
* Test case
|
||||
* @author Nicolas Maisonneuve
|
||||
*/
|
||||
|
||||
public class TestSpellChecker
|
||||
extends TestCase {
|
||||
private SpellChecker spellChecker;
|
||||
Directory userindex, spellindex;
|
||||
|
||||
protected void setUp () throws Exception {
|
||||
super.setUp();
|
||||
|
||||
//create a user index
|
||||
userindex=new RAMDirectory();
|
||||
IndexWriter writer=new IndexWriter(userindex, new SimpleAnalyzer(), true);
|
||||
|
||||
for (int i=0; i<1000; i++) {
|
||||
Document doc=new Document();
|
||||
doc.add(Field.Text("field1", English.intToEnglish(i)));
|
||||
doc.add(Field.Text("field2", English.intToEnglish(i+1))); // + word thousand
|
||||
writer.addDocument(doc);
|
||||
}
|
||||
writer.close();
|
||||
|
||||
// create the spellChecker
|
||||
File file=new File("d://test");
|
||||
spellindex=FSDirectory.getDirectory(file, true);
|
||||
spellChecker=new SpellChecker(spellindex);
|
||||
}
|
||||
|
||||
|
||||
public void testBuild () {
|
||||
try {
|
||||
IndexReader r=IndexReader.open(userindex);
|
||||
|
||||
spellChecker.clearIndex();
|
||||
|
||||
addwords(r, "field1");
|
||||
int num_field1=this.numdoc();
|
||||
|
||||
addwords(r, "field2");
|
||||
int num_field2=this.numdoc();
|
||||
|
||||
this.assertTrue(num_field2==num_field1+1);
|
||||
|
||||
// test small word
|
||||
String[] l=spellChecker.suggestSimilar("fvie", 2);
|
||||
this.assertTrue(l[0].equals("five"));
|
||||
|
||||
l=spellChecker.suggestSimilar("fiv", 2);
|
||||
this.assertTrue(l[0].equals("five"));
|
||||
|
||||
l=spellChecker.suggestSimilar("ive", 2);
|
||||
this.assertTrue(l[0].equals("five"));
|
||||
|
||||
l=spellChecker.suggestSimilar("fives", 2);
|
||||
this.assertTrue(l[0].equals("five"));
|
||||
|
||||
l=spellChecker.suggestSimilar("fie", 2);
|
||||
this.assertTrue(l[0].equals("five"));
|
||||
|
||||
l=spellChecker.suggestSimilar("fi", 2);
|
||||
this.assertEquals(0,l.length);
|
||||
|
||||
// test restreint to a field
|
||||
l=spellChecker.suggestSimilar("tousand", 10, r, "field1", false);
|
||||
this.assertEquals(0,l.length); // there isn't the term thousand in the field field1
|
||||
|
||||
l=spellChecker.suggestSimilar("tousand", 10, r, "field2", false);
|
||||
this.assertEquals(1,l.length); // there is the term thousand in the field field2
|
||||
}
|
||||
catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
this.assertTrue(false);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
private void addwords (IndexReader r, String field) throws IOException {
|
||||
long time=System.currentTimeMillis();
|
||||
spellChecker.indexDictionnary(new LuceneDictionary(r, field));
|
||||
time=System.currentTimeMillis()-time;
|
||||
System.out.println("time to build "+field+": "+time);
|
||||
}
|
||||
|
||||
|
||||
private int numdoc () throws IOException {
|
||||
IndexReader rs=IndexReader.open(spellindex);
|
||||
int num=rs.numDocs();
|
||||
this.assertTrue(num!=0);
|
||||
System.out.println("num docs: "+num);
|
||||
rs.close();
|
||||
return num;
|
||||
}
|
||||
|
||||
|
||||
protected void tearDown () throws Exception {
|
||||
spellChecker=null;
|
||||
super.tearDown();
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue