Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

convert Hashtable to Set, to avoid unnecessary synchronization issues 

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@150225 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Erik Hatcher 2004-03-10 00:18:02 +00:00
parent c436cd8914
commit c230388a92
2 changed files with 110 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
src/java/org/apache/lucene/analysis/StopFilter.java
View File

@ -55,31 +55,55 @@ package org.apache.lucene.analysis;
*/
import java.io.IOException;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Set;
/** Removes stop words from a token stream. */
/**
* Removes stop words from a token stream.
*/
public final class StopFilter extends TokenFilter {
private Hashtable table;
private Set table;
/** Constructs a filter which removes words from the input
TokenStream that are named in the array of words. */
/**
* Constructs a filter which removes words from the input
* TokenStream that are named in the array of words.
*/
public StopFilter(TokenStream in, String[] stopWords) {
super(in);
table = makeStopTable(stopWords);
table = makeStopSet(stopWords);
}
/** Constructs a filter which removes words from the input
TokenStream that are named in the Hashtable. */
/**
* Constructs a filter which removes words from the input
* TokenStream that are named in the Hashtable.
*
* @deprecated Use {@link #StopFilter(TokenStream, Set)} StopFilter(TokenStream,Map)} instead
*/
public StopFilter(TokenStream in, Hashtable stopTable) {
super(in);
table = stopTable.keySet();
}
/**
* Constructs a filter which removes words from the input
* TokenStream that are named in the Set.
*/
public StopFilter(TokenStream in, Set stopTable) {
super(in);
table = stopTable;
}
/** Builds a Hashtable from an array of stop words, appropriate for passing
into the StopFilter constructor. This permits this table construction to
be cached once when an Analyzer is constructed. */
/**
* Builds a Hashtable from an array of stop words,
* appropriate for passing into the StopFilter constructor.
* This permits this table construction to be cached once when
* an Analyzer is constructed.
*
* @deprecated Use {@link #makeStopSet(String[] makeStopSet) instead.
*/
public static final Hashtable makeStopTable(String[] stopWords) {
Hashtable stopTable = new Hashtable(stopWords.length);
for (int i = 0; i < stopWords.length; i++)
@ -87,11 +111,26 @@ public final class StopFilter extends TokenFilter {
return stopTable;
}
/** Returns the next input Token whose termText() is not a stop word. */
/**
* Builds a Set from an array of stop words,
* appropriate for passing into the StopFilter constructor.
* This permits this table construction to be cached once when
* an Analyzer is constructed.
*/
public static final Set makeStopSet(String[] stopWords) {
Set stopTable = new HashSet(stopWords.length);
for (int i = 0; i < stopWords.length; i++)
stopTable.add(stopWords[i]);
return stopTable;
}
/**
* Returns the next input Token whose termText() is not a stop word.
*/
public final Token next() throws IOException {
// return the first non-stop word found
for (Token token = input.next(); token != null; token = input.next())
if (table.get(token.termText) == null)
if (!table.contains(token.termText))
return token;
// reached EOS -- return null
return null;

119
src/java/org/apache/lucene/analysis/de/WordlistLoader.java
View File

@ -68,71 +68,70 @@ import java.util.Hashtable;
* @author Gerhard Schwarz
* @version $Id$
*/
public class WordlistLoader
{
/**
* @param path Path to the wordlist
* @param wordfile Name of the wordlist
*/
public static Hashtable getWordtable( String path, String wordfile ) {
if ( path == null || wordfile == null ) {
return new Hashtable();
}
return getWordtable( new File( path, wordfile ) );
public class WordlistLoader {
/**
* @param path Path to the wordlist
* @param wordfile Name of the wordlist
*/
public static Hashtable getWordtable(String path, String wordfile) {
if (path == null || wordfile == null) {
return new Hashtable();
}
return getWordtable(new File(path, wordfile));
}
/**
* @param wordfile Complete path to the wordlist
*/
public static Hashtable getWordtable( String wordfile ) {
if ( wordfile == null ) {
return new Hashtable();
}
return getWordtable( new File( wordfile ) );
/**
* @param wordfile Complete path to the wordlist
*/
public static Hashtable getWordtable(String wordfile) {
if (wordfile == null) {
return new Hashtable();
}
return getWordtable(new File(wordfile));
}
/**
* @param wordfile File containing the wordlist
*/
public static Hashtable getWordtable( File wordfile ) {
if ( wordfile == null ) {
return new Hashtable();
}
Hashtable result = null;
try {
LineNumberReader lnr = new LineNumberReader( new FileReader( wordfile ) );
String word = null;
String[] stopwords = new String[100];
int wordcount = 0;
while ( ( word = lnr.readLine() ) != null ) {
wordcount++;
if ( wordcount == stopwords.length ) {
String[] tmp = new String[stopwords.length + 50];
System.arraycopy( stopwords, 0, tmp, 0, wordcount );
stopwords = tmp;
}
stopwords[wordcount-1] = word;
}
result = makeWordTable( stopwords, wordcount );
}
// On error, use an empty table
catch ( IOException e ) {
result = new Hashtable();
}
return result;
/**
* @param wordfile File containing the wordlist
*/
public static Hashtable getWordtable(File wordfile) {
if (wordfile == null) {
return new Hashtable();
}
Hashtable result = null;
try {
LineNumberReader lnr = new LineNumberReader(new FileReader(wordfile));
String word = null;
String[] stopwords = new String[100];
int wordcount = 0;
while ((word = lnr.readLine()) != null) {
wordcount++;
if (wordcount == stopwords.length) {
String[] tmp = new String[stopwords.length + 50];
System.arraycopy(stopwords, 0, tmp, 0, wordcount);
stopwords = tmp;
}
stopwords[wordcount - 1] = word;
}
result = makeWordTable(stopwords, wordcount);
}
// On error, use an empty table
catch (IOException e) {
result = new Hashtable();
}
return result;
}
/**
* Builds the wordlist table.
*
* @param words Word that where read
* @param length Amount of words that where read into <tt>words</tt>
*/
private static Hashtable makeWordTable( String[] words, int length ) {
Hashtable table = new Hashtable( length );
for ( int i = 0; i < length; i++ ) {
table.put( words[i], words[i] );
}
return table;
/**
* Builds the wordlist table.
*
* @param words Word that where read
* @param length Amount of words that where read into <tt>words</tt>
*/
private static Hashtable makeWordTable(String[] words, int length) {
Hashtable table = new Hashtable(length);
for (int i = 0; i < length; i++) {
table.put(words[i], words[i]);
}
return table;
}
}