Apache/lucene
1
0
Fork 0
mirror of https://github.com/apache/lucene.git synced 2025-02-18 07:55:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

LUCENE-1943: Improve performance of ChineseFilter

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@821322 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2009-10-03 13:54:12 +00:00
parent 891570478d
commit 1f9088b038
1 changed files with 9 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseFilter.java
View File

@ -18,9 +18,9 @@ package org.apache.lucene.analysis.cn;
*/
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Arrays;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
@ -56,33 +56,32 @@ public final class ChineseFilter extends TokenFilter {
};
private Map stopTable;
private CharArraySet stopTable;
private TermAttribute termAtt;
public ChineseFilter(TokenStream in) {
super(in);
stopTable = new HashMap(STOP_WORDS.length);
for (int i = 0; i < STOP_WORDS.length; i++)
stopTable.put(STOP_WORDS[i], STOP_WORDS[i]);
stopTable = new CharArraySet(Arrays.asList(STOP_WORDS), false);
termAtt = addAttribute(TermAttribute.class);
}
public boolean incrementToken() throws IOException {
while (input.incrementToken()) {
String text = termAtt.term();
char text[] = termAtt.termBuffer();
int termLength = termAtt.termLength();
// why not key off token type here assuming ChineseTokenizer comes first?
if (stopTable.get(text) == null) {
switch (Character.getType(text.charAt(0))) {
if (!stopTable.contains(text, 0, termLength)) {
switch (Character.getType(text[0])) {
case Character.LOWERCASE_LETTER:
case Character.UPPERCASE_LETTER:
// English word/token should larger than 1 character.
if (text.length()>1) {
if (termLength>1) {
return true;
}
break;