Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-1490: fix latin1 conversion of HALFWIDTH_AND_FULLWIDTH_FORMS characters to only apply to the correct subset 

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@755666 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2009-03-18 17:28:53 +00:00
parent 68f4d8b1a8
commit 96863198a5
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
contrib/analyzers/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java
View File

@ -148,11 +148,13 @@ public final class CJKTokenizer extends Tokenizer {
|| (ub == Character.UnicodeBlock.HALFWIDTH_AND_FULLWIDTH_FORMS)
) {
if (ub == Character.UnicodeBlock.HALFWIDTH_AND_FULLWIDTH_FORMS) {
/** convert HALFWIDTH_AND_FULLWIDTH_FORMS to BASIC_LATIN */
int i = (int) c;
if (i >= 65281 && i <= 65374) {
/** convert certain HALFWIDTH_AND_FULLWIDTH_FORMS to BASIC_LATIN */
i = i - 65248;
c = (char) i;
}
}
// if the current character is a letter or "_" "+" "#"
if (Character.isLetterOrDigit(c)