Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-8966: The Korean analyzer split tokens on boundaries between digits and alphabetic characters.

This commit is contained in:
jimczi 2019-09-13 09:57:21 +02:00
parent 3ed96026d3
commit c8f36238ab
3 changed files with 24 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lucene/CHANGES.txt
View File

@ -135,6 +135,8 @@ Improvements
* LUCENE-8976: Use exact distance between point and bounding rectangle in FloatPointNearestNeighbor. (Ignacio Vera)
* LUCENE-8966: The Korean analyzer now split tokens on boundaries between digits and alphabetic characters. (Jim Ferenczi)
Optimizations
* LUCENE-8922: DisjunctionMaxQuery more efficiently leverages impacts to skip

8
lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java
View File

@ -754,6 +754,7 @@ public final class KoreanTokenizer extends Tokenizer {
unknownWordLength = 1;
UnicodeScript scriptCode = UnicodeScript.of(firstCharacter);
final boolean isPunct = isPunctuation(firstCharacter);
final boolean isDigit = Character.isDigit(firstCharacter);
for (int posAhead = pos + 1; unknownWordLength < MAX_UNKNOWN_WORD_LENGTH; posAhead++) {
int next = buffer.get(posAhead);
if (next == -1) {
@ -768,8 +769,11 @@ public final class KoreanTokenizer extends Tokenizer {
|| chType == Character.NON_SPACING_MARK;
if (sameScript
&& isPunctuation(ch, chType) == isPunct
&& characterDefinition.isGroup(ch)) {
// split on punctuation
&& isPunctuation(ch, chType) == isPunct
// split on digit
&& Character.isDigit(ch) == isDigit
&& characterDefinition.isGroup(ch)) {
unknownWordLength++;
} else {
break;

16
lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/TestKoreanTokenizer.java
View File

@ -108,6 +108,22 @@ public class TestKoreanTokenizer extends BaseTokenStreamTestCase {
};
}
public void testSeparateNumber() throws IOException {
assertAnalyzesTo(analyzer, "44사이즈",
new String[]{"44", "사이즈"},
new int[]{0, 2},
new int[]{2, 5},
new int[]{1, 1}
);
assertAnalyzesTo(analyzer, "９.９사이즈",
new String[]{"９", "９", "사이즈"},
new int[]{0, 2, 3},
new int[]{1, 3, 6},
new int[]{1, 1, 1}
);
}
public void testSpaces() throws IOException {
assertAnalyzesTo(analyzer, "화학 이외의 것",
new String[]{"화학", "이외", "의", "것"},