honeymoose/ik-analyzer-solr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: honeymoose:master
Branches Tags
honeymoose:master
honeymoose:dev
honeymoose:v8.5.0
honeymoose:v8.4.0
honeymoose:v8.3.1
honeymoose:v8.3.0
honeymoose:v8.1.1
honeymoose:v8.1.0
honeymoose:v8.0.0
honeymoose:v7.7.1
honeymoose:v7.7.0
honeymoose:v7.6.0
honeymoose:v7.5.0
..
compare: honeymoose:v8.5.0
Branches Tags
honeymoose:master
honeymoose:dev
honeymoose:v8.5.0
honeymoose:v8.4.0
honeymoose:v8.3.1
honeymoose:v8.3.0
honeymoose:v8.1.1
honeymoose:v8.1.0
honeymoose:v8.0.0
honeymoose:v7.7.1
honeymoose:v7.7.0
honeymoose:v7.6.0
honeymoose:v7.5.0

No commits in common. "master" and "v8.5.0" have entirely different histories.
master ... v8.5.0

22 changed files with 918 additions and 998 deletions
Show Stats Expand all files Collapse all files

1
README.md
View File

@ -6,6 +6,7 @@ ik-analyzer for solr 7.x-8.x
[![Open Source Love](https://badges.frapsoft.com/os/v1/open-source.svg?v=103)](https://github.com/magese/ik-analyzer-solr/releases) [![Open Source Love](https://badges.frapsoft.com/os/v1/open-source.svg?v=103)](https://github.com/magese/ik-analyzer-solr/releases)
[![Crates.io](https://img.shields.io/crates/l/rustc-serialize.svg)](./LICENSE) [![Crates.io](https://img.shields.io/crates/l/rustc-serialize.svg)](./LICENSE)
[![Build Status](https://travis-ci.org/magese/ik-analyzer-solr.svg?branch=master)](https://travis-ci.org/magese/ik-analyzer-solr) [![Build Status](https://travis-ci.org/magese/ik-analyzer-solr.svg?branch=master)](https://travis-ci.org/magese/ik-analyzer-solr)
[![HitCount](http://hits.dwyl.io/magese/ik-analyzer-solr.svg)](http://hits.dwyl.io/magese/ik-analyzer-solr)
[![GitHub forks](https://img.shields.io/github/forks/magese/ik-analyzer-solr.svg?style=social&label=Fork)](https://github.com/magese/ik-analyzer-solr/network/members) [![GitHub forks](https://img.shields.io/github/forks/magese/ik-analyzer-solr.svg?style=social&label=Fork)](https://github.com/magese/ik-analyzer-solr/network/members)
[![GitHub stars](https://img.shields.io/github/stars/magese/ik-analyzer-solr.svg?style=social&label=Star)](https://github.com/magese/ik-analyzer-solr/stargazers) [![GitHub stars](https://img.shields.io/github/stars/magese/ik-analyzer-solr.svg?style=social&label=Star)](https://github.com/magese/ik-analyzer-solr/stargazers)

2
src/main/java/org/wltea/analyzer/cfg/Configuration.java
View File

@ -76,7 +76,7 @@ public interface Configuration {
* *
* @return String 量词词典路径 * @return String 量词词典路径
*/ */
String getQuantifierDictionary(); String getQuantifierDicionary();
/** /**
* 获取扩展字典配置路径 * 获取扩展字典配置路径

2
src/main/java/org/wltea/analyzer/cfg/DefaultConfig.java
View File

@ -145,7 +145,7 @@ public class DefaultConfig implements Configuration {
* *
* @return String 量词词典路径 * @return String 量词词典路径
*/ */
public String getQuantifierDictionary() { public String getQuantifierDicionary() {
return PATH_DIC_QUANTIFIER; return PATH_DIC_QUANTIFIER;
} }

19
src/main/java/org/wltea/analyzer/core/CJKSegmenter.java
View File

@ -27,12 +27,12 @@
*/ */
package org.wltea.analyzer.core; package org.wltea.analyzer.core;
import org.wltea.analyzer.dic.Dictionary;
import org.wltea.analyzer.dic.Hit;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import org.wltea.analyzer.dic.Dictionary;
import org.wltea.analyzer.dic.Hit;
/** /**
* 中文-日韩文子分词器 * 中文-日韩文子分词器
@ -42,7 +42,7 @@ class CJKSegmenter implements ISegmenter {
//子分词器标签 //子分词器标签
private static final String SEGMENTER_NAME = "CJK_SEGMENTER"; private static final String SEGMENTER_NAME = "CJK_SEGMENTER";
//待处理的分词hit队列 //待处理的分词hit队列
private final List<Hit> tmpHits; private List<Hit> tmpHits;
CJKSegmenter(){ CJKSegmenter(){
@ -80,19 +80,20 @@ class CJKSegmenter implements ISegmenter {
//********************************* //*********************************
//再对当前指针位置的字符进行单字匹配 //再对当前指针位置的字符进行单字匹配
Hit singleCharHit = Dictionary.getSingleton().matchInMainDict(context.getSegmentBuff(), context.getCursor(), 1); Hit singleCharHit = Dictionary.getSingleton().matchInMainDict(context.getSegmentBuff(), context.getCursor(), 1);
if(singleCharHit.isMatch()){//首字成词
// 首字为词前缀
if (singleCharHit.isMatch()) {
//输出当前的词 //输出当前的词
Lexeme newLexeme = new Lexeme(context.getBufferOffset() , context.getCursor() , 1 , Lexeme.TYPE_CNWORD); Lexeme newLexeme = new Lexeme(context.getBufferOffset() , context.getCursor() , 1 , Lexeme.TYPE_CNWORD);
context.addLexeme(newLexeme); context.addLexeme(newLexeme);
}
// 前缀匹配则放入hit列表 //同时也是词前缀
if(singleCharHit.isPrefix()){ if(singleCharHit.isPrefix()){
//前缀匹配则放入hit列表 //前缀匹配则放入hit列表
this.tmpHits.add(singleCharHit); this.tmpHits.add(singleCharHit);
} }
}else if(singleCharHit.isPrefix()){//首字为词前缀
//前缀匹配则放入hit列表
this.tmpHits.add(singleCharHit);
}
}else{ }else{

27
src/main/java/org/wltea/analyzer/core/CN_QuantifierSegmenter.java
View File

@ -36,6 +36,7 @@ import java.util.List;
import java.util.Set; import java.util.Set;
/** /**
*
* 中文数量词子分词器 * 中文数量词子分词器
*/ */
class CN_QuantifierSegmenter implements ISegmenter{ class CN_QuantifierSegmenter implements ISegmenter{
@ -43,14 +44,14 @@ class CN_QuantifierSegmenter implements ISegmenter {
//子分词器标签 //子分词器标签
private static final String SEGMENTER_NAME = "QUAN_SEGMENTER"; private static final String SEGMENTER_NAME = "QUAN_SEGMENTER";
private static final Set<Character> CHN_NUMBER_CHARS = new HashSet<>(); private static Set<Character> ChnNumberChars = new HashSet<>();
static{ static{
//中文数词 //中文数词
//Cnum
String chn_Num = "一二两三四五六七八九十零壹贰叁肆伍陆柒捌玖拾百千万亿拾佰仟萬億兆卅廿"; String chn_Num = "一二两三四五六七八九十零壹贰叁肆伍陆柒捌玖拾百千万亿拾佰仟萬億兆卅廿";
char[] ca = chn_Num.toCharArray(); char[] ca = chn_Num.toCharArray();
for(char nChar : ca){ for(char nChar : ca){
CHN_NUMBER_CHARS.add(nChar); ChnNumberChars.add(nChar);
} }
} }
@ -67,7 +68,7 @@ class CN_QuantifierSegmenter implements ISegmenter {
private int nEnd; private int nEnd;
//待处理的量词hit队列 //待处理的量词hit队列
private final List<Hit> countHits; private List<Hit> countHits;
CN_QuantifierSegmenter(){ CN_QuantifierSegmenter(){
@ -110,14 +111,14 @@ class CN_QuantifierSegmenter implements ISegmenter {
private void processCNumber(AnalyzeContext context){ private void processCNumber(AnalyzeContext context){
if(nStart == -1 && nEnd == -1){//初始状态 if(nStart == -1 && nEnd == -1){//初始状态
if(CharacterUtil.CHAR_CHINESE == context.getCurrentCharType() if(CharacterUtil.CHAR_CHINESE == context.getCurrentCharType()
&& CHN_NUMBER_CHARS.contains(context.getCurrentChar())) { && ChnNumberChars.contains(context.getCurrentChar())){
//记录数词的起始、结束位置 //记录数词的起始、结束位置
nStart = context.getCursor(); nStart = context.getCursor();
nEnd = context.getCursor(); nEnd = context.getCursor();
} }
}else{//正在处理状态 }else{//正在处理状态
if(CharacterUtil.CHAR_CHINESE == context.getCurrentCharType() if(CharacterUtil.CHAR_CHINESE == context.getCurrentCharType()
&& CHN_NUMBER_CHARS.contains(context.getCurrentChar())) { && ChnNumberChars.contains(context.getCurrentChar())){
//记录数词的结束位置 //记录数词的结束位置
nEnd = context.getCursor(); nEnd = context.getCursor();
}else{ }else{
@ -143,7 +144,6 @@ class CN_QuantifierSegmenter implements ISegmenter {
/** /**
* 处理中文量词 * 处理中文量词
*
* @param context 需要处理的内容 * @param context 需要处理的内容
*/ */
private void processCount(AnalyzeContext context){ private void processCount(AnalyzeContext context){
@ -179,19 +179,21 @@ class CN_QuantifierSegmenter implements ISegmenter {
//********************************* //*********************************
//对当前指针位置的字符进行单字匹配 //对当前指针位置的字符进行单字匹配
Hit singleCharHit = Dictionary.getSingleton().matchInQuantifierDict(context.getSegmentBuff(), context.getCursor(), 1); Hit singleCharHit = Dictionary.getSingleton().matchInQuantifierDict(context.getSegmentBuff(), context.getCursor(), 1);
if(singleCharHit.isMatch()){//首字成量词词
// 首字为量词前缀
if (singleCharHit.isMatch()) {
//输出当前的词 //输出当前的词
Lexeme newLexeme = new Lexeme(context.getBufferOffset() , context.getCursor() , 1 , Lexeme.TYPE_COUNT); Lexeme newLexeme = new Lexeme(context.getBufferOffset() , context.getCursor() , 1 , Lexeme.TYPE_COUNT);
context.addLexeme(newLexeme); context.addLexeme(newLexeme);
}
// 前缀匹配则放入hit列表 //同时也是词前缀
if(singleCharHit.isPrefix()){ if(singleCharHit.isPrefix()){
//前缀匹配则放入hit列表 //前缀匹配则放入hit列表
this.countHits.add(singleCharHit); this.countHits.add(singleCharHit);
} }
}else if(singleCharHit.isPrefix()){//首字为量词前缀
//前缀匹配则放入hit列表
this.countHits.add(singleCharHit);
}
}else{ }else{
//输入的不是中文字符 //输入的不是中文字符
@ -227,7 +229,6 @@ class CN_QuantifierSegmenter implements ISegmenter {
/** /**
* 添加数词词元到结果集 * 添加数词词元到结果集
*
* @param context 需要添加的词元 * @param context 需要添加的词元
*/ */
private void outputNumLexeme(AnalyzeContext context){ private void outputNumLexeme(AnalyzeContext context){

3
src/main/java/org/wltea/analyzer/core/CharacterUtil.java
View File

@ -28,6 +28,7 @@
package org.wltea.analyzer.core; package org.wltea.analyzer.core;
/** /**
*
* 字符集识别工具类 * 字符集识别工具类
*/ */
class CharacterUtil { class CharacterUtil {
@ -45,7 +46,6 @@ class CharacterUtil {
/** /**
* 识别字符类型 * 识别字符类型
*
* @param input 需要识别的字符 * @param input 需要识别的字符
* @return int CharacterUtil定义的字符类型常量 * @return int CharacterUtil定义的字符类型常量
*/ */
@ -85,7 +85,6 @@ class CharacterUtil {
/** /**
* 进行字符规格化（全角转半角，大写转小写处理） * 进行字符规格化（全角转半角，大写转小写处理）
*
* @param input 需要转换的字符 * @param input 需要转换的字符
* @return char * @return char
*/ */

4
src/main/java/org/wltea/analyzer/core/IKArbitrator.java
View File

@ -35,7 +35,9 @@ import java.util.TreeSet;
*/ */
class IKArbitrator { class IKArbitrator {
IKArbitrator() {} IKArbitrator() {
}
/** /**
* 分词歧义处理 * 分词歧义处理

22
src/main/java/org/wltea/analyzer/core/IKSegmenter.java
View File

@ -41,25 +41,15 @@ import java.util.List;
*/ */
public final class IKSegmenter { public final class IKSegmenter {
/** //字符窜reader
* 字符窜reader
*/
private Reader input; private Reader input;
/** //分词器配置项
* 分词器配置项 private Configuration cfg;
*/ //分词器上下文
private final Configuration cfg;
/**
* 分词器上下文
*/
private AnalyzeContext context; private AnalyzeContext context;
/** //分词处理器列表
* 分词处理器列表
*/
private List<ISegmenter> segmenters; private List<ISegmenter> segmenters;
/** //分词歧义裁决器
* 分词歧义裁决器
*/
private IKArbitrator arbitrator; private IKArbitrator arbitrator;

2
src/main/java/org/wltea/analyzer/core/ISegmenter.java
View File

@ -29,13 +29,13 @@ package org.wltea.analyzer.core;
/** /**
*
* 子分词器接口 * 子分词器接口
*/ */
interface ISegmenter { interface ISegmenter {
/** /**
* 从分析器读取下一个可能分解的词元对象 * 从分析器读取下一个可能分解的词元对象
*
* @param context 分词算法上下文 * @param context 分词算法上下文
*/ */
void analyze(AnalyzeContext context); void analyze(AnalyzeContext context);

18
src/main/java/org/wltea/analyzer/core/LetterSegmenter.java
View File

@ -34,18 +34,14 @@ import java.util.Arrays;
*/ */
class LetterSegmenter implements ISegmenter { class LetterSegmenter implements ISegmenter {
/** //子分词器标签
* 子分词器标签
*/
private static final String SEGMENTER_NAME = "LETTER_SEGMENTER"; private static final String SEGMENTER_NAME = "LETTER_SEGMENTER";
/** //链接符号
* 链接符号
*/
private static final char[] Letter_Connector = new char[]{'#', '&', '+', '-', '.', '@', '_'}; private static final char[] Letter_Connector = new char[]{'#', '&', '+', '-', '.', '@', '_'};
/**
* 数字符号 //数字符号
*/
private static final char[] Num_Connector = new char[]{',', '.'}; private static final char[] Num_Connector = new char[]{',', '.'};
/* /*
* 词元的开始位置， * 词元的开始位置，
* 同时作为子分词器状态标识 * 同时作为子分词器状态标识
@ -57,18 +53,22 @@ class LetterSegmenter implements ISegmenter {
* end记录的是在词元中最后一个出现的Letter但非Sign_Connector的字符的位置 * end记录的是在词元中最后一个出现的Letter但非Sign_Connector的字符的位置
*/ */
private int end; private int end;
/* /*
* 字母起始位置 * 字母起始位置
*/ */
private int englishStart; private int englishStart;
/* /*
* 字母结束位置 * 字母结束位置
*/ */
private int englishEnd; private int englishEnd;
/* /*
* 阿拉伯数字起始位置 * 阿拉伯数字起始位置
*/ */
private int arabicStart; private int arabicStart;
/* /*
* 阿拉伯数字结束位置 * 阿拉伯数字结束位置
*/ */

72
src/main/java/org/wltea/analyzer/core/Lexeme.java
View File

@ -32,61 +32,34 @@ package org.wltea.analyzer.core;
*/ */
@SuppressWarnings("unused") @SuppressWarnings("unused")
public class Lexeme implements Comparable<Lexeme>{ public class Lexeme implements Comparable<Lexeme>{
/** //英文
* 英文
*/
static final int TYPE_ENGLISH = 1; static final int TYPE_ENGLISH = 1;
/** //数字
* 数字
*/
static final int TYPE_ARABIC = 2; static final int TYPE_ARABIC = 2;
/** //英文数字混合
* 英文数字混合
*/
static final int TYPE_LETTER = 3; static final int TYPE_LETTER = 3;
/** //中文词元
* 中文词元
*/
static final int TYPE_CNWORD = 4; static final int TYPE_CNWORD = 4;
/** //中文单字
* 中文单字
*/
static final int TYPE_CNCHAR = 64; static final int TYPE_CNCHAR = 64;
/** //日韩文字
* 日韩文字
*/
static final int TYPE_OTHER_CJK = 8; static final int TYPE_OTHER_CJK = 8;
/** //中文数词
* 中文数词
*/
static final int TYPE_CNUM = 16; static final int TYPE_CNUM = 16;
/** //中文量词
* 中文量词
*/
static final int TYPE_COUNT = 32; static final int TYPE_COUNT = 32;
/** //中文数量词
* 中文数量词
*/
static final int TYPE_CQUAN = 48; static final int TYPE_CQUAN = 48;
/**
* 词元的起始位移 //词元的起始位移
*/
private int offset; private int offset;
/** //词元的相对起始位置
* 词元的相对起始位置
*/
private int begin; private int begin;
/** //词元的长度
* 词元的长度
*/
private int length; private int length;
/** //词元文本
* 词元文本
*/
private String lexemeText; private String lexemeText;
/** //词元类型
* 词元类型
*/
private int lexemeType; private int lexemeType;
@ -147,7 +120,7 @@ public class Lexeme implements Comparable<Lexeme> {
//this.length < other.getLength() //this.length < other.getLength()
return Integer.compare(other.getLength(), this.length); return Integer.compare(other.getLength(), this.length);
} else { }else{//this.begin > other.getBegin()
return 1; return 1;
} }
} }
@ -163,10 +136,8 @@ public class Lexeme implements Comparable<Lexeme> {
int getBegin() { int getBegin() {
return begin; return begin;
} }
/** /**
* 获取词元在文本中的起始位置 * 获取词元在文本中的起始位置
*
* @return int * @return int
*/ */
public int getBeginPosition(){ public int getBeginPosition(){
@ -179,7 +150,6 @@ public class Lexeme implements Comparable<Lexeme> {
/** /**
* 获取词元在文本中的结束位置 * 获取词元在文本中的结束位置
*
* @return int * @return int
*/ */
public int getEndPosition(){ public int getEndPosition(){
@ -188,7 +158,6 @@ public class Lexeme implements Comparable<Lexeme> {
/** /**
* 获取词元的字符长度 * 获取词元的字符长度
*
* @return int * @return int
*/ */
public int getLength(){ public int getLength(){
@ -204,7 +173,6 @@ public class Lexeme implements Comparable<Lexeme> {
/** /**
* 获取词元的文本内容 * 获取词元的文本内容
*
* @return String * @return String
*/ */
public String getLexemeText() { public String getLexemeText() {
@ -226,7 +194,6 @@ public class Lexeme implements Comparable<Lexeme> {
/** /**
* 获取词元类型 * 获取词元类型
*
* @return int * @return int
*/ */
int getLexemeType() { int getLexemeType() {
@ -235,7 +202,6 @@ public class Lexeme implements Comparable<Lexeme> {
/** /**
* 获取词元类型标示字符串 * 获取词元类型标示字符串
*
* @return String * @return String
*/ */
public String getLexemeTypeString(){ public String getLexemeTypeString(){
@ -269,7 +235,7 @@ public class Lexeme implements Comparable<Lexeme> {
return "TYPE_CQUAN"; return "TYPE_CQUAN";
default : default :
return "UNKNOWN"; return "UNKONW";
} }
} }
@ -280,7 +246,6 @@ public class Lexeme implements Comparable<Lexeme> {
/** /**
* 合并两个相邻的词元 * 合并两个相邻的词元
*
* @return boolean 词元是否成功合并 * @return boolean 词元是否成功合并
*/ */
boolean append(Lexeme l, int lexemeType){ boolean append(Lexeme l, int lexemeType){
@ -293,10 +258,9 @@ public class Lexeme implements Comparable<Lexeme> {
} }
} }
/** /**
* ToString 方法
* *
* @return 字符串输出
*/ */
public String toString(){ public String toString(){
return this.getBeginPosition() + "-" + this.getEndPosition() + return this.getBeginPosition() + "-" + this.getEndPosition() +

36
src/main/java/org/wltea/analyzer/core/LexemePath.java
View File

@ -34,17 +34,11 @@ package org.wltea.analyzer.core;
@SuppressWarnings("unused") @SuppressWarnings("unused")
class LexemePath extends QuickSortSet implements Comparable<LexemePath> { class LexemePath extends QuickSortSet implements Comparable<LexemePath> {
/** //起始位置
* 起始位置
*/
private int pathBegin; private int pathBegin;
/** //结束
* 结束
*/
private int pathEnd; private int pathEnd;
/** //词元链的有效字符长度
* 词元链的有效字符长度
*/
private int payloadLength; private int payloadLength;
LexemePath() { LexemePath() {
@ -106,6 +100,7 @@ class LexemePath extends QuickSortSet implements Comparable<LexemePath> {
/** /**
* 移除尾部的Lexeme * 移除尾部的Lexeme
*
*/ */
void removeTail() { void removeTail() {
Lexeme tail = this.pollLast(); Lexeme tail = this.pollLast();
@ -122,6 +117,7 @@ class LexemePath extends QuickSortSet implements Comparable<LexemePath> {
/** /**
* 检测词元位置交叉（有歧义的切分） * 检测词元位置交叉（有歧义的切分）
*
*/ */
boolean checkCross(Lexeme lexeme) { boolean checkCross(Lexeme lexeme) {
return (lexeme.getBegin() >= this.pathBegin && lexeme.getBegin() < this.pathEnd) return (lexeme.getBegin() >= this.pathBegin && lexeme.getBegin() < this.pathEnd)
@ -145,6 +141,7 @@ class LexemePath extends QuickSortSet implements Comparable<LexemePath> {
/** /**
* 获取LexemePath的路径长度 * 获取LexemePath的路径长度
*
*/ */
private int getPathLength() { private int getPathLength() {
return this.pathEnd - this.pathBegin; return this.pathEnd - this.pathBegin;
@ -153,6 +150,7 @@ class LexemePath extends QuickSortSet implements Comparable<LexemePath> {
/** /**
* X权重（词元长度积） * X权重（词元长度积）
*
*/ */
private int getXWeight() { private int getXWeight() {
int product = 1; int product = 1;
@ -198,36 +196,31 @@ class LexemePath extends QuickSortSet implements Comparable<LexemePath> {
return -1; return -1;
} else if (this.payloadLength < o.payloadLength) { } else if (this.payloadLength < o.payloadLength) {
return 1; return 1;
} } else {
//比较词元个数，越少越好 //比较词元个数，越少越好
if (this.size() < o.size()) { if (this.size() < o.size()) {
return -1; return -1;
} else if (this.size() > o.size()) { } else if (this.size() > o.size()) {
return 1; return 1;
} } else {
//路径跨度越大越好 //路径跨度越大越好
if (this.getPathLength() > o.getPathLength()) { if (this.getPathLength() > o.getPathLength()) {
return -1; return -1;
} else if (this.getPathLength() < o.getPathLength()) { } else if (this.getPathLength() < o.getPathLength()) {
return 1; return 1;
} } else {
//根据统计学结论，逆向切分概率高于正向切分，因此位置越靠后的优先 //根据统计学结论，逆向切分概率高于正向切分，因此位置越靠后的优先
if (this.pathEnd > o.pathEnd) { if (this.pathEnd > o.pathEnd) {
return -1; return -1;
} else if (pathEnd < o.pathEnd) { } else if (pathEnd < o.pathEnd) {
return 1; return 1;
} } else {
//词长越平均越好 //词长越平均越好
if (this.getXWeight() > o.getXWeight()) { if (this.getXWeight() > o.getXWeight()) {
return -1; return -1;
} else if (this.getXWeight() < o.getXWeight()) { } else if (this.getXWeight() < o.getXWeight()) {
return 1; return 1;
} } else {
//词元位置权重比较 //词元位置权重比较
if (this.getPWeight() > o.getPWeight()) { if (this.getPWeight() > o.getPWeight()) {
return -1; return -1;
@ -235,6 +228,11 @@ class LexemePath extends QuickSortSet implements Comparable<LexemePath> {
return 1; return 1;
} }
}
}
}
}
}
return 0; return 0;
} }

28
src/main/java/org/wltea/analyzer/core/QuickSortSet.java
View File

@ -28,20 +28,14 @@
package org.wltea.analyzer.core; package org.wltea.analyzer.core;
/** /**
* IK分词器专用的Lexeme快速排序集合 * IK分词器专用的Lexem快速排序集合
*/ */
class QuickSortSet { class QuickSortSet {
/** //链表头
* 链表头
*/
private Cell head; private Cell head;
/** //链表尾
* 链表尾
*/
private Cell tail; private Cell tail;
/** //链表的实际大小
* 链表的实际大小
*/
private int size; private int size;
QuickSortSet() { QuickSortSet() {
@ -59,15 +53,16 @@ class QuickSortSet {
this.size++; this.size++;
} else { } else {
if (this.tail.compareTo(newCell) < 0) { /*if(this.tail.compareTo(newCell) == 0){//词元与尾部词元相同，不放入集合
// 词元接入链表尾部
}else */
if (this.tail.compareTo(newCell) < 0) {//词元接入链表尾部
this.tail.next = newCell; this.tail.next = newCell;
newCell.prev = this.tail; newCell.prev = this.tail;
this.tail = newCell; this.tail = newCell;
this.size++; this.size++;
} else if (this.head.compareTo(newCell) > 0) { } else if (this.head.compareTo(newCell) > 0) {//词元接入链表头部
// 词元接入链表头部
this.head.prev = newCell; this.head.prev = newCell;
newCell.next = this.head; newCell.next = this.head;
this.head = newCell; this.head = newCell;
@ -79,9 +74,10 @@ class QuickSortSet {
while (index != null && index.compareTo(newCell) > 0) { while (index != null && index.compareTo(newCell) > 0) {
index = index.prev; index = index.prev;
} }
/*if(index.compareTo(newCell) == 0){//词元与集合中的词元重复，不放入集合
// 词元插入链表中的某个位置 }else */
if ((index != null ? index.compareTo(newCell) : 1) < 0) { if ((index != null ? index.compareTo(newCell) : 1) < 0) {//词元插入链表中的某个位置
newCell.prev = index; newCell.prev = index;
newCell.next = index.next; newCell.next = index.next;
index.next.prev = newCell; index.next.prev = newCell;

36
src/main/java/org/wltea/analyzer/dic/DictSegment.java
View File

@ -37,38 +37,24 @@ import java.util.Map;
@SuppressWarnings("unused") @SuppressWarnings("unused")
class DictSegment implements Comparable<DictSegment> { class DictSegment implements Comparable<DictSegment> {
/** //公用字典表，存储汉字
* 公用字典表，存储汉字
*/
private static final Map<Character, Character> charMap = new HashMap<>(16, 0.95f); private static final Map<Character, Character> charMap = new HashMap<>(16, 0.95f);
/** //数组大小上限
* 数组大小上限
*/
private static final int ARRAY_LENGTH_LIMIT = 3; private static final int ARRAY_LENGTH_LIMIT = 3;
/** //Map存储结构
* Map存储结构 private Map<Character, DictSegment> childrenMap;
*/ //数组方式存储结构
private volatile Map<Character, DictSegment> childrenMap; private DictSegment[] childrenArray;
/**
* 数组方式存储结构
*/
private volatile DictSegment[] childrenArray;
/** //当前节点上存储的字符
* 当前节点上存储的字符 private Character nodeChar;
*/ //当前节点存储的Segment数目
private final Character nodeChar; //storeSize <=ARRAY_LENGTH_LIMIT ，使用数组存储， storeSize >ARRAY_LENGTH_LIMIT ,则使用Map存储
/**
* 当前节点存储的Segment数目
* storeSize <=ARRAY_LENGTH_LIMIT ，使用数组存储， storeSize >ARRAY_LENGTH_LIMIT ,则使用Map存储
*/
private int storeSize = 0; private int storeSize = 0;
/** //当前DictSegment状态 ,默认 0 , 1表示从根节点到当前节点的路径表示一个词
* 当前DictSegment状态 ,默认 0 , 1表示从根节点到当前节点的路径表示一个词
*/
private int nodeState = 0; private int nodeState = 0;

12
src/main/java/org/wltea/analyzer/dic/Dictionary.java
View File

@ -27,14 +27,14 @@
*/ */
package org.wltea.analyzer.dic; package org.wltea.analyzer.dic;
import org.wltea.analyzer.cfg.Configuration;
import org.wltea.analyzer.cfg.DefaultConfig;
import java.io.*; import java.io.*;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.util.Collection; import java.util.Collection;
import java.util.List; import java.util.List;
import org.wltea.analyzer.cfg.Configuration;
import org.wltea.analyzer.cfg.DefaultConfig;
/** /**
* 词典管理类，单例模式 * 词典管理类，单例模式
*/ */
@ -44,7 +44,7 @@ public class Dictionary {
/* /*
* 词典单子实例 * 词典单子实例
*/ */
private static volatile Dictionary singleton; private static Dictionary singleton;
/* /*
* 主词典对象 * 主词典对象
@ -63,7 +63,7 @@ public class Dictionary {
/** /**
* 配置对象 * 配置对象
*/ */
private final Configuration cfg; private Configuration cfg;
/** /**
* 私有构造方法，阻止外部直接实例化本类 * 私有构造方法，阻止外部直接实例化本类
@ -326,7 +326,7 @@ public class Dictionary {
// 建立一个量词典实例 // 建立一个量词典实例
_QuantifierDict = new DictSegment((char) 0); _QuantifierDict = new DictSegment((char) 0);
// 读取量词词典文件 // 读取量词词典文件
InputStream is = this.getClass().getClassLoader().getResourceAsStream(cfg.getQuantifierDictionary()); InputStream is = this.getClass().getClassLoader().getResourceAsStream(cfg.getQuantifierDicionary());
if (is == null) { if (is == null) {
throw new RuntimeException("Quantifier Dictionary not found!!!"); throw new RuntimeException("Quantifier Dictionary not found!!!");
} }

29
src/main/java/org/wltea/analyzer/dic/Hit.java
View File

@ -32,33 +32,24 @@ package org.wltea.analyzer.dic;
*/ */
@SuppressWarnings("unused") @SuppressWarnings("unused")
public class Hit { public class Hit {
/** //Hit不匹配
* Hit不匹配
*/
private static final int UNMATCH = 0x00000000; private static final int UNMATCH = 0x00000000;
/** //Hit完全匹配
* Hit完全匹配
*/
private static final int MATCH = 0x00000001; private static final int MATCH = 0x00000001;
/** //Hit前缀匹配
* Hit前缀匹配
*/
private static final int PREFIX = 0x00000010; private static final int PREFIX = 0x00000010;
/** //该HIT当前状态，默认未匹配
* 该HIT当前状态，默认未匹配
*/
private int hitState = UNMATCH; private int hitState = UNMATCH;
/**
* 记录词典匹配过程中，当前匹配到的词典分支节点 //记录词典匹配过程中，当前匹配到的词典分支节点
*/
private DictSegment matchedDictSegment; private DictSegment matchedDictSegment;
/** /*
* 词段开始位置 * 词段开始位置
*/ */
private int begin; private int begin;
/** /*
* 词段的结束位置 * 词段的结束位置
*/ */
private int end; private int end;
@ -95,7 +86,9 @@ public class Hit {
public boolean isUnmatch() { public boolean isUnmatch() {
return this.hitState == UNMATCH ; return this.hitState == UNMATCH ;
} }
/**
*
*/
void setUnmatch() { void setUnmatch() {
this.hitState = UNMATCH; this.hitState = UNMATCH;
} }

2
src/main/java/org/wltea/analyzer/lucene/IKAnalyzer.java
View File

@ -36,7 +36,7 @@ import org.apache.lucene.analysis.Tokenizer;
@SuppressWarnings("unused") @SuppressWarnings("unused")
public final class IKAnalyzer extends Analyzer { public final class IKAnalyzer extends Analyzer {
private final boolean useSmart; private boolean useSmart;
private boolean useSmart() { private boolean useSmart() {
return useSmart; return useSmart;

26
src/main/java/org/wltea/analyzer/lucene/IKTokenizer.java
View File

@ -39,30 +39,21 @@ import java.io.IOException;
/** /**
* IK分词器 Lucene Tokenizer适配器类 * IK分词器 Lucene Tokenizer适配器类
* 兼容Lucene 4.0版本
*/ */
@SuppressWarnings({"unused", "FinalMethodInFinalClass"}) @SuppressWarnings("unused")
public final class IKTokenizer extends Tokenizer { public final class IKTokenizer extends Tokenizer {
/** //IK分词器实现
* IK分词器实现
*/
private IKSegmenter _IKImplement; private IKSegmenter _IKImplement;
/** //词元文本属性
* 词元文本属性
*/
private CharTermAttribute termAtt; private CharTermAttribute termAtt;
/** //词元位移属性
* 词元位移属性
*/
private OffsetAttribute offsetAtt; private OffsetAttribute offsetAtt;
/** //词元分类属性（该属性分类参考org.wltea.analyzer.core.Lexeme中的分类常量）
* 词元分类属性（该属性分类参考org.wltea.analyzer.core.Lexeme中的分类常量）
*/
private TypeAttribute typeAtt; private TypeAttribute typeAtt;
/** //记录最后一个词元的结束位置
* 记录最后一个词元的结束位置
*/
private int endPosition; private int endPosition;
/** /**
@ -93,8 +84,7 @@ public final class IKTokenizer extends Tokenizer {
_IKImplement = new IKSegmenter(input, useSmart); _IKImplement = new IKSegmenter(input, useSmart);
} }
/* /* (non-Javadoc)
* (non-Javadoc)
* @see org.apache.lucene.analysis.TokenStream#incrementToken() * @see org.apache.lucene.analysis.TokenStream#incrementToken()
*/ */
@Override @Override

6
src/main/java/org/wltea/analyzer/lucene/IKTokenizerFactory.java
View File

@ -21,8 +21,8 @@
* 版权声明 2012，乌龙茶工作室 * 版权声明 2012，乌龙茶工作室
* provided by Linliangyi and copyright 2012 by Oolong studio * provided by Linliangyi and copyright 2012 by Oolong studio
* *
* 8.5.0版本 由 Magese (magese@live.cn) 更新 * 8.3.1版本 由 Magese (magese@live.cn) 更新
* release 8.5.0 update by Magese(magese@live.cn) * release 8.3.1 update by Magese(magese@live.cn)
* *
*/ */
package org.wltea.analyzer.lucene; package org.wltea.analyzer.lucene;
@ -44,8 +44,6 @@ import java.nio.charset.StandardCharsets;
import java.util.*; import java.util.*;
/** /**
* 分词器工厂类
*
* @author <a href="magese@live.cn">Magese</a> * @author <a href="magese@live.cn">Magese</a>
*/ */
public class IKTokenizerFactory extends TokenizerFactory implements ResourceLoaderAware, UpdateThread.UpdateJob { public class IKTokenizerFactory extends TokenizerFactory implements ResourceLoaderAware, UpdateThread.UpdateJob {

304
src/main/java/org/wltea/analyzer/query/IKQueryExpressionParser.java
View File

@ -46,11 +46,11 @@ import java.util.Stack;
public class IKQueryExpressionParser { public class IKQueryExpressionParser {
private final List<Element> elements = new ArrayList<>(); private List<Element> elements = new ArrayList<>();
private final Stack<Query> querys = new Stack<>(); private Stack<Query> querys = new Stack<>();
private final Stack<Element> operates = new Stack<>(); private Stack<Element> operates = new Stack<>();
/** /**
* 解析查询表达式，生成Lucene Query对象 * 解析查询表达式，生成Lucene Query对象
@ -87,263 +87,263 @@ public class IKQueryExpressionParser {
if (expression == null) { if (expression == null) {
return; return;
} }
Element currentElement = null; Element curretElement = null;
char[] expChars = expression.toCharArray(); char[] expChars = expression.toCharArray();
for (char expChar : expChars) { for (char expChar : expChars) {
switch (expChar) { switch (expChar) {
case '&': case '&':
if (currentElement == null) { if (curretElement == null) {
currentElement = new Element(); curretElement = new Element();
currentElement.type = '&'; curretElement.type = '&';
currentElement.append(expChar); curretElement.append(expChar);
} else if (currentElement.type == '&') { } else if (curretElement.type == '&') {
currentElement.append(expChar); curretElement.append(expChar);
this.elements.add(currentElement); this.elements.add(curretElement);
currentElement = null; curretElement = null;
} else if (currentElement.type == '\'') { } else if (curretElement.type == '\'') {
currentElement.append(expChar); curretElement.append(expChar);
} else { } else {
this.elements.add(currentElement); this.elements.add(curretElement);
currentElement = new Element(); curretElement = new Element();
currentElement.type = '&'; curretElement.type = '&';
currentElement.append(expChar); curretElement.append(expChar);
} }
break; break;
case '|': case '|':
if (currentElement == null) { if (curretElement == null) {
currentElement = new Element(); curretElement = new Element();
currentElement.type = '|'; curretElement.type = '|';
currentElement.append(expChar); curretElement.append(expChar);
} else if (currentElement.type == '|') { } else if (curretElement.type == '|') {
currentElement.append(expChar); curretElement.append(expChar);
this.elements.add(currentElement); this.elements.add(curretElement);
currentElement = null; curretElement = null;
} else if (currentElement.type == '\'') { } else if (curretElement.type == '\'') {
currentElement.append(expChar); curretElement.append(expChar);
} else { } else {
this.elements.add(currentElement); this.elements.add(curretElement);
currentElement = new Element(); curretElement = new Element();
currentElement.type = '|'; curretElement.type = '|';
currentElement.append(expChar); curretElement.append(expChar);
} }
break; break;
case '-': case '-':
if (currentElement != null) { if (curretElement != null) {
if (currentElement.type == '\'') { if (curretElement.type == '\'') {
currentElement.append(expChar); curretElement.append(expChar);
continue; continue;
} else { } else {
this.elements.add(currentElement); this.elements.add(curretElement);
} }
} }
currentElement = new Element(); curretElement = new Element();
currentElement.type = '-'; curretElement.type = '-';
currentElement.append(expChar); curretElement.append(expChar);
this.elements.add(currentElement); this.elements.add(curretElement);
currentElement = null; curretElement = null;
break; break;
case '(': case '(':
if (currentElement != null) { if (curretElement != null) {
if (currentElement.type == '\'') { if (curretElement.type == '\'') {
currentElement.append(expChar); curretElement.append(expChar);
continue; continue;
} else { } else {
this.elements.add(currentElement); this.elements.add(curretElement);
} }
} }
currentElement = new Element(); curretElement = new Element();
currentElement.type = '('; curretElement.type = '(';
currentElement.append(expChar); curretElement.append(expChar);
this.elements.add(currentElement); this.elements.add(curretElement);
currentElement = null; curretElement = null;
break; break;
case ')': case ')':
if (currentElement != null) { if (curretElement != null) {
if (currentElement.type == '\'') { if (curretElement.type == '\'') {
currentElement.append(expChar); curretElement.append(expChar);
continue; continue;
} else { } else {
this.elements.add(currentElement); this.elements.add(curretElement);
} }
} }
currentElement = new Element(); curretElement = new Element();
currentElement.type = ')'; curretElement.type = ')';
currentElement.append(expChar); curretElement.append(expChar);
this.elements.add(currentElement); this.elements.add(curretElement);
currentElement = null; curretElement = null;
break; break;
case ':': case ':':
if (currentElement != null) { if (curretElement != null) {
if (currentElement.type == '\'') { if (curretElement.type == '\'') {
currentElement.append(expChar); curretElement.append(expChar);
continue; continue;
} else { } else {
this.elements.add(currentElement); this.elements.add(curretElement);
} }
} }
currentElement = new Element(); curretElement = new Element();
currentElement.type = ':'; curretElement.type = ':';
currentElement.append(expChar); curretElement.append(expChar);
this.elements.add(currentElement); this.elements.add(curretElement);
currentElement = null; curretElement = null;
break; break;
case '=': case '=':
if (currentElement != null) { if (curretElement != null) {
if (currentElement.type == '\'') { if (curretElement.type == '\'') {
currentElement.append(expChar); curretElement.append(expChar);
continue; continue;
} else { } else {
this.elements.add(currentElement); this.elements.add(curretElement);
} }
} }
currentElement = new Element(); curretElement = new Element();
currentElement.type = '='; curretElement.type = '=';
currentElement.append(expChar); curretElement.append(expChar);
this.elements.add(currentElement); this.elements.add(curretElement);
currentElement = null; curretElement = null;
break; break;
case ' ': case ' ':
if (currentElement != null) { if (curretElement != null) {
if (currentElement.type == '\'') { if (curretElement.type == '\'') {
currentElement.append(expChar); curretElement.append(expChar);
} else { } else {
this.elements.add(currentElement); this.elements.add(curretElement);
currentElement = null; curretElement = null;
} }
} }
break; break;
case '\'': case '\'':
if (currentElement == null) { if (curretElement == null) {
currentElement = new Element(); curretElement = new Element();
currentElement.type = '\''; curretElement.type = '\'';
} else if (currentElement.type == '\'') { } else if (curretElement.type == '\'') {
this.elements.add(currentElement); this.elements.add(curretElement);
currentElement = null; curretElement = null;
} else { } else {
this.elements.add(currentElement); this.elements.add(curretElement);
currentElement = new Element(); curretElement = new Element();
currentElement.type = '\''; curretElement.type = '\'';
} }
break; break;
case '[': case '[':
if (currentElement != null) { if (curretElement != null) {
if (currentElement.type == '\'') { if (curretElement.type == '\'') {
currentElement.append(expChar); curretElement.append(expChar);
continue; continue;
} else { } else {
this.elements.add(currentElement); this.elements.add(curretElement);
} }
} }
currentElement = new Element(); curretElement = new Element();
currentElement.type = '['; curretElement.type = '[';
currentElement.append(expChar); curretElement.append(expChar);
this.elements.add(currentElement); this.elements.add(curretElement);
currentElement = null; curretElement = null;
break; break;
case ']': case ']':
if (currentElement != null) { if (curretElement != null) {
if (currentElement.type == '\'') { if (curretElement.type == '\'') {
currentElement.append(expChar); curretElement.append(expChar);
continue; continue;
} else { } else {
this.elements.add(currentElement); this.elements.add(curretElement);
} }
} }
currentElement = new Element(); curretElement = new Element();
currentElement.type = ']'; curretElement.type = ']';
currentElement.append(expChar); curretElement.append(expChar);
this.elements.add(currentElement); this.elements.add(curretElement);
currentElement = null; curretElement = null;
break; break;
case '{': case '{':
if (currentElement != null) { if (curretElement != null) {
if (currentElement.type == '\'') { if (curretElement.type == '\'') {
currentElement.append(expChar); curretElement.append(expChar);
continue; continue;
} else { } else {
this.elements.add(currentElement); this.elements.add(curretElement);
} }
} }
currentElement = new Element(); curretElement = new Element();
currentElement.type = '{'; curretElement.type = '{';
currentElement.append(expChar); curretElement.append(expChar);
this.elements.add(currentElement); this.elements.add(curretElement);
currentElement = null; curretElement = null;
break; break;
case '}': case '}':
if (currentElement != null) { if (curretElement != null) {
if (currentElement.type == '\'') { if (curretElement.type == '\'') {
currentElement.append(expChar); curretElement.append(expChar);
continue; continue;
} else { } else {
this.elements.add(currentElement); this.elements.add(curretElement);
} }
} }
currentElement = new Element(); curretElement = new Element();
currentElement.type = '}'; curretElement.type = '}';
currentElement.append(expChar); curretElement.append(expChar);
this.elements.add(currentElement); this.elements.add(curretElement);
currentElement = null; curretElement = null;
break; break;
case ',': case ',':
if (currentElement != null) { if (curretElement != null) {
if (currentElement.type == '\'') { if (curretElement.type == '\'') {
currentElement.append(expChar); curretElement.append(expChar);
continue; continue;
} else { } else {
this.elements.add(currentElement); this.elements.add(curretElement);
} }
} }
currentElement = new Element(); curretElement = new Element();
currentElement.type = ','; curretElement.type = ',';
currentElement.append(expChar); curretElement.append(expChar);
this.elements.add(currentElement); this.elements.add(curretElement);
currentElement = null; curretElement = null;
break; break;
default: default:
if (currentElement == null) { if (curretElement == null) {
currentElement = new Element(); curretElement = new Element();
currentElement.type = 'F'; curretElement.type = 'F';
currentElement.append(expChar); curretElement.append(expChar);
} else if (currentElement.type == 'F') { } else if (curretElement.type == 'F') {
currentElement.append(expChar); curretElement.append(expChar);
} else if (currentElement.type == '\'') { } else if (curretElement.type == '\'') {
currentElement.append(expChar); curretElement.append(expChar);
} else { } else {
this.elements.add(currentElement); this.elements.add(curretElement);
currentElement = new Element(); curretElement = new Element();
currentElement.type = 'F'; curretElement.type = 'F';
currentElement.append(expChar); curretElement.append(expChar);
} }
} }
} }
if (currentElement != null) { if (curretElement != null) {
this.elements.add(currentElement); this.elements.add(curretElement);
} }
} }
@ -673,7 +673,7 @@ public class IKQueryExpressionParser {
* @author linliangyi * @author linliangyi
* May 20, 2010 * May 20, 2010
*/ */
private static class Element { private class Element {
char type = 0; char type = 0;
StringBuffer eleTextBuff; StringBuffer eleTextBuff;
@ -692,9 +692,11 @@ public class IKQueryExpressionParser {
public static void main(String[] args) { public static void main(String[] args) {
IKQueryExpressionParser parser = new IKQueryExpressionParser(); IKQueryExpressionParser parser = new IKQueryExpressionParser();
//String ikQueryExp = "newsTitle:'的两款《魔兽世界》插件Bigfoot和月光宝盒'";
String ikQueryExp = "(id='ABcdRf' && date:{'20010101','20110101'} && keyword:'魔兽中国') || (content:'KSHT-KSH-A001-18' || ulr='www.ik.com') - name:'林良益'"; String ikQueryExp = "(id='ABcdRf' && date:{'20010101','20110101'} && keyword:'魔兽中国') || (content:'KSHT-KSH-A001-18' || ulr='www.ik.com') - name:'林良益'";
Query result = parser.parseExp(ikQueryExp); Query result = parser.parseExp(ikQueryExp);
System.out.println(result); System.out.println(result);
} }
} }

3
src/main/java/org/wltea/analyzer/query/SWMCQueryBuilder.java
View File

@ -45,7 +45,6 @@ import java.util.List;
* *
* @author linliangyi * @author linliangyi
*/ */
@SuppressWarnings("unused")
class SWMCQueryBuilder { class SWMCQueryBuilder {
/** /**
@ -119,8 +118,8 @@ class SWMCQueryBuilder {
//借助lucene queryparser 生成SWMC Query //借助lucene queryparser 生成SWMC Query
QueryParser qp = new QueryParser(fieldName, new StandardAnalyzer()); QueryParser qp = new QueryParser(fieldName, new StandardAnalyzer());
qp.setAutoGeneratePhraseQueries(false);
qp.setDefaultOperator(QueryParser.AND_OPERATOR); qp.setDefaultOperator(QueryParser.AND_OPERATOR);
qp.setAutoGeneratePhraseQueries(true);
if ((shortCount * 1.0f / totalCount) > 0.5f) { if ((shortCount * 1.0f / totalCount) > 0.5f) {
try { try {