honeymoose/ik-analyzer-solr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

增加配置文件名称读取，更换使用的IO流。

Browse Source
This commit is contained in:
magese 2018-12-26 14:03:37 +08:00
parent bac4768ed7
commit 720278edd1
1 changed files with 34 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
src/main/java/org/wltea/analyzer/lucene/IKTokenizerFactory.java
View File

@ -21,8 +21,8 @@
* 版权声明 2012，乌龙茶工作室 * 版权声明 2012，乌龙茶工作室
* provided by Linliangyi and copyright 2012 by Oolong studio * provided by Linliangyi and copyright 2012 by Oolong studio
* *
* 7.5版本 由 Magese (magese@live.cn) 更新 * 7.6版本 由 Magese (magese@live.cn) 更新
* release 7.5 update by Magese(magese@live.cn) * release 7.6 update by Magese(magese@live.cn)
* *
*/ */
package org.wltea.analyzer.lucene; package org.wltea.analyzer.lucene;
@ -36,6 +36,11 @@ import org.wltea.analyzer.dic.Dictionary;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
import java.nio.charset.StandardCharsets;
import java.util.*; import java.util.*;
/** /**
@ -50,7 +55,9 @@ public class IKTokenizerFactory extends TokenizerFactory implements ResourceLoad
public IKTokenizerFactory(Map<String, String> args) { public IKTokenizerFactory(Map<String, String> args) {
super(args); super(args);
String useSmartArg = args.get("useSmart"); String useSmartArg = args.get("useSmart");
String confArg = args.get("conf");
this.setUseSmart(Boolean.parseBoolean(useSmartArg)); this.setUseSmart(Boolean.parseBoolean(useSmartArg));
this.setConf(confArg);
} }
@Override @Override
@ -67,10 +74,10 @@ public class IKTokenizerFactory extends TokenizerFactory implements ResourceLoad
*/ */
@Override @Override
public void inform(ResourceLoader resourceLoader) throws IOException { public void inform(ResourceLoader resourceLoader) throws IOException {
System.out.println(String.format("IKTokenizerFactory "+ this.hashCode() +" inform conf: %s", this.conf)); System.out.println(String.format("IKTokenizerFactory " + this.hashCode() + " inform conf: %s", getConf()));
this.loader = resourceLoader; this.loader = resourceLoader;
update(); update();
if ((this.conf != null) && (!this.conf.trim().isEmpty())) { if ((getConf() != null) && (!getConf().trim().isEmpty())) {
UpdateThread.getInstance().register(this); UpdateThread.getInstance().register(this);
} }
} }
@ -82,24 +89,27 @@ public class IKTokenizerFactory extends TokenizerFactory implements ResourceLoad
*/ */
@Override @Override
public void update() throws IOException { public void update() throws IOException {
// 默认UTF-8解码
CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder()
.onMalformedInput(CodingErrorAction.REPORT)
.onUnmappableCharacter(CodingErrorAction.REPORT);
// 获取ik.conf配置文件信息 // 获取ik.conf配置文件信息
Properties p = canUpdate(); Properties p = canUpdate();
if (p != null) { if (p != null) {
// 获取词典表名称集合 // 获取词典表名称集合
List<String> dicPaths = SplitFileNames(p.getProperty("files")); List<String> dicPaths = SplitFileNames(p.getProperty("files"));
// 获取词典文件的IO流 // 获取词典文件的IO流
List<InputStream> inputStreamList = new ArrayList<>(); List<Reader> inputStreamReaderList = new ArrayList<>();
for (String path : dicPaths) { for (String path : dicPaths) {
if ((path != null) && (!path.isEmpty())) { if ((path != null) && (!path.isEmpty())) {
InputStream is = this.loader.openResource(path); Reader isr = new InputStreamReader(loader.openResource(path), decoder);
if (is != null) { inputStreamReaderList.add(isr);
inputStreamList.add(is);
}
} }
} }
// 如果IO流集合不为空则执行加载词典 // 如果IO流集合不为空则执行加载词典
if (!inputStreamList.isEmpty()) if (!inputStreamReaderList.isEmpty())
Dictionary.reloadDic(inputStreamList); Dictionary.reloadDic(inputStreamReaderList);
} }
} }
@ -108,10 +118,10 @@ public class IKTokenizerFactory extends TokenizerFactory implements ResourceLoad
*/ */
private Properties canUpdate() { private Properties canUpdate() {
try { try {
if (this.conf == null) if (getConf() == null)
return null; return null;
Properties p = new Properties(); Properties p = new Properties();
InputStream confStream = this.loader.openResource(this.conf); // 获取配置文件流 InputStream confStream = this.loader.openResource(getConf()); // 获取配置文件流
p.load(confStream); // 读取配置文件 p.load(confStream); // 读取配置文件
confStream.close(); // 关闭文件流 confStream.close(); // 关闭文件流
String lastupdate = p.getProperty("lastupdate", "0"); // 获取最后更新数字 String lastupdate = p.getProperty("lastupdate", "0"); // 获取最后更新数字
@ -122,13 +132,13 @@ public class IKTokenizerFactory extends TokenizerFactory implements ResourceLoad
String paths = p.getProperty("files"); // 获取词典文件名 String paths = p.getProperty("files"); // 获取词典文件名
if ((paths == null) || (paths.trim().isEmpty())) if ((paths == null) || (paths.trim().isEmpty()))
return null; return null;
System.out.println("loading ik.conf files success."); System.out.println("loading " + getConf() + " files success.");
return p; return p;
} }
this.lastUpdateTime = t; this.lastUpdateTime = t;
return null; return null;
} catch (Exception e) { } catch (Exception e) {
System.err.println("parsing ik.conf NullPointerException!!!" + Arrays.toString(e.getStackTrace())); System.err.println("parsing " + getConf() + " NullPointerException!!!" + Arrays.toString(e.getStackTrace()));
} }
return null; return null;
} }
@ -148,6 +158,7 @@ public class IKTokenizerFactory extends TokenizerFactory implements ResourceLoad
return result; return result;
} }
/* getter & setter */
private boolean useSmart() { private boolean useSmart() {
return useSmart; return useSmart;
} }
@ -155,4 +166,12 @@ public class IKTokenizerFactory extends TokenizerFactory implements ResourceLoad
private void setUseSmart(boolean useSmart) { private void setUseSmart(boolean useSmart) {
this.useSmart = useSmart; this.useSmart = useSmart;
} }
private String getConf() {
return conf;
}
private void setConf(String conf) {
this.conf = conf;
}
} }