Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-10493: Unify TokenInfoFST in kuromoji and nori (#795)

This commit is contained in:
Tomoko Uchida 2022-04-07 21:29:44 +09:00 committed by GitHub
parent 4d2b08554a
commit 9aa8ec9d06
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 95 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/TokenInfoFST.java Normal file
View File

@ -0,0 +1,91 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.analysis.morph;
import java.io.IOException;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.FST.Arc;
/**
* Thin wrapper around an FST with root-arc caching.
*
* <p>Root arcs between <code>cacheFloor</code> and <code>cacheFloor</code> are cached.
*/
public abstract class TokenInfoFST {
protected final FST<Long> fst;
private final int cacheCeiling;
private final int cacheFloor;
private final Arc<Long>[] rootCache;
public final Long NO_OUTPUT;
protected TokenInfoFST(FST<Long> fst, int cacheCeiling, int cacheFloor) throws IOException {
if (cacheCeiling < cacheFloor) {
throw new IllegalArgumentException(
"cacheCeiling must be larger than cacheFloor; cacheCeiling="
+ cacheCeiling
+ ", cacheFloor="
+ cacheFloor);
}
this.fst = fst;
this.cacheCeiling = cacheCeiling;
this.cacheFloor = cacheFloor;
NO_OUTPUT = fst.outputs.getNoOutput();
rootCache = cacheRootArcs();
}
@SuppressWarnings({"rawtypes", "unchecked"})
private Arc<Long>[] cacheRootArcs() throws IOException {
Arc<Long>[] rootCache = new Arc[1 + (cacheCeiling - cacheFloor)];
Arc<Long> firstArc = new Arc<>();
fst.getFirstArc(firstArc);
Arc<Long> arc = new Arc<>();
final FST.BytesReader fstReader = fst.getBytesReader();
// TODO: jump to cacheFloor, readNextRealArc to ceiling? (just be careful we don't add bugs)
for (int i = 0; i < rootCache.length; i++) {
if (fst.findTargetArc(cacheFloor + i, firstArc, arc, fstReader) != null) {
rootCache[i] = new Arc<Long>().copyFrom(arc);
}
}
return rootCache;
}
public Arc<Long> findTargetArc(
int ch, Arc<Long> follow, Arc<Long> arc, boolean useCache, FST.BytesReader fstReader)
throws IOException {
if (useCache && ch >= cacheFloor && ch <= cacheCeiling) {
assert ch != FST.END_LABEL;
final Arc<Long> result = rootCache[ch - cacheFloor];
if (result == null) {
return null;
} else {
arc.copyFrom(result);
return arc;
}
} else {
return fst.findTargetArc(ch, follow, arc, fstReader);
}
}
public Arc<Long> getFirstArc(Arc<Long> arc) {
return fst.getFirstArc(arc);
}
public FST.BytesReader getBytesReader() {
return fst.getBytesReader();
}
}

56
lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoFST.java
View File

@ -18,7 +18,6 @@ package org.apache.lucene.analysis.ja.dict;
import java.io.IOException;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.FST.Arc;
/**
* Thin wrapper around an FST with root-arc caching for Japanese.
@ -26,64 +25,13 @@ import org.apache.lucene.util.fst.FST.Arc;
* <p>Depending upon fasterButMoreRam, either just kana (191 arcs), or kana and han (28,607 arcs)
* are cached. The latter offers additional performance at the cost of more RAM.
*/
public final class TokenInfoFST {
private final FST<Long> fst;
public final class TokenInfoFST extends org.apache.lucene.analysis.morph.TokenInfoFST {
// depending upon fasterButMoreRam, we cache root arcs for either
// kana (0x3040-0x30FF) or kana + han (0x3040-0x9FFF)
// false: 191 arcs
// true: 28,607 arcs (costs ~1.5MB)
private final int cacheCeiling;
private final FST.Arc<Long>[] rootCache;
public final Long NO_OUTPUT;
public TokenInfoFST(FST<Long> fst, boolean fasterButMoreRam) throws IOException {
this.fst = fst;
this.cacheCeiling = fasterButMoreRam ? 0x9FFF : 0x30FF;
NO_OUTPUT = fst.outputs.getNoOutput();
rootCache = cacheRootArcs();
}
@SuppressWarnings({"rawtypes", "unchecked"})
private FST.Arc<Long>[] cacheRootArcs() throws IOException {
FST.Arc<Long>[] rootCache = new FST.Arc[1 + (cacheCeiling - 0x3040)];
FST.Arc<Long> firstArc = new FST.Arc<>();
fst.getFirstArc(firstArc);
FST.Arc<Long> arc = new FST.Arc<>();
final FST.BytesReader fstReader = fst.getBytesReader();
// TODO: jump to 3040, readNextRealArc to ceiling? (just be careful we don't add bugs)
for (int i = 0; i < rootCache.length; i++) {
if (fst.findTargetArc(0x3040 + i, firstArc, arc, fstReader) != null) {
rootCache[i] = new FST.Arc<Long>().copyFrom(arc);
}
}
return rootCache;
}
public FST.Arc<Long> findTargetArc(
int ch, FST.Arc<Long> follow, FST.Arc<Long> arc, boolean useCache, FST.BytesReader fstReader)
throws IOException {
if (useCache && ch >= 0x3040 && ch <= cacheCeiling) {
assert ch != FST.END_LABEL;
final Arc<Long> result = rootCache[ch - 0x3040];
if (result == null) {
return null;
} else {
arc.copyFrom(result);
return arc;
}
} else {
return fst.findTargetArc(ch, follow, arc, fstReader);
}
}
public Arc<Long> getFirstArc(FST.Arc<Long> arc) {
return fst.getFirstArc(arc);
}
public FST.BytesReader getBytesReader() {
return fst.getBytesReader();
super(fst, fasterButMoreRam ? 0x9FFF : 0x30FF, 0x3040);
}
/** @lucene.internal for testing only */

55
lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoFST.java
View File

@ -18,63 +18,12 @@ package org.apache.lucene.analysis.ko.dict;
import java.io.IOException;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.FST.Arc;
/** Thin wrapper around an FST with root-arc caching for Hangul syllables (11,172 arcs). */
public final class TokenInfoFST {
private final FST<Long> fst;
private final int cacheCeiling;
private final FST.Arc<Long>[] rootCache;
public final Long NO_OUTPUT;
public final class TokenInfoFST extends org.apache.lucene.analysis.morph.TokenInfoFST {
public TokenInfoFST(FST<Long> fst) throws IOException {
this.fst = fst;
this.cacheCeiling = 0xD7A3;
NO_OUTPUT = fst.outputs.getNoOutput();
rootCache = cacheRootArcs();
}
@SuppressWarnings({"rawtypes", "unchecked"})
private FST.Arc<Long>[] cacheRootArcs() throws IOException {
FST.Arc<Long>[] rootCache = new FST.Arc[1 + (cacheCeiling - 0xAC00)];
FST.Arc<Long> firstArc = new FST.Arc<>();
fst.getFirstArc(firstArc);
FST.Arc<Long> arc = new FST.Arc<>();
final FST.BytesReader fstReader = fst.getBytesReader();
// TODO: jump to AC00, readNextRealArc to ceiling? (just be careful we don't add bugs)
for (int i = 0; i < rootCache.length; i++) {
if (fst.findTargetArc(0xAC00 + i, firstArc, arc, fstReader) != null) {
rootCache[i] = new FST.Arc<Long>().copyFrom(arc);
}
}
return rootCache;
}
public FST.Arc<Long> findTargetArc(
int ch, FST.Arc<Long> follow, FST.Arc<Long> arc, boolean useCache, FST.BytesReader fstReader)
throws IOException {
if (useCache && ch >= 0xAC00 && ch <= cacheCeiling) {
assert ch != FST.END_LABEL;
final Arc<Long> result = rootCache[ch - 0xAC00];
if (result == null) {
return null;
} else {
arc.copyFrom(result);
return arc;
}
} else {
return fst.findTargetArc(ch, follow, arc, fstReader);
}
}
public Arc<Long> getFirstArc(FST.Arc<Long> arc) {
return fst.getFirstArc(arc);
}
public FST.BytesReader getBytesReader() {
return fst.getBytesReader();
super(fst, 0xD7A3, 0xAC00);
}
/** @lucene.internal for testing only */