LUCENE-10493: Unify TokenInfoFST in kuromoji and nori (#795)

lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/TokenInfoFST.java

@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.morph;
+
+import java.io.IOException;
+import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.FST.Arc;
+
+/**
+ * Thin wrapper around an FST with root-arc caching.
+ *
+ * <p>Root arcs between <code>cacheFloor</code> and <code>cacheFloor</code> are cached.
+ */
+public abstract class TokenInfoFST {
+  protected final FST<Long> fst;
+  private final int cacheCeiling;
+  private final int cacheFloor;
+  private final Arc<Long>[] rootCache;
+
+  public final Long NO_OUTPUT;
+
+  protected TokenInfoFST(FST<Long> fst, int cacheCeiling, int cacheFloor) throws IOException {
+    if (cacheCeiling < cacheFloor) {
+      throw new IllegalArgumentException(
+          "cacheCeiling must be larger than cacheFloor; cacheCeiling="
+              + cacheCeiling
+              + ", cacheFloor="
+              + cacheFloor);
+    }
+    this.fst = fst;
+    this.cacheCeiling = cacheCeiling;
+    this.cacheFloor = cacheFloor;
+    NO_OUTPUT = fst.outputs.getNoOutput();
+    rootCache = cacheRootArcs();
+  }
+
+  @SuppressWarnings({"rawtypes", "unchecked"})
+  private Arc<Long>[] cacheRootArcs() throws IOException {
+    Arc<Long>[] rootCache = new Arc[1 + (cacheCeiling - cacheFloor)];
+    Arc<Long> firstArc = new Arc<>();
+    fst.getFirstArc(firstArc);
+    Arc<Long> arc = new Arc<>();
+    final FST.BytesReader fstReader = fst.getBytesReader();
+    // TODO: jump to cacheFloor, readNextRealArc to ceiling? (just be careful we don't add bugs)
+    for (int i = 0; i < rootCache.length; i++) {
+      if (fst.findTargetArc(cacheFloor + i, firstArc, arc, fstReader) != null) {
+        rootCache[i] = new Arc<Long>().copyFrom(arc);
+      }
+    }
+    return rootCache;
+  }
+
+  public Arc<Long> findTargetArc(
+      int ch, Arc<Long> follow, Arc<Long> arc, boolean useCache, FST.BytesReader fstReader)
+      throws IOException {
+    if (useCache && ch >= cacheFloor && ch <= cacheCeiling) {
+      assert ch != FST.END_LABEL;
+      final Arc<Long> result = rootCache[ch - cacheFloor];
+      if (result == null) {
+        return null;
+      } else {
+        arc.copyFrom(result);
+        return arc;
+      }
+    } else {
+      return fst.findTargetArc(ch, follow, arc, fstReader);
+    }
+  }
+
+  public Arc<Long> getFirstArc(Arc<Long> arc) {
+    return fst.getFirstArc(arc);
+  }
+
+  public FST.BytesReader getBytesReader() {
+    return fst.getBytesReader();
+  }
+}

lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoFST.java

@@ -18,7 +18,6 @@ package org.apache.lucene.analysis.ja.dict;
 
 import java.io.IOException;
 import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.FST.Arc;
 
 /**
  * Thin wrapper around an FST with root-arc caching for Japanese.
@@ -26,64 +25,13 @@ import org.apache.lucene.util.fst.FST.Arc;
  * <p>Depending upon fasterButMoreRam, either just kana (191 arcs), or kana and han (28,607 arcs)
  * are cached. The latter offers additional performance at the cost of more RAM.
  */
-public final class TokenInfoFST {
+public final class TokenInfoFST extends org.apache.lucene.analysis.morph.TokenInfoFST {
-  private final FST<Long> fst;
-
   // depending upon fasterButMoreRam, we cache root arcs for either
   // kana (0x3040-0x30FF) or kana + han (0x3040-0x9FFF)
   // false: 191 arcs
   // true:  28,607 arcs (costs ~1.5MB)
-  private final int cacheCeiling;
-  private final FST.Arc<Long>[] rootCache;
-
-  public final Long NO_OUTPUT;
-
   public TokenInfoFST(FST<Long> fst, boolean fasterButMoreRam) throws IOException {
-    this.fst = fst;
+    super(fst, fasterButMoreRam ? 0x9FFF : 0x30FF, 0x3040);
-    this.cacheCeiling = fasterButMoreRam ? 0x9FFF : 0x30FF;
-    NO_OUTPUT = fst.outputs.getNoOutput();
-    rootCache = cacheRootArcs();
-  }
-
-  @SuppressWarnings({"rawtypes", "unchecked"})
-  private FST.Arc<Long>[] cacheRootArcs() throws IOException {
-    FST.Arc<Long>[] rootCache = new FST.Arc[1 + (cacheCeiling - 0x3040)];
-    FST.Arc<Long> firstArc = new FST.Arc<>();
-    fst.getFirstArc(firstArc);
-    FST.Arc<Long> arc = new FST.Arc<>();
-    final FST.BytesReader fstReader = fst.getBytesReader();
-    // TODO: jump to 3040, readNextRealArc to ceiling? (just be careful we don't add bugs)
-    for (int i = 0; i < rootCache.length; i++) {
-      if (fst.findTargetArc(0x3040 + i, firstArc, arc, fstReader) != null) {
-        rootCache[i] = new FST.Arc<Long>().copyFrom(arc);
-      }
-    }
-    return rootCache;
-  }
-
-  public FST.Arc<Long> findTargetArc(
-      int ch, FST.Arc<Long> follow, FST.Arc<Long> arc, boolean useCache, FST.BytesReader fstReader)
-      throws IOException {
-    if (useCache && ch >= 0x3040 && ch <= cacheCeiling) {
-      assert ch != FST.END_LABEL;
-      final Arc<Long> result = rootCache[ch - 0x3040];
-      if (result == null) {
-        return null;
-      } else {
-        arc.copyFrom(result);
-        return arc;
-      }
-    } else {
-      return fst.findTargetArc(ch, follow, arc, fstReader);
-    }
-  }
-
-  public Arc<Long> getFirstArc(FST.Arc<Long> arc) {
-    return fst.getFirstArc(arc);
-  }
-
-  public FST.BytesReader getBytesReader() {
-    return fst.getBytesReader();
   }
 
   /** @lucene.internal for testing only */

lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoFST.java

@@ -18,63 +18,12 @@ package org.apache.lucene.analysis.ko.dict;
 
 import java.io.IOException;
 import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.FST.Arc;
 
 /** Thin wrapper around an FST with root-arc caching for Hangul syllables (11,172 arcs). */
-public final class TokenInfoFST {
+public final class TokenInfoFST extends org.apache.lucene.analysis.morph.TokenInfoFST {
-  private final FST<Long> fst;
-
-  private final int cacheCeiling;
-  private final FST.Arc<Long>[] rootCache;
-
-  public final Long NO_OUTPUT;
 
   public TokenInfoFST(FST<Long> fst) throws IOException {
-    this.fst = fst;
+    super(fst, 0xD7A3, 0xAC00);
-    this.cacheCeiling = 0xD7A3;
-    NO_OUTPUT = fst.outputs.getNoOutput();
-    rootCache = cacheRootArcs();
-  }
-
-  @SuppressWarnings({"rawtypes", "unchecked"})
-  private FST.Arc<Long>[] cacheRootArcs() throws IOException {
-    FST.Arc<Long>[] rootCache = new FST.Arc[1 + (cacheCeiling - 0xAC00)];
-    FST.Arc<Long> firstArc = new FST.Arc<>();
-    fst.getFirstArc(firstArc);
-    FST.Arc<Long> arc = new FST.Arc<>();
-    final FST.BytesReader fstReader = fst.getBytesReader();
-    // TODO: jump to AC00, readNextRealArc to ceiling? (just be careful we don't add bugs)
-    for (int i = 0; i < rootCache.length; i++) {
-      if (fst.findTargetArc(0xAC00 + i, firstArc, arc, fstReader) != null) {
-        rootCache[i] = new FST.Arc<Long>().copyFrom(arc);
-      }
-    }
-    return rootCache;
-  }
-
-  public FST.Arc<Long> findTargetArc(
-      int ch, FST.Arc<Long> follow, FST.Arc<Long> arc, boolean useCache, FST.BytesReader fstReader)
-      throws IOException {
-    if (useCache && ch >= 0xAC00 && ch <= cacheCeiling) {
-      assert ch != FST.END_LABEL;
-      final Arc<Long> result = rootCache[ch - 0xAC00];
-      if (result == null) {
-        return null;
-      } else {
-        arc.copyFrom(result);
-        return arc;
-      }
-    } else {
-      return fst.findTargetArc(ch, follow, arc, fstReader);
-    }
-  }
-
-  public Arc<Long> getFirstArc(FST.Arc<Long> arc) {
-    return fst.getFirstArc(arc);
-  }
-
-  public FST.BytesReader getBytesReader() {
-    return fst.getBytesReader();
   }
 
   /** @lucene.internal for testing only */