Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

don't pass pos to FST.getBytesReader 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1433109 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2013-01-14 20:38:38 +00:00
parent e87c7a4ae7
commit 9050df8046
16 changed files with 37 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilter.java
View File

@ -59,7 +59,7 @@ public class MappingCharFilter extends BaseCharFilter {
cachedRootArcs = normMap.cachedRootArcs; cachedRootArcs = normMap.cachedRootArcs;
if (map != null) { if (map != null) {
fstReader = map.getBytesReader(0); fstReader = map.getBytesReader();
} else { } else {
fstReader = null; fstReader = null;
} }

2
lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/NormalizeCharMap.java
View File

@ -49,7 +49,7 @@ public class NormalizeCharMap {
try { try {
// Pre-cache root arcs: // Pre-cache root arcs:
final FST.Arc<CharsRef> scratchArc = new FST.Arc<CharsRef>(); final FST.Arc<CharsRef> scratchArc = new FST.Arc<CharsRef>();
final FST.BytesReader fstReader = map.getBytesReader(0); final FST.BytesReader fstReader = map.getBytesReader();
map.getFirstArc(scratchArc); map.getFirstArc(scratchArc);
if (FST.targetHasArcs(scratchArc)) { if (FST.targetHasArcs(scratchArc)) {
map.readFirstRealTargetArc(scratchArc.target, scratchArc, fstReader); map.readFirstRealTargetArc(scratchArc.target, scratchArc, fstReader);

2
lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java
View File

@ -263,7 +263,7 @@ public final class SynonymFilter extends TokenFilter {
this.synonyms = synonyms; this.synonyms = synonyms;
this.ignoreCase = ignoreCase; this.ignoreCase = ignoreCase;
this.fst = synonyms.fst; this.fst = synonyms.fst;
this.fstReader = fst.getBytesReader(0); this.fstReader = fst.getBytesReader();
if (fst == null) { if (fst == null) {
throw new IllegalArgumentException("fst must be non-null"); throw new IllegalArgumentException("fst must be non-null");
} }

4
lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
View File

@ -201,10 +201,10 @@ public final class JapaneseTokenizer extends Tokenizer {
characterDefinition = unkDictionary.getCharacterDefinition(); characterDefinition = unkDictionary.getCharacterDefinition();
this.userDictionary = userDictionary; this.userDictionary = userDictionary;
costs = ConnectionCosts.getInstance(); costs = ConnectionCosts.getInstance();
fstReader = fst.getBytesReader(0); fstReader = fst.getBytesReader();
if (userDictionary != null) { if (userDictionary != null) {
userFST = userDictionary.getFST(); userFST = userDictionary.getFST();
userFSTReader = userFST.getBytesReader(0); userFSTReader = userFST.getBytesReader();
} else { } else {
userFST = null; userFST = null;
userFSTReader = null; userFSTReader = null;

6
lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoFST.java
View File

@ -54,7 +54,7 @@ public final class TokenInfoFST {
FST.Arc<Long> firstArc = new FST.Arc<Long>(); FST.Arc<Long> firstArc = new FST.Arc<Long>();
fst.getFirstArc(firstArc); fst.getFirstArc(firstArc);
FST.Arc<Long> arc = new FST.Arc<Long>(); FST.Arc<Long> arc = new FST.Arc<Long>();
final FST.BytesReader fstReader = fst.getBytesReader(0); final FST.BytesReader fstReader = fst.getBytesReader();
// TODO: jump to 3040, readNextRealArc to ceiling? (just be careful we don't add bugs) // TODO: jump to 3040, readNextRealArc to ceiling? (just be careful we don't add bugs)
for (int i = 0; i < rootCache.length; i++) { for (int i = 0; i < rootCache.length; i++) {
if (fst.findTargetArc(0x3040 + i, firstArc, arc, fstReader) != null) { if (fst.findTargetArc(0x3040 + i, firstArc, arc, fstReader) != null) {
@ -83,8 +83,8 @@ public final class TokenInfoFST {
return fst.getFirstArc(arc); return fst.getFirstArc(arc);
} }
public FST.BytesReader getBytesReader(int pos) { public FST.BytesReader getBytesReader() {
return fst.getBytesReader(pos); return fst.getBytesReader();
} }
/** @lucene.internal for testing only */ /** @lucene.internal for testing only */

2
lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java
View File

@ -139,7 +139,7 @@ public final class UserDictionary implements Dictionary {
TreeMap<Integer, int[]> result = new TreeMap<Integer, int[]>(); // index, [length, length...] TreeMap<Integer, int[]> result = new TreeMap<Integer, int[]>(); // index, [length, length...]
boolean found = false; // true if we found any results boolean found = false; // true if we found any results
final FST.BytesReader fstReader = fst.getBytesReader(0); final FST.BytesReader fstReader = fst.getBytesReader();
FST.Arc<Long> arc = new FST.Arc<Long>(); FST.Arc<Long> arc = new FST.Arc<Long>();
int end = off + len; int end = off + len;

4
lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java
View File

@ -833,7 +833,7 @@ public class BlockTreeTermsReader extends FieldsProducer {
if (index == null) { if (index == null) {
fstReader = null; fstReader = null;
} else { } else {
fstReader = index.getBytesReader(0); fstReader = index.getBytesReader();
} }
// TODO: if the automaton is "smallish" we really // TODO: if the automaton is "smallish" we really
@ -1277,7 +1277,7 @@ public class BlockTreeTermsReader extends FieldsProducer {
if (index == null) { if (index == null) {
fstReader = null; fstReader = null;
} else { } else {
fstReader = index.getBytesReader(0); fstReader = index.getBytesReader();
} }
// Init w/ root block; don't use index since it may // Init w/ root block; don't use index since it may

14
lucene/core/src/java/org/apache/lucene/util/fst/FST.java
View File

@ -417,7 +417,7 @@ public final class FST<T> {
cachedRootArcs = (Arc<T>[]) new Arc[0x80]; cachedRootArcs = (Arc<T>[]) new Arc[0x80];
final Arc<T> arc = new Arc<T>(); final Arc<T> arc = new Arc<T>();
getFirstArc(arc); getFirstArc(arc);
final BytesReader in = getBytesReader(0); final BytesReader in = getBytesReader();
if (targetHasArcs(arc)) { if (targetHasArcs(arc)) {
readFirstRealTargetArc(arc.target, arc, in); readFirstRealTargetArc(arc.target, arc, in);
while(true) { while(true) {
@ -1246,22 +1246,12 @@ public final class FST<T> {
/** Returns a {@link BytesReader} for this FST, positioned at /** Returns a {@link BytesReader} for this FST, positioned at
* position 0. */ * position 0. */
public BytesReader getBytesReader() { public BytesReader getBytesReader() {
return getBytesReader(0);
}
/** Returns a {@link BytesReader} for this FST, positioned at
* the provided position. */
public BytesReader getBytesReader(long pos) {
// TODO: maybe re-use via ThreadLocal?
BytesReader in; BytesReader in;
if (packed) { if (packed) {
in = bytes.getForwardReader(); in = bytes.getForwardReader();
} else { } else {
in = bytes.getReverseReader(); in = bytes.getReverseReader();
} }
if (pos != 0) {
in.setPosition(pos);
}
return in; return in;
} }
@ -1448,7 +1438,7 @@ public final class FST<T> {
Arc<T> arc = new Arc<T>(); Arc<T> arc = new Arc<T>();
final BytesReader r = getBytesReader(0); final BytesReader r = getBytesReader();
final int topN = Math.min(maxDerefNodes, inCounts.size()); final int topN = Math.min(maxDerefNodes, inCounts.size());

8
lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java
View File

@ -46,7 +46,7 @@ abstract class FSTEnum<T> {
* term before target. */ * term before target. */
protected FSTEnum(FST<T> fst) { protected FSTEnum(FST<T> fst) {
this.fst = fst; this.fst = fst;
fstReader = fst.getBytesReader(0); fstReader = fst.getBytesReader();
NO_OUTPUT = fst.outputs.getNoOutput(); NO_OUTPUT = fst.outputs.getNoOutput();
fst.getFirstArc(getArc(0)); fst.getFirstArc(getArc(0));
output[0] = NO_OUTPUT; output[0] = NO_OUTPUT;
@ -145,7 +145,7 @@ abstract class FSTEnum<T> {
// Arcs are fixed array -- use binary search to find // Arcs are fixed array -- use binary search to find
// the target. // the target.
final FST.BytesReader in = fst.getBytesReader(0); final FST.BytesReader in = fst.getBytesReader();
int low = arc.arcIdx; int low = arc.arcIdx;
int high = arc.numArcs-1; int high = arc.numArcs-1;
int mid = 0; int mid = 0;
@ -284,7 +284,7 @@ abstract class FSTEnum<T> {
// Arcs are fixed array -- use binary search to find // Arcs are fixed array -- use binary search to find
// the target. // the target.
final FST.BytesReader in = fst.getBytesReader(0); final FST.BytesReader in = fst.getBytesReader();
int low = arc.arcIdx; int low = arc.arcIdx;
int high = arc.numArcs-1; int high = arc.numArcs-1;
int mid = 0; int mid = 0;
@ -434,7 +434,7 @@ abstract class FSTEnum<T> {
FST.Arc<T> arc = getArc(upto-1); FST.Arc<T> arc = getArc(upto-1);
int targetLabel = getTargetLabel(); int targetLabel = getTargetLabel();
final FST.BytesReader fstReader = fst.getBytesReader(0); final FST.BytesReader fstReader = fst.getBytesReader();
while(true) { while(true) {
//System.out.println(" cycle target=" + (targetLabel == -1 ? "-1" : (char) targetLabel)); //System.out.println(" cycle target=" + (targetLabel == -1 ? "-1" : (char) targetLabel));

12
lucene/core/src/java/org/apache/lucene/util/fst/Util.java
View File

@ -39,7 +39,7 @@ public final class Util {
// TODO: would be nice not to alloc this on every lookup // TODO: would be nice not to alloc this on every lookup
final FST.Arc<T> arc = fst.getFirstArc(new FST.Arc<T>()); final FST.Arc<T> arc = fst.getFirstArc(new FST.Arc<T>());
final BytesReader fstReader = fst.getBytesReader(0); final BytesReader fstReader = fst.getBytesReader();
// Accumulate output as we go // Accumulate output as we go
T output = fst.outputs.getNoOutput(); T output = fst.outputs.getNoOutput();
@ -64,7 +64,7 @@ public final class Util {
public static<T> T get(FST<T> fst, BytesRef input) throws IOException { public static<T> T get(FST<T> fst, BytesRef input) throws IOException {
assert fst.inputType == FST.INPUT_TYPE.BYTE1; assert fst.inputType == FST.INPUT_TYPE.BYTE1;
final BytesReader fstReader = fst.getBytesReader(0); final BytesReader fstReader = fst.getBytesReader();
// TODO: would be nice not to alloc this on every lookup // TODO: would be nice not to alloc this on every lookup
final FST.Arc<T> arc = fst.getFirstArc(new FST.Arc<T>()); final FST.Arc<T> arc = fst.getFirstArc(new FST.Arc<T>());
@ -101,7 +101,7 @@ public final class Util {
* fit this. */ * fit this. */
public static IntsRef getByOutput(FST<Long> fst, long targetOutput) throws IOException { public static IntsRef getByOutput(FST<Long> fst, long targetOutput) throws IOException {
final BytesReader in = fst.getBytesReader(0); final BytesReader in = fst.getBytesReader();
// TODO: would be nice not to alloc this on every lookup // TODO: would be nice not to alloc this on every lookup
FST.Arc<Long> arc = fst.getFirstArc(new FST.Arc<Long>()); FST.Arc<Long> arc = fst.getFirstArc(new FST.Arc<Long>());
@ -285,7 +285,7 @@ public final class Util {
public TopNSearcher(FST<T> fst, int topN, int maxQueueDepth, Comparator<T> comparator) { public TopNSearcher(FST<T> fst, int topN, int maxQueueDepth, Comparator<T> comparator) {
this.fst = fst; this.fst = fst;
this.bytesReader = fst.getBytesReader(0); this.bytesReader = fst.getBytesReader();
this.topN = topN; this.topN = topN;
this.maxQueueDepth = maxQueueDepth; this.maxQueueDepth = maxQueueDepth;
this.comparator = comparator; this.comparator = comparator;
@ -374,7 +374,7 @@ public final class Util {
//System.out.println("search topN=" + topN); //System.out.println("search topN=" + topN);
final BytesReader fstReader = fst.getBytesReader(0); final BytesReader fstReader = fst.getBytesReader();
final T NO_OUTPUT = fst.outputs.getNoOutput(); final T NO_OUTPUT = fst.outputs.getNoOutput();
// TODO: we could enable FST to sorting arcs by weight // TODO: we could enable FST to sorting arcs by weight
@ -597,7 +597,7 @@ public final class Util {
emitDotState(out, "initial", "point", "white", ""); emitDotState(out, "initial", "point", "white", "");
final T NO_OUTPUT = fst.outputs.getNoOutput(); final T NO_OUTPUT = fst.outputs.getNoOutput();
final BytesReader r = fst.getBytesReader(0); final BytesReader r = fst.getBytesReader();
// final FST.Arc<T> scratchArc = new FST.Arc<T>(); // final FST.Arc<T> scratchArc = new FST.Arc<T>();

10
lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
View File

@ -1033,7 +1033,7 @@ public class TestFSTs extends LuceneTestCase {
throws IOException { throws IOException {
if (FST.targetHasArcs(arc)) { if (FST.targetHasArcs(arc)) {
int childCount = 0; int childCount = 0;
BytesReader fstReader = fst.getBytesReader(0); BytesReader fstReader = fst.getBytesReader();
for (arc = fst.readFirstTargetArc(arc, arc, fstReader);; for (arc = fst.readFirstTargetArc(arc, arc, fstReader);;
arc = fst.readNextArc(arc, fstReader), childCount++) arc = fst.readNextArc(arc, fstReader), childCount++)
{ {
@ -1168,12 +1168,12 @@ public class TestFSTs extends LuceneTestCase {
assertEquals(nothing, startArc.nextFinalOutput); assertEquals(nothing, startArc.nextFinalOutput);
FST.Arc<Long> arc = fst.readFirstTargetArc(startArc, new FST.Arc<Long>(), FST.Arc<Long> arc = fst.readFirstTargetArc(startArc, new FST.Arc<Long>(),
fst.getBytesReader(0)); fst.getBytesReader());
assertEquals('a', arc.label); assertEquals('a', arc.label);
assertEquals(17, arc.nextFinalOutput.longValue()); assertEquals(17, arc.nextFinalOutput.longValue());
assertTrue(arc.isFinal()); assertTrue(arc.isFinal());
arc = fst.readNextArc(arc, fst.getBytesReader(0)); arc = fst.readNextArc(arc, fst.getBytesReader());
assertEquals('b', arc.label); assertEquals('b', arc.label);
assertFalse(arc.isFinal()); assertFalse(arc.isFinal());
assertEquals(42, arc.output.longValue()); assertEquals(42, arc.output.longValue());
@ -1303,7 +1303,7 @@ public class TestFSTs extends LuceneTestCase {
//Util.toDot(fst, w, false, false); //Util.toDot(fst, w, false, false);
//w.close(); //w.close();
BytesReader reader = fst.getBytesReader(0); BytesReader reader = fst.getBytesReader();
//System.out.println("testing: " + allPrefixes.size() + " prefixes"); //System.out.println("testing: " + allPrefixes.size() + " prefixes");
for (String prefix : allPrefixes) { for (String prefix : allPrefixes) {
@ -1424,7 +1424,7 @@ public class TestFSTs extends LuceneTestCase {
//Util.toDot(fst, w, false, false); //Util.toDot(fst, w, false, false);
//w.close(); //w.close();
BytesReader reader = fst.getBytesReader(0); BytesReader reader = fst.getBytesReader();
//System.out.println("testing: " + allPrefixes.size() + " prefixes"); //System.out.println("testing: " + allPrefixes.size() + " prefixes");
for (String prefix : allPrefixes) { for (String prefix : allPrefixes) {

2
lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
View File

@ -587,7 +587,7 @@ public class AnalyzingSuggester extends Lookup {
//System.out.println(" prefixPaths: " + prefixPaths.size()); //System.out.println(" prefixPaths: " + prefixPaths.size());
BytesReader bytesReader = fst.getBytesReader(0); BytesReader bytesReader = fst.getBytesReader();
FST.Arc<Pair<Long,BytesRef>> scratchArc = new FST.Arc<Pair<Long,BytesRef>>(); FST.Arc<Pair<Long,BytesRef>> scratchArc = new FST.Arc<Pair<Long,BytesRef>>();

2
lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FSTUtil.java
View File

@ -77,7 +77,7 @@ public class FSTUtil {
new IntsRef())); new IntsRef()));
final FST.Arc<T> scratchArc = new FST.Arc<T>(); final FST.Arc<T> scratchArc = new FST.Arc<T>();
final FST.BytesReader fstReader = fst.getBytesReader(0); final FST.BytesReader fstReader = fst.getBytesReader();
while (queue.size() != 0) { while (queue.size() != 0) {
final Path<T> path = queue.remove(queue.size() - 1); final Path<T> path = queue.remove(queue.size() - 1);

8
lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java
View File

@ -139,7 +139,7 @@ public class FSTCompletion {
try { try {
List<Arc<Object>> rootArcs = new ArrayList<Arc<Object>>(); List<Arc<Object>> rootArcs = new ArrayList<Arc<Object>>();
Arc<Object> arc = automaton.getFirstArc(new Arc<Object>()); Arc<Object> arc = automaton.getFirstArc(new Arc<Object>());
FST.BytesReader fstReader = automaton.getBytesReader(0); FST.BytesReader fstReader = automaton.getBytesReader();
automaton.readFirstTargetArc(arc, arc, fstReader); automaton.readFirstTargetArc(arc, arc, fstReader);
while (true) { while (true) {
rootArcs.add(new Arc<Object>().copyFrom(arc)); rootArcs.add(new Arc<Object>().copyFrom(arc));
@ -173,7 +173,7 @@ public class FSTCompletion {
// Get the UTF-8 bytes representation of the input key. // Get the UTF-8 bytes representation of the input key.
try { try {
final FST.Arc<Object> scratch = new FST.Arc<Object>(); final FST.Arc<Object> scratch = new FST.Arc<Object>();
FST.BytesReader fstReader = automaton.getBytesReader(0); FST.BytesReader fstReader = automaton.getBytesReader();
for (; rootArcIndex < rootArcs.length; rootArcIndex++) { for (; rootArcIndex < rootArcs.length; rootArcIndex++) {
final FST.Arc<Object> rootArc = rootArcs[rootArcIndex]; final FST.Arc<Object> rootArc = rootArcs[rootArcIndex];
final FST.Arc<Object> arc = scratch.copyFrom(rootArc); final FST.Arc<Object> arc = scratch.copyFrom(rootArc);
@ -338,7 +338,7 @@ public class FSTCompletion {
final int max = utf8.offset + utf8.length; final int max = utf8.offset + utf8.length;
// Cannot save as instance var since multiple threads // Cannot save as instance var since multiple threads
// can use FSTCompletion at once... // can use FSTCompletion at once...
final FST.BytesReader fstReader = automaton.getBytesReader(0); final FST.BytesReader fstReader = automaton.getBytesReader();
for (int i = utf8.offset; i < max; i++) { for (int i = utf8.offset; i < max; i++) {
if (automaton.findTargetArc(utf8.bytes[i] & 0xff, arc, arc, fstReader) == null) { if (automaton.findTargetArc(utf8.bytes[i] & 0xff, arc, arc, fstReader) == null) {
// No matching prefixes, return an empty result. // No matching prefixes, return an empty result.
@ -362,7 +362,7 @@ public class FSTCompletion {
} }
assert output.offset == 0; assert output.offset == 0;
output.bytes[output.length++] = (byte) arc.label; output.bytes[output.length++] = (byte) arc.label;
FST.BytesReader fstReader = automaton.getBytesReader(0); FST.BytesReader fstReader = automaton.getBytesReader();
automaton.readFirstTargetArc(arc, arc, fstReader); automaton.readFirstTargetArc(arc, arc, fstReader);
while (true) { while (true) {
if (arc.label == FST.END_LABEL) { if (arc.label == FST.END_LABEL) {

2
lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
View File

@ -200,7 +200,7 @@ public class WFSTCompletionLookup extends Lookup {
private Long lookupPrefix(BytesRef scratch, Arc<Long> arc) throws /*Bogus*/IOException { private Long lookupPrefix(BytesRef scratch, Arc<Long> arc) throws /*Bogus*/IOException {
assert 0 == fst.outputs.getNoOutput().longValue(); assert 0 == fst.outputs.getNoOutput().longValue();
long output = 0; long output = 0;
BytesReader bytesReader = fst.getBytesReader(0); BytesReader bytesReader = fst.getBytesReader();
fst.getFirstArc(arc); fst.getFirstArc(arc);

4
lucene/test-framework/src/java/org/apache/lucene/util/fst/FSTTester.java
View File

@ -204,7 +204,7 @@ public class FSTTester<T> {
final FST.Arc<T> arc = fst.getFirstArc(new FST.Arc<T>()); final FST.Arc<T> arc = fst.getFirstArc(new FST.Arc<T>());
final T NO_OUTPUT = fst.outputs.getNoOutput(); final T NO_OUTPUT = fst.outputs.getNoOutput();
T output = NO_OUTPUT; T output = NO_OUTPUT;
final FST.BytesReader fstReader = fst.getBytesReader(0); final FST.BytesReader fstReader = fst.getBytesReader();
for(int i=0;i<=term.length;i++) { for(int i=0;i<=term.length;i++) {
final int label; final int label;
@ -241,7 +241,7 @@ public class FSTTester<T> {
in.offset = 0; in.offset = 0;
final T NO_OUTPUT = fst.outputs.getNoOutput(); final T NO_OUTPUT = fst.outputs.getNoOutput();
T output = NO_OUTPUT; T output = NO_OUTPUT;
final FST.BytesReader fstReader = fst.getBytesReader(0); final FST.BytesReader fstReader = fst.getBytesReader();
while(true) { while(true) {
// read all arcs: // read all arcs: