Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-3069: accumulate metadata lazily 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene3069@1506612 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Han Jiang 2013-07-24 16:15:01 +00:00
parent 39b9082e0c
commit dc1bee7529
1 changed files with 46 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
lucene/core/src/java/org/apache/lucene/codecs/temp/TempFSTTermsReader.java
View File

@ -389,13 +389,17 @@ public class TempFSTTermsReader extends FieldsProducer {
/* True when there is pending term when calling next() */ /* True when there is pending term when calling next() */
boolean pending; boolean pending;
/* stack to record how current term is constructed, used to accumulate /* stack to record how current term is constructed,
* metadata or rewind term: * used to accumulate metadata or rewind term:
* level == term.length + 1, * level == term.length + 1,
* == 0 when term is null */ * == 0 when term is null */
Frame[] stack; Frame[] stack;
int level; int level;
/* to which level the metadata is accumulated
* so that we can accumulate metadata lazily */
int metaUpto;
/* term dict fst */ /* term dict fst */
final FST<TempTermOutputs.TempMetaData> fst; final FST<TempTermOutputs.TempMetaData> fst;
final FST.BytesReader fstReader; final FST.BytesReader fstReader;
@ -436,7 +440,6 @@ public class TempFSTTermsReader extends FieldsProducer {
pw2.write(compiled.toDot()); pw2.write(compiled.toDot());
pw2.close(); pw2.close();
*/ */
this.meta = fstOutputs.getNoOutput();
this.level = -1; this.level = -1;
this.stack = new Frame[16]; this.stack = new Frame[16];
for (int i = 0 ; i < stack.length; i++) { for (int i = 0 ; i < stack.length; i++) {
@ -449,6 +452,8 @@ public class TempFSTTermsReader extends FieldsProducer {
frame = loadFirstFrame(newFrame()); frame = loadFirstFrame(newFrame());
pushFrame(frame); pushFrame(frame);
this.meta = null;
this.metaUpto = 1;
this.decoded = false; this.decoded = false;
this.pending = false; this.pending = false;
@ -472,10 +477,30 @@ public class TempFSTTermsReader extends FieldsProducer {
} }
} }
/** Lazily accumulate meta data, when we got a accepted term */
void loadMetaData() throws IOException {
FST.Arc<TempTermOutputs.TempMetaData> last, next;
last = stack[metaUpto].fstArc;
while (metaUpto != level) {
metaUpto++;
next = stack[metaUpto].fstArc;
next.output = fstOutputs.add(next.output, last.output);
last = next;
}
if (last.isFinal()) {
meta = fstOutputs.add(last.output, last.nextFinalOutput);
} else {
meta = last.output;
}
state.docFreq = meta.docFreq;
state.totalTermFreq = meta.totalTermFreq;
}
@Override @Override
public SeekStatus seekCeil(BytesRef target, boolean useCache) throws IOException { public SeekStatus seekCeil(BytesRef target, boolean useCache) throws IOException {
decoded = false; decoded = false;
term = doSeekCeil(target); term = doSeekCeil(target);
loadMetaData();
if (term == null) { if (term == null) {
return SeekStatus.END; return SeekStatus.END;
} else { } else {
@ -488,6 +513,7 @@ public class TempFSTTermsReader extends FieldsProducer {
//if (DEBUG) System.out.println("Enum next()"); //if (DEBUG) System.out.println("Enum next()");
if (pending) { if (pending) {
pending = false; pending = false;
loadMetaData();
return term; return term;
} }
decoded = false; decoded = false;
@ -497,7 +523,7 @@ public class TempFSTTermsReader extends FieldsProducer {
if (loadExpandFrame(topFrame(), frame) != null) { // has valid target if (loadExpandFrame(topFrame(), frame) != null) { // has valid target
pushFrame(frame); pushFrame(frame);
if (isAccept(frame)) { // gotcha if (isAccept(frame)) { // gotcha
return term; break;
} }
continue; // check next target continue; // check next target
} }
@ -506,7 +532,7 @@ public class TempFSTTermsReader extends FieldsProducer {
if (loadNextFrame(topFrame(), frame) != null) { // has valid sibling if (loadNextFrame(topFrame(), frame) != null) { // has valid sibling
pushFrame(frame); pushFrame(frame);
if (isAccept(frame)) { // gotcha if (isAccept(frame)) { // gotcha
return term; break DFS;
} }
continue DFS; // check next target continue DFS; // check next target
} }
@ -514,7 +540,8 @@ public class TempFSTTermsReader extends FieldsProducer {
} }
return null; return null;
} }
return null; loadMetaData();
return term;
} }
private BytesRef doSeekCeil(BytesRef target) throws IOException { private BytesRef doSeekCeil(BytesRef target) throws IOException {
@ -642,47 +669,33 @@ public class TempFSTTermsReader extends FieldsProducer {
return !frame.fstArc.isLast(); return !frame.fstArc.isLast();
} }
void pushFrame(Frame frame) throws IOException { void pushFrame(Frame frame) {
final FST.Arc<TempTermOutputs.TempMetaData> arc = frame.fstArc; term = grow(frame.fstArc.label);
arc.output = fstOutputs.add(topFrame().fstArc.output, arc.output);
if (arc.isFinal()) {
arc.nextFinalOutput = fstOutputs.add(arc.output, arc.nextFinalOutput);
meta = arc.nextFinalOutput;
} else {
meta = arc.output;
}
term = grow(arc.label);
state.docFreq = meta.docFreq;
state.totalTermFreq = meta.totalTermFreq;
level++; level++;
//if (DEBUG) System.out.println(" term=" + term + " level=" + level); //if (DEBUG) System.out.println(" term=" + term + " level=" + level);
} }
Frame popFrame() throws IOException { Frame popFrame() {
final Frame pop = stack[level--], top = topFrame();
if (top.fstArc.isFinal()) {
meta = top.fstArc.nextFinalOutput;
} else {
meta = top.fstArc.output;
}
term = shrink(); term = shrink();
level--;
metaUpto = metaUpto > level ? level : metaUpto;
//if (DEBUG) System.out.println(" term=" + term + " level=" + level); //if (DEBUG) System.out.println(" term=" + term + " level=" + level);
return pop; return stack[level+1];
} }
Frame newFrame() throws IOException { Frame newFrame() {
if (level+1 == stack.length) { if (level+1 == stack.length) {
final Frame[] next = new Frame[ArrayUtil.oversize(level+2, RamUsageEstimator.NUM_BYTES_OBJECT_REF)]; final Frame[] temp = new Frame[ArrayUtil.oversize(level+2, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
System.arraycopy(stack, 0, next, 0, stack.length); System.arraycopy(stack, 0, temp, 0, stack.length);
for (int i = stack.length; i < next.length; i++) { for (int i = stack.length; i < temp.length; i++) {
next[i] = new Frame(); temp[i] = new Frame();
} }
stack = next; stack = temp;
} }
return stack[level+1]; return stack[level+1];
} }
Frame topFrame() throws IOException { Frame topFrame() {
return stack[level]; return stack[level];
} }