Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-4678: add BytesStore.truncate and use it to not double-buffer during pack 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1432646 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2013-01-13 15:49:35 +00:00
parent 55c5b77cf0
commit aff120659c
5 changed files with 93 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
lucene/analysis/kuromoji/src/resources/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary$fst.dat
View File

Binary file not shown.

33
lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java
View File

@ -69,6 +69,14 @@ class BytesStore extends DataOutput {
nextWrite = blocks.get(blocks.size()-1).length; nextWrite = blocks.get(blocks.size()-1).length;
} }
/** Absolute write byte; you must ensure dest is < max
* position written so far. */
public void writeByte(int dest, byte b) {
int blockIndex = dest >> blockBits;
byte[] block = blocks.get(blockIndex);
block[dest & blockMask] = b;
}
@Override @Override
public void writeByte(byte b) { public void writeByte(byte b) {
if (nextWrite == blockSize) { if (nextWrite == blockSize) {
@ -237,9 +245,10 @@ class BytesStore extends DataOutput {
} }
} }
/** Reverse the last numBytes. */ /** Reverse from srcPos, inclusive, to destPos, inclusive. */
public void reverse(int srcPos, int destPos) { public void reverse(int srcPos, int destPos) {
assert srcPos < destPos; assert srcPos < destPos;
assert destPos < getPosition();
//System.out.println("reverse src=" + srcPos + " dest=" + destPos); //System.out.println("reverse src=" + srcPos + " dest=" + destPos);
int srcBlockIndex = srcPos >> blockBits; int srcBlockIndex = srcPos >> blockBits;
@ -275,7 +284,7 @@ class BytesStore extends DataOutput {
} }
} }
public void skip(int len) { public void skipBytes(int len) {
while (len > 0) { while (len > 0) {
int chunk = blockSize - nextWrite; int chunk = blockSize - nextWrite;
if (len <= chunk) { if (len <= chunk) {
@ -294,6 +303,26 @@ class BytesStore extends DataOutput {
return (blocks.size()-1) * blockSize + nextWrite; return (blocks.size()-1) * blockSize + nextWrite;
} }
/** Pos must be less than the max position written so far!
* Ie, you cannot "grow" the file with this! */
public void truncate(int newLen) {
assert newLen <= getPosition();
assert newLen >= 0;
int blockIndex = newLen >> blockBits;
nextWrite = newLen & blockMask;
if (nextWrite == 0) {
blockIndex--;
nextWrite = blockSize;
}
blocks.subList(blockIndex+1, blocks.size()).clear();
if (newLen == 0) {
current = null;
} else {
current = blocks.get(blockIndex);
}
assert newLen == getPosition();
}
public void finish() { public void finish() {
if (current != null) { if (current != null) {
byte[] lastBuffer = new byte[nextWrite]; byte[] lastBuffer = new byte[nextWrite];

48
lucene/core/src/java/org/apache/lucene/util/fst/FST.java
View File

@ -730,7 +730,7 @@ public final class FST<T> {
int destPos = fixedArrayStart + nodeIn.numArcs*maxBytesPerArc; int destPos = fixedArrayStart + nodeIn.numArcs*maxBytesPerArc;
assert destPos >= srcPos; assert destPos >= srcPos;
if (destPos > srcPos) { if (destPos > srcPos) {
bytes.skip(destPos - srcPos); bytes.skipBytes(destPos - srcPos);
for(int arcIdx=nodeIn.numArcs-1;arcIdx>=0;arcIdx--) { for(int arcIdx=nodeIn.numArcs-1;arcIdx>=0;arcIdx--) {
destPos -= maxBytesPerArc; destPos -= maxBytesPerArc;
srcPos -= bytesPerArc[arcIdx]; srcPos -= bytesPerArc[arcIdx];
@ -1444,9 +1444,6 @@ public final class FST<T> {
throw new IllegalArgumentException("this FST was not built with willPackFST=true"); throw new IllegalArgumentException("this FST was not built with willPackFST=true");
} }
final RAMOutputStream buffer = new RAMOutputStream();
byte[] bufferBytes = new byte[64];
Arc<T> arc = new Arc<T>(); Arc<T> arc = new Arc<T>();
final BytesReader r = getBytesReader(0); final BytesReader r = getBytesReader(0);
@ -1553,7 +1550,7 @@ public final class FST<T> {
// this is an array'd node and bytesPerArc changes: // this is an array'd node and bytesPerArc changes:
writeNode: writeNode:
while(true) { // retry writing this node while(true) { // retry writing this node
assert buffer.getFilePointer() == 0;
//System.out.println(" cycle: retry"); //System.out.println(" cycle: retry");
readFirstRealTargetArc(node, arc, r); readFirstRealTargetArc(node, arc, r);
@ -1563,9 +1560,9 @@ public final class FST<T> {
if (bytesPerArc == 0) { if (bytesPerArc == 0) {
bytesPerArc = arc.bytesPerArc; bytesPerArc = arc.bytesPerArc;
} }
buffer.writeByte(ARCS_AS_FIXED_ARRAY); writer.writeByte(ARCS_AS_FIXED_ARRAY);
buffer.writeVInt(arc.numArcs); writer.writeVInt(arc.numArcs);
buffer.writeVInt(bytesPerArc); writer.writeVInt(bytesPerArc);
//System.out.println("node " + node + ": " + arc.numArcs + " arcs"); //System.out.println("node " + node + ": " + arc.numArcs + " arcs");
} }
@ -1574,7 +1571,7 @@ public final class FST<T> {
while(true) { // iterate over all arcs for this node while(true) { // iterate over all arcs for this node
//System.out.println(" cycle next arc"); //System.out.println(" cycle next arc");
final int arcStartPos = (int) buffer.getFilePointer(); final int arcStartPos = writer.getPosition();
nodeArcCount++; nodeArcCount++;
byte flags = 0; byte flags = 0;
@ -1621,7 +1618,7 @@ public final class FST<T> {
absPtr = topNodeMap.size() + (int) newNodeAddress.get(arc.target) + addressError; absPtr = topNodeMap.size() + (int) newNodeAddress.get(arc.target) + addressError;
} }
int delta = (int) (newNodeAddress.get(arc.target) + addressError - buffer.getFilePointer() - address - 2); int delta = (int) (newNodeAddress.get(arc.target) + addressError - writer.getPosition() - 2);
if (delta < 0) { if (delta < 0) {
//System.out.println("neg: " + delta); //System.out.println("neg: " + delta);
anyNegDelta = true; anyNegDelta = true;
@ -1637,23 +1634,23 @@ public final class FST<T> {
} }
assert flags != ARCS_AS_FIXED_ARRAY; assert flags != ARCS_AS_FIXED_ARRAY;
buffer.writeByte(flags); writer.writeByte(flags);
fst.writeLabel(buffer, arc.label); fst.writeLabel(writer, arc.label);
if (arc.output != NO_OUTPUT) { if (arc.output != NO_OUTPUT) {
outputs.write(arc.output, buffer); outputs.write(arc.output, writer);
if (!retry) { if (!retry) {
fst.arcWithOutputCount++; fst.arcWithOutputCount++;
} }
} }
if (arc.nextFinalOutput != NO_OUTPUT) { if (arc.nextFinalOutput != NO_OUTPUT) {
outputs.writeFinalOutput(arc.nextFinalOutput, buffer); outputs.writeFinalOutput(arc.nextFinalOutput, writer);
} }
if (doWriteTarget) { if (doWriteTarget) {
int delta = (int) (newNodeAddress.get(arc.target) + addressError - buffer.getFilePointer() - address); int delta = (int) (newNodeAddress.get(arc.target) + addressError - writer.getPosition());
if (delta < 0) { if (delta < 0) {
anyNegDelta = true; anyNegDelta = true;
//System.out.println("neg: " + delta); //System.out.println("neg: " + delta);
@ -1662,7 +1659,7 @@ public final class FST<T> {
if (flag(flags, BIT_TARGET_DELTA)) { if (flag(flags, BIT_TARGET_DELTA)) {
//System.out.println(" delta"); //System.out.println(" delta");
buffer.writeVInt(delta); writer.writeVInt(delta);
if (!retry) { if (!retry) {
deltaCount++; deltaCount++;
} }
@ -1674,7 +1671,7 @@ public final class FST<T> {
System.out.println(" abs"); System.out.println(" abs");
} }
*/ */
buffer.writeVInt(absPtr); writer.writeVInt(absPtr);
if (!retry) { if (!retry) {
if (absPtr >= topNodeMap.size()) { if (absPtr >= topNodeMap.size()) {
absCount++; absCount++;
@ -1686,7 +1683,7 @@ public final class FST<T> {
} }
if (useArcArray) { if (useArcArray) {
final int arcBytes = (int) (buffer.getFilePointer() - arcStartPos); final int arcBytes = writer.getPosition() - arcStartPos;
//System.out.println(" " + arcBytes + " bytes"); //System.out.println(" " + arcBytes + " bytes");
maxBytesPerArc = Math.max(maxBytesPerArc, arcBytes); maxBytesPerArc = Math.max(maxBytesPerArc, arcBytes);
// NOTE: this may in fact go "backwards", if // NOTE: this may in fact go "backwards", if
@ -1696,11 +1693,7 @@ public final class FST<T> {
// will retry (below) so it's OK to ovewrite // will retry (below) so it's OK to ovewrite
// bytes: // bytes:
//wasted += bytesPerArc - arcBytes; //wasted += bytesPerArc - arcBytes;
int skip = (int) (arcStartPos + bytesPerArc - buffer.getFilePointer()); writer.skipBytes(arcStartPos + bytesPerArc - writer.getPosition());
while(skip > 0) {
buffer.writeByte((byte) 0);
skip--;
}
} }
if (arc.isLast()) { if (arc.isLast()) {
@ -1725,19 +1718,12 @@ public final class FST<T> {
// Retry: // Retry:
bytesPerArc = maxBytesPerArc; bytesPerArc = maxBytesPerArc;
buffer.reset(); writer.truncate(address);
nodeArcCount = 0; nodeArcCount = 0;
retry = true; retry = true;
anyNegDelta = false; anyNegDelta = false;
} }
if (bufferBytes.length < (int) buffer.getFilePointer()) {
bufferBytes = ArrayUtil.grow(bufferBytes, (int) buffer.getFilePointer());
}
buffer.writeTo(bufferBytes, 0);
writer.writeBytes(bufferBytes, 0, (int) buffer.getFilePointer());
buffer.reset();
negDelta |= anyNegDelta; negDelta |= anyNegDelta;
fst.arcCount += nodeArcCount; fst.arcCount += nodeArcCount;

48
lucene/core/src/test/org/apache/lucene/util/fst/TestBytesStore.java
View File

@ -42,7 +42,7 @@ public class TestBytesStore extends LuceneTestCase {
int pos = 0; int pos = 0;
while(pos < numBytes) { while(pos < numBytes) {
int op = random().nextInt(7); int op = random().nextInt(8);
if (VERBOSE) { if (VERBOSE) {
System.out.println(" cycle pos=" + pos); System.out.println(" cycle pos=" + pos);
} }
@ -97,21 +97,21 @@ public class TestBytesStore extends LuceneTestCase {
case 3: case 3:
{ {
// reverse bytes // reverse bytes
if (pos > 0) { if (pos > 1) {
int len = _TestUtil.nextInt(random(), 1, Math.min(100, pos)); int len = _TestUtil.nextInt(random(), 2, Math.min(100, pos));
int start; int start;
if (len == pos) { if (len == pos) {
start = 0; start = 0;
} else { } else {
start = random().nextInt(pos - len); start = random().nextInt(pos - len);
} }
int end = start + len; int end = start + len - 1;
if (VERBOSE) { if (VERBOSE) {
System.out.println(" reverse start=" + start + " end=" + end + " len=" + len); System.out.println(" reverse start=" + start + " end=" + end + " len=" + len + " pos=" + pos);
} }
bytes.reverse(start, end); bytes.reverse(start, end);
while(start < end) { while(start <= end) {
byte b = expected[end]; byte b = expected[end];
expected[end] = expected[start]; expected[end] = expected[start];
expected[start] = b; expected[start] = b;
@ -159,17 +159,49 @@ public class TestBytesStore extends LuceneTestCase {
{ {
// skip // skip
int len = random().nextInt(Math.min(100, numBytes - pos)); int len = random().nextInt(Math.min(100, numBytes - pos));
pos += len;
bytes.skip(len);
if (VERBOSE) { if (VERBOSE) {
System.out.println(" skip len=" + len); System.out.println(" skip len=" + len);
} }
pos += len;
bytes.skipBytes(len);
// NOTE: must fill in zeros in case truncate was
// used, else we get false fails:
if (len > 0) {
byte[] zeros = new byte[len];
bytes.writeBytes(pos-len, zeros, 0, len);
}
}
break;
case 7:
{
// absWriteByte
if (pos > 0) {
int dest = random().nextInt(pos);
byte b = (byte) random().nextInt(256);
expected[dest] = b;
bytes.writeByte(dest, b);
} }
break; break;
} }
}
assertEquals(pos, bytes.getPosition()); assertEquals(pos, bytes.getPosition());
if (pos > 0 && random().nextInt(50) == 17) {
// truncate
int len = _TestUtil.nextInt(random(), 1, Math.min(pos, 100));
bytes.truncate(pos - len);
pos -= len;
Arrays.fill(expected, pos, pos+len, (byte) 0);
if (VERBOSE) {
System.out.println(" truncate len=" + len + " newPos=" + pos);
}
}
if ((pos > 0 && random().nextInt(200) == 17)) { if ((pos > 0 && random().nextInt(200) == 17)) {
verify(bytes, expected, pos); verify(bytes, expected, pos);
} }

7
lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
View File

@ -483,11 +483,14 @@ public class TestFSTs extends LuceneTestCase {
break; break;
} }
} }
long t = System.currentTimeMillis() - tStart;
System.out.println((t / 1000.0) + " sec to build"); long tMid = System.currentTimeMillis();
System.out.println(((tMid-tStart) / 1000.0) + " sec to add all terms");
assert builder.getTermCount() == ord; assert builder.getTermCount() == ord;
FST<T> fst = builder.finish(); FST<T> fst = builder.finish();
long tEnd = System.currentTimeMillis();
System.out.println(((tEnd-tMid) / 1000.0) + " sec to finish/pack");
if (fst == null) { if (fst == null) {
System.out.println("FST was fully pruned!"); System.out.println("FST was fully pruned!");
System.exit(0); System.exit(0);