Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

strengthen param checking for block tree's term block min/max sizes 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1651362 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2015-01-13 14:44:02 +00:00
parent c0707926df
commit 847cd8aa59
7 changed files with 41 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java
View File

@ -24,6 +24,7 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
import org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter;
import org.apache.lucene.codecs.lucene50.Lucene50PostingsReader;
import org.apache.lucene.codecs.lucene50.Lucene50PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
@ -56,9 +57,8 @@ public class BlockTreeOrdsPostingsFormat extends PostingsFormat {
public BlockTreeOrdsPostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
super("BlockTreeOrds");
this.minTermBlockSize = minTermBlockSize;
assert minTermBlockSize > 1;
this.maxTermBlockSize = maxTermBlockSize;
assert minTermBlockSize <= maxTermBlockSize;
BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
}
@Override

21
lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java
View File

@ -180,18 +180,7 @@ public final class OrdsBlockTreeTermsWriter extends FieldsConsumer {
int maxItemsInBlock)
throws IOException
{
if (minItemsInBlock <= 1) {
throw new IllegalArgumentException("minItemsInBlock must be >= 2; got " + minItemsInBlock);
}
if (maxItemsInBlock <= 0) {
throw new IllegalArgumentException("maxItemsInBlock must be >= 1; got " + maxItemsInBlock);
}
if (minItemsInBlock > maxItemsInBlock) {
throw new IllegalArgumentException("maxItemsInBlock must be >= minItemsInBlock; got maxItemsInBlock=" + maxItemsInBlock + " minItemsInBlock=" + minItemsInBlock);
}
if (2*(minItemsInBlock-1) > maxItemsInBlock) {
throw new IllegalArgumentException("maxItemsInBlock must be at least 2*(minItemsInBlock-1); got maxItemsInBlock=" + maxItemsInBlock + " minItemsInBlock=" + minItemsInBlock);
}
BlockTreeTermsWriter.validateSettings(minItemsInBlock, maxItemsInBlock);
maxDoc = state.segmentInfo.getDocCount();
@ -427,7 +416,7 @@ public final class OrdsBlockTreeTermsWriter extends FieldsConsumer {
// System.out.println(" add sub=" + indexEnt.input + " " + indexEnt.input + " output=" + indexEnt.output);
//}
Output output = indexEnt.output;
long blockTermCount = output.endOrd - output.startOrd + 1;
//long blockTermCount = output.endOrd - output.startOrd + 1;
Output newOutput = FST_OUTPUTS.newOutput(output.bytes, termOrdOffset+output.startOrd, output.endOrd-termOrdOffset);
//System.out.println(" append sub=" + indexEnt.input + " output=" + indexEnt.output + " termOrdOffset=" + termOrdOffset + " blockTermCount=" + blockTermCount + " newOutput=" + newOutput + " endOrd=" + (termOrdOffset+Long.MAX_VALUE-output.endOrd));
builder.add(Util.toIntsRef(indexEnt.input, scratchIntsRef), newOutput);
@ -612,9 +601,6 @@ public final class OrdsBlockTreeTermsWriter extends FieldsConsumer {
// compact format in this case:
boolean isLeafBlock = hasSubBlocks == false;
// Number of terms in this block
int termCount;
// Number of terms in this block and all sub-blocks (recursively)
long totalTermCount;
@ -661,12 +647,10 @@ public final class OrdsBlockTreeTermsWriter extends FieldsConsumer {
bytesWriter.reset();
absolute = false;
}
termCount = end-start;
totalTermCount = end-start;
} else {
// Mixed terms and sub-blocks:
subIndices = new ArrayList<>();
termCount = 0;
totalTermCount = 0;
for (int i=start;i<end;i++) {
PendingEntry ent = pending.get(i);
@ -714,7 +698,6 @@ public final class OrdsBlockTreeTermsWriter extends FieldsConsumer {
bytesWriter.reset();
absolute = false;
termCount++;
totalTermCount++;
} else {
PendingBlock block = (PendingBlock) ent;

21
lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java
View File

@ -255,16 +255,15 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
int maxItemsInBlock)
throws IOException
{
validateSettings(minItemsInBlock, maxItemsInBlock);
if (minItemsInBlock <= 1) {
throw new IllegalArgumentException("minItemsInBlock must be >= 2; got " + minItemsInBlock);
}
if (maxItemsInBlock <= 0) {
throw new IllegalArgumentException("maxItemsInBlock must be >= 1; got " + maxItemsInBlock);
}
if (minItemsInBlock > maxItemsInBlock) {
throw new IllegalArgumentException("maxItemsInBlock must be >= minItemsInBlock; got maxItemsInBlock=" + maxItemsInBlock + " minItemsInBlock=" + minItemsInBlock);
}
if (2*(minItemsInBlock-1) > maxItemsInBlock) {
if (maxItemsInBlock < 2*(minItemsInBlock-1)) {
throw new IllegalArgumentException("maxItemsInBlock must be at least 2*(minItemsInBlock-1); got maxItemsInBlock=" + maxItemsInBlock + " minItemsInBlock=" + minItemsInBlock);
}
@ -308,6 +307,20 @@ public final class BlockTreeTermsWriter extends FieldsConsumer {
indexOut.writeLong(dirStart);
}
/** Throws {@code IllegalArgumentException} if any of these settings
* is invalid. */
public static void validateSettings(int minItemsInBlock, int maxItemsInBlock) {
if (minItemsInBlock <= 1) {
throw new IllegalArgumentException("minItemsInBlock must be >= 2; got " + minItemsInBlock);
}
if (minItemsInBlock > maxItemsInBlock) {
throw new IllegalArgumentException("maxItemsInBlock must be >= minItemsInBlock; got maxItemsInBlock=" + maxItemsInBlock + " minItemsInBlock=" + minItemsInBlock);
}
if (2*(minItemsInBlock-1) > maxItemsInBlock) {
throw new IllegalArgumentException("maxItemsInBlock must be at least 2*(minItemsInBlock-1); got maxItemsInBlock=" + maxItemsInBlock + " minItemsInBlock=" + minItemsInBlock);
}
}
@Override
public void write(Fields fields) throws IOException {

3
lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsFormat.java
View File

@ -413,10 +413,9 @@ public final class Lucene50PostingsFormat extends PostingsFormat {
* @see BlockTreeTermsWriter#BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int) */
public Lucene50PostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
super("Lucene50");
BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
this.minTermBlockSize = minTermBlockSize;
assert minTermBlockSize > 1;
this.maxTermBlockSize = maxTermBlockSize;
assert minTermBlockSize <= maxTermBlockSize;
}
@Override

17
lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat.java
View File

@ -64,4 +64,21 @@ public class TestBlockPostingsFormat extends BasePostingsFormatTestCase {
w.close();
d.close();
}
private void shouldFail(int minItemsInBlock, int maxItemsInBlock) {
try {
new Lucene50PostingsFormat(minItemsInBlock, maxItemsInBlock);
fail("did not hit exception");
} catch (IllegalArgumentException iae) {
// expected
}
}
public void testInvalidBlockSizes() throws Exception {
shouldFail(0, 0);
shouldFail(10, 8);
shouldFail(-1, 10);
shouldFail(10, -1);
shouldFail(10, 12);
}
}

1
lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsFormat.java
View File

@ -77,6 +77,7 @@ public class IDVersionPostingsFormat extends PostingsFormat {
super("IDVersion");
this.minTermsInBlock = minTermsInBlock;
this.maxTermsInBlock = maxTermsInBlock;
BlockTreeTermsWriter.validateSettings(minTermsInBlock, maxTermsInBlock);
}
@Override

13
lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java
View File

@ -176,18 +176,7 @@ public final class VersionBlockTreeTermsWriter extends FieldsConsumer {
int maxItemsInBlock)
throws IOException
{
if (minItemsInBlock <= 1) {
throw new IllegalArgumentException("minItemsInBlock must be >= 2; got " + minItemsInBlock);
}
if (maxItemsInBlock <= 0) {
throw new IllegalArgumentException("maxItemsInBlock must be >= 1; got " + maxItemsInBlock);
}
if (minItemsInBlock > maxItemsInBlock) {
throw new IllegalArgumentException("maxItemsInBlock must be >= minItemsInBlock; got maxItemsInBlock=" + maxItemsInBlock + " minItemsInBlock=" + minItemsInBlock);
}
if (2*(minItemsInBlock-1) > maxItemsInBlock) {
throw new IllegalArgumentException("maxItemsInBlock must be at least 2*(minItemsInBlock-1); got maxItemsInBlock=" + maxItemsInBlock + " minItemsInBlock=" + minItemsInBlock);
}
BlockTreeTermsWriter.validateSettings(minItemsInBlock, maxItemsInBlock);
maxDoc = state.segmentInfo.getDocCount();