Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-6382: enforce max allowed indexed position 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1673508 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2015-04-14 18:54:10 +00:00
parent 382f9e45e0
commit 82b3106f6c
15 changed files with 270 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/CHANGES.txt
View File

@ -113,6 +113,9 @@ Other
* LUCENE-6399: Benchmark module's QueryMaker.resetInputs should call setConfig
so queries can react to property changes in new rounds. (David Smiley)
* LUCENE-6382: Lucene now enforces that positions never exceed the
maximum value IndexWriter.MAX_POSITION. (Robert Muir, Mike McCandless)
Build
* LUCENE-6420: Update forbiddenapis to v1.8 (Uwe Schindler)

106
lucene/backward-codecs/src/test/org/apache/lucene/index/TestMaxPositionInOldIndex.java Normal file
View File

@ -0,0 +1,106 @@
package org.apache.lucene.index;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.InputStream;
import java.nio.file.Path;
import org.apache.lucene.document.Document;
import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
// LUCENE-6382
public class TestMaxPositionInOldIndex extends LuceneTestCase {
// Save this to BuildMaxPositionIndex.java and follow the compile/run instructions to regenerate the .zip:
/*
import java.io.IOException;
import java.nio.file.Paths;
import org.apache.lucene.analysis.CannedTokenStream;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
// Compile:
// javac -cp lucene/build/core/lucene-core-5.1.0-SNAPSHOT.jar:lucene/build/test-framework/lucene-test-framework-5.1.0-SNAPSHOT.jar:lucene/build/analysis/common/lucene-analyzers-common-5.1.0-SNAPSHOT.jar BuildMaxPositionIndex.java
// Run:
// java -cp .:lucene/build/core/lucene-core-5.1.0-SNAPSHOT.jar:lucene/build/test-framework/lucene-test-framework-5.1.0-SNAPSHOT.jar:lucene/build/analysis/common/lucene-analyzers-common-5.1.0-SNAPSHOT.jar:lucene/build/codecs/lucene-codecs-5.1.0-SNAPSHOT.jar BuildMaxPositionIndex
// cd maxposindex
// zip maxposindex.zip *
public class BuildMaxPositionIndex {
public static void main(String[] args) throws IOException {
Directory dir = FSDirectory.open(Paths.get("maxposindex"));
IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(new WhitespaceAnalyzer()));
Document doc = new Document();
// This is at position 1:
Token t1 = new Token("foo", 0, 3);
t1.setPositionIncrement(2);
Token t2 = new Token("foo", 4, 7);
// This overflows max position:
t2.setPositionIncrement(Integer.MAX_VALUE-1);
t2.setPayload(new BytesRef(new byte[] { 0x1 } ));
doc.add(new TextField("foo", new CannedTokenStream(new Token[] {t1, t2})));
iw.addDocument(doc);
iw.close();
dir.close();
}
}
*/
public void testCorruptIndex() throws Exception {
Path path = createTempDir("maxposindex");
InputStream resource = getClass().getResourceAsStream("maxposindex.zip");
assertNotNull("maxposindex not found", resource);
TestUtil.unzip(resource, path);
BaseDirectoryWrapper dir = newFSDirectory(path);
dir.setCheckIndexOnClose(false);
try {
TestUtil.checkIndex(dir, false, true);
fail("corruption was not detected");
} catch (RuntimeException re) {
// expected
assertTrue(re.getMessage().contains("pos 2147483647 > IndexWriter.MAX_POSITION=2147483519"));
}
// Also confirm merging detects this:
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergeScheduler(new SerialMergeScheduler()));
w.addDocument(new Document());
try {
w.forceMerge(1);
} catch (CorruptIndexException cie) {
// SerialMergeScheduler
assertTrue(cie.getMessage().contains("position=2147483647 is too large (> IndexWriter.MAX_POSITION=2147483519), field=\"foo\" doc=0 (resource=PerFieldPostings(segment=_0 formats=1)"));
}
w.close();
dir.close();
}
}

BIN
lucene/backward-codecs/src/test/org/apache/lucene/index/maxposindex.zip Normal file
View File

Binary file not shown.

7
lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java
View File

@ -36,6 +36,7 @@ import org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.IntBlockTermStat
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexOutput;
@ -250,6 +251,12 @@ public final class Lucene50PostingsWriter extends PushPostingsWriterBase {
@Override
public void addPosition(int position, BytesRef payload, int startOffset, int endOffset) throws IOException {
if (position > IndexWriter.MAX_POSITION) {
throw new CorruptIndexException("position=" + position + " is too large (> IndexWriter.MAX_POSITION=" + IndexWriter.MAX_POSITION + ")", docOut);
}
if (position < 0) {
throw new CorruptIndexException("position=" + position + " is < 0", docOut);
}
posDeltaBuffer[posBufferUpto] = position - lastPosition;
if (writePayloads) {
if (payload == null || payload.length == 0) {

7
lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
View File

@ -217,6 +217,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat {
private final Map<String,FieldsProducer> fields = new TreeMap<>();
private final Map<String,FieldsProducer> formats = new HashMap<>();
private final String segment;
// clone for merge
FieldsReader(FieldsReader other) throws IOException {
@ -234,6 +235,8 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat {
assert producer != null;
fields.put(ent.getKey(), producer);
}
segment = other.segment;
}
public FieldsReader(final SegmentReadState readState) throws IOException {
@ -267,6 +270,8 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat {
IOUtils.closeWhileHandlingException(formats.values());
}
}
this.segment = readState.segmentInfo.name;
}
@Override
@ -320,7 +325,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat {
@Override
public String toString() {
return "PerFieldPostings(formats=" + formats.size() + ")";
return "PerFieldPostings(segment=" + segment + " formats=" + formats.size() + ")";
}
}

3
lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
View File

@ -1322,6 +1322,9 @@ public class CheckIndex implements Closeable {
if (pos < 0) {
throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " is out of bounds");
}
if (pos > IndexWriter.MAX_POSITION) {
throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " > IndexWriter.MAX_POSITION=" + IndexWriter.MAX_POSITION);
}
if (pos < lastPos) {
throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " < lastPos " + lastPos);
}

5
lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
View File

@ -612,8 +612,11 @@ final class DefaultIndexingChain extends DocConsumer {
if (invertState.position < invertState.lastPosition) {
if (posIncr == 0) {
throw new IllegalArgumentException("first position increment must be > 0 (got 0) for field '" + field.name() + "'");
} else {
throw new IllegalArgumentException("position increments (and gaps) must be >= 0 (got " + posIncr + ") for field '" + field.name() + "'");
}
throw new IllegalArgumentException("position increments (and gaps) must be >= 0 (got " + posIncr + ") for field '" + field.name() + "'");
} else if (invertState.position > IndexWriter.MAX_POSITION) {
throw new IllegalArgumentException("position " + invertState.position + " is too large for field '" + field.name() + "': max allowed position is " + IndexWriter.MAX_POSITION);
}
invertState.lastPosition = invertState.position;
if (posIncr == 0) {

3
lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -203,6 +203,9 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
// ArrayUtil.MAX_ARRAY_LENGTH here because this can vary across JVMs:
public static final int MAX_DOCS = Integer.MAX_VALUE - 128;
/** Maximum value of the token position in an indexed field. */
public static final int MAX_POSITION = Integer.MAX_VALUE - 128;
// Use package-private instance var to enforce the limit so testing
// can use less electricity:
private static int actualMaxDocs = MAX_DOCS;

33
lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java
View File

@ -46,21 +46,23 @@ public class MappedMultiFields extends FilterFields {
if (terms == null) {
return null;
} else {
return new MappedMultiTerms(mergeState, terms);
return new MappedMultiTerms(field, mergeState, terms);
}
}
private static class MappedMultiTerms extends FilterTerms {
final MergeState mergeState;
final String field;
public MappedMultiTerms(MergeState mergeState, MultiTerms multiTerms) {
public MappedMultiTerms(String field, MergeState mergeState, MultiTerms multiTerms) {
super(multiTerms);
this.field = field;
this.mergeState = mergeState;
}
@Override
public TermsEnum iterator() throws IOException {
return new MappedMultiTermsEnum(mergeState, (MultiTermsEnum) in.iterator());
return new MappedMultiTermsEnum(field, mergeState, (MultiTermsEnum) in.iterator());
}
@Override
@ -86,9 +88,11 @@ public class MappedMultiFields extends FilterFields {
private static class MappedMultiTermsEnum extends FilterTermsEnum {
final MergeState mergeState;
final String field;
public MappedMultiTermsEnum(MergeState mergeState, MultiTermsEnum multiTermsEnum) {
public MappedMultiTermsEnum(String field, MergeState mergeState, MultiTermsEnum multiTermsEnum) {
super(multiTermsEnum);
this.field = field;
this.mergeState = mergeState;
}
@ -110,26 +114,19 @@ public class MappedMultiFields extends FilterFields {
MappingMultiPostingsEnum mappingDocsAndPositionsEnum;
if (reuse instanceof MappingMultiPostingsEnum) {
mappingDocsAndPositionsEnum = (MappingMultiPostingsEnum) reuse;
MappingMultiPostingsEnum postings = (MappingMultiPostingsEnum) reuse;
if (postings.field.equals(this.field)) {
mappingDocsAndPositionsEnum = postings;
} else {
mappingDocsAndPositionsEnum = new MappingMultiPostingsEnum(field, mergeState);
}
} else {
mappingDocsAndPositionsEnum = new MappingMultiPostingsEnum(mergeState);
mappingDocsAndPositionsEnum = new MappingMultiPostingsEnum(field, mergeState);
}
MultiPostingsEnum docsAndPositionsEnum = (MultiPostingsEnum) in.postings(liveDocs, mappingDocsAndPositionsEnum.multiDocsAndPositionsEnum, flags);
mappingDocsAndPositionsEnum.reset(docsAndPositionsEnum);
return mappingDocsAndPositionsEnum;
/*
MappingMultiDocsEnum mappingDocsEnum;
if (reuse instanceof MappingMultiDocsEnum) {
mappingDocsEnum = (MappingMultiDocsEnum) reuse;
} else {
mappingDocsEnum = new MappingMultiDocsEnum(mergeState);
}
MultiDocsEnum docsEnum = (MultiDocsEnum) in.docs(liveDocs, mappingDocsEnum.multiDocsEnum, flags);
mappingDocsEnum.reset(docsEnum);
return mappingDocsEnum;*/
}
}
}

16
lucene/core/src/java/org/apache/lucene/index/MappingMultiPostingsEnum.java
View File

@ -39,9 +39,11 @@ final class MappingMultiPostingsEnum extends PostingsEnum {
int doc = -1;
private MergeState mergeState;
MultiPostingsEnum multiDocsAndPositionsEnum;
final String field;
/** Sole constructor. */
public MappingMultiPostingsEnum(MergeState mergeState) {
public MappingMultiPostingsEnum(String field, MergeState mergeState) {
this.field = field;
this.mergeState = mergeState;
}
@ -112,9 +114,17 @@ final class MappingMultiPostingsEnum extends PostingsEnum {
@Override
public int nextPosition() throws IOException {
return current.nextPosition();
int pos = current.nextPosition();
if (pos < 0) {
throw new CorruptIndexException("position=" + pos + " is negative, field=\"" + field + " doc=" + doc,
mergeState.fieldsProducers[upto].toString());
} else if (pos > IndexWriter.MAX_POSITION) {
throw new CorruptIndexException("position=" + pos + " is too large (> IndexWriter.MAX_POSITION=" + IndexWriter.MAX_POSITION + "), field=\"" + field + "\" doc=" + doc,
mergeState.fieldsProducers[upto].toString());
}
return pos;
}
@Override
public int startOffset() throws IOException {
return current.startOffset();

2
lucene/core/src/java/org/apache/lucene/util/Version.java
View File

@ -202,7 +202,7 @@ public final class Version {
/** Returns a new version based on raw numbers
*
* @lucene.internal */
public static final Version fromBits(int major, int minor, int bugfix) {
public static Version fromBits(int major, int minor, int bugfix) {
return new Version(major, minor, bugfix);
}

7
lucene/core/src/test/org/apache/lucene/index/Test2BPositions.java
View File

@ -29,12 +29,9 @@ import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.TimeUnits;
import org.apache.lucene.util.LuceneTestCase.Monster;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
/**
* Test indexes ~82M docs with 52 positions each, so you get &gt; Integer.MAX_VALUE positions
* @lucene.experimental
@ -60,8 +57,8 @@ public class Test2BPositions extends LuceneTestCase {
MergePolicy mp = w.getConfig().getMergePolicy();
if (mp instanceof LogByteSizeMergePolicy) {
// 1 petabyte:
((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024*1024*1024);
// 1 petabyte:
((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024*1024*1024);
}
Document doc = new Document();

46
lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
View File

@ -30,10 +30,8 @@ import java.util.Random;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CannedTokenStream;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@ -1727,49 +1725,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
uoe.doFail = false;
d.close();
}
public void testIllegalPositions() throws Exception {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
Document doc = new Document();
Token t1 = new Token("foo", 0, 3);
t1.setPositionIncrement(Integer.MAX_VALUE);
Token t2 = new Token("bar", 4, 7);
t2.setPositionIncrement(200);
TokenStream overflowingTokenStream = new CannedTokenStream(
new Token[] { t1, t2 }
);
Field field = new TextField("foo", overflowingTokenStream);
doc.add(field);
try {
iw.addDocument(doc);
fail();
} catch (IllegalArgumentException expected) {
// expected exception
}
iw.close();
dir.close();
}
public void testLegalbutVeryLargePositions() throws Exception {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
Document doc = new Document();
Token t1 = new Token("foo", 0, 3);
t1.setPositionIncrement(Integer.MAX_VALUE-500);
if (random().nextBoolean()) {
t1.setPayload(new BytesRef(new byte[] { 0x1 } ));
}
TokenStream overflowingTokenStream = new CannedTokenStream(
new Token[] { t1 }
);
Field field = new TextField("foo", overflowingTokenStream);
doc.add(field);
iw.addDocument(doc);
iw.close();
dir.close();
}
public void testBoostOmitNorms() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));

103
lucene/core/src/test/org/apache/lucene/index/TestMaxPosition.java Normal file
View File

@ -0,0 +1,103 @@
package org.apache.lucene.index;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.analysis.CannedTokenStream;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.TextField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
// LUCENE-6382
public class TestMaxPosition extends LuceneTestCase {
public void testTooBigPosition() throws Exception {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
Document doc = new Document();
// This is at position 1:
Token t1 = new Token("foo", 0, 3);
t1.setPositionIncrement(2);
if (random().nextBoolean()) {
t1.setPayload(new BytesRef(new byte[] { 0x1 } ));
}
Token t2 = new Token("foo", 4, 7);
// This should overflow max:
t2.setPositionIncrement(IndexWriter.MAX_POSITION);
if (random().nextBoolean()) {
t2.setPayload(new BytesRef(new byte[] { 0x1 } ));
}
doc.add(new TextField("foo", new CannedTokenStream(new Token[] {t1, t2})));
try {
iw.addDocument(doc);
fail("did not hit exception");
} catch (IllegalArgumentException iae) {
// expected
}
// Document should not be visible:
IndexReader r = DirectoryReader.open(iw, true);
assertEquals(0, r.numDocs());
r.close();
iw.close();
dir.close();
}
public void testMaxPosition() throws Exception {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
Document doc = new Document();
// This is at position 0:
Token t1 = new Token("foo", 0, 3);
if (random().nextBoolean()) {
t1.setPayload(new BytesRef(new byte[] { 0x1 } ));
}
Token t2 = new Token("foo", 4, 7);
t2.setPositionIncrement(IndexWriter.MAX_POSITION);
if (random().nextBoolean()) {
t2.setPayload(new BytesRef(new byte[] { 0x1 } ));
}
doc.add(new TextField("foo", new CannedTokenStream(new Token[] {t1, t2})));
iw.addDocument(doc);
// Document should be visible:
IndexReader r = DirectoryReader.open(iw, true);
assertEquals(1, r.numDocs());
PostingsEnum postings = MultiFields.getTermPositionsEnum(r, null, "foo", new BytesRef("foo"));
// "foo" appears in docID=0
assertEquals(0, postings.nextDoc());
// "foo" appears 2 times in the doc
assertEquals(2, postings.freq());
// first at pos=0
assertEquals(0, postings.nextPosition());
// next at pos=MAX
assertEquals(IndexWriter.MAX_POSITION, postings.nextPosition());
r.close();
iw.close();
dir.close();
}
}

4
lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
View File

@ -25,10 +25,11 @@ import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.index.AssertingLeafReader;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.Terms;
@ -219,6 +220,7 @@ public final class AssertingPostingsFormat extends PostingsFormat {
for(int i=0;i<freq;i++) {
int pos = postingsEnum.nextPosition();
assert pos >= lastPos: "pos=" + pos + " vs lastPos=" + lastPos + " i=" + i + " freq=" + freq;
assert pos <= IndexWriter.MAX_POSITION: "pos=" + pos + " is > IndexWriter.MAX_POSITION=" + IndexWriter.MAX_POSITION;
lastPos = pos;
if (hasOffsets) {