Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-6766: MultiXXX now refuse to merge if there is an index sort

This commit is contained in:
Mike McCandless 2016-05-06 19:02:41 -04:00
parent 8fe78da23c
commit eb8b1a92d8
11 changed files with 179 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene50/Lucene50RWSegmentInfoFormat.java
View File

@ -94,10 +94,8 @@ public class Lucene50RWSegmentInfoFormat extends Lucene50SegmentInfoFormat {
@Override
public void write(Directory dir, SegmentInfo si, IOContext ioContext) throws IOException {
final String fileName = IndexFileNames.segmentFileName(si.name, "", Lucene50SegmentInfoFormat.SI_EXTENSION);
// nocommit indexSort
if (si.getIndexSort() != null) {
throw new IllegalArgumentException("teach me to write indexSort");
}
assert si.getIndexSort() == null;
try (IndexOutput output = dir.createOutput(fileName, ioContext)) {
// Only add the file once we've successfully created it, else IFD assert can trip:

4
lucene/core/src/java/org/apache/lucene/index/DocIDMerger.java
View File

@ -84,7 +84,9 @@ public class DocIDMerger<T extends DocIDMerger.Sub> {
/** Reuse API, currently only used by postings during merge */
public void reset() {
if (queue != null) {
assert queue.size() == 0;
queue.clear();
// nocommit why does bloom filter wrapper trip this?
// assert queue.size() == 0: "queue.size() = " + queue.size();
for(T sub : subs) {
while (true) {
int docID = sub.nextDoc();

21
lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
View File

@ -78,6 +78,9 @@ public class MultiDocValues {
final int[] starts = new int[size+1];
for (int i = 0; i < size; i++) {
LeafReaderContext context = leaves.get(i);
if (context.reader().getIndexSort() != null) {
throw new IllegalArgumentException("cannot handle index sort: reader=" + context.reader());
}
NumericDocValues v = context.reader().getNormValues(field);
if (v == null) {
v = DocValues.emptyNumeric();
@ -120,6 +123,9 @@ public class MultiDocValues {
final int[] starts = new int[size+1];
for (int i = 0; i < size; i++) {
LeafReaderContext context = leaves.get(i);
if (context.reader().getIndexSort() != null) {
throw new IllegalArgumentException("cannot handle index sort: reader=" + context.reader());
}
NumericDocValues v = context.reader().getNumericDocValues(field);
if (v == null) {
v = DocValues.emptyNumeric();
@ -165,6 +171,9 @@ public class MultiDocValues {
final int[] starts = new int[size+1];
for (int i = 0; i < size; i++) {
LeafReaderContext context = leaves.get(i);
if (context.reader().getIndexSort() != null) {
throw new IllegalArgumentException("cannot handle index sort: reader=" + context.reader());
}
Bits v = context.reader().getDocsWithField(field);
if (v == null) {
v = new Bits.MatchNoBits(context.reader().maxDoc());
@ -210,6 +219,9 @@ public class MultiDocValues {
final int[] starts = new int[size+1];
for (int i = 0; i < size; i++) {
LeafReaderContext context = leaves.get(i);
if (context.reader().getIndexSort() != null) {
throw new IllegalArgumentException("cannot handle index sort: reader=" + context.reader());
}
BinaryDocValues v = context.reader().getBinaryDocValues(field);
if (v == null) {
v = DocValues.emptyBinary();
@ -254,6 +266,9 @@ public class MultiDocValues {
final int[] starts = new int[size+1];
for (int i = 0; i < size; i++) {
LeafReaderContext context = leaves.get(i);
if (context.reader().getIndexSort() != null) {
throw new IllegalArgumentException("cannot handle index sort: reader=" + context.reader());
}
SortedNumericDocValues v = context.reader().getSortedNumericDocValues(field);
if (v == null) {
v = DocValues.emptySortedNumeric(context.reader().maxDoc());
@ -312,6 +327,9 @@ public class MultiDocValues {
final int[] starts = new int[size+1];
for (int i = 0; i < size; i++) {
LeafReaderContext context = leaves.get(i);
if (context.reader().getIndexSort() != null) {
throw new IllegalArgumentException("cannot handle index sort: reader=" + context.reader());
}
SortedDocValues v = context.reader().getSortedDocValues(field);
if (v == null) {
v = DocValues.emptySorted();
@ -352,6 +370,9 @@ public class MultiDocValues {
final int[] starts = new int[size+1];
for (int i = 0; i < size; i++) {
LeafReaderContext context = leaves.get(i);
if (context.reader().getIndexSort() != null) {
throw new IllegalArgumentException("cannot handle index sort: reader=" + context.reader());
}
SortedSetDocValues v = context.reader().getSortedSetDocValues(field);
if (v == null) {
v = DocValues.emptySortedSet();

9
lucene/core/src/java/org/apache/lucene/index/MultiFields.java
View File

@ -51,7 +51,7 @@ public final class MultiFields extends Fields {
private final ReaderSlice[] subSlices;
private final Map<String,Terms> terms = new ConcurrentHashMap<>();
// nocommit should we somehow throw exc if you try to pass in "sorted" Fields?
// nocommit make test for sorted fields
/** Returns a single {@link Fields} instance for this
* reader, merging fields/terms/docs/positions on the
@ -72,6 +72,9 @@ public final class MultiFields extends Fields {
final List<ReaderSlice> slices = new ArrayList<>(leaves.size());
for (final LeafReaderContext ctx : leaves) {
final LeafReader r = ctx.reader();
if (r.getIndexSort() != null) {
throw new IllegalArgumentException("cannot handle index sort: reader=" + r);
}
final Fields f = r.fields();
fields.add(f);
slices.add(new ReaderSlice(ctx.docBase, r.maxDoc(), fields.size()-1));
@ -107,6 +110,10 @@ public final class MultiFields extends Fields {
for (int i = 0; i < size; i++) {
// record all liveDocs, even if they are null
final LeafReaderContext ctx = leaves.get(i);
if (ctx.reader().getIndexSort() != null) {
throw new IllegalArgumentException("cannot handle index sort: reader=" + ctx.reader());
}
liveDocs[i] = ctx.reader().getLiveDocs();
starts[i] = ctx.docBase;
}

2
lucene/core/src/java/org/apache/lucene/index/MultiReader.java
View File

@ -65,6 +65,8 @@ public class MultiReader extends BaseCompositeReader<IndexReader> {
}
}
// nocommit what if there is an indexSort?
@Override
protected synchronized void doClose() throws IOException {
IOException ioe = null;

40
lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
View File

@ -37,7 +37,6 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.IndexWriter; // javadocs
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.Term;
@ -803,21 +802,34 @@ public class IndexSearcher {
* @lucene.experimental
*/
public CollectionStatistics collectionStatistics(String field) throws IOException {
final int docCount;
final long sumTotalTermFreq;
final long sumDocFreq;
int docCount = 0;
long sumTotalTermFreq = 0;
long sumDocFreq = 0;
assert field != null;
Terms terms = MultiFields.getTerms(reader, field);
if (terms == null) {
docCount = 0;
sumTotalTermFreq = 0;
sumDocFreq = 0;
} else {
docCount = terms.getDocCount();
sumTotalTermFreq = terms.getSumTotalTermFreq();
sumDocFreq = terms.getSumDocFreq();
for(LeafReaderContext ctx : reader.leaves()) {
Terms terms = ctx.reader().fields().terms(field);
if (terms != null) {
int subDocCount = terms.getDocCount();
if (subDocCount == -1) {
docCount = -1;
} else if (docCount != -1) {
docCount += subDocCount;
}
long subSumDocFreq = terms.getSumDocFreq();
if (subSumDocFreq == -1) {
sumDocFreq = -1;
} else if (sumDocFreq != -1) {
sumDocFreq += subSumDocFreq;
}
long subSumTotalTermFreq = terms.getSumTotalTermFreq();
if (subSumTotalTermFreq == -1) {
sumTotalTermFreq = -1;
} else if (sumTotalTermFreq != -1) {
sumTotalTermFreq += subSumTotalTermFreq;
}
}
}
return new CollectionStatistics(field, reader.maxDoc(), docCount, sumTotalTermFreq, sumDocFreq);
}

18
lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
View File

@ -137,7 +137,7 @@ public class TestIndexSorting extends LuceneTestCase {
assertEquals(0, topDocs.totalHits);
} else {
assertEquals(1, topDocs.totalHits);
assertEquals(i, MultiDocValues.getNumericValues(reader, "id").get(topDocs.scoreDocs[0].doc));
assertEquals(i, getNumericDocValue(reader, "id", topDocs.scoreDocs[0].doc));
Document document = reader.document(topDocs.scoreDocs[0].doc);
assertEquals(Integer.toString(i), document.get("id"));
}
@ -148,6 +148,14 @@ public class TestIndexSorting extends LuceneTestCase {
dir.close();
}
private static long getNumericDocValue(IndexReader reader, String field, int docID) throws IOException {
// We can't use MultiDocValues because it gets angry about the sorting:
List<LeafReaderContext> leaves = reader.leaves();
int sub = ReaderUtil.subIndex(docID, leaves);
LeafReaderContext leaf = leaves.get(sub);
return leaf.reader().getNumericDocValues(field).get(docID - leaf.docBase);
}
public void testSortOnMerge() throws IOException {
testSortOnMerge(false);
}
@ -241,7 +249,7 @@ public class TestIndexSorting extends LuceneTestCase {
assertEquals(0, topDocs.totalHits);
} else {
assertEquals(1, topDocs.totalHits);
assertEquals(values.get(i).longValue(), MultiDocValues.getNumericValues(reader, "foo").get(topDocs.scoreDocs[0].doc));
assertEquals(values.get(i).longValue(), getNumericDocValue(reader, "foo", topDocs.scoreDocs[0].doc));
}
}
reader.close();
@ -335,7 +343,7 @@ public class TestIndexSorting extends LuceneTestCase {
for (int i = 0; i < numDocs; ++i) {
final TopDocs topDocs = searcher.search(new TermQuery(new Term("id", Integer.toString(i))), 1);
assertEquals(1, topDocs.totalHits);
assertEquals(values.get(i).longValue(), MultiDocValues.getNumericValues(reader, "foo").get(topDocs.scoreDocs[0].doc));
assertEquals(values.get(i).longValue(), getNumericDocValue(reader, "foo", topDocs.scoreDocs[0].doc));
}
reader.close();
w.close();
@ -380,8 +388,8 @@ public class TestIndexSorting extends LuceneTestCase {
assertEquals(topDocs.totalHits, topDocs2.totalHits);
if (topDocs.totalHits == 1) {
assertEquals(
MultiDocValues.getNumericValues(reader, "foo").get(topDocs.scoreDocs[0].doc),
MultiDocValues.getNumericValues(reader2, "foo").get(topDocs2.scoreDocs[0].doc));
getNumericDocValue(reader, "foo", topDocs.scoreDocs[0].doc),
getNumericDocValue(reader2, "foo", topDocs2.scoreDocs[0].doc));
}
}

57
lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java
View File

@ -26,6 +26,8 @@ import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@ -412,4 +414,59 @@ public class TestMultiDocValues extends LuceneTestCase {
ir2.close();
dir.close();
}
public void testNoIndexSort() throws Exception {
IndexWriterConfig iwc = newIndexWriterConfig();
iwc.setIndexSort(new Sort(new SortField("foo", SortField.Type.INT)));
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, iwc);
w.addDocument(new Document());
DirectoryReader.open(w).close();
w.addDocument(new Document());
// this makes a sorted segment:
w.forceMerge(1);
// this makes another segment, so that MultiDocValues isn't just a no-op:
w.addDocument(new Document());
IndexReader r = DirectoryReader.open(w);
String message = expectThrows(IllegalArgumentException.class, () -> {
MultiDocValues.getDocsWithField(r, "foo");
}).getMessage();
assertTrue(message.contains("cannot handle index sort"));
assertTrue(message.contains("indexSort=<int: \"foo\">"));
message = expectThrows(IllegalArgumentException.class, () -> {
MultiDocValues.getNumericValues(r, "foo");
}).getMessage();
assertTrue(message.contains("cannot handle index sort"));
assertTrue(message.contains("indexSort=<int: \"foo\">"));
message = expectThrows(IllegalArgumentException.class, () -> {
MultiDocValues.getBinaryValues(r, "foo");
}).getMessage();
assertTrue(message.contains("cannot handle index sort"));
assertTrue(message.contains("indexSort=<int: \"foo\">"));
message = expectThrows(IllegalArgumentException.class, () -> {
MultiDocValues.getSortedValues(r, "foo");
}).getMessage();
assertTrue(message.contains("cannot handle index sort"));
assertTrue(message.contains("indexSort=<int: \"foo\">"));
message = expectThrows(IllegalArgumentException.class, () -> {
MultiDocValues.getSortedSetValues(r, "foo");
}).getMessage();
assertTrue(message.contains("cannot handle index sort"));
assertTrue(message.contains("indexSort=<int: \"foo\">"));
message = expectThrows(IllegalArgumentException.class, () -> {
MultiDocValues.getSortedNumericValues(r, "foo");
}).getMessage();
assertTrue(message.contains("cannot handle index sort"));
assertTrue(message.contains("indexSort=<int: \"foo\">"));
r.close();
w.close();
dir.close();
}
}

26
lucene/core/src/test/org/apache/lucene/index/TestMultiFields.java
View File

@ -29,6 +29,8 @@ import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@ -199,4 +201,28 @@ public class TestMultiFields extends LuceneTestCase {
r.close();
dir.close();
}
public void testNoIndexSort() throws Exception {
IndexWriterConfig iwc = newIndexWriterConfig();
iwc.setIndexSort(new Sort(new SortField("foo", SortField.Type.INT)));
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, iwc);
w.addDocument(new Document());
DirectoryReader.open(w).close();
w.addDocument(new Document());
// this makes a sorted segment:
w.forceMerge(1);
// this makes another segment, so that MultiFields.getFields isn't just a no-op:
w.addDocument(new Document());
IndexReader r = DirectoryReader.open(w);
Exception e = expectThrows(IllegalArgumentException.class, () -> {
MultiFields.getFields(r);
});
assertTrue(e.getMessage().contains("cannot handle index sort"));
assertTrue(e.getMessage().contains("indexSort=<int: \"foo\">"));
r.close();
w.close();
dir.close();
}
}

30
lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
View File

@ -48,6 +48,7 @@ import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FilterLeafReader;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReader;
@ -586,8 +587,8 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
// We sorted postings by weight during indexing, so we
// only retrieve the first num hits now:
Collector c2 = new EarlyTerminatingSortingCollector(c, SORT, num);
IndexSearcher searcher = searcherMgr.acquire();
List<LookupResult> results = null;
IndexSearcher searcher = searcherMgr.acquire();
try {
//System.out.println("got searcher=" + searcher);
searcher.search(finalQuery, c2);
@ -607,6 +608,19 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
return results;
}
private static BytesRef getBinaryDocValue(IndexReader reader, String field, int docID) throws IOException {
// We can't use MultiDocValues because it gets angry about the sorting:
List<LeafReaderContext> leaves = reader.leaves();
int sub = ReaderUtil.subIndex(docID, leaves);
LeafReaderContext leaf = leaves.get(sub);
BinaryDocValues bdv = leaf.reader().getBinaryDocValues(field);
if (bdv == null) {
return null;
} else {
return bdv.get(docID - leaf.docBase);
}
}
/**
* Create the results based on the search hits.
* Can be overridden by subclass to add particular behavior (e.g. weight transformation).
@ -621,24 +635,20 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
boolean doHighlight, Set<String> matchedTokens, String prefixToken)
throws IOException {
BinaryDocValues textDV = MultiDocValues.getBinaryValues(searcher.getIndexReader(), TEXT_FIELD_NAME);
// This will just be null if app didn't pass payloads to build():
// TODO: maybe just stored fields? they compress...
BinaryDocValues payloadsDV = MultiDocValues.getBinaryValues(searcher.getIndexReader(), "payloads");
List<LeafReaderContext> leaves = searcher.getIndexReader().leaves();
List<LookupResult> results = new ArrayList<>();
for (int i=0;i<hits.scoreDocs.length;i++) {
FieldDoc fd = (FieldDoc) hits.scoreDocs[i];
BytesRef term = textDV.get(fd.doc);
BytesRef term = getBinaryDocValue(searcher.getIndexReader(), TEXT_FIELD_NAME, fd.doc);
String text = term.utf8ToString();
long score = (Long) fd.fields[0];
BytesRef payload;
if (payloadsDV != null) {
payload = BytesRef.deepCopyOf(payloadsDV.get(fd.doc));
} else {
payload = null;
BytesRef payload = getBinaryDocValue(searcher.getIndexReader(), "payloads", fd.doc);
if (payload != null) {
payload = BytesRef.deepCopyOf(payload);
}
// Must look up sorted-set by segment:

2
lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java
View File

@ -66,7 +66,7 @@ public class AnalyzingInfixSuggesterTest extends LuceneTestCase {
assertEquals("a penny saved is a penny earned", results.get(0).key);
assertEquals("a penny saved is a penny <b>ear</b>ned", results.get(0).highlightKey);
assertEquals(10, results.get(0).value);
assertEquals(new BytesRef("foobaz"), results.get(0).payload);
assertEquals("foobaz", results.get(0).payload.utf8ToString());
assertEquals("lend me your ear", results.get(1).key);
assertEquals("lend me your <b>ear</b>", results.get(1).highlightKey);