Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-7537: fix some 6.x backport issues

This commit is contained in:
Mike McCandless 2016-11-16 10:35:50 -05:00
parent e357f957f3
commit 64b9eefaa9
5 changed files with 74 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
View File

@ -221,21 +221,26 @@ public abstract class DocValuesConsumer implements Closeable {
mergeSortedSetField(mergeFieldInfo, mergeState, toMerge);
} else if (type == DocValuesType.SORTED_NUMERIC) {
List<SortedNumericDocValues> toMerge = new ArrayList<>();
List<SortedNumericDocValues> toMerge2 = new ArrayList<>();
for (int i=0;i<mergeState.docValuesProducers.length;i++) {
SortedNumericDocValues values = null;
SortedNumericDocValues values2 = null;
DocValuesProducer docValuesProducer = mergeState.docValuesProducers[i];
if (docValuesProducer != null) {
FieldInfo fieldInfo = mergeState.fieldInfos[i].fieldInfo(mergeFieldInfo.name);
if (fieldInfo != null && fieldInfo.getDocValuesType() == DocValuesType.SORTED_NUMERIC) {
values = docValuesProducer.getSortedNumeric(fieldInfo);
values2 = docValuesProducer.getSortedNumeric(fieldInfo);
}
}
if (values == null) {
values = DocValues.emptySortedNumeric(mergeState.maxDocs[i]);
values2 = values;
}
toMerge.add(values);
toMerge2.add(values2);
}
mergeSortedNumericField(mergeFieldInfo, mergeState, toMerge);
mergeSortedNumericField(mergeFieldInfo, mergeState, toMerge, toMerge2);
} else {
throw new AssertionError("type=" + type);
}
@ -445,6 +450,11 @@ public abstract class DocValuesConsumer implements Closeable {
return docID;
}
}
@Override
public String toString() {
return "SortedNumericDocValuesSub values=" + values + " docID=" + docID + " mappedDocID=" + mappedDocID;
}
}
/**
@ -452,8 +462,10 @@ public abstract class DocValuesConsumer implements Closeable {
* <p>
* The default implementation calls {@link #addSortedNumericField}, passing
* iterables that filter deleted documents.
* <p>
* We require two <code>toMerge</code> lists because we need to separately iterate the values for each segment concurrently.
*/
public void mergeSortedNumericField(FieldInfo fieldInfo, final MergeState mergeState, List<SortedNumericDocValues> toMerge) throws IOException {
public void mergeSortedNumericField(FieldInfo fieldInfo, final MergeState mergeState, List<SortedNumericDocValues> toMerge, List<SortedNumericDocValues> toMerge2) throws IOException {
addSortedNumericField(fieldInfo,
// doc -> value count
@ -514,9 +526,9 @@ public abstract class DocValuesConsumer implements Closeable {
public Iterator<Number> iterator() {
// We must make a new DocIDMerger for each iterator:
List<SortedNumericDocValuesSub> subs = new ArrayList<>();
assert mergeState.docMaps.length == toMerge.size();
for(int i=0;i<toMerge.size();i++) {
subs.add(new SortedNumericDocValuesSub(mergeState.docMaps[i], toMerge.get(i), mergeState.maxDocs[i]));
assert mergeState.docMaps.length == toMerge2.size();
for(int i=0;i<toMerge2.size();i++) {
subs.add(new SortedNumericDocValuesSub(mergeState.docMaps[i], toMerge2.get(i), mergeState.maxDocs[i]));
}
final DocIDMerger<SortedNumericDocValuesSub> docIDMerger = new DocIDMerger<>(subs, mergeState.segmentInfo.getIndexSort() != null);

29
lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
View File

@ -143,7 +143,7 @@ public class MultiDocValues {
};
}
}
/** Returns a Bits for a reader's docsWithField (potentially merging on-the-fly)
* <p>
* This is a slow way to access this bitset. Instead, access them per-segment
@ -330,6 +330,33 @@ public class MultiDocValues {
return new MultiSortedDocValues(values, starts, mapping);
}
}
/** Expert: returns a SortedDocValues from an array of leaf reader's sorted doc values (potentially doing extremely slow things).
* <p>
* This is an extremely slow way to access sorted values. Instead, access them per-segment
* with {@link LeafReader#getSortedDocValues(String)}
* </p>
*/
public static SortedDocValues getSortedValues(IndexReader r, final SortedDocValues[] leafValues, final int[] docStarts) throws IOException {
final List<LeafReaderContext> leaves = r.leaves();
final int size = leaves.size();
if (leafValues.length != size) {
throw new IllegalArgumentException("leafValues must match the number of leaves; got leafValues.length=" + leafValues.length + " vs leaves.size()=" + leaves.size());
}
if (docStarts.length != size+1) {
throw new IllegalArgumentException("docStarts must match the number of leaves, plus one; got docStarts.length=" + docStarts.length + " vs leaves.size()=" + leaves.size());
}
if (leafValues.length == 0) {
return null;
} else if (leafValues.length == 1) {
return leafValues[0];
}
OrdinalMap mapping = OrdinalMap.build(r.getCoreCacheKey(), leafValues, PackedInts.DEFAULT);
return new MultiSortedDocValues(leafValues, docStarts, mapping);
}
/** Returns a SortedSetDocValues for a reader's docvalues (potentially doing extremely slow things).
* <p>

11
lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
View File

@ -128,18 +128,21 @@ final class MultiSorter {
final int reverseMul = sortField.getReverse() ? -1 : 1;
final SortField.Type sortType = Sorter.getSortFieldType(sortField);
switch(sortField.getType()) {
switch(sortType) {
case STRING:
{
// this uses the efficient segment-local ordinal map:
MultiReader multiReader = new MultiReader(readers.toArray(new LeafReader[readers.size()]));
final SortedDocValues sorted = MultiDocValues.getSortedValues(multiReader, sortField.getField());
final int[] docStarts = new int[readers.size()];
final int[] docStarts = new int[readers.size()+1];
List<LeafReaderContext> leaves = multiReader.leaves();
final SortedDocValues[] leafValues = new SortedDocValues[readers.size()];
for(int i=0;i<readers.size();i++) {
leafValues[i] = Sorter.getOrWrapSorted(readers.get(i), sortField);
docStarts[i] = leaves.get(i).docBase;
}
docStarts[readers.size()] = multiReader.maxDoc();
final SortedDocValues sorted = MultiDocValues.getSortedValues(multiReader, leafValues, docStarts);
final int missingOrd;
if (sortField.getMissingValue() == SortField.STRING_LAST) {
missingOrd = sortField.getReverse() ? Integer.MIN_VALUE : Integer.MAX_VALUE;
@ -176,7 +179,7 @@ final class MultiSorter {
if (sortField.getMissingValue() != null) {
missingValue = (Long) sortField.getMissingValue();
} else {
missingValue = 0;
missingValue = 0l;
}
return new CrossReaderComparator() {

27
lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
View File

@ -411,6 +411,9 @@ public class TestIndexSorting extends LuceneTestCase {
assertFalse(docsWithField.get(0));
assertEquals(7, values.get(1));
assertEquals(18, values.get(2));
r.close();
w.close();
dir.close();
}
public void testMissingMultiValuedLongFirst() throws Exception {
@ -1016,9 +1019,9 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
assertEquals(-1.0, Double.longBitsToDouble(values.get(0)), 0.0);
assertEquals(7.0, Double.longBitsToDouble(values.get(1)), 0.0);
assertEquals(18.0, Double.longBitsToDouble(values.get(2)), 0.0);
assertEquals(-1.0, Float.intBitsToFloat((int) values.get(0)), 0.0);
assertEquals(7.0, Float.intBitsToFloat((int) values.get(1)), 0.0);
assertEquals(18.0, Float.intBitsToFloat((int) values.get(2)), 0.0);
r.close();
w.close();
dir.close();
@ -1298,11 +1301,21 @@ public class TestIndexSorting extends LuceneTestCase {
IndexWriter w = new IndexWriter(dir, iwc);
final int numDocs = atLeast(1000);
final FixedBitSet deleted = new FixedBitSet(numDocs);
if (VERBOSE) {
System.out.println("TEST: " + numDocs + " docs");
}
for (int i = 0; i < numDocs; ++i) {
Document doc = new Document();
int num = random().nextInt(10);
if (VERBOSE) {
System.out.println("doc id=" + i + " count=" + num);
}
for (int j = 0; j < num; j++) {
doc.add(new SortedNumericDocValuesField("foo", random().nextInt(2000)));
int n = random().nextInt(2000);
if (VERBOSE) {
System.out.println(" " + n);
}
doc.add(new SortedNumericDocValuesField("foo", n));
}
doc.add(new StringField("id", Integer.toString(i), Store.YES));
doc.add(new NumericDocValuesField("id", i));
@ -1315,6 +1328,9 @@ public class TestIndexSorting extends LuceneTestCase {
final int id = TestUtil.nextInt(random(), 0, i);
deleted.set(id);
w.deleteDocuments(new Term("id", Integer.toString(id)));
if (VERBOSE) {
System.out.println(" delete doc id=" + id);
}
}
}
@ -1329,8 +1345,7 @@ public class TestIndexSorting extends LuceneTestCase {
} else {
assertEquals(1, topDocs.totalHits);
NumericDocValues values = MultiDocValues.getNumericValues(reader, "id");
assertEquals(topDocs.scoreDocs[0].doc, values.advance(topDocs.scoreDocs[0].doc));
assertEquals(i, values.longValue());
assertEquals(i, MultiDocValues.getNumericValues(reader, "id").get(topDocs.scoreDocs[0].doc));
Document document = reader.document(topDocs.scoreDocs[0].doc);
assertEquals(Integer.toString(i), document.get("id"));
}

2
lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
View File

@ -149,7 +149,7 @@ public class AssertingDocValuesFormat extends DocValuesFormat {
Number next = valueIterator.next();
assert next != null;
long nextValue = next.longValue();
assert nextValue >= previous;
assert nextValue >= previous: "nextValue=" + nextValue + " vs previous=" + previous;
previous = nextValue;
}
}