Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5703: BinaryDocValues producers don't allocate or copy bytes on each access anymore 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1600688 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2014-06-05 15:54:49 +00:00
parent 9c4695bcc1
commit 8f267c8560
77 changed files with 1184 additions and 1184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lucene/CHANGES.txt
View File

@ -205,6 +205,10 @@ API Changes
removed, because buffering and checksumming is provided by FilterOutputStreams,
provided by the JDK. (Uwe Schindler, Mike McCandless)
* LUCENE-5703: BinaryDocValues API changed to work like TermsEnum and not allocate/
copy bytes on each access, you are responsible for cloning if you want to keep
data around. (Adrien Grand)
Optimizations
* LUCENE-5603: hunspell stemmer more efficiently strips prefixes
@ -247,6 +251,9 @@ Optimizations
* LUCENE-5730: FSDirectory.open returns MMapDirectory for 64-bit operating
systems, not just Linux and Windows. (Robert Muir)
* LUCENE-5703: BinaryDocValues producers don't allocate or copy bytes on
each access anymore. (Adrien Grand)
Bug fixes
* LUCENE-5673: MMapDirectory: Work around a "bug" in the JDK that throws

99
lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java
View File

@ -57,9 +57,9 @@ class DirectDocValuesProducer extends DocValuesProducer {
// ram instances we have already loaded
private final Map<Integer,NumericDocValues> numericInstances =
new HashMap<>();
private final Map<Integer,BinaryDocValues> binaryInstances =
private final Map<Integer,BinaryRawValues> binaryInstances =
new HashMap<>();
private final Map<Integer,SortedDocValues> sortedInstances =
private final Map<Integer,SortedRawValues> sortedInstances =
new HashMap<>();
private final Map<Integer,SortedSetRawValues> sortedSetInstances =
new HashMap<>();
@ -178,9 +178,13 @@ class DirectDocValuesProducer extends DocValuesProducer {
} else if (fieldType == BYTES) {
binaries.put(fieldNumber, readBinaryEntry(meta));
} else if (fieldType == SORTED) {
sorteds.put(fieldNumber, readSortedEntry(meta));
SortedEntry entry = readSortedEntry(meta);
sorteds.put(fieldNumber, entry);
binaries.put(fieldNumber, entry.values);
} else if (fieldType == SORTED_SET) {
sortedSets.put(fieldNumber, readSortedSetEntry(meta));
SortedSetEntry entry = readSortedSetEntry(meta);
sortedSets.put(fieldNumber, entry);
binaries.put(fieldNumber, entry.values);
} else {
throw new CorruptIndexException("invalid entry type: " + fieldType + ", input=" + meta);
}
@ -279,16 +283,29 @@ class DirectDocValuesProducer extends DocValuesProducer {
@Override
public synchronized BinaryDocValues getBinary(FieldInfo field) throws IOException {
BinaryDocValues instance = binaryInstances.get(field.number);
BinaryRawValues instance = binaryInstances.get(field.number);
if (instance == null) {
// Lazy load
instance = loadBinary(binaries.get(field.number));
binaryInstances.put(field.number, instance);
}
return instance;
final byte[] bytes = instance.bytes;
final int[] address = instance.address;
return new BinaryDocValues() {
final BytesRef term = new BytesRef();
@Override
public BytesRef get(int docID) {
term.bytes = bytes;
term.offset = address[docID];
term.length = address[docID+1] - term.offset;
return term;
}
};
}
private BinaryDocValues loadBinary(BinaryEntry entry) throws IOException {
private BinaryRawValues loadBinary(BinaryEntry entry) throws IOException {
data.seek(entry.offset);
final byte[] bytes = new byte[entry.numBytes];
data.readBytes(bytes, 0, entry.numBytes);
@ -302,31 +319,26 @@ class DirectDocValuesProducer extends DocValuesProducer {
ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(bytes) + RamUsageEstimator.sizeOf(address));
return new BinaryDocValues() {
@Override
public void get(int docID, BytesRef result) {
result.bytes = bytes;
result.offset = address[docID];
result.length = address[docID+1] - result.offset;
};
};
BinaryRawValues values = new BinaryRawValues();
values.bytes = bytes;
values.address = address;
return values;
}
@Override
public synchronized SortedDocValues getSorted(FieldInfo field) throws IOException {
SortedDocValues instance = sortedInstances.get(field.number);
if (instance == null) {
// Lazy load
instance = loadSorted(field);
sortedInstances.put(field.number, instance);
}
return instance;
}
private SortedDocValues loadSorted(FieldInfo field) throws IOException {
public SortedDocValues getSorted(FieldInfo field) throws IOException {
final SortedEntry entry = sorteds.get(field.number);
final NumericDocValues docToOrd = loadNumeric(entry.docToOrd);
final BinaryDocValues values = loadBinary(entry.values);
SortedRawValues instance;
synchronized (this) {
instance = sortedInstances.get(field.number);
if (instance == null) {
// Lazy load
instance = loadSorted(field);
sortedInstances.put(field.number, instance);
}
}
final NumericDocValues docToOrd = instance.docToOrd;
final BinaryDocValues values = getBinary(field);
return new SortedDocValues() {
@ -336,8 +348,8 @@ class DirectDocValuesProducer extends DocValuesProducer {
}
@Override
public void lookupOrd(int ord, BytesRef result) {
values.get(ord, result);
public BytesRef lookupOrd(int ord) {
return values.get(ord);
}
@Override
@ -351,6 +363,14 @@ class DirectDocValuesProducer extends DocValuesProducer {
};
}
private SortedRawValues loadSorted(FieldInfo field) throws IOException {
final SortedEntry entry = sorteds.get(field.number);
final NumericDocValues docToOrd = loadNumeric(entry.docToOrd);
final SortedRawValues values = new SortedRawValues();
values.docToOrd = docToOrd;
return values;
}
@Override
public synchronized SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
SortedSetRawValues instance = sortedSetInstances.get(field.number);
@ -363,7 +383,7 @@ class DirectDocValuesProducer extends DocValuesProducer {
final NumericDocValues docToOrdAddress = instance.docToOrdAddress;
final NumericDocValues ords = instance.ords;
final BinaryDocValues values = instance.values;
final BinaryDocValues values = getBinary(field);
// Must make a new instance since the iterator has state:
return new RandomAccessOrds() {
@ -387,8 +407,8 @@ class DirectDocValuesProducer extends DocValuesProducer {
}
@Override
public void lookupOrd(long ord, BytesRef result) {
values.get((int) ord, result);
public BytesRef lookupOrd(long ord) {
return values.get((int) ord);
}
@Override
@ -416,7 +436,6 @@ class DirectDocValuesProducer extends DocValuesProducer {
SortedSetRawValues instance = new SortedSetRawValues();
instance.docToOrdAddress = loadNumeric(entry.docToOrdAddress);
instance.ords = loadNumeric(entry.ords);
instance.values = loadBinary(entry.values);
return instance;
}
@ -465,11 +484,19 @@ class DirectDocValuesProducer extends DocValuesProducer {
public void close() throws IOException {
data.close();
}
static class BinaryRawValues {
byte[] bytes;
int[] address;
}
static class SortedRawValues {
NumericDocValues docToOrd;
}
static class SortedSetRawValues {
NumericDocValues docToOrdAddress;
NumericDocValues ords;
BinaryDocValues values;
}
static class NumericEntry {

110
lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
View File

@ -71,7 +71,7 @@ class MemoryDocValuesProducer extends DocValuesProducer {
// ram instances we have already loaded
private final Map<Integer,NumericDocValues> numericInstances =
new HashMap<>();
private final Map<Integer,BinaryDocValues> binaryInstances =
private final Map<Integer,BytesAndAddresses> pagedBytesInstances =
new HashMap<>();
private final Map<Integer,FST<Long>> fstInstances =
new HashMap<>();
@ -279,50 +279,68 @@ class MemoryDocValuesProducer extends DocValuesProducer {
}
@Override
public synchronized BinaryDocValues getBinary(FieldInfo field) throws IOException {
BinaryDocValues instance = binaryInstances.get(field.number);
if (instance == null) {
instance = loadBinary(field);
binaryInstances.put(field.number, instance);
public BinaryDocValues getBinary(FieldInfo field) throws IOException {
BinaryEntry entry = binaries.get(field.number);
BytesAndAddresses instance;
synchronized (this) {
instance = pagedBytesInstances.get(field.number);
if (instance == null) {
instance = loadBinary(field);
pagedBytesInstances.put(field.number, instance);
}
}
final PagedBytes.Reader bytesReader = instance.reader;
final MonotonicBlockPackedReader addresses = instance.addresses;
if (addresses == null) {
assert entry.minLength == entry.maxLength;
final int fixedLength = entry.minLength;
return new BinaryDocValues() {
final BytesRef term = new BytesRef();
@Override
public BytesRef get(int docID) {
bytesReader.fillSlice(term, fixedLength * (long)docID, fixedLength);
return term;
}
};
} else {
return new BinaryDocValues() {
final BytesRef term = new BytesRef();
@Override
public BytesRef get(int docID) {
long startAddress = docID == 0 ? 0 : addresses.get(docID-1);
long endAddress = addresses.get(docID);
bytesReader.fillSlice(term, startAddress, (int) (endAddress - startAddress));
return term;
}
};
}
return instance;
}
private BinaryDocValues loadBinary(FieldInfo field) throws IOException {
private BytesAndAddresses loadBinary(FieldInfo field) throws IOException {
BytesAndAddresses bytesAndAddresses = new BytesAndAddresses();
BinaryEntry entry = binaries.get(field.number);
data.seek(entry.offset);
PagedBytes bytes = new PagedBytes(16);
bytes.copy(data, entry.numBytes);
final PagedBytes.Reader bytesReader = bytes.freeze(true);
if (entry.minLength == entry.maxLength) {
final int fixedLength = entry.minLength;
ramBytesUsed.addAndGet(bytes.ramBytesUsed());
return new BinaryDocValues() {
@Override
public void get(int docID, BytesRef result) {
bytesReader.fillSlice(result, fixedLength * (long)docID, fixedLength);
}
};
} else {
bytesAndAddresses.reader = bytes.freeze(true);
ramBytesUsed.addAndGet(bytesAndAddresses.reader.ramBytesUsed());
if (entry.minLength != entry.maxLength) {
data.seek(data.getFilePointer() + entry.missingBytes);
final MonotonicBlockPackedReader addresses = new MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false);
ramBytesUsed.addAndGet(bytes.ramBytesUsed() + addresses.ramBytesUsed());
return new BinaryDocValues() {
@Override
public void get(int docID, BytesRef result) {
long startAddress = docID == 0 ? 0 : addresses.get(docID-1);
long endAddress = addresses.get(docID);
bytesReader.fillSlice(result, startAddress, (int) (endAddress - startAddress));
}
};
bytesAndAddresses.addresses = new MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false);
ramBytesUsed.addAndGet(bytesAndAddresses.addresses.ramBytesUsed());
}
return bytesAndAddresses;
}
@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
final FSTEntry entry = fsts.get(field.number);
if (entry.numOrds == 0) {
return DocValues.EMPTY_SORTED;
return DocValues.emptySorted();
}
FST<Long> instance;
synchronized(this) {
@ -345,21 +363,21 @@ class MemoryDocValuesProducer extends DocValuesProducer {
final BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<>(fst);
return new SortedDocValues() {
final BytesRef term = new BytesRef();
@Override
public int getOrd(int docID) {
return (int) docToOrd.get(docID);
}
@Override
public void lookupOrd(int ord, BytesRef result) {
public BytesRef lookupOrd(int ord) {
try {
in.setPosition(0);
fst.getFirstArc(firstArc);
IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts);
result.bytes = new byte[output.length];
result.offset = 0;
result.length = 0;
Util.toBytesRef(output, result);
Util.toBytesRef(output, term);
return term;
} catch (IOException bogus) {
throw new RuntimeException(bogus);
}
@ -397,7 +415,7 @@ class MemoryDocValuesProducer extends DocValuesProducer {
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
final FSTEntry entry = fsts.get(field.number);
if (entry.numOrds == 0) {
return DocValues.EMPTY_SORTED_SET; // empty FST!
return DocValues.emptySortedSet(); // empty FST!
}
FST<Long> instance;
synchronized(this) {
@ -418,9 +436,10 @@ class MemoryDocValuesProducer extends DocValuesProducer {
final Arc<Long> scratchArc = new Arc<>();
final IntsRef scratchInts = new IntsRef();
final BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<>(fst);
final BytesRef ref = new BytesRef();
final ByteArrayDataInput input = new ByteArrayDataInput();
return new SortedSetDocValues() {
final BytesRef term = new BytesRef();
BytesRef ref;
long currentOrd;
@Override
@ -435,21 +454,19 @@ class MemoryDocValuesProducer extends DocValuesProducer {
@Override
public void setDocument(int docID) {
docToOrds.get(docID, ref);
ref = docToOrds.get(docID);
input.reset(ref.bytes, ref.offset, ref.length);
currentOrd = 0;
}
@Override
public void lookupOrd(long ord, BytesRef result) {
public BytesRef lookupOrd(long ord) {
try {
in.setPosition(0);
fst.getFirstArc(firstArc);
IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts);
result.bytes = new byte[output.length];
result.offset = 0;
result.length = 0;
Util.toBytesRef(output, result);
Util.toBytesRef(output, term);
return term;
} catch (IOException bogus) {
throw new RuntimeException(bogus);
}
@ -552,7 +569,12 @@ class MemoryDocValuesProducer extends DocValuesProducer {
long offset;
long numOrds;
}
static class BytesAndAddresses {
PagedBytes.Reader reader;
MonotonicBlockPackedReader addresses;
}
// exposes FSTEnum directly as a TermsEnum: avoids binary-search next()
static class FSTTermsEnum extends TermsEnum {
final BytesRefFSTEnum<Long> in;

38
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
View File

@ -216,8 +216,10 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
final DecimalFormat decoder = new DecimalFormat(field.pattern, new DecimalFormatSymbols(Locale.ROOT));
return new BinaryDocValues() {
final BytesRef term = new BytesRef();
@Override
public void get(int docID, BytesRef result) {
public BytesRef get(int docID) {
try {
if (docID < 0 || docID >= maxDoc) {
throw new IndexOutOfBoundsException("docID must be 0 .. " + (maxDoc-1) + "; got " + docID);
@ -231,10 +233,11 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
} catch (ParseException pe) {
throw new CorruptIndexException("failed to parse int length (resource=" + in + ")", pe);
}
result.bytes = new byte[len];
result.offset = 0;
result.length = len;
in.readBytes(result.bytes, 0, len);
term.grow(len);
term.offset = 0;
term.length = len;
in.readBytes(term.bytes, 0, len);
return term;
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
@ -293,6 +296,8 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
final DecimalFormat ordDecoder = new DecimalFormat(field.ordPattern, new DecimalFormatSymbols(Locale.ROOT));
return new SortedDocValues() {
final BytesRef term = new BytesRef();
@Override
public int getOrd(int docID) {
if (docID < 0 || docID >= maxDoc) {
@ -312,7 +317,7 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
}
@Override
public void lookupOrd(int ord, BytesRef result) {
public BytesRef lookupOrd(int ord) {
try {
if (ord < 0 || ord >= field.numValues) {
throw new IndexOutOfBoundsException("ord must be 0 .. " + (field.numValues-1) + "; got " + ord);
@ -326,10 +331,11 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
} catch (ParseException pe) {
throw new CorruptIndexException("failed to parse int length (resource=" + in + ")", pe);
}
result.bytes = new byte[len];
result.offset = 0;
result.length = len;
in.readBytes(result.bytes, 0, len);
term.grow(len);
term.offset = 0;
term.length = len;
in.readBytes(term.bytes, 0, len);
return term;
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
@ -357,6 +363,7 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
return new SortedSetDocValues() {
String[] currentOrds = new String[0];
int currentIndex = 0;
final BytesRef term = new BytesRef();
@Override
public long nextOrd() {
@ -388,7 +395,7 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
}
@Override
public void lookupOrd(long ord, BytesRef result) {
public BytesRef lookupOrd(long ord) {
try {
if (ord < 0 || ord >= field.numValues) {
throw new IndexOutOfBoundsException("ord must be 0 .. " + (field.numValues-1) + "; got " + ord);
@ -402,10 +409,11 @@ class SimpleTextDocValuesReader extends DocValuesProducer {
} catch (ParseException pe) {
throw new CorruptIndexException("failed to parse int length (resource=" + in + ")", pe);
}
result.bytes = new byte[len];
result.offset = 0;
result.length = len;
in.readBytes(result.bytes, 0, len);
term.grow(len);
term.offset = 0;
term.length = len;
in.readBytes(term.bytes, 0, len);
return term;
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}

14
lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
View File

@ -199,7 +199,7 @@ public abstract class DocValuesConsumer implements Closeable {
return new Iterator<BytesRef>() {
int readerUpto = -1;
int docIDUpto;
BytesRef nextValue = new BytesRef();
BytesRef nextValue;
BytesRef nextPointer; // points to null if missing, or nextValue
AtomicReader currentReader;
BinaryDocValues currentValues;
@ -248,7 +248,7 @@ public abstract class DocValuesConsumer implements Closeable {
if (currentLiveDocs == null || currentLiveDocs.get(docIDUpto)) {
nextIsSet = true;
if (currentDocsWithField.get(docIDUpto)) {
currentValues.get(docIDUpto, nextValue);
nextValue = currentValues.get(docIDUpto);
nextPointer = nextValue;
} else {
nextPointer = null;
@ -308,7 +308,6 @@ public abstract class DocValuesConsumer implements Closeable {
@Override
public Iterator<BytesRef> iterator() {
return new Iterator<BytesRef>() {
final BytesRef scratch = new BytesRef();
int currentOrd;
@Override
@ -323,9 +322,9 @@ public abstract class DocValuesConsumer implements Closeable {
}
int segmentNumber = map.getFirstSegmentNumber(currentOrd);
int segmentOrd = (int)map.getFirstSegmentOrd(currentOrd);
dvs[segmentNumber].lookupOrd(segmentOrd, scratch);
final BytesRef term = dvs[segmentNumber].lookupOrd(segmentOrd);
currentOrd++;
return scratch;
return term;
}
@Override
@ -444,7 +443,6 @@ public abstract class DocValuesConsumer implements Closeable {
@Override
public Iterator<BytesRef> iterator() {
return new Iterator<BytesRef>() {
final BytesRef scratch = new BytesRef();
long currentOrd;
@Override
@ -459,9 +457,9 @@ public abstract class DocValuesConsumer implements Closeable {
}
int segmentNumber = map.getFirstSegmentNumber(currentOrd);
long segmentOrd = map.getFirstSegmentOrd(currentOrd);
dvs[segmentNumber].lookupOrd(segmentOrd, scratch);
final BytesRef term = dvs[segmentNumber].lookupOrd(segmentOrd);
currentOrd++;
return scratch;
return term;
}
@Override

44
lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java
View File

@ -329,9 +329,12 @@ final class Lucene40DocValuesReader extends DocValuesProducer {
success = true;
ramBytesUsed.addAndGet(bytes.ramBytesUsed());
return new BinaryDocValues() {
@Override
public void get(int docID, BytesRef result) {
bytesReader.fillSlice(result, fixedLength * (long)docID, fixedLength);
public BytesRef get(int docID) {
final BytesRef term = new BytesRef();
bytesReader.fillSlice(term, fixedLength * (long)docID, fixedLength);
return term;
}
};
} finally {
@ -369,10 +372,12 @@ final class Lucene40DocValuesReader extends DocValuesProducer {
ramBytesUsed.addAndGet(bytes.ramBytesUsed() + reader.ramBytesUsed());
return new BinaryDocValues() {
@Override
public void get(int docID, BytesRef result) {
public BytesRef get(int docID) {
final BytesRef term = new BytesRef();
long startAddress = reader.get(docID);
long endAddress = reader.get(docID+1);
bytesReader.fillSlice(result, startAddress, (int)(endAddress - startAddress));
bytesReader.fillSlice(term, startAddress, (int)(endAddress - startAddress));
return term;
}
};
} finally {
@ -412,9 +417,11 @@ final class Lucene40DocValuesReader extends DocValuesProducer {
success = true;
return new BinaryDocValues() {
@Override
public void get(int docID, BytesRef result) {
public BytesRef get(int docID) {
final BytesRef term = new BytesRef();
final long offset = fixedLength * reader.get(docID);
bytesReader.fillSlice(result, offset, fixedLength);
bytesReader.fillSlice(term, offset, fixedLength);
return term;
}
};
} finally {
@ -452,20 +459,23 @@ final class Lucene40DocValuesReader extends DocValuesProducer {
ramBytesUsed.addAndGet(bytes.ramBytesUsed() + reader.ramBytesUsed());
success = true;
return new BinaryDocValues() {
@Override
public void get(int docID, BytesRef result) {
public BytesRef get(int docID) {
final BytesRef term = new BytesRef();
long startAddress = reader.get(docID);
BytesRef lengthBytes = new BytesRef();
bytesReader.fillSlice(lengthBytes, startAddress, 1);
byte code = lengthBytes.bytes[lengthBytes.offset];
if ((code & 128) == 0) {
// length is 1 byte
bytesReader.fillSlice(result, startAddress + 1, (int) code);
bytesReader.fillSlice(term, startAddress + 1, (int) code);
} else {
bytesReader.fillSlice(lengthBytes, startAddress + 1, 1);
int length = ((code & 0x7f) << 8) | (lengthBytes.bytes[lengthBytes.offset] & 0xff);
bytesReader.fillSlice(result, startAddress + 2, length);
bytesReader.fillSlice(term, startAddress + 2, length);
}
return term;
}
};
} finally {
@ -538,8 +548,10 @@ final class Lucene40DocValuesReader extends DocValuesProducer {
}
@Override
public void lookupOrd(int ord, BytesRef result) {
bytesReader.fillSlice(result, fixedLength * (long) ord, fixedLength);
public BytesRef lookupOrd(int ord) {
final BytesRef term = new BytesRef();
bytesReader.fillSlice(term, fixedLength * (long) ord, fixedLength);
return term;
}
@Override
@ -574,10 +586,12 @@ final class Lucene40DocValuesReader extends DocValuesProducer {
}
@Override
public void lookupOrd(int ord, BytesRef result) {
public BytesRef lookupOrd(int ord) {
final BytesRef term = new BytesRef();
long startAddress = addressReader.get(ord);
long endAddress = addressReader.get(ord+1);
bytesReader.fillSlice(result, startAddress, (int)(endAddress - startAddress));
bytesReader.fillSlice(term, startAddress, (int)(endAddress - startAddress));
return term;
}
@Override
@ -604,8 +618,8 @@ final class Lucene40DocValuesReader extends DocValuesProducer {
}
@Override
public void lookupOrd(int ord, BytesRef result) {
in.lookupOrd(ord+1, result);
public BytesRef lookupOrd(int ord) {
return in.lookupOrd(ord+1);
}
@Override

46
lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java
View File

@ -40,6 +40,7 @@ import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
@ -292,19 +293,24 @@ class Lucene42DocValuesProducer extends DocValuesProducer {
ramBytesUsed.addAndGet(bytes.ramBytesUsed());
return new BinaryDocValues() {
@Override
public void get(int docID, BytesRef result) {
bytesReader.fillSlice(result, fixedLength * (long)docID, fixedLength);
public BytesRef get(int docID) {
final BytesRef term = new BytesRef();
bytesReader.fillSlice(term, fixedLength * (long)docID, fixedLength);
return term;
}
};
} else {
final MonotonicBlockPackedReader addresses = new MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false);
ramBytesUsed.addAndGet(bytes.ramBytesUsed() + addresses.ramBytesUsed());
return new BinaryDocValues() {
@Override
public void get(int docID, BytesRef result) {
public BytesRef get(int docID) {
long startAddress = docID == 0 ? 0 : addresses.get(docID-1);
long endAddress = addresses.get(docID);
bytesReader.fillSlice(result, startAddress, (int) (endAddress - startAddress));
final BytesRef term = new BytesRef();
bytesReader.fillSlice(term, startAddress, (int) (endAddress - startAddress));
return term;
}
};
}
@ -334,21 +340,24 @@ class Lucene42DocValuesProducer extends DocValuesProducer {
final BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<>(fst);
return new SortedDocValues() {
final BytesRef term = new BytesRef();
@Override
public int getOrd(int docID) {
return (int) docToOrd.get(docID);
}
@Override
public void lookupOrd(int ord, BytesRef result) {
public BytesRef lookupOrd(int ord) {
try {
in.setPosition(0);
fst.getFirstArc(firstArc);
IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts);
result.bytes = new byte[output.length];
result.offset = 0;
result.length = 0;
Util.toBytesRef(output, result);
term.bytes = ArrayUtil.grow(term.bytes, output.length);
term.offset = 0;
term.length = 0;
return Util.toBytesRef(output, term);
} catch (IOException bogus) {
throw new RuntimeException(bogus);
}
@ -386,7 +395,7 @@ class Lucene42DocValuesProducer extends DocValuesProducer {
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
final FSTEntry entry = fsts.get(field.number);
if (entry.numOrds == 0) {
return DocValues.EMPTY_SORTED_SET; // empty FST!
return DocValues.emptySortedSet(); // empty FST!
}
FST<Long> instance;
synchronized(this) {
@ -407,9 +416,10 @@ class Lucene42DocValuesProducer extends DocValuesProducer {
final Arc<Long> scratchArc = new Arc<>();
final IntsRef scratchInts = new IntsRef();
final BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<>(fst);
final BytesRef ref = new BytesRef();
final ByteArrayDataInput input = new ByteArrayDataInput();
return new SortedSetDocValues() {
final BytesRef term = new BytesRef();
BytesRef ordsRef;
long currentOrd;
@Override
@ -424,21 +434,21 @@ class Lucene42DocValuesProducer extends DocValuesProducer {
@Override
public void setDocument(int docID) {
docToOrds.get(docID, ref);
input.reset(ref.bytes, ref.offset, ref.length);
ordsRef = docToOrds.get(docID);
input.reset(ordsRef.bytes, ordsRef.offset, ordsRef.length);
currentOrd = 0;
}
@Override
public void lookupOrd(long ord, BytesRef result) {
public BytesRef lookupOrd(long ord) {
try {
in.setPosition(0);
fst.getFirstArc(firstArc);
IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts);
result.bytes = new byte[output.length];
result.offset = 0;
result.length = 0;
Util.toBytesRef(output, result);
term.bytes = ArrayUtil.grow(term.bytes, output.length);
term.offset = 0;
term.length = 0;
return Util.toBytesRef(output, term);
} catch (IOException bogus) {
throw new RuntimeException(bogus);
}

98
lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java
View File

@ -383,18 +383,20 @@ public class Lucene45DocValuesProducer extends DocValuesProducer implements Clos
final IndexInput data = this.data.clone();
return new LongBinaryDocValues() {
final BytesRef term;
{
term = new BytesRef(bytes.maxLength);
term.offset = 0;
term.length = bytes.maxLength;
}
@Override
public void get(long id, BytesRef result) {
public BytesRef get(long id) {
long address = bytes.offset + id * bytes.maxLength;
try {
data.seek(address);
// NOTE: we could have one buffer, but various consumers (e.g. FieldComparatorSource)
// assume "they" own the bytes after calling this!
final byte[] buffer = new byte[bytes.maxLength];
data.readBytes(buffer, 0, buffer.length);
result.bytes = buffer;
result.offset = 0;
result.length = buffer.length;
data.readBytes(term.bytes, 0, term.length);
return term;
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -425,20 +427,18 @@ public class Lucene45DocValuesProducer extends DocValuesProducer implements Clos
final MonotonicBlockPackedReader addresses = getAddressInstance(data, field, bytes);
return new LongBinaryDocValues() {
final BytesRef term = new BytesRef(Math.max(0, bytes.maxLength));
@Override
public void get(long id, BytesRef result) {
public BytesRef get(long id) {
long startAddress = bytes.offset + (id == 0 ? 0 : addresses.get(id-1));
long endAddress = bytes.offset + addresses.get(id);
int length = (int) (endAddress - startAddress);
try {
data.seek(startAddress);
// NOTE: we could have one buffer, but various consumers (e.g. FieldComparatorSource)
// assume "they" own the bytes after calling this!
final byte[] buffer = new byte[length];
data.readBytes(buffer, 0, buffer.length);
result.bytes = buffer;
result.offset = 0;
result.length = length;
data.readBytes(term.bytes, 0, length);
term.length = length;
return term;
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -496,8 +496,8 @@ public class Lucene45DocValuesProducer extends DocValuesProducer implements Clos
}
@Override
public void lookupOrd(int ord, BytesRef result) {
binary.get(ord, result);
public BytesRef lookupOrd(int ord) {
return binary.get(ord);
}
@Override
@ -583,8 +583,8 @@ public class Lucene45DocValuesProducer extends DocValuesProducer implements Clos
}
@Override
public void lookupOrd(long ord, BytesRef result) {
binary.get(ord, result);
public BytesRef lookupOrd(long ord) {
return binary.get(ord);
}
@Override
@ -723,11 +723,11 @@ public class Lucene45DocValuesProducer extends DocValuesProducer implements Clos
// internally we compose complex dv (sorted/sortedset) from other ones
static abstract class LongBinaryDocValues extends BinaryDocValues {
@Override
public final void get(int docID, BytesRef result) {
get((long)docID, result);
public final BytesRef get(int docID) {
return get((long) docID);
}
abstract void get(long id, BytesRef Result);
abstract BytesRef get(long id);
}
// in the compressed case, we add a few additional operations for
@ -752,13 +752,10 @@ public class Lucene45DocValuesProducer extends DocValuesProducer implements Clos
}
@Override
public void get(long id, BytesRef result) {
public BytesRef get(long id) {
try {
termsEnum.seekExact(id);
BytesRef term = termsEnum.term();
result.bytes = term.bytes;
result.offset = term.offset;
result.length = term.length;
return termsEnum.term();
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -793,28 +790,18 @@ public class Lucene45DocValuesProducer extends DocValuesProducer implements Clos
return new TermsEnum() {
private long currentOrd = -1;
// TODO: maxLength is negative when all terms are merged away...
private final BytesRef termBuffer = new BytesRef(bytes.maxLength < 0 ? 0 : bytes.maxLength);
private final BytesRef term = new BytesRef(); // TODO: paranoia?
private final BytesRef term = new BytesRef(bytes.maxLength < 0 ? 0 : bytes.maxLength);
@Override
public BytesRef next() throws IOException {
if (doNext() == null) {
return null;
} else {
setTerm();
return term;
}
}
private BytesRef doNext() throws IOException {
if (++currentOrd >= numValues) {
return null;
} else {
int start = input.readVInt();
int suffix = input.readVInt();
input.readBytes(termBuffer.bytes, start, suffix);
termBuffer.length = start + suffix;
return termBuffer;
input.readBytes(term.bytes, start, suffix);
term.length = start + suffix;
return term;
}
}
@ -827,8 +814,8 @@ public class Lucene45DocValuesProducer extends DocValuesProducer implements Clos
while (low <= high) {
long mid = (low + high) >>> 1;
doSeek(mid * interval);
int cmp = termBuffer.compareTo(text);
seekExact(mid * interval);
int cmp = term.compareTo(text);
if (cmp < 0) {
low = mid + 1;
@ -836,7 +823,6 @@ public class Lucene45DocValuesProducer extends DocValuesProducer implements Clos
high = mid - 1;
} else {
// we got lucky, found an indexed term
setTerm();
return SeekStatus.FOUND;
}
}
@ -847,15 +833,13 @@ public class Lucene45DocValuesProducer extends DocValuesProducer implements Clos
// block before insertion point
long block = low-1;
doSeek(block < 0 ? -1 : block * interval);
seekExact(block < 0 ? -1 : block * interval);
while (doNext() != null) {
int cmp = termBuffer.compareTo(text);
while (next() != null) {
int cmp = term.compareTo(text);
if (cmp == 0) {
setTerm();
return SeekStatus.FOUND;
} else if (cmp > 0) {
setTerm();
return SeekStatus.NOT_FOUND;
}
}
@ -865,11 +849,6 @@ public class Lucene45DocValuesProducer extends DocValuesProducer implements Clos
@Override
public void seekExact(long ord) throws IOException {
doSeek(ord);
setTerm();
}
private void doSeek(long ord) throws IOException {
long block = ord / interval;
if (ord >= currentOrd && block == currentOrd / interval) {
@ -881,16 +860,9 @@ public class Lucene45DocValuesProducer extends DocValuesProducer implements Clos
}
while (currentOrd < ord) {
doNext();
next();
}
}
private void setTerm() {
// TODO: is there a cleaner way
term.bytes = new byte[termBuffer.length];
term.offset = 0;
term.copyBytes(termBuffer);
}
@Override
public BytesRef term() throws IOException {

98
lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesProducer.java
View File

@ -363,18 +363,20 @@ public class Lucene49DocValuesProducer extends DocValuesProducer implements Clos
final IndexInput data = this.data.clone();
return new LongBinaryDocValues() {
final BytesRef term;
{
term = new BytesRef(bytes.maxLength);
term.offset = 0;
term.length = bytes.maxLength;
}
@Override
public void get(long id, BytesRef result) {
public BytesRef get(long id) {
long address = bytes.offset + id * bytes.maxLength;
try {
data.seek(address);
// NOTE: we could have one buffer, but various consumers (e.g. FieldComparatorSource)
// assume "they" own the bytes after calling this!
final byte[] buffer = new byte[bytes.maxLength];
data.readBytes(buffer, 0, buffer.length);
result.bytes = buffer;
result.offset = 0;
result.length = buffer.length;
data.readBytes(term.bytes, 0, term.length);
return term;
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -405,20 +407,18 @@ public class Lucene49DocValuesProducer extends DocValuesProducer implements Clos
final MonotonicBlockPackedReader addresses = getAddressInstance(data, field, bytes);
return new LongBinaryDocValues() {
final BytesRef term = new BytesRef(Math.max(0, bytes.maxLength));
@Override
public void get(long id, BytesRef result) {
public BytesRef get(long id) {
long startAddress = bytes.offset + (id == 0 ? 0 : addresses.get(id-1));
long endAddress = bytes.offset + addresses.get(id);
int length = (int) (endAddress - startAddress);
try {
data.seek(startAddress);
// NOTE: we could have one buffer, but various consumers (e.g. FieldComparatorSource)
// assume "they" own the bytes after calling this!
final byte[] buffer = new byte[length];
data.readBytes(buffer, 0, buffer.length);
result.bytes = buffer;
result.offset = 0;
result.length = length;
data.readBytes(term.bytes, 0, length);
term.length = length;
return term;
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -474,8 +474,8 @@ public class Lucene49DocValuesProducer extends DocValuesProducer implements Clos
}
@Override
public void lookupOrd(int ord, BytesRef result) {
binary.get(ord, result);
public BytesRef lookupOrd(int ord) {
return binary.get(ord);
}
@Override
@ -561,8 +561,8 @@ public class Lucene49DocValuesProducer extends DocValuesProducer implements Clos
}
@Override
public void lookupOrd(long ord, BytesRef result) {
binary.get(ord, result);
public BytesRef lookupOrd(long ord) {
return binary.get(ord);
}
@Override
@ -704,11 +704,11 @@ public class Lucene49DocValuesProducer extends DocValuesProducer implements Clos
// internally we compose complex dv (sorted/sortedset) from other ones
static abstract class LongBinaryDocValues extends BinaryDocValues {
@Override
public final void get(int docID, BytesRef result) {
get((long)docID, result);
public final BytesRef get(int docID) {
return get((long)docID);
}
abstract void get(long id, BytesRef Result);
abstract BytesRef get(long id);
}
// in the compressed case, we add a few additional operations for
@ -733,13 +733,10 @@ public class Lucene49DocValuesProducer extends DocValuesProducer implements Clos
}
@Override
public void get(long id, BytesRef result) {
public BytesRef get(long id) {
try {
termsEnum.seekExact(id);
BytesRef term = termsEnum.term();
result.bytes = term.bytes;
result.offset = term.offset;
result.length = term.length;
return termsEnum.term();
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -774,28 +771,18 @@ public class Lucene49DocValuesProducer extends DocValuesProducer implements Clos
return new TermsEnum() {
private long currentOrd = -1;
// TODO: maxLength is negative when all terms are merged away...
private final BytesRef termBuffer = new BytesRef(bytes.maxLength < 0 ? 0 : bytes.maxLength);
private final BytesRef term = new BytesRef(); // TODO: paranoia?
private final BytesRef term = new BytesRef(bytes.maxLength < 0 ? 0 : bytes.maxLength);
@Override
public BytesRef next() throws IOException {
if (doNext() == null) {
return null;
} else {
setTerm();
return term;
}
}
private BytesRef doNext() throws IOException {
if (++currentOrd >= numValues) {
return null;
} else {
int start = input.readVInt();
int suffix = input.readVInt();
input.readBytes(termBuffer.bytes, start, suffix);
termBuffer.length = start + suffix;
return termBuffer;
input.readBytes(term.bytes, start, suffix);
term.length = start + suffix;
return term;
}
}
@ -808,8 +795,8 @@ public class Lucene49DocValuesProducer extends DocValuesProducer implements Clos
while (low <= high) {
long mid = (low + high) >>> 1;
doSeek(mid * interval);
int cmp = termBuffer.compareTo(text);
seekExact(mid * interval);
int cmp = term.compareTo(text);
if (cmp < 0) {
low = mid + 1;
@ -817,7 +804,6 @@ public class Lucene49DocValuesProducer extends DocValuesProducer implements Clos
high = mid - 1;
} else {
// we got lucky, found an indexed term
setTerm();
return SeekStatus.FOUND;
}
}
@ -828,15 +814,13 @@ public class Lucene49DocValuesProducer extends DocValuesProducer implements Clos
// block before insertion point
long block = low-1;
doSeek(block < 0 ? -1 : block * interval);
seekExact(block < 0 ? -1 : block * interval);
while (doNext() != null) {
int cmp = termBuffer.compareTo(text);
while (next() != null) {
int cmp = term.compareTo(text);
if (cmp == 0) {
setTerm();
return SeekStatus.FOUND;
} else if (cmp > 0) {
setTerm();
return SeekStatus.NOT_FOUND;
}
}
@ -846,11 +830,6 @@ public class Lucene49DocValuesProducer extends DocValuesProducer implements Clos
@Override
public void seekExact(long ord) throws IOException {
doSeek(ord);
setTerm();
}
private void doSeek(long ord) throws IOException {
long block = ord / interval;
if (ord >= currentOrd && block == currentOrd / interval) {
@ -862,16 +841,9 @@ public class Lucene49DocValuesProducer extends DocValuesProducer implements Clos
}
while (currentOrd < ord) {
doNext();
next();
}
}
private void setTerm() {
// TODO: is there a cleaner way
term.bytes = new byte[termBuffer.length];
term.offset = 0;
term.copyBytes(termBuffer);
}
@Override
public BytesRef term() throws IOException {

7
lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java
View File

@ -28,6 +28,9 @@ public abstract class BinaryDocValues {
* constructors, typically implicit.) */
protected BinaryDocValues() {}
/** Lookup the value for document. */
public abstract void get(int docID, BytesRef result);
/** Lookup the value for document. The returned {@link BytesRef} may be
* re-used across calls to {@link #get(int)} so make sure to
* {@link BytesRef#deepCopyOf(BytesRef) copy it} if you want to keep it
* around. */
public abstract BytesRef get(int docID);
}

31
lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
View File

@ -1392,12 +1392,11 @@ public class CheckIndex {
}
private static void checkBinaryDocValues(String fieldName, AtomicReader reader, BinaryDocValues dv, Bits docsWithField) {
BytesRef scratch = new BytesRef();
for (int i = 0; i < reader.maxDoc(); i++) {
dv.get(i, scratch);
assert scratch.isValid();
if (docsWithField.get(i) == false && scratch.length > 0) {
throw new RuntimeException("dv for field: " + fieldName + " is missing but has value=" + scratch + " for doc: " + i);
final BytesRef term = dv.get(i);
assert term.isValid();
if (docsWithField.get(i) == false && term.length > 0) {
throw new RuntimeException("dv for field: " + fieldName + " is missing but has value=" + term + " for doc: " + i);
}
}
}
@ -1430,16 +1429,15 @@ public class CheckIndex {
throw new RuntimeException("dv for field: " + fieldName + " has holes in its ords, valueCount=" + dv.getValueCount() + " but only used: " + seenOrds.cardinality());
}
BytesRef lastValue = null;
BytesRef scratch = new BytesRef();
for (int i = 0; i <= maxOrd; i++) {
dv.lookupOrd(i, scratch);
assert scratch.isValid();
final BytesRef term = dv.lookupOrd(i);
assert term.isValid();
if (lastValue != null) {
if (scratch.compareTo(lastValue) <= 0) {
throw new RuntimeException("dv for field: " + fieldName + " has ords out of order: " + lastValue + " >=" + scratch);
if (term.compareTo(lastValue) <= 0) {
throw new RuntimeException("dv for field: " + fieldName + " has ords out of order: " + lastValue + " >=" + term);
}
}
lastValue = BytesRef.deepCopyOf(scratch);
lastValue = BytesRef.deepCopyOf(term);
}
}
@ -1501,16 +1499,15 @@ public class CheckIndex {
}
BytesRef lastValue = null;
BytesRef scratch = new BytesRef();
for (long i = 0; i <= maxOrd; i++) {
dv.lookupOrd(i, scratch);
assert scratch.isValid();
final BytesRef term = dv.lookupOrd(i);
assert term.isValid();
if (lastValue != null) {
if (scratch.compareTo(lastValue) <= 0) {
throw new RuntimeException("dv for field: " + fieldName + " has ords out of order: " + lastValue + " >=" + scratch);
if (term.compareTo(lastValue) <= 0) {
throw new RuntimeException("dv for field: " + fieldName + " has ords out of order: " + lastValue + " >=" + term);
}
}
lastValue = BytesRef.deepCopyOf(scratch);
lastValue = BytesRef.deepCopyOf(term);
}
}

145
lucene/core/src/java/org/apache/lucene/index/DocValues.java
View File

@ -26,87 +26,92 @@ import org.apache.lucene.util.BytesRef;
* This class contains utility methods and constants for DocValues
*/
public final class DocValues {
/* no instantiation */
private DocValues() {}
/**
* An empty BinaryDocValues which returns {@link BytesRef#EMPTY_BYTES} for every document
*/
public static final BinaryDocValues EMPTY_BINARY = new BinaryDocValues() {
@Override
public void get(int docID, BytesRef result) {
result.bytes = BytesRef.EMPTY_BYTES;
result.offset = 0;
result.length = 0;
}
};
public static final BinaryDocValues emptyBinary() {
final BytesRef empty = new BytesRef();
return new BinaryDocValues() {
@Override
public BytesRef get(int docID) {
return empty;
}
};
}
/**
* An empty NumericDocValues which returns zero for every document
*/
public static final NumericDocValues EMPTY_NUMERIC = new NumericDocValues() {
@Override
public long get(int docID) {
return 0;
}
};
public static final NumericDocValues emptyNumeric() {
return new NumericDocValues() {
@Override
public long get(int docID) {
return 0;
}
};
}
/**
* An empty SortedDocValues which returns {@link BytesRef#EMPTY_BYTES} for every document
*/
public static final SortedDocValues EMPTY_SORTED = new SortedDocValues() {
@Override
public int getOrd(int docID) {
return -1;
}
public static final SortedDocValues emptySorted() {
final BytesRef empty = new BytesRef();
return new SortedDocValues() {
@Override
public int getOrd(int docID) {
return -1;
}
@Override
public void lookupOrd(int ord, BytesRef result) {
result.bytes = BytesRef.EMPTY_BYTES;
result.offset = 0;
result.length = 0;
}
@Override
public BytesRef lookupOrd(int ord) {
return empty;
}
@Override
public int getValueCount() {
return 0;
}
};
@Override
public int getValueCount() {
return 0;
}
};
}
/**
* An empty SortedDocValues which returns {@link SortedSetDocValues#NO_MORE_ORDS} for every document
*/
public static final SortedSetDocValues EMPTY_SORTED_SET = new RandomAccessOrds() {
@Override
public long nextOrd() {
return NO_MORE_ORDS;
}
@Override
public void setDocument(int docID) {}
@Override
public void lookupOrd(long ord, BytesRef result) {
throw new IndexOutOfBoundsException();
}
@Override
public long getValueCount() {
return 0;
}
@Override
public long ordAt(int index) {
throw new IndexOutOfBoundsException();
}
@Override
public int cardinality() {
return 0;
}
};
public static final SortedSetDocValues emptySortedSet() {
return new RandomAccessOrds() {
@Override
public long nextOrd() {
return NO_MORE_ORDS;
}
@Override
public void setDocument(int docID) {}
@Override
public BytesRef lookupOrd(long ord) {
throw new IndexOutOfBoundsException();
}
@Override
public long getValueCount() {
return 0;
}
@Override
public long ordAt(int index) {
throw new IndexOutOfBoundsException();
}
@Override
public int cardinality() {
return 0;
}
};
}
/**
* Returns a multi-valued view over the provided SortedDocValues
@ -166,52 +171,52 @@ public final class DocValues {
// as opposed to the AtomicReader apis (which must be strict for consistency), these are lenient
/**
* Returns NumericDocValues for the reader, or {@link #EMPTY_NUMERIC} if it has none.
* Returns NumericDocValues for the reader, or {@link #emptyNumeric()} if it has none.
*/
public static NumericDocValues getNumeric(AtomicReader in, String field) throws IOException {
NumericDocValues dv = in.getNumericDocValues(field);
if (dv == null) {
return EMPTY_NUMERIC;
return emptyNumeric();
} else {
return dv;
}
}
/**
* Returns BinaryDocValues for the reader, or {@link #EMPTY_BINARY} if it has none.
* Returns BinaryDocValues for the reader, or {@link #emptyBinary} if it has none.
*/
public static BinaryDocValues getBinary(AtomicReader in, String field) throws IOException {
BinaryDocValues dv = in.getBinaryDocValues(field);
if (dv == null) {
dv = in.getSortedDocValues(field);
if (dv == null) {
return EMPTY_BINARY;
return emptyBinary();
}
}
return dv;
}
/**
* Returns SortedDocValues for the reader, or {@link #EMPTY_SORTED} if it has none.
* Returns SortedDocValues for the reader, or {@link #emptySorted} if it has none.
*/
public static SortedDocValues getSorted(AtomicReader in, String field) throws IOException {
SortedDocValues dv = in.getSortedDocValues(field);
if (dv == null) {
return EMPTY_SORTED;
return emptySorted();
} else {
return dv;
}
}
/**
* Returns SortedSetDocValues for the reader, or {@link #EMPTY_SORTED_SET} if it has none.
* Returns SortedSetDocValues for the reader, or {@link #emptySortedSet} if it has none.
*/
public static SortedSetDocValues getSortedSet(AtomicReader in, String field) throws IOException {
SortedSetDocValues dv = in.getSortedSetDocValues(field);
if (dv == null) {
SortedDocValues sorted = in.getSortedDocValues(field);
if (sorted == null) {
return EMPTY_SORTED_SET;
return emptySortedSet();
}
return singleton(sorted);
}

22
lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
View File

@ -74,7 +74,7 @@ public class MultiDocValues {
AtomicReaderContext context = leaves.get(i);
NumericDocValues v = context.reader().getNormValues(field);
if (v == null) {
v = DocValues.EMPTY_NUMERIC;
v = DocValues.emptyNumeric();
} else {
anyReal = true;
}
@ -116,7 +116,7 @@ public class MultiDocValues {
AtomicReaderContext context = leaves.get(i);
NumericDocValues v = context.reader().getNumericDocValues(field);
if (v == null) {
v = DocValues.EMPTY_NUMERIC;
v = DocValues.emptyNumeric();
} else {
anyReal = true;
}
@ -206,7 +206,7 @@ public class MultiDocValues {
AtomicReaderContext context = leaves.get(i);
BinaryDocValues v = context.reader().getBinaryDocValues(field);
if (v == null) {
v = DocValues.EMPTY_BINARY;
v = DocValues.emptyBinary();
} else {
anyReal = true;
}
@ -220,9 +220,9 @@ public class MultiDocValues {
} else {
return new BinaryDocValues() {
@Override
public void get(int docID, BytesRef result) {
public BytesRef get(int docID) {
int subIndex = ReaderUtil.subIndex(docID, starts);
values[subIndex].get(docID - starts[subIndex], result);
return values[subIndex].get(docID - starts[subIndex]);
}
};
}
@ -251,7 +251,7 @@ public class MultiDocValues {
AtomicReaderContext context = leaves.get(i);
SortedDocValues v = context.reader().getSortedDocValues(field);
if (v == null) {
v = DocValues.EMPTY_SORTED;
v = DocValues.emptySorted();
} else {
anyReal = true;
}
@ -295,7 +295,7 @@ public class MultiDocValues {
AtomicReaderContext context = leaves.get(i);
SortedSetDocValues v = context.reader().getSortedSetDocValues(field);
if (v == null) {
v = DocValues.EMPTY_SORTED_SET;
v = DocValues.emptySortedSet();
} else {
anyReal = true;
}
@ -453,10 +453,10 @@ public class MultiDocValues {
}
@Override
public void lookupOrd(int ord, BytesRef result) {
public BytesRef lookupOrd(int ord) {
int subIndex = mapping.getFirstSegmentNumber(ord);
int segmentOrd = (int) mapping.getFirstSegmentOrd(ord);
values[subIndex].lookupOrd(segmentOrd, result);
return values[subIndex].lookupOrd(segmentOrd);
}
@Override
@ -504,10 +504,10 @@ public class MultiDocValues {
}
@Override
public void lookupOrd(long ord, BytesRef result) {
public BytesRef lookupOrd(long ord) {
int subIndex = mapping.getFirstSegmentNumber(ord);
long segmentOrd = mapping.getFirstSegmentOrd(ord);
values[subIndex].lookupOrd(segmentOrd, result);
return values[subIndex].lookupOrd(segmentOrd);
}
@Override

4
lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
View File

@ -400,7 +400,6 @@ class ReadersAndUpdates {
int curDoc = -1;
int updateDoc = updatesIter.nextDoc();
BytesRef scratch = new BytesRef();
@Override
public boolean hasNext() {
@ -421,8 +420,7 @@ class ReadersAndUpdates {
assert curDoc < updateDoc;
if (currentValues != null && docsWithField.get(curDoc)) {
// only read the current value if the document had a value before
currentValues.get(curDoc, scratch);
return scratch;
return currentValues.get(curDoc);
} else {
return null;
}

10
lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
View File

@ -168,7 +168,7 @@ final class SegmentMerger {
NumericDocValues values = reader.getNumericDocValues(field.name);
Bits bits = reader.getDocsWithField(field.name);
if (values == null) {
values = DocValues.EMPTY_NUMERIC;
values = DocValues.emptyNumeric();
bits = new Bits.MatchNoBits(reader.maxDoc());
}
toMerge.add(values);
@ -182,7 +182,7 @@ final class SegmentMerger {
BinaryDocValues values = reader.getBinaryDocValues(field.name);
Bits bits = reader.getDocsWithField(field.name);
if (values == null) {
values = DocValues.EMPTY_BINARY;
values = DocValues.emptyBinary();
bits = new Bits.MatchNoBits(reader.maxDoc());
}
toMerge.add(values);
@ -194,7 +194,7 @@ final class SegmentMerger {
for (AtomicReader reader : mergeState.readers) {
SortedDocValues values = reader.getSortedDocValues(field.name);
if (values == null) {
values = DocValues.EMPTY_SORTED;
values = DocValues.emptySorted();
}
toMerge.add(values);
}
@ -204,7 +204,7 @@ final class SegmentMerger {
for (AtomicReader reader : mergeState.readers) {
SortedSetDocValues values = reader.getSortedSetDocValues(field.name);
if (values == null) {
values = DocValues.EMPTY_SORTED_SET;
values = DocValues.emptySortedSet();
}
toMerge.add(values);
}
@ -235,7 +235,7 @@ final class SegmentMerger {
for (AtomicReader reader : mergeState.readers) {
NumericDocValues norms = reader.getNormValues(field.name);
if (norms == null) {
norms = DocValues.EMPTY_NUMERIC;
norms = DocValues.emptyNumeric();
}
toMerge.add(norms);
docsWithField.add(new Bits.MatchAllBits(reader.maxDoc()));

4
lucene/core/src/java/org/apache/lucene/index/SingletonSortedSetDocValues.java
View File

@ -58,9 +58,9 @@ final class SingletonSortedSetDocValues extends SortedSetDocValues {
}
@Override
public void lookupOrd(long ord, BytesRef result) {
public BytesRef lookupOrd(long ord) {
// cast is ok: single-valued cannot exceed Integer.MAX_VALUE
in.lookupOrd((int)ord, result);
return in.lookupOrd((int) ord);
}
@Override

4
lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
View File

@ -140,7 +140,7 @@ public final class SlowCompositeReaderWrapper extends AtomicReader {
AtomicReaderContext context = in.leaves().get(i);
SortedDocValues v = context.reader().getSortedDocValues(field);
if (v == null) {
v = DocValues.EMPTY_SORTED;
v = DocValues.emptySorted();
}
values[i] = v;
starts[i] = context.docBase;
@ -179,7 +179,7 @@ public final class SlowCompositeReaderWrapper extends AtomicReader {
AtomicReaderContext context = in.leaves().get(i);
SortedSetDocValues v = context.reader().getSortedSetDocValues(field);
if (v == null) {
v = DocValues.EMPTY_SORTED_SET;
v = DocValues.emptySortedSet();
}
values[i] = v;
starts[i] = context.docBase;

25
lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java
View File

@ -28,7 +28,7 @@ import org.apache.lucene.util.BytesRef;
* are dense and in increasing sorted order.
*/
public abstract class SortedDocValues extends BinaryDocValues {
/** Sole constructor. (For invocation by subclass
* constructors, typically implicit.) */
protected SortedDocValues() {}
@ -42,12 +42,14 @@ public abstract class SortedDocValues extends BinaryDocValues {
*/
public abstract int getOrd(int docID);
/** Retrieves the value for the specified ordinal.
/** Retrieves the value for the specified ordinal. The returned
* {@link BytesRef} may be re-used across calls to {@link #lookupOrd(int)}
* so make sure to {@link BytesRef#deepCopyOf(BytesRef) copy it} if you want
* to keep it around.
* @param ord ordinal to lookup (must be &gt;= 0 and &lt {@link #getValueCount()})
* @param result will be populated with the ordinal's value
* @see #getOrd(int)
*/
public abstract void lookupOrd(int ord, BytesRef result);
public abstract BytesRef lookupOrd(int ord);
/**
* Returns the number of unique values.
@ -56,15 +58,15 @@ public abstract class SortedDocValues extends BinaryDocValues {
*/
public abstract int getValueCount();
private final BytesRef empty = new BytesRef();
@Override
public void get(int docID, BytesRef result) {
public BytesRef get(int docID) {
int ord = getOrd(docID);
if (ord == -1) {
result.bytes = BytesRef.EMPTY_BYTES;
result.length = 0;
result.offset = 0;
return empty;
} else {
lookupOrd(ord, result);
return lookupOrd(ord);
}
}
@ -75,14 +77,13 @@ public abstract class SortedDocValues extends BinaryDocValues {
* @param key Key to look up
**/
public int lookupTerm(BytesRef key) {
BytesRef spare = new BytesRef();
int low = 0;
int high = getValueCount()-1;
while (low <= high) {
int mid = (low + high) >>> 1;
lookupOrd(mid, spare);
int cmp = spare.compareTo(key);
final BytesRef term = lookupOrd(mid);
int cmp = term.compareTo(key);
if (cmp < 0) {
low = mid + 1;

26
lucene/core/src/java/org/apache/lucene/index/SortedDocValuesTermsEnum.java
View File

@ -28,11 +28,13 @@ import org.apache.lucene.util.BytesRef;
class SortedDocValuesTermsEnum extends TermsEnum {
private final SortedDocValues values;
private int currentOrd = -1;
private final BytesRef term = new BytesRef();
private BytesRef term;
private final BytesRef scratch;
/** Creates a new TermsEnum over the provided values */
public SortedDocValuesTermsEnum(SortedDocValues values) {
this.values = values;
scratch = new BytesRef();
}
@Override
@ -40,12 +42,8 @@ class SortedDocValuesTermsEnum extends TermsEnum {
int ord = values.lookupTerm(text);
if (ord >= 0) {
currentOrd = ord;
term.offset = 0;
// TODO: is there a cleaner way?
// term.bytes may be pointing to codec-private byte[]
// storage, so we must force new byte[] allocation:
term.bytes = new byte[text.length];
term.copyBytes(text);
scratch.copyBytes(text);
term = scratch;
return SeekStatus.FOUND;
} else {
currentOrd = -ord-1;
@ -53,7 +51,7 @@ class SortedDocValuesTermsEnum extends TermsEnum {
return SeekStatus.END;
} else {
// TODO: hmm can we avoid this "extra" lookup?:
values.lookupOrd(currentOrd, term);
term = values.lookupOrd(currentOrd);
return SeekStatus.NOT_FOUND;
}
}
@ -63,13 +61,9 @@ class SortedDocValuesTermsEnum extends TermsEnum {
public boolean seekExact(BytesRef text) throws IOException {
int ord = values.lookupTerm(text);
if (ord >= 0) {
term.offset = 0;
// TODO: is there a cleaner way?
// term.bytes may be pointing to codec-private byte[]
// storage, so we must force new byte[] allocation:
term.bytes = new byte[text.length];
term.copyBytes(text);
currentOrd = ord;
scratch.copyBytes(text);
term = scratch;
return true;
} else {
return false;
@ -80,7 +74,7 @@ class SortedDocValuesTermsEnum extends TermsEnum {
public void seekExact(long ord) throws IOException {
assert ord >= 0 && ord < values.getValueCount();
currentOrd = (int) ord;
values.lookupOrd(currentOrd, term);
term = values.lookupOrd(currentOrd);
}
@Override
@ -89,7 +83,7 @@ class SortedDocValuesTermsEnum extends TermsEnum {
if (currentOrd >= values.getValueCount()) {
return null;
}
values.lookupOrd(currentOrd, term);
term = values.lookupOrd(currentOrd);
return term;
}

13
lucene/core/src/java/org/apache/lucene/index/SortedSetDocValues.java
View File

@ -53,12 +53,14 @@ public abstract class SortedSetDocValues {
*/
public abstract void setDocument(int docID);
/** Retrieves the value for the specified ordinal.
/** Retrieves the value for the specified ordinal. The returned
* {@link BytesRef} may be re-used across calls to lookupOrd so make sure to
* {@link BytesRef#deepCopyOf(BytesRef) copy it} if you want to keep it
* around.
* @param ord ordinal to lookup
* @param result will be populated with the ordinal's value
* @see #nextOrd
*/
public abstract void lookupOrd(long ord, BytesRef result);
public abstract BytesRef lookupOrd(long ord);
/**
* Returns the number of unique values.
@ -74,14 +76,13 @@ public abstract class SortedSetDocValues {
* @param key Key to look up
**/
public long lookupTerm(BytesRef key) {
BytesRef spare = new BytesRef();
long low = 0;
long high = getValueCount()-1;
while (low <= high) {
long mid = (low + high) >>> 1;
lookupOrd(mid, spare);
int cmp = spare.compareTo(key);
final BytesRef term = lookupOrd(mid);
int cmp = term.compareTo(key);
if (cmp < 0) {
low = mid + 1;

26
lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesTermsEnum.java
View File

@ -28,11 +28,13 @@ import org.apache.lucene.util.BytesRef;
class SortedSetDocValuesTermsEnum extends TermsEnum {
private final SortedSetDocValues values;
private long currentOrd = -1;
private final BytesRef term = new BytesRef();
private BytesRef term;
private final BytesRef scratch;
/** Creates a new TermsEnum over the provided values */
public SortedSetDocValuesTermsEnum(SortedSetDocValues values) {
this.values = values;
scratch = new BytesRef();
}
@Override
@ -40,12 +42,8 @@ class SortedSetDocValuesTermsEnum extends TermsEnum {
long ord = values.lookupTerm(text);
if (ord >= 0) {
currentOrd = ord;
term.offset = 0;
// TODO: is there a cleaner way?
// term.bytes may be pointing to codec-private byte[]
// storage, so we must force new byte[] allocation:
term.bytes = new byte[text.length];
term.copyBytes(text);
scratch.copyBytes(text);
term = scratch;
return SeekStatus.FOUND;
} else {
currentOrd = -ord-1;
@ -53,7 +51,7 @@ class SortedSetDocValuesTermsEnum extends TermsEnum {
return SeekStatus.END;
} else {
// TODO: hmm can we avoid this "extra" lookup?:
values.lookupOrd(currentOrd, term);
term = values.lookupOrd(currentOrd);
return SeekStatus.NOT_FOUND;
}
}
@ -63,13 +61,9 @@ class SortedSetDocValuesTermsEnum extends TermsEnum {
public boolean seekExact(BytesRef text) throws IOException {
long ord = values.lookupTerm(text);
if (ord >= 0) {
term.offset = 0;
// TODO: is there a cleaner way?
// term.bytes may be pointing to codec-private byte[]
// storage, so we must force new byte[] allocation:
term.bytes = new byte[text.length];
term.copyBytes(text);
currentOrd = ord;
scratch.copyBytes(text);
term = scratch;
return true;
} else {
return false;
@ -80,7 +74,7 @@ class SortedSetDocValuesTermsEnum extends TermsEnum {
public void seekExact(long ord) throws IOException {
assert ord >= 0 && ord < values.getValueCount();
currentOrd = (int) ord;
values.lookupOrd(currentOrd, term);
term = values.lookupOrd(currentOrd);
}
@Override
@ -89,7 +83,7 @@ class SortedSetDocValuesTermsEnum extends TermsEnum {
if (currentOrd >= values.getValueCount()) {
return null;
}
values.lookupOrd(currentOrd, term);
term = values.lookupOrd(currentOrd);
return term;
}

73
lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
View File

@ -831,7 +831,7 @@ public abstract class FieldComparator<T> {
if (values[slot] == null) {
values[slot] = new BytesRef();
}
termsIndex.lookupOrd(ord, values[slot]);
values[slot].copyBytes(termsIndex.lookupOrd(ord));
}
ords[slot] = ord;
readerGen[slot] = currentReaderGen;
@ -960,21 +960,25 @@ public abstract class FieldComparator<T> {
// sentinels, just used internally in this comparator
private static final byte[] MISSING_BYTES = new byte[0];
private static final byte[] NON_MISSING_BYTES = new byte[0];
private BytesRef[] values;
// TODO: this is seriously not good, we should nuke this comparator, or
// instead we should represent missing as null, or use missingValue from the user...
// but it was always this way...
private final BytesRef MISSING_BYTESREF = new BytesRef(MISSING_BYTES);
private final BytesRef[] values;
private final BytesRef[] tempBRs;
private BinaryDocValues docTerms;
private Bits docsWithField;
private final String field;
private BytesRef bottom;
private BytesRef topValue;
private final BytesRef tempBR = new BytesRef();
// TODO: add missing first/last support here?
/** Sole constructor. */
TermValComparator(int numHits, String field) {
values = new BytesRef[numHits];
tempBRs = new BytesRef[numHits];
this.field = field;
}
@ -982,32 +986,27 @@ public abstract class FieldComparator<T> {
public int compare(int slot1, int slot2) {
final BytesRef val1 = values[slot1];
final BytesRef val2 = values[slot2];
if (val1.bytes == MISSING_BYTES) {
if (val2.bytes == MISSING_BYTES) {
return 0;
}
return -1;
} else if (val2.bytes == MISSING_BYTES) {
return 1;
}
return val1.compareTo(val2);
return compareValues(val1, val2);
}
@Override
public int compareBottom(int doc) {
docTerms.get(doc, tempBR);
setMissingBytes(doc, tempBR);
return compareValues(bottom, tempBR);
final BytesRef comparableBytes = getComparableBytes(doc, docTerms.get(doc));
return compareValues(bottom, comparableBytes);
}
@Override
public void copy(int slot, int doc) {
if (values[slot] == null) {
values[slot] = new BytesRef();
final BytesRef comparableBytes = getComparableBytes(doc, docTerms.get(doc));
if (comparableBytes == MISSING_BYTESREF) {
values[slot] = MISSING_BYTESREF;
} else {
if (tempBRs[slot] == null) {
tempBRs[slot] = new BytesRef();
}
values[slot] = tempBRs[slot];
values[slot].copyBytes(comparableBytes);
}
docTerms.get(doc, values[slot]);
setMissingBytes(doc, values[slot]);
}
@Override
@ -1027,6 +1026,9 @@ public abstract class FieldComparator<T> {
if (value == null) {
throw new IllegalArgumentException("value cannot be null");
}
if (value.bytes == MISSING_BYTES) {
value = MISSING_BYTESREF;
}
topValue = value;
}
@ -1038,12 +1040,12 @@ public abstract class FieldComparator<T> {
@Override
public int compareValues(BytesRef val1, BytesRef val2) {
// missing always sorts first:
if (val1.bytes == MISSING_BYTES) {
if (val2.bytes == MISSING_BYTES) {
if (val1 == MISSING_BYTESREF) {
if (val2 == MISSING_BYTESREF) {
return 0;
}
return -1;
} else if (val2.bytes == MISSING_BYTES) {
} else if (val2 == MISSING_BYTESREF) {
return 1;
}
return val1.compareTo(val2);
@ -1051,20 +1053,19 @@ public abstract class FieldComparator<T> {
@Override
public int compareTop(int doc) {
docTerms.get(doc, tempBR);
setMissingBytes(doc, tempBR);
return compareValues(topValue, tempBR);
final BytesRef comparableBytes = getComparableBytes(doc, docTerms.get(doc));
return compareValues(topValue, comparableBytes);
}
private void setMissingBytes(int doc, BytesRef br) {
if (br.length == 0) {
br.offset = 0;
if (docsWithField.get(doc) == false) {
br.bytes = MISSING_BYTES;
} else {
br.bytes = NON_MISSING_BYTES;
}
/**
* Given a document and a term, return the term itself if it exists or
* {@link #MISSING_BYTESREF} otherwise.
*/
private BytesRef getComparableBytes(int doc, BytesRef term) {
if (term.length == 0 && docsWithField.get(doc) == false) {
return MISSING_BYTESREF;
}
return term;
}
}
}

16
lucene/core/src/java/org/apache/lucene/search/SortedSetSelector.java
View File

@ -106,8 +106,8 @@ public class SortedSetSelector {
}
@Override
public void lookupOrd(int ord, BytesRef result) {
in.lookupOrd(ord, result);
public BytesRef lookupOrd(int ord) {
return in.lookupOrd(ord);
}
@Override
@ -141,8 +141,8 @@ public class SortedSetSelector {
}
@Override
public void lookupOrd(int ord, BytesRef result) {
in.lookupOrd(ord, result);
public BytesRef lookupOrd(int ord) {
return in.lookupOrd(ord);
}
@Override
@ -176,8 +176,8 @@ public class SortedSetSelector {
}
@Override
public void lookupOrd(int ord, BytesRef result) {
in.lookupOrd(ord, result);
public BytesRef lookupOrd(int ord) {
return in.lookupOrd(ord);
}
@Override
@ -211,8 +211,8 @@ public class SortedSetSelector {
}
@Override
public void lookupOrd(int ord, BytesRef result) {
in.lookupOrd(ord, result);
public BytesRef lookupOrd(int ord) {
return in.lookupOrd(ord);
}
@Override

5
lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldDocValuesFormat.java
View File

@ -110,7 +110,6 @@ public class TestPerFieldDocValuesFormat extends BaseDocValuesFormatTestCase {
Query query = new TermQuery(new Term("fieldname", "text"));
TopDocs hits = isearcher.search(query, null, 1);
assertEquals(1, hits.totalHits);
BytesRef scratch = new BytesRef();
// Iterate through the results:
for (int i = 0; i < hits.scoreDocs.length; i++) {
StoredDocument hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
@ -119,8 +118,8 @@ public class TestPerFieldDocValuesFormat extends BaseDocValuesFormatTestCase {
NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv1");
assertEquals(5, dv.get(hits.scoreDocs[i].doc));
BinaryDocValues dv2 = ireader.leaves().get(0).reader().getBinaryDocValues("dv2");
dv2.get(hits.scoreDocs[i].doc, scratch);
assertEquals(new BytesRef("hello world"), scratch);
final BytesRef term = dv2.get(hits.scoreDocs[i].doc);
assertEquals(new BytesRef("hello world"), term);
}
ireader.close();

10
lucene/core/src/test/org/apache/lucene/index/Test2BBinaryDocValues.java
View File

@ -80,15 +80,14 @@ public class Test2BBinaryDocValues extends LuceneTestCase {
int expectedValue = 0;
for (AtomicReaderContext context : r.leaves()) {
AtomicReader reader = context.reader();
BytesRef scratch = new BytesRef();
BinaryDocValues dv = reader.getBinaryDocValues("dv");
for (int i = 0; i < reader.maxDoc(); i++) {
bytes[0] = (byte)(expectedValue >> 24);
bytes[1] = (byte)(expectedValue >> 16);
bytes[2] = (byte)(expectedValue >> 8);
bytes[3] = (byte) expectedValue;
dv.get(i, scratch);
assertEquals(data, scratch);
final BytesRef term = dv.get(i);
assertEquals(data, term);
expectedValue++;
}
}
@ -141,11 +140,10 @@ public class Test2BBinaryDocValues extends LuceneTestCase {
ByteArrayDataInput input = new ByteArrayDataInput();
for (AtomicReaderContext context : r.leaves()) {
AtomicReader reader = context.reader();
BytesRef scratch = new BytesRef(bytes);
BinaryDocValues dv = reader.getBinaryDocValues("dv");
for (int i = 0; i < reader.maxDoc(); i++) {
dv.get(i, scratch);
input.reset(scratch.bytes, scratch.offset, scratch.length);
final BytesRef term = dv.get(i);
input.reset(term.bytes, term.offset, term.length);
assertEquals(expectedValue % 65535, input.readVInt());
assertTrue(input.eof());
expectedValue++;

9
lucene/core/src/test/org/apache/lucene/index/Test2BSortedDocValues.java
View File

@ -78,13 +78,12 @@ public class Test2BSortedDocValues extends LuceneTestCase {
int expectedValue = 0;
for (AtomicReaderContext context : r.leaves()) {
AtomicReader reader = context.reader();
BytesRef scratch = new BytesRef();
BinaryDocValues dv = reader.getSortedDocValues("dv");
for (int i = 0; i < reader.maxDoc(); i++) {
bytes[0] = (byte)(expectedValue >> 8);
bytes[1] = (byte) expectedValue;
dv.get(i, scratch);
assertEquals(data, scratch);
final BytesRef term = dv.get(i);
assertEquals(data, term);
expectedValue++;
}
}
@ -144,8 +143,8 @@ public class Test2BSortedDocValues extends LuceneTestCase {
bytes[2] = (byte) (counter >> 8);
bytes[3] = (byte) counter;
counter++;
dv.get(i, scratch);
assertEquals(data, scratch);
final BytesRef term = dv.get(i);
assertEquals(data, term);
}
}

32
lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
View File

@ -524,20 +524,19 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
(byte)(id >>> 24), (byte)(id >>> 16),(byte)(id >>> 8),(byte)id
};
BytesRef expectedRef = new BytesRef(bytes);
BytesRef scratch = new BytesRef();
dvBytesDerefFixed.get(i, scratch);
assertEquals(expectedRef, scratch);
dvBytesDerefVar.get(i, scratch);
assertEquals(expectedRef, scratch);
dvBytesSortedFixed.get(i, scratch);
assertEquals(expectedRef, scratch);
dvBytesSortedVar.get(i, scratch);
assertEquals(expectedRef, scratch);
dvBytesStraightFixed.get(i, scratch);
assertEquals(expectedRef, scratch);
dvBytesStraightVar.get(i, scratch);
assertEquals(expectedRef, scratch);
BytesRef term = dvBytesDerefFixed.get(i);
assertEquals(expectedRef, term);
term = dvBytesDerefVar.get(i);
assertEquals(expectedRef, term);
term = dvBytesSortedFixed.get(i);
assertEquals(expectedRef, term);
term = dvBytesSortedVar.get(i);
assertEquals(expectedRef, term);
term = dvBytesStraightFixed.get(i);
assertEquals(expectedRef, term);
term = dvBytesStraightVar.get(i);
assertEquals(expectedRef, term);
assertEquals((double)id, Double.longBitsToDouble(dvDouble.get(i)), 0D);
assertEquals((float)id, Float.intBitsToFloat((int)dvFloat.get(i)), 0F);
@ -549,8 +548,8 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
dvSortedSet.setDocument(i);
long ord = dvSortedSet.nextOrd();
assertEquals(SortedSetDocValues.NO_MORE_ORDS, dvSortedSet.nextOrd());
dvSortedSet.lookupOrd(ord, scratch);
assertEquals(expectedRef, scratch);
term = dvSortedSet.lookupOrd(ord);
assertEquals(expectedRef, term);
}
}
}
@ -1055,9 +1054,8 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
private void assertBinaryDocValues(AtomicReader r, String f, String cf) throws IOException {
BinaryDocValues bdvf = r.getBinaryDocValues(f);
BinaryDocValues bdvcf = r.getBinaryDocValues(cf);
BytesRef scratch = new BytesRef();
for (int i = 0; i < r.maxDoc(); i++) {
assertEquals(TestBinaryDocValuesUpdates.getValue(bdvcf, i, scratch ), TestBinaryDocValuesUpdates.getValue(bdvf, i, scratch)*2);
assertEquals(TestBinaryDocValuesUpdates.getValue(bdvcf, i), TestBinaryDocValuesUpdates.getValue(bdvf, i)*2);
}
}

114
lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
View File

@ -59,13 +59,13 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks;
@SuppressWarnings("resource")
public class TestBinaryDocValuesUpdates extends LuceneTestCase {
static long getValue(BinaryDocValues bdv, int idx, BytesRef scratch) {
bdv.get(idx, scratch);
idx = scratch.offset;
byte b = scratch.bytes[idx++];
static long getValue(BinaryDocValues bdv, int idx) {
BytesRef term = bdv.get(idx);
idx = term.offset;
byte b = term.bytes[idx++];
long value = b & 0x7FL;
for (int shift = 7; (b & 0x80L) != 0; shift += 7) {
b = scratch.bytes[idx++];
b = term.bytes[idx++];
value |= (b & 0x7FL) << shift;
}
return value;
@ -139,9 +139,8 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
assertEquals(1, reader.leaves().size());
AtomicReader r = reader.leaves().get(0).reader();
BinaryDocValues bdv = r.getBinaryDocValues("val");
BytesRef scratch = new BytesRef();
assertEquals(2, getValue(bdv, 0, scratch));
assertEquals(2, getValue(bdv, 1, scratch));
assertEquals(2, getValue(bdv, 0));
assertEquals(2, getValue(bdv, 1));
reader.close();
dir.close();
@ -179,14 +178,13 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
writer.shutdown();
}
BytesRef scratch = new BytesRef();
for (AtomicReaderContext context : reader.leaves()) {
AtomicReader r = context.reader();
BinaryDocValues bdv = r.getBinaryDocValues("val");
assertNotNull(bdv);
for (int i = 0; i < r.maxDoc(); i++) {
long expected = expectedValues[i + context.docBase];
long actual = getValue(bdv, i, scratch);
long actual = getValue(bdv, i);
assertEquals(expected, actual);
}
}
@ -222,11 +220,10 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
assertNotNull(reader2);
assertTrue(reader1 != reader2);
BytesRef scratch = new BytesRef();
BinaryDocValues bdv1 = reader1.leaves().get(0).reader().getBinaryDocValues("val");
BinaryDocValues bdv2 = reader2.leaves().get(0).reader().getBinaryDocValues("val");
assertEquals(1, getValue(bdv1, 0, scratch));
assertEquals(10, getValue(bdv2, 0, scratch));
assertEquals(1, getValue(bdv1, 0));
assertEquals(10, getValue(bdv2, 0));
writer.shutdown();
IOUtils.close(reader1, reader2, dir);
@ -274,9 +271,8 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
long[] expectedValues = new long[] { 1, 2, 3, 17, 5, 17};
BinaryDocValues bdv = slow.getBinaryDocValues("val");
BytesRef scratch = new BytesRef();
for (int i = 0; i < expectedValues.length; i++) {
assertEquals(expectedValues[i], getValue(bdv, i, scratch));
assertEquals(expectedValues[i], getValue(bdv, i));
}
reader.close();
@ -311,7 +307,7 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
AtomicReader r = reader.leaves().get(0).reader();
assertFalse(r.getLiveDocs().get(0));
assertEquals(17, getValue(r.getBinaryDocValues("val"), 1, new BytesRef()));
assertEquals(17, getValue(r.getBinaryDocValues("val"), 1));
reader.close();
dir.close();
@ -345,7 +341,7 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
AtomicReader r = reader.leaves().get(0).reader();
assertFalse(r.getLiveDocs().get(0));
assertEquals(1, getValue(r.getBinaryDocValues("val"), 0, new BytesRef())); // deletes are currently applied first
assertEquals(1, getValue(r.getBinaryDocValues("val"), 0)); // deletes are currently applied first
reader.close();
dir.close();
@ -379,20 +375,19 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
BinaryDocValues bdv = r.getBinaryDocValues("bdv");
SortedDocValues sdv = r.getSortedDocValues("sdv");
SortedSetDocValues ssdv = r.getSortedSetDocValues("ssdv");
BytesRef scratch = new BytesRef();
for (int i = 0; i < r.maxDoc(); i++) {
assertEquals(i, ndv.get(i));
assertEquals(17, getValue(bdv, i, scratch));
sdv.get(i, scratch);
assertEquals(new BytesRef(Integer.toString(i)), scratch);
assertEquals(17, getValue(bdv, i));
BytesRef term = sdv.get(i);
assertEquals(new BytesRef(Integer.toString(i)), term);
ssdv.setDocument(i);
long ord = ssdv.nextOrd();
ssdv.lookupOrd(ord, scratch);
assertEquals(i, Integer.parseInt(scratch.utf8ToString()));
term = ssdv.lookupOrd(ord);
assertEquals(i, Integer.parseInt(term.utf8ToString()));
if (i != 0) {
ord = ssdv.nextOrd();
ssdv.lookupOrd(ord, scratch);
assertEquals(i * 2, Integer.parseInt(scratch.utf8ToString()));
term = ssdv.lookupOrd(ord);
assertEquals(i * 2, Integer.parseInt(term.utf8ToString()));
}
assertEquals(SortedSetDocValues.NO_MORE_ORDS, ssdv.nextOrd());
}
@ -425,10 +420,9 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
BinaryDocValues bdv1 = r.getBinaryDocValues("bdv1");
BinaryDocValues bdv2 = r.getBinaryDocValues("bdv2");
BytesRef scratch = new BytesRef();
for (int i = 0; i < r.maxDoc(); i++) {
assertEquals(17, getValue(bdv1, i, scratch));
assertEquals(i, getValue(bdv2, i, scratch));
assertEquals(17, getValue(bdv1, i));
assertEquals(i, getValue(bdv2, i));
}
reader.close();
@ -457,9 +451,8 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
final DirectoryReader reader = DirectoryReader.open(dir);
AtomicReader r = reader.leaves().get(0).reader();
BinaryDocValues bdv = r.getBinaryDocValues("bdv");
BytesRef scratch = new BytesRef();
for (int i = 0; i < r.maxDoc(); i++) {
assertEquals(17, getValue(bdv, i, scratch));
assertEquals(17, getValue(bdv, i));
}
reader.close();
@ -525,11 +518,10 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
AtomicReader r = SlowCompositeReaderWrapper.wrap(reader);
BinaryDocValues bdv = r.getBinaryDocValues("bdv");
SortedDocValues sdv = r.getSortedDocValues("sorted");
BytesRef scratch = new BytesRef();
for (int i = 0; i < r.maxDoc(); i++) {
assertEquals(17, getValue(bdv, i, scratch));
sdv.get(i, scratch);
assertEquals(new BytesRef("value"), scratch);
assertEquals(17, getValue(bdv, i));
BytesRef term = sdv.get(i);
assertEquals(new BytesRef("value"), term);
}
reader.close();
@ -555,9 +547,8 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
final DirectoryReader reader = DirectoryReader.open(dir);
final AtomicReader r = SlowCompositeReaderWrapper.wrap(reader);
BinaryDocValues bdv = r.getBinaryDocValues("bdv");
BytesRef scratch = new BytesRef();
for (int i = 0; i < r.maxDoc(); i++) {
assertEquals(3, getValue(bdv, i, scratch));
assertEquals(3, getValue(bdv, i));
}
reader.close();
dir.close();
@ -624,9 +615,8 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
assertNull("index should have no deletes after forceMerge", r.getLiveDocs());
BinaryDocValues bdv = r.getBinaryDocValues("bdv");
assertNotNull(bdv);
BytesRef scratch = new BytesRef();
for (int i = 0; i < r.maxDoc(); i++) {
assertEquals(value, getValue(bdv, i, scratch));
assertEquals(value, getValue(bdv, i));
}
reader.close();
}
@ -656,9 +646,8 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
final DirectoryReader reader = DirectoryReader.open(dir);
final AtomicReader r = SlowCompositeReaderWrapper.wrap(reader);
BinaryDocValues bdv = r.getBinaryDocValues("bdv");
BytesRef scratch = new BytesRef();
for (int i = 0; i < r.maxDoc(); i++) {
assertEquals(3, getValue(bdv, i, scratch));
assertEquals(3, getValue(bdv, i));
}
reader.close();
dir.close();
@ -728,7 +717,6 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
reader = newReader;
// System.out.println("[" + Thread.currentThread().getName() + "]: reopened reader: " + reader);
assertTrue(reader.numDocs() > 0); // we delete at most one document per round
BytesRef scratch = new BytesRef();
for (AtomicReaderContext context : reader.leaves()) {
AtomicReader r = context.reader();
// System.out.println(((SegmentReader) r).getSegmentName());
@ -743,7 +731,7 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
if (liveDocs == null || liveDocs.get(doc)) {
// System.out.println("doc=" + (doc + context.docBase) + " f='" + f + "' vslue=" + getValue(bdv, doc, scratch));
assertTrue(docsWithField.get(doc));
assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], getValue(bdv, doc, scratch));
assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], getValue(bdv, doc));
}
}
}
@ -793,17 +781,16 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
writer.shutdown();
DirectoryReader reader = DirectoryReader.open(dir);
BytesRef scratch = new BytesRef();
for (AtomicReaderContext context : reader.leaves()) {
AtomicReader r = context.reader();
BinaryDocValues bdv = r.getBinaryDocValues("bdv");
Bits docsWithField = r.getDocsWithField("bdv");
assertNotNull(docsWithField);
assertTrue(docsWithField.get(0));
assertEquals(5L, getValue(bdv, 0, scratch));
assertEquals(5L, getValue(bdv, 0));
assertFalse(docsWithField.get(1));
bdv.get(1, scratch);
assertEquals(0, scratch.length);
BytesRef term = bdv.get(1);
assertEquals(0, term.length);
}
reader.close();
@ -839,12 +826,11 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
writer.shutdown();
DirectoryReader reader = DirectoryReader.open(dir);
BytesRef scratch = new BytesRef();
for (AtomicReaderContext context : reader.leaves()) {
AtomicReader r = context.reader();
BinaryDocValues bdv = r.getBinaryDocValues("bdv");
for (int i = 0; i < r.maxDoc(); i++) {
assertEquals(5L, getValue(bdv, i, scratch));
assertEquals(5L, getValue(bdv, i));
}
}
reader.close();
@ -869,7 +855,7 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
DirectoryReader r = DirectoryReader.open(dir);
BinaryDocValues bdv = r.leaves().get(0).reader().getBinaryDocValues("f");
assertEquals(17, getValue(bdv, 0, new BytesRef()));
assertEquals(17, getValue(bdv, 0));
r.close();
dir.close();
@ -1013,7 +999,6 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
writer.shutdown();
DirectoryReader reader = DirectoryReader.open(dir);
BytesRef scratch = new BytesRef();
for (AtomicReaderContext context : reader.leaves()) {
AtomicReader r = context.reader();
for (int i = 0; i < numFields; i++) {
@ -1026,7 +1011,7 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
if (liveDocs == null || liveDocs.get(j)) {
assertTrue(docsWithBdv.get(j));
assertTrue(docsWithControl.get(j));
assertEquals(getValue(control, j, scratch), getValue(bdv, j, scratch) * 2);
assertEquals(getValue(control, j), getValue(bdv, j) * 2);
}
}
}
@ -1053,7 +1038,6 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
}
int numGens = atLeast(5);
BytesRef scratch = new BytesRef();
for (int i = 0; i < numGens; i++) {
int doc = random().nextInt(numDocs);
Term t = new Term("id", "doc" + doc);
@ -1065,7 +1049,7 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
BinaryDocValues fbdv = r.getBinaryDocValues("f");
BinaryDocValues cfbdv = r.getBinaryDocValues("cf");
for (int j = 0; j < r.maxDoc(); j++) {
assertEquals(getValue(cfbdv, j, scratch), getValue(fbdv, j, scratch) * 2);
assertEquals(getValue(cfbdv, j), getValue(fbdv, j) * 2);
}
}
reader.close();
@ -1114,11 +1098,10 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
AtomicReader r = SlowCompositeReaderWrapper.wrap(reader);
BinaryDocValues f1 = r.getBinaryDocValues("f1");
BinaryDocValues f2 = r.getBinaryDocValues("f2");
BytesRef scratch = new BytesRef();
assertEquals(12L, getValue(f1, 0, scratch));
assertEquals(13L, getValue(f2, 0, scratch));
assertEquals(17L, getValue(f1, 1, scratch));
assertEquals(2L, getValue(f2, 1, scratch));
assertEquals(12L, getValue(f1, 0));
assertEquals(13L, getValue(f2, 0));
assertEquals(17L, getValue(f1, 1));
assertEquals(2L, getValue(f2, 1));
reader.close();
dir.close();
}
@ -1167,13 +1150,12 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
writer.shutdown();
DirectoryReader reader = DirectoryReader.open(dir2);
BytesRef scratch = new BytesRef();
for (AtomicReaderContext context : reader.leaves()) {
AtomicReader r = context.reader();
BinaryDocValues bdv = r.getBinaryDocValues("bdv");
BinaryDocValues control = r.getBinaryDocValues("control");
for (int i = 0; i < r.maxDoc(); i++) {
assertEquals(getValue(bdv, i, scratch)*2, getValue(control, i, scratch));
assertEquals(getValue(bdv, i)*2, getValue(control, i));
}
}
reader.close();
@ -1263,14 +1245,13 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
writer.shutdown();
DirectoryReader reader = DirectoryReader.open(dir);
BytesRef scratch = new BytesRef();
for (AtomicReaderContext context : reader.leaves()) {
for (int i = 0; i < numBinaryFields; i++) {
AtomicReader r = context.reader();
BinaryDocValues f = r.getBinaryDocValues("f" + i);
BinaryDocValues cf = r.getBinaryDocValues("cf" + i);
for (int j = 0; j < r.maxDoc(); j++) {
assertEquals("reader=" + r + ", field=f" + i + ", doc=" + j, getValue(cf, j, scratch), getValue(f, j, scratch) * 2);
assertEquals("reader=" + r + ", field=f" + i + ", doc=" + j, getValue(cf, j), getValue(f, j) * 2);
}
}
}
@ -1298,9 +1279,8 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
writer.shutdown();
DirectoryReader reader = DirectoryReader.open(dir);
BytesRef scratch = new BytesRef();
assertEquals(4, getValue(reader.leaves().get(0).reader().getBinaryDocValues("f1"), 0, scratch));
assertEquals(3, getValue(reader.leaves().get(0).reader().getBinaryDocValues("f2"), 0, scratch));
assertEquals(4, getValue(reader.leaves().get(0).reader().getBinaryDocValues("f1"), 0));
assertEquals(3, getValue(reader.leaves().get(0).reader().getBinaryDocValues("f2"), 0));
reader.close();
dir.close();
@ -1324,7 +1304,7 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
DirectoryReader reader = DirectoryReader.open(dir);
assertEquals(1, reader.leaves().size());
assertEquals(2L, getValue(reader.leaves().get(0).reader().getBinaryDocValues("f1"), 0, new BytesRef()));
assertEquals(2L, getValue(reader.leaves().get(0).reader().getBinaryDocValues("f1"), 0));
reader.close();
dir.close();
@ -1346,7 +1326,7 @@ public class TestBinaryDocValuesUpdates extends LuceneTestCase {
DirectoryReader reader = DirectoryReader.open(dir);
assertEquals(1, reader.leaves().size());
assertEquals(1L, getValue(reader.leaves().get(0).reader().getBinaryDocValues("f1"), 0, new BytesRef()));
assertEquals(1L, getValue(reader.leaves().get(0).reader().getBinaryDocValues("f1"), 0));
reader.close();
dir.close();

1
lucene/core/src/test/org/apache/lucene/index/TestDocValuesFormat.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.index;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.TestUtil;
/** Tests the codec configuration defined by LuceneTestCase randomly
* (typically a mix across different fields).

5
lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
View File

@ -177,13 +177,12 @@ public class TestDocValuesIndexing extends LuceneTestCase {
DirectoryReader r = w.getReader();
BinaryDocValues s = DocValues.getSorted(getOnlySegmentReader(r), "field");
BytesRef bytes1 = new BytesRef();
s.get(0, bytes1);
BytesRef bytes1 = s.get(0);
assertEquals(bytes.length, bytes1.length);
bytes[0] = 0;
assertEquals(b, bytes1);
s.get(1, bytes1);
bytes1 = s.get(1);
assertEquals(bytes.length, bytes1.length);
bytes[0] = 1;
assertEquals(b, bytes1);

2
lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
View File

@ -2091,7 +2091,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase {
for (int i = 0; i < reader.maxDoc(); i++) {
if (liveDocs == null || liveDocs.get(i)) {
assertEquals("doc=" + (docBase + i), cf.get(i), f.get(i) * 2);
assertEquals("doc=" + (docBase + i), TestBinaryDocValuesUpdates.getValue(bcf, i, scratch), TestBinaryDocValuesUpdates.getValue(bf, i, scratch) * 2);
assertEquals("doc=" + (docBase + i), TestBinaryDocValuesUpdates.getValue(bcf, i), TestBinaryDocValuesUpdates.getValue(bf, i) * 2);
}
}
}

11
lucene/core/src/test/org/apache/lucene/index/TestMixedDocValuesUpdates.java
View File

@ -116,7 +116,6 @@ public class TestMixedDocValuesUpdates extends LuceneTestCase {
reader = newReader;
// System.out.println("[" + Thread.currentThread().getName() + "]: reopened reader: " + reader);
assertTrue(reader.numDocs() > 0); // we delete at most one document per round
BytesRef scratch = new BytesRef();
for (AtomicReaderContext context : reader.leaves()) {
AtomicReader r = context.reader();
// System.out.println(((SegmentReader) r).getSegmentName());
@ -141,7 +140,7 @@ public class TestMixedDocValuesUpdates extends LuceneTestCase {
if (field < numNDVFields) {
assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], ndv.get(doc));
} else {
assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], TestBinaryDocValuesUpdates.getValue(bdv, doc, scratch));
assertEquals("invalid value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], TestBinaryDocValuesUpdates.getValue(bdv, doc));
}
}
}
@ -275,7 +274,7 @@ public class TestMixedDocValuesUpdates extends LuceneTestCase {
assertTrue(docsWithBdv.get(j));
assertTrue(docsWithControl.get(j));
long ctrlValue = control.get(j);
long bdvValue = TestBinaryDocValuesUpdates.getValue(bdv, j, scratch) * 2;
long bdvValue = TestBinaryDocValuesUpdates.getValue(bdv, j) * 2;
// if (ctrlValue != bdvValue) {
// System.out.println("seg=" + r + ", f=f" + i + ", doc=" + j + ", group=" + r.document(j).get("updKey") + ", ctrlValue=" + ctrlValue + ", bdvBytes=" + scratch);
// }
@ -306,7 +305,6 @@ public class TestMixedDocValuesUpdates extends LuceneTestCase {
}
int numGens = atLeast(5);
BytesRef scratch = new BytesRef();
for (int i = 0; i < numGens; i++) {
int doc = random().nextInt(numDocs);
Term t = new Term("id", "doc" + doc);
@ -319,7 +317,7 @@ public class TestMixedDocValuesUpdates extends LuceneTestCase {
BinaryDocValues fbdv = r.getBinaryDocValues("f");
NumericDocValues cfndv = r.getNumericDocValues("cf");
for (int j = 0; j < r.maxDoc(); j++) {
assertEquals(cfndv.get(j), TestBinaryDocValuesUpdates.getValue(fbdv, j, scratch) * 2);
assertEquals(cfndv.get(j), TestBinaryDocValuesUpdates.getValue(fbdv, j) * 2);
}
}
reader.close();
@ -381,14 +379,13 @@ public class TestMixedDocValuesUpdates extends LuceneTestCase {
writer.shutdown();
DirectoryReader reader = DirectoryReader.open(dir);
BytesRef scratch = new BytesRef();
for (AtomicReaderContext context : reader.leaves()) {
for (int i = 0; i < numBinaryFields; i++) {
AtomicReader r = context.reader();
BinaryDocValues f = r.getBinaryDocValues("f" + i);
NumericDocValues cf = r.getNumericDocValues("cf" + i);
for (int j = 0; j < r.maxDoc(); j++) {
assertEquals("reader=" + r + ", field=f" + i + ", doc=" + j, cf.get(j), TestBinaryDocValuesUpdates.getValue(f, j, scratch) * 2);
assertEquals("reader=" + r + ", field=f" + i + ", doc=" + j, cf.get(j), TestBinaryDocValuesUpdates.getValue(f, j) * 2);
}
}
}

30
lucene/core/src/test/org/apache/lucene/index/TestMultiDocValues.java
View File

@ -96,11 +96,9 @@ public class TestMultiDocValues extends LuceneTestCase {
BinaryDocValues multi = MultiDocValues.getBinaryValues(ir, "bytes");
BinaryDocValues single = merged.getBinaryDocValues("bytes");
BytesRef actual = new BytesRef();
BytesRef expected = new BytesRef();
for (int i = 0; i < numDocs; i++) {
single.get(i, expected);
multi.get(i, actual);
final BytesRef expected = BytesRef.deepCopyOf(single.get(i));
final BytesRef actual = multi.get(i);
assertEquals(expected, actual);
}
ir.close();
@ -139,14 +137,12 @@ public class TestMultiDocValues extends LuceneTestCase {
SortedDocValues multi = MultiDocValues.getSortedValues(ir, "bytes");
SortedDocValues single = merged.getSortedDocValues("bytes");
assertEquals(single.getValueCount(), multi.getValueCount());
BytesRef actual = new BytesRef();
BytesRef expected = new BytesRef();
for (int i = 0; i < numDocs; i++) {
// check ord
assertEquals(single.getOrd(i), multi.getOrd(i));
// check value
single.get(i, expected);
multi.get(i, actual);
final BytesRef expected = BytesRef.deepCopyOf(single.get(i));
final BytesRef actual = multi.get(i);
assertEquals(expected, actual);
}
ir.close();
@ -183,14 +179,12 @@ public class TestMultiDocValues extends LuceneTestCase {
SortedDocValues multi = MultiDocValues.getSortedValues(ir, "bytes");
SortedDocValues single = merged.getSortedDocValues("bytes");
assertEquals(single.getValueCount(), multi.getValueCount());
BytesRef actual = new BytesRef();
BytesRef expected = new BytesRef();
for (int i = 0; i < numDocs; i++) {
// check ord
assertEquals(single.getOrd(i), multi.getOrd(i));
// check ord value
single.get(i, expected);
multi.get(i, actual);
final BytesRef expected = BytesRef.deepCopyOf(single.get(i));
final BytesRef actual = multi.get(i);
assertEquals(expected, actual);
}
ir.close();
@ -230,12 +224,10 @@ public class TestMultiDocValues extends LuceneTestCase {
assertNull(single);
} else {
assertEquals(single.getValueCount(), multi.getValueCount());
BytesRef actual = new BytesRef();
BytesRef expected = new BytesRef();
// check values
for (long i = 0; i < single.getValueCount(); i++) {
single.lookupOrd(i, expected);
multi.lookupOrd(i, actual);
final BytesRef expected = BytesRef.deepCopyOf(single.lookupOrd(i));
final BytesRef actual = multi.lookupOrd(i);
assertEquals(expected, actual);
}
// check ord list
@ -295,12 +287,10 @@ public class TestMultiDocValues extends LuceneTestCase {
assertNull(single);
} else {
assertEquals(single.getValueCount(), multi.getValueCount());
BytesRef actual = new BytesRef();
BytesRef expected = new BytesRef();
// check values
for (long i = 0; i < single.getValueCount(); i++) {
single.lookupOrd(i, expected);
multi.lookupOrd(i, actual);
final BytesRef expected = BytesRef.deepCopyOf(single.lookupOrd(i));
final BytesRef actual = multi.lookupOrd(i);
assertEquals(expected, actual);
}
// check ord list

22
lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
View File

@ -356,21 +356,20 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
BinaryDocValues bdv = r.getBinaryDocValues("bdv");
SortedDocValues sdv = r.getSortedDocValues("sdv");
SortedSetDocValues ssdv = r.getSortedSetDocValues("ssdv");
BytesRef scratch = new BytesRef();
for (int i = 0; i < r.maxDoc(); i++) {
assertEquals(17, ndv.get(i));
bdv.get(i, scratch);
assertEquals(new BytesRef(Integer.toString(i)), scratch);
sdv.get(i, scratch);
assertEquals(new BytesRef(Integer.toString(i)), scratch);
BytesRef term = bdv.get(i);
assertEquals(new BytesRef(Integer.toString(i)), term);
term = sdv.get(i);
assertEquals(new BytesRef(Integer.toString(i)), term);
ssdv.setDocument(i);
long ord = ssdv.nextOrd();
ssdv.lookupOrd(ord, scratch);
assertEquals(i, Integer.parseInt(scratch.utf8ToString()));
term = ssdv.lookupOrd(ord);
assertEquals(i, Integer.parseInt(term.utf8ToString()));
if (i != 0) {
ord = ssdv.nextOrd();
ssdv.lookupOrd(ord, scratch);
assertEquals(i * 2, Integer.parseInt(scratch.utf8ToString()));
term = ssdv.lookupOrd(ord);
assertEquals(i * 2, Integer.parseInt(term.utf8ToString()));
}
assertEquals(SortedSetDocValues.NO_MORE_ORDS, ssdv.nextOrd());
}
@ -504,11 +503,10 @@ public class TestNumericDocValuesUpdates extends LuceneTestCase {
AtomicReader r = SlowCompositeReaderWrapper.wrap(reader);
NumericDocValues ndv = r.getNumericDocValues("ndv");
SortedDocValues sdv = r.getSortedDocValues("sorted");
BytesRef scratch = new BytesRef();
for (int i = 0; i < r.maxDoc(); i++) {
assertEquals(17, ndv.get(i));
sdv.get(i, scratch);
assertEquals(new BytesRef("value"), scratch);
final BytesRef term = sdv.get(i);
assertEquals(new BytesRef("value"), term);
}
reader.close();

5
lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java
View File

@ -148,7 +148,6 @@ class ElevationComparatorSource extends FieldComparatorSource {
SortedDocValues idIndex;
private final int[] values = new int[numHits];
private final BytesRef tempBR = new BytesRef();
int bottomVal;
@Override
@ -171,8 +170,8 @@ class ElevationComparatorSource extends FieldComparatorSource {
if (ord == -1) {
return 0;
} else {
idIndex.lookupOrd(ord, tempBR);
Integer prio = priority.get(tempBR);
final BytesRef term = idIndex.lookupOrd(ord);
Integer prio = priority.get(term);
return prio == null ? 0 : prio.intValue();
}
}

7
lucene/facet/src/java/org/apache/lucene/facet/sortedset/DefaultSortedSetDocValuesReaderState.java
View File

@ -73,17 +73,16 @@ public class DefaultSortedSetDocValuesReaderState extends SortedSetDocValuesRead
// each term/ord it's assigning as it goes...
String lastDim = null;
int startOrd = -1;
BytesRef spare = new BytesRef();
// TODO: this approach can work for full hierarchy?;
// TaxoReader can't do this since ords are not in
// "sorted order" ... but we should generalize this to
// support arbitrary hierarchy:
for(int ord=0;ord<valueCount;ord++) {
dv.lookupOrd(ord, spare);
String[] components = FacetsConfig.stringToPath(spare.utf8ToString());
final BytesRef term = dv.lookupOrd(ord);
String[] components = FacetsConfig.stringToPath(term.utf8ToString());
if (components.length != 2) {
throw new IllegalArgumentException("this class can only handle 2 level hierarchy (dim/value); got: " + Arrays.toString(components) + " " + spare.utf8ToString());
throw new IllegalArgumentException("this class can only handle 2 level hierarchy (dim/value); got: " + Arrays.toString(components) + " " + term.utf8ToString());
}
if (!components[0].equals(lastDim)) {
if (lastDim != null) {

6
lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetCounts.java
View File

@ -131,13 +131,11 @@ public class SortedSetDocValuesFacetCounts extends Facets {
return null;
}
BytesRef scratch = new BytesRef();
LabelAndValue[] labelValues = new LabelAndValue[q.size()];
for(int i=labelValues.length-1;i>=0;i--) {
TopOrdAndIntQueue.OrdAndValue ordAndValue = q.pop();
dv.lookupOrd(ordAndValue.ord, scratch);
String[] parts = FacetsConfig.stringToPath(scratch.utf8ToString());
final BytesRef term = dv.lookupOrd(ordAndValue.ord);
String[] parts = FacetsConfig.stringToPath(term.utf8ToString());
labelValues[i] = new LabelAndValue(parts[1], ordAndValue.value);
}

6
lucene/facet/src/java/org/apache/lucene/facet/taxonomy/DocValuesOrdinalsReader.java
View File

@ -46,17 +46,15 @@ public class DocValuesOrdinalsReader extends OrdinalsReader {
public OrdinalsSegmentReader getReader(AtomicReaderContext context) throws IOException {
BinaryDocValues values0 = context.reader().getBinaryDocValues(field);
if (values0 == null) {
values0 = DocValues.EMPTY_BINARY;
values0 = DocValues.emptyBinary();
}
final BinaryDocValues values = values0;
return new OrdinalsSegmentReader() {
private final BytesRef bytes = new BytesRef(32);
@Override
public void get(int docID, IntsRef ordinals) throws IOException {
values.get(docID, bytes);
final BytesRef bytes = values.get(docID);
decode(bytes, ordinals);
}
};

11
lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java
View File

@ -55,17 +55,16 @@ public class FastTaxonomyFacetCounts extends IntTaxonomyFacets {
if (dv == null) { // this reader does not have DocValues for the requested category list
continue;
}
BytesRef scratch = new BytesRef();
DocIdSetIterator docs = hits.bits.iterator();
int doc;
while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
dv.get(doc, scratch);
byte[] bytes = scratch.bytes;
int end = scratch.offset + scratch.length;
final BytesRef bytesRef = dv.get(doc);
byte[] bytes = bytesRef.bytes;
int end = bytesRef.offset + bytesRef.length;
int ord = 0;
int offset = scratch.offset;
int offset = bytesRef.offset;
int prev = 0;
while (offset < end) {
byte b = bytes[offset++];

11
lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumFloatAssociations.java
View File

@ -54,8 +54,7 @@ public class TaxonomyFacetSumFloatAssociations extends FloatTaxonomyFacets {
if (dv == null) { // this reader does not have DocValues for the requested category list
continue;
}
BytesRef scratch = new BytesRef();
DocIdSetIterator docs = hits.bits.iterator();
int doc;
@ -63,10 +62,10 @@ public class TaxonomyFacetSumFloatAssociations extends FloatTaxonomyFacets {
//System.out.println(" doc=" + doc);
// TODO: use OrdinalsReader? we'd need to add a
// BytesRef getAssociation()?
dv.get(doc, scratch);
byte[] bytes = scratch.bytes;
int end = scratch.offset + scratch.length;
int offset = scratch.offset;
final BytesRef bytesRef = dv.get(doc);
byte[] bytes = bytesRef.bytes;
int end = bytesRef.offset + bytesRef.length;
int offset = bytesRef.offset;
while (offset < end) {
int ord = ((bytes[offset]&0xFF) << 24) |
((bytes[offset+1]&0xFF) << 16) |

11
lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java
View File

@ -54,8 +54,7 @@ public class TaxonomyFacetSumIntAssociations extends IntTaxonomyFacets {
if (dv == null) { // this reader does not have DocValues for the requested category list
continue;
}
BytesRef scratch = new BytesRef();
DocIdSetIterator docs = hits.bits.iterator();
int doc;
@ -63,10 +62,10 @@ public class TaxonomyFacetSumIntAssociations extends IntTaxonomyFacets {
//System.out.println(" doc=" + doc);
// TODO: use OrdinalsReader? we'd need to add a
// BytesRef getAssociation()?
dv.get(doc, scratch);
byte[] bytes = scratch.bytes;
int end = scratch.offset + scratch.length;
int offset = scratch.offset;
final BytesRef bytesRef = dv.get(doc);
byte[] bytes = bytesRef.bytes;
int end = bytesRef.offset + bytesRef.length;
int offset = bytesRef.offset;
while (offset < end) {
int ord = ((bytes[offset]&0xFF) << 24) |
((bytes[offset+1]&0xFF) << 16) |

59
lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupHeadsCollector.java
View File

@ -47,7 +47,6 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
private static final int DEFAULT_INITIAL_SIZE = 128;
final String groupField;
final BytesRef scratchBytesRef = new BytesRef();
SortedDocValues groupIndex;
AtomicReaderContext readerContext;
@ -134,17 +133,17 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
@Override
protected void retrieveGroupHeadAndAddIfNotExist(int doc) throws IOException {
final int ord = groupIndex.getOrd(doc);
final BytesRef groupValue;
BytesRef groupValue;
if (ord == -1) {
groupValue = null;
} else {
groupIndex.lookupOrd(ord, scratchBytesRef);
groupValue = scratchBytesRef;
groupValue = groupIndex.lookupOrd(ord);
}
GroupHead groupHead = groups.get(groupValue);
if (groupHead == null) {
groupValue = groupValue == null ? null : BytesRef.deepCopyOf(groupValue);
groupHead = new GroupHead(groupValue, sortWithinGroup, doc);
groups.put(groupValue == null ? null : BytesRef.deepCopyOf(groupValue), groupHead);
groups.put(groupValue, groupHead);
temporalResult.stop = true;
} else {
temporalResult.stop = false;
@ -183,7 +182,6 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
final FieldComparator<?>[] comparators;
@SuppressWarnings({"unchecked","rawtypes"})
private GroupHead(BytesRef groupValue, Sort sort, int doc) throws IOException {
super(groupValue, doc + readerContext.docBase);
final SortField[] sortFields = sort.getSort();
@ -254,12 +252,11 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
GroupHead groupHead;
if (!ordSet.exists(key)) {
ordSet.put(key);
BytesRef term;
final BytesRef term;
if (key == -1) {
term = null;
} else {
term = new BytesRef();
groupIndex.lookupOrd(key, term);
term = BytesRef.deepCopyOf(groupIndex.lookupOrd(key));
}
groupHead = new GroupHead(doc, term);
collectedGroups.add(groupHead);
@ -332,7 +329,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
sortOrds[i] = sortsIndex[i].getOrd(doc);
sortValues[i] = new BytesRef();
if (sortOrds[i] != -1) {
sortsIndex[i].get(doc, sortValues[i]);
sortValues[i].copyBytes(sortsIndex[i].get(doc));
}
}
}
@ -351,12 +348,8 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
} else {
if (sortOrds[compIDX] < 0) {
// The current segment doesn't contain the sort value we encountered before. Therefore the ord is negative.
if (sortsIndex[compIDX].getOrd(doc) == -1) {
scratchBytesRef.length = 0;
} else {
sortsIndex[compIDX].get(doc, scratchBytesRef);
}
return sortValues[compIDX].compareTo(scratchBytesRef);
final BytesRef term = sortsIndex[compIDX].get(doc);
return sortValues[compIDX].compareTo(term);
} else {
return sortOrds[compIDX] - sortsIndex[compIDX].getOrd(doc);
}
@ -370,11 +363,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
scores[i] = scorer.score();
} else {
sortOrds[i] = sortsIndex[i].getOrd(doc);
if (sortOrds[i] == -1) {
sortValues[i].length = 0;
} else {
sortsIndex[i].get(doc, sortValues[i]);
}
sortValues[i].copyBytes(sortsIndex[i].get(doc));
}
}
this.doc = doc + readerContext.docBase;
@ -422,12 +411,11 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
GroupHead groupHead;
if (!ordSet.exists(key)) {
ordSet.put(key);
BytesRef term;
final BytesRef term;
if (key == -1) {
term = null;
} else {
term = new BytesRef();
groupIndex.lookupOrd(key, term);
term = BytesRef.deepCopyOf(groupIndex.lookupOrd(key));
}
groupHead = new GroupHead(doc, term);
collectedGroups.add(groupHead);
@ -487,9 +475,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
for (int i = 0; i < sortsIndex.length; i++) {
sortOrds[i] = sortsIndex[i].getOrd(doc);
sortValues[i] = new BytesRef();
if (sortOrds[i] != -1) {
sortsIndex[i].get(doc, sortValues[i]);
}
sortValues[i].copyBytes(sortsIndex[i].get(doc));
}
}
@ -497,12 +483,8 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
public int compare(int compIDX, int doc) throws IOException {
if (sortOrds[compIDX] < 0) {
// The current segment doesn't contain the sort value we encountered before. Therefore the ord is negative.
if (sortsIndex[compIDX].getOrd(doc) == -1) {
scratchBytesRef.length = 0;
} else {
sortsIndex[compIDX].get(doc, scratchBytesRef);
}
return sortValues[compIDX].compareTo(scratchBytesRef);
final BytesRef term = sortsIndex[compIDX].get(doc);
return sortValues[compIDX].compareTo(term);
} else {
return sortOrds[compIDX] - sortsIndex[compIDX].getOrd(doc);
}
@ -512,11 +494,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
public void updateDocHead(int doc) throws IOException {
for (int i = 0; i < sortsIndex.length; i++) {
sortOrds[i] = sortsIndex[i].getOrd(doc);
if (sortOrds[i] == -1) {
sortValues[i].length = 0;
} else {
sortsIndex[i].lookupOrd(sortOrds[i], sortValues[i]);
}
sortValues[i].copyBytes(sortsIndex[i].get(doc));
}
this.doc = doc + readerContext.docBase;
}
@ -565,12 +543,11 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
GroupHead groupHead;
if (!ordSet.exists(key)) {
ordSet.put(key);
BytesRef term;
final BytesRef term;
if (key == -1) {
term = null;
} else {
term = new BytesRef();
groupIndex.lookupOrd(key, term);
term = BytesRef.deepCopyOf(groupIndex.lookupOrd(key));
}
groupHead = new GroupHead(doc, term);
collectedGroups.add(groupHead);

6
lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupsCollector.java
View File

@ -20,7 +20,6 @@ package org.apache.lucene.search.grouping.term;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.grouping.AbstractAllGroupsCollector;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SentinelIntSet;
@ -87,12 +86,11 @@ public class TermAllGroupsCollector extends AbstractAllGroupsCollector<BytesRef>
int key = index.getOrd(doc);
if (!ordSet.exists(key)) {
ordSet.put(key);
BytesRef term;
final BytesRef term;
if (key == -1) {
term = null;
} else {
term = new BytesRef();
index.lookupOrd(key, term);
term = BytesRef.deepCopyOf(index.lookupOrd(key));
}
groups.add(term);
}

6
lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermDistinctValuesCollector.java
View File

@ -20,7 +20,6 @@ package org.apache.lucene.search.grouping.term;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.grouping.AbstractDistinctValuesCollector;
import org.apache.lucene.search.grouping.SearchGroup;
import org.apache.lucene.util.BytesRef;
@ -80,9 +79,8 @@ public class TermDistinctValuesCollector extends AbstractDistinctValuesCollector
if (countOrd == -1) {
gc.uniqueValues.add(null);
} else {
BytesRef br = new BytesRef();
countFieldTermIndex.lookupOrd(countOrd, br);
gc.uniqueValues.add(br);
BytesRef term = BytesRef.deepCopyOf(countFieldTermIndex.lookupOrd(countOrd));
gc.uniqueValues.add(term);
}
gc.ords = Arrays.copyOf(gc.ords, gc.ords.length + 1);

5
lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java
View File

@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.grouping.AbstractFirstPassGroupingCollector;
import org.apache.lucene.util.BytesRef;
@ -36,7 +35,6 @@ import org.apache.lucene.util.BytesRef;
*/
public class TermFirstPassGroupingCollector extends AbstractFirstPassGroupingCollector<BytesRef> {
private final BytesRef scratchBytesRef = new BytesRef();
private SortedDocValues index;
private String groupField;
@ -68,8 +66,7 @@ public class TermFirstPassGroupingCollector extends AbstractFirstPassGroupingCol
if (ord == -1) {
return null;
} else {
index.lookupOrd(ord, scratchBytesRef);
return scratchBytesRef;
return index.lookupOrd(ord);
}
}

15
lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java
View File

@ -107,16 +107,14 @@ public abstract class TermGroupFacetCollector extends AbstractGroupFacetCollecto
if (groupOrd == -1) {
groupKey = null;
} else {
groupKey = new BytesRef();
groupFieldTermsIndex.lookupOrd(groupOrd, groupKey);
groupKey = BytesRef.deepCopyOf(groupFieldTermsIndex.lookupOrd(groupOrd));
}
BytesRef facetKey;
if (facetOrd == -1) {
facetKey = null;
} else {
facetKey = new BytesRef();
facetFieldTermsIndex.lookupOrd(facetOrd, facetKey);
facetKey = BytesRef.deepCopyOf(facetFieldTermsIndex.lookupOrd(facetOrd));
}
groupedFacetHits.add(new GroupedFacetHit(groupKey, facetKey));
@ -224,8 +222,7 @@ public abstract class TermGroupFacetCollector extends AbstractGroupFacetCollecto
if (groupOrd == -1) {
groupKey = null;
} else {
groupKey = new BytesRef();
groupFieldTermsIndex.lookupOrd(groupOrd, groupKey);
groupKey = BytesRef.deepCopyOf(groupFieldTermsIndex.lookupOrd(groupOrd));
}
groupedFacetHits.add(new GroupedFacetHit(groupKey, null));
return;
@ -263,16 +260,14 @@ public abstract class TermGroupFacetCollector extends AbstractGroupFacetCollecto
if (groupOrd == -1) {
groupKey = null;
} else {
groupKey = new BytesRef();
groupFieldTermsIndex.lookupOrd(groupOrd, groupKey);
groupKey = BytesRef.deepCopyOf(groupFieldTermsIndex.lookupOrd(groupOrd));
}
final BytesRef facetValue;
if (facetOrd == facetFieldNumTerms) {
facetValue = null;
} else {
facetFieldDocTermOrds.lookupOrd(facetOrd, scratch);
facetValue = BytesRef.deepCopyOf(scratch); // must we?
facetValue = BytesRef.deepCopyOf(facetFieldDocTermOrds.lookupOrd(facetOrd));
}
groupedFacetHits.add(new GroupedFacetHit(groupKey, facetValue));
}

11
lucene/join/src/java/org/apache/lucene/search/join/TermsCollector.java
View File

@ -23,9 +23,6 @@ import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.SimpleCollector;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
@ -78,8 +75,8 @@ abstract class TermsCollector extends SimpleCollector {
docTermOrds.setDocument(doc);
long ord;
while ((ord = docTermOrds.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
docTermOrds.lookupOrd(ord, scratch);
collectorTerms.add(scratch);
final BytesRef term = docTermOrds.lookupOrd(ord);
collectorTerms.add(term);
}
}
@ -101,8 +98,8 @@ abstract class TermsCollector extends SimpleCollector {
@Override
public void collect(int doc) throws IOException {
fromDocTerms.get(doc, spare);
collectorTerms.add(spare);
final BytesRef term = fromDocTerms.get(doc);
collectorTerms.add(term);
}
@Override

16
lucene/join/src/java/org/apache/lucene/search/join/TermsWithScoreCollector.java
View File

@ -93,7 +93,6 @@ abstract class TermsWithScoreCollector extends SimpleCollector {
// impl that works with single value per document
static class SV extends TermsWithScoreCollector {
final BytesRef spare = new BytesRef();
BinaryDocValues fromDocTerms;
SV(String field, ScoreMode scoreMode) {
@ -102,8 +101,7 @@ abstract class TermsWithScoreCollector extends SimpleCollector {
@Override
public void collect(int doc) throws IOException {
fromDocTerms.get(doc, spare);
int ord = collectedTerms.add(spare);
int ord = collectedTerms.add(fromDocTerms.get(doc));
if (ord < 0) {
ord = -ord - 1;
} else {
@ -144,8 +142,7 @@ abstract class TermsWithScoreCollector extends SimpleCollector {
@Override
public void collect(int doc) throws IOException {
fromDocTerms.get(doc, spare);
int ord = collectedTerms.add(spare);
int ord = collectedTerms.add(fromDocTerms.get(doc));
if (ord < 0) {
ord = -ord - 1;
} else {
@ -183,7 +180,6 @@ abstract class TermsWithScoreCollector extends SimpleCollector {
static class MV extends TermsWithScoreCollector {
SortedSetDocValues fromDocTermOrds;
final BytesRef scratch = new BytesRef();
MV(String field, ScoreMode scoreMode) {
super(field, scoreMode);
@ -194,9 +190,7 @@ abstract class TermsWithScoreCollector extends SimpleCollector {
fromDocTermOrds.setDocument(doc);
long ord;
while ((ord = fromDocTermOrds.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
fromDocTermOrds.lookupOrd(ord, scratch);
int termID = collectedTerms.add(scratch);
int termID = collectedTerms.add(fromDocTermOrds.lookupOrd(ord));
if (termID < 0) {
termID = -termID - 1;
} else {
@ -233,9 +227,7 @@ abstract class TermsWithScoreCollector extends SimpleCollector {
fromDocTermOrds.setDocument(doc);
long ord;
while ((ord = fromDocTermOrds.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
fromDocTermOrds.lookupOrd(ord, scratch);
int termID = collectedTerms.add(scratch);
int termID = collectedTerms.add(fromDocTermOrds.lookupOrd(ord));
if (termID < 0) {
termID = -termID - 1;
} else {

17
lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
View File

@ -662,14 +662,13 @@ public class TestJoinUtil extends LuceneTestCase {
private Scorer scorer;
private SortedSetDocValues docTermOrds;
final BytesRef joinValue = new BytesRef();
@Override
public void collect(int doc) throws IOException {
docTermOrds.setDocument(doc);
long ord;
while ((ord = docTermOrds.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
docTermOrds.lookupOrd(ord, joinValue);
final BytesRef joinValue = docTermOrds.lookupOrd(ord);
JoinScore joinScore = joinValueToJoinScores.get(joinValue);
if (joinScore == null) {
joinValueToJoinScores.put(BytesRef.deepCopyOf(joinValue), joinScore = new JoinScore());
@ -699,12 +698,10 @@ public class TestJoinUtil extends LuceneTestCase {
private Scorer scorer;
private BinaryDocValues terms;
private Bits docsWithField;
private final BytesRef spare = new BytesRef();
@Override
public void collect(int doc) throws IOException {
terms.get(doc, spare);
BytesRef joinValue = spare;
final BytesRef joinValue = terms.get(doc);
if (joinValue.length == 0 && !docsWithField.get(doc)) {
return;
}
@ -764,7 +761,6 @@ public class TestJoinUtil extends LuceneTestCase {
toSearcher.search(new MatchAllDocsQuery(), new SimpleCollector() {
private SortedSetDocValues docTermOrds;
private final BytesRef scratch = new BytesRef();
private int docBase;
@Override
@ -772,8 +768,8 @@ public class TestJoinUtil extends LuceneTestCase {
docTermOrds.setDocument(doc);
long ord;
while ((ord = docTermOrds.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
docTermOrds.lookupOrd(ord, scratch);
JoinScore joinScore = joinValueToJoinScores.get(scratch);
final BytesRef joinValue = docTermOrds.lookupOrd(ord);
JoinScore joinScore = joinValueToJoinScores.get(joinValue);
if (joinScore == null) {
continue;
}
@ -803,12 +799,11 @@ public class TestJoinUtil extends LuceneTestCase {
private BinaryDocValues terms;
private int docBase;
private final BytesRef spare = new BytesRef();
@Override
public void collect(int doc) {
terms.get(doc, spare);
JoinScore joinScore = joinValueToJoinScores.get(spare);
final BytesRef joinValue = terms.get(doc);
JoinScore joinScore = joinValueToJoinScores.get(joinValue);
if (joinScore == null) {
return;
}

16
lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java
View File

@ -201,8 +201,8 @@ public class SortingAtomicReader extends FilterAtomicReader {
}
@Override
public void get(int docID, BytesRef result) {
in.get(docMap.newToOld(docID), result);
public BytesRef get(int docID) {
return in.get(docMap.newToOld(docID));
}
}
@ -259,8 +259,8 @@ public class SortingAtomicReader extends FilterAtomicReader {
}
@Override
public void lookupOrd(int ord, BytesRef result) {
in.lookupOrd(ord, result);
public BytesRef lookupOrd(int ord) {
return in.lookupOrd(ord);
}
@Override
@ -269,8 +269,8 @@ public class SortingAtomicReader extends FilterAtomicReader {
}
@Override
public void get(int docID, BytesRef result) {
in.get(docMap.newToOld(docID), result);
public BytesRef get(int docID) {
return in.get(docMap.newToOld(docID));
}
@Override
@ -300,8 +300,8 @@ public class SortingAtomicReader extends FilterAtomicReader {
}
@Override
public void lookupOrd(long ord, BytesRef result) {
in.lookupOrd(ord, result);
public BytesRef lookupOrd(long ord) {
return in.lookupOrd(ord);
}
@Override

11
lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java
View File

@ -32,7 +32,6 @@ import org.apache.lucene.index.Fields;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@ -773,7 +772,7 @@ public class DocTermOrds {
/** Returns a SortedSetDocValues view of this instance */
public SortedSetDocValues iterator(AtomicReader reader) throws IOException {
if (isEmpty()) {
return DocValues.EMPTY_SORTED_SET;
return DocValues.emptySortedSet();
} else {
return new Iterator(reader);
}
@ -874,16 +873,12 @@ public class DocTermOrds {
}
@Override
public void lookupOrd(long ord, BytesRef result) {
BytesRef ref = null;
public BytesRef lookupOrd(long ord) {
try {
ref = DocTermOrds.this.lookupTerm(te, (int) ord);
return DocTermOrds.this.lookupTerm(te, (int) ord);
} catch (IOException e) {
throw new RuntimeException(e);
}
result.bytes = ref.bytes;
result.offset = ref.offset;
result.length = ref.length;
}
@Override

75
lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheImpl.java
View File

@ -431,11 +431,11 @@ class FieldCacheImpl implements FieldCache {
} else {
final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
if (info == null) {
return DocValues.EMPTY_NUMERIC;
return DocValues.emptyNumeric();
} else if (info.hasDocValues()) {
throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
} else if (!info.isIndexed()) {
return DocValues.EMPTY_NUMERIC;
return DocValues.emptyNumeric();
}
return (NumericDocValues) caches.get(Long.TYPE).get(reader, new CacheKey(field, parser), setDocsWithField);
}
@ -523,7 +523,7 @@ class FieldCacheImpl implements FieldCache {
}
}
public static class SortedDocValuesImpl extends SortedDocValues {
public static class SortedDocValuesImpl {
private final PagedBytes.Reader bytes;
private final MonotonicAppendingLongBuffer termOrdToBytesOffset;
private final PackedInts.Reader docToTermOrd;
@ -535,26 +535,33 @@ class FieldCacheImpl implements FieldCache {
this.termOrdToBytesOffset = termOrdToBytesOffset;
this.numOrd = numOrd;
}
public SortedDocValues iterator() {
final BytesRef term = new BytesRef();
return new SortedDocValues() {
@Override
public int getValueCount() {
return numOrd;
}
@Override
public int getValueCount() {
return numOrd;
}
@Override
public int getOrd(int docID) {
// Subtract 1, matching the 1+ord we did when
// storing, so that missing values, which are 0 in the
// packed ints, are returned as -1 ord:
return (int) docToTermOrd.get(docID)-1;
}
@Override
public int getOrd(int docID) {
// Subtract 1, matching the 1+ord we did when
// storing, so that missing values, which are 0 in the
// packed ints, are returned as -1 ord:
return (int) docToTermOrd.get(docID)-1;
}
@Override
public void lookupOrd(int ord, BytesRef ret) {
if (ord < 0) {
throw new IllegalArgumentException("ord must be >=0 (got ord=" + ord + ")");
}
bytes.fill(ret, termOrdToBytesOffset.get(ord));
@Override
public BytesRef lookupOrd(int ord) {
if (ord < 0) {
throw new IllegalArgumentException("ord must be >=0 (got ord=" + ord + ")");
}
bytes.fill(term, termOrdToBytesOffset.get(ord));
return term;
}
};
}
}
@ -571,15 +578,16 @@ class FieldCacheImpl implements FieldCache {
} else {
final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
if (info == null) {
return DocValues.EMPTY_SORTED;
return DocValues.emptySorted();
} else if (info.hasDocValues()) {
// we don't try to build a sorted instance from numeric/binary doc
// values because dedup can be very costly
throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
} else if (!info.isIndexed()) {
return DocValues.EMPTY_SORTED;
return DocValues.emptySorted();
}
return (SortedDocValues) caches.get(SortedDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio), false);
SortedDocValuesImpl impl = (SortedDocValuesImpl) caches.get(SortedDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio), false);
return impl.iterator();
}
}
@ -674,22 +682,23 @@ class FieldCacheImpl implements FieldCache {
private static class BinaryDocValuesImpl extends BinaryDocValues {
private final PagedBytes.Reader bytes;
private final PackedInts.Reader docToOffset;
private final BytesRef term;
public BinaryDocValuesImpl(PagedBytes.Reader bytes, PackedInts.Reader docToOffset) {
this.bytes = bytes;
this.docToOffset = docToOffset;
term = new BytesRef();
}
@Override
public void get(int docID, BytesRef ret) {
public BytesRef get(int docID) {
final int pointer = (int) docToOffset.get(docID);
if (pointer == 0) {
ret.bytes = BytesRef.EMPTY_BYTES;
ret.offset = 0;
ret.length = 0;
term.length = 0;
} else {
bytes.fill(ret, pointer);
bytes.fill(term, pointer);
}
return term;
}
}
@ -713,11 +722,11 @@ class FieldCacheImpl implements FieldCache {
final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
if (info == null) {
return DocValues.EMPTY_BINARY;
return DocValues.emptyBinary();
} else if (info.hasDocValues()) {
throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
} else if (!info.isIndexed()) {
return DocValues.EMPTY_BINARY;
return DocValues.emptyBinary();
}
return (BinaryDocValues) caches.get(BinaryDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio), setDocsWithField);
@ -835,18 +844,18 @@ class FieldCacheImpl implements FieldCache {
final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
if (info == null) {
return DocValues.EMPTY_SORTED_SET;
return DocValues.emptySortedSet();
} else if (info.hasDocValues()) {
throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
} else if (!info.isIndexed()) {
return DocValues.EMPTY_SORTED_SET;
return DocValues.emptySortedSet();
}
// ok we need to uninvert. check if we can optimize a bit.
Terms terms = reader.terms(field);
if (terms == null) {
return DocValues.EMPTY_SORTED_SET;
return DocValues.emptySortedSet();
} else {
// if #postings = #docswithfield we know that the field is "single valued enough".
// its possible the same term might appear twice in the same document, but SORTED_SET discards frequency.

11
lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java
View File

@ -239,9 +239,8 @@ public abstract class SorterTestBase extends LuceneTestCase {
@Test
public void testBinaryDocValuesField() throws Exception {
BinaryDocValues dv = reader.getBinaryDocValues(BINARY_DV_FIELD);
BytesRef bytes = new BytesRef();
for (int i = 0; i < reader.maxDoc(); i++) {
dv.get(i, bytes);
final BytesRef bytes = dv.get(i);
assertEquals("incorrect binary DocValues for doc " + i, sortedValues[i].toString(), bytes.utf8ToString());
}
}
@ -367,9 +366,8 @@ public abstract class SorterTestBase extends LuceneTestCase {
public void testSortedDocValuesField() throws Exception {
SortedDocValues dv = reader.getSortedDocValues(SORTED_DV_FIELD);
int maxDoc = reader.maxDoc();
BytesRef bytes = new BytesRef();
for (int i = 0; i < maxDoc; i++) {
dv.get(i, bytes);
final BytesRef bytes = dv.get(i);
assertEquals("incorrect sorted DocValues for doc " + i, sortedValues[i].toString(), bytes.utf8ToString());
}
}
@ -379,13 +377,12 @@ public abstract class SorterTestBase extends LuceneTestCase {
assumeTrue("default codec does not support SORTED_SET", defaultCodecSupportsSortedSet());
SortedSetDocValues dv = reader.getSortedSetDocValues(SORTED_SET_DV_FIELD);
int maxDoc = reader.maxDoc();
BytesRef bytes = new BytesRef();
for (int i = 0; i < maxDoc; i++) {
dv.setDocument(i);
dv.lookupOrd(dv.nextOrd(), bytes);
BytesRef bytes = dv.lookupOrd(dv.nextOrd());
int value = sortedValues[i].intValue();
assertEquals("incorrect sorted-set DocValues for doc " + i, Integer.valueOf(value).toString(), bytes.utf8ToString());
dv.lookupOrd(dv.nextOrd(), bytes);
bytes = dv.lookupOrd(dv.nextOrd());
assertEquals("incorrect sorted-set DocValues for doc " + i, Integer.valueOf(value + 1).toString(), bytes.utf8ToString());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, dv.nextOrd());
}

15
lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java
View File

@ -456,11 +456,10 @@ public class TestDocTermOrds extends LuceneTestCase {
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
BytesRef value = new BytesRef();
v.lookupOrd(0, value);
BytesRef value = v.lookupOrd(0);
assertEquals(-3, NumericUtils.prefixCodedToInt(value));
v.lookupOrd(1, value);
value = v.lookupOrd(1);
assertEquals(5, NumericUtils.prefixCodedToInt(value));
ir.close();
@ -498,11 +497,10 @@ public class TestDocTermOrds extends LuceneTestCase {
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
BytesRef value = new BytesRef();
v.lookupOrd(0, value);
BytesRef value = v.lookupOrd(0);
assertEquals(-3, NumericUtils.prefixCodedToLong(value));
v.lookupOrd(1, value);
value = v.lookupOrd(1);
assertEquals(5, NumericUtils.prefixCodedToLong(value));
ir.close();
@ -640,11 +638,10 @@ public class TestDocTermOrds extends LuceneTestCase {
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
BytesRef value = new BytesRef();
v.lookupOrd(0, value);
BytesRef value = v.lookupOrd(0);
assertEquals("bar", value.utf8ToString());
v.lookupOrd(1, value);
value = v.lookupOrd(1);
assertEquals("baz", value.utf8ToString());
ir.close();

50
lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCache.java
View File

@ -201,28 +201,23 @@ public class TestFieldCache extends LuceneTestCase {
// getTermsIndex
SortedDocValues termsIndex = cache.getTermsIndex(reader, "theRandomUnicodeString");
assertSame("Second request to cache return same array", termsIndex, cache.getTermsIndex(reader, "theRandomUnicodeString"));
final BytesRef br = new BytesRef();
for (int i = 0; i < NUM_DOCS; i++) {
final BytesRef term;
final String s;
final int ord = termsIndex.getOrd(i);
if (ord == -1) {
term = null;
s = null;
} else {
termsIndex.lookupOrd(ord, br);
term = br;
s = termsIndex.lookupOrd(ord).utf8ToString();
}
final String s = term == null ? null : term.utf8ToString();
assertTrue("for doc " + i + ": " + s + " does not equal: " + unicodeStrings[i], unicodeStrings[i] == null || unicodeStrings[i].equals(s));
}
int nTerms = termsIndex.getValueCount();
TermsEnum tenum = termsIndex.termsEnum();
BytesRef val = new BytesRef();
for (int i=0; i<nTerms; i++) {
BytesRef val1 = tenum.next();
termsIndex.lookupOrd(i, val);
BytesRef val1 = BytesRef.deepCopyOf(tenum.next());
final BytesRef val = termsIndex.lookupOrd(i);
// System.out.println("i="+i);
assertEquals(val, val1);
}
@ -231,13 +226,13 @@ public class TestFieldCache extends LuceneTestCase {
int num = atLeast(100);
for (int i = 0; i < num; i++) {
int k = random().nextInt(nTerms);
termsIndex.lookupOrd(k, val);
final BytesRef val = BytesRef.deepCopyOf(termsIndex.lookupOrd(k));
assertEquals(TermsEnum.SeekStatus.FOUND, tenum.seekCeil(val));
assertEquals(val, tenum.term());
}
for(int i=0;i<nTerms;i++) {
termsIndex.lookupOrd(i, val);
final BytesRef val = BytesRef.deepCopyOf(termsIndex.lookupOrd(i));
assertEquals(TermsEnum.SeekStatus.FOUND, tenum.seekCeil(val));
assertEquals(val, tenum.term());
}
@ -250,14 +245,12 @@ public class TestFieldCache extends LuceneTestCase {
assertSame("Second request to cache return same array", terms, cache.getTerms(reader, "theRandomUnicodeString", true));
Bits bits = cache.getDocsWithField(reader, "theRandomUnicodeString");
for (int i = 0; i < NUM_DOCS; i++) {
terms.get(i, br);
final BytesRef term;
final String s;
if (!bits.get(i)) {
term = null;
s = null;
} else {
term = br;
s = terms.get(i).utf8ToString();
}
final String s = term == null ? null : term.utf8ToString();
assertTrue("for doc " + i + ": " + s + " does not equal: " + unicodeStrings[i], unicodeStrings[i] == null || unicodeStrings[i].equals(s));
}
@ -282,8 +275,7 @@ public class TestFieldCache extends LuceneTestCase {
}
long ord = termOrds.nextOrd();
assert ord != SortedSetDocValues.NO_MORE_ORDS;
BytesRef scratch = new BytesRef();
termOrds.lookupOrd(ord, scratch);
BytesRef scratch = termOrds.lookupOrd(ord);
assertEquals(v, scratch);
}
assertEquals(SortedSetDocValues.NO_MORE_ORDS, termOrds.nextOrd());
@ -448,8 +440,6 @@ public class TestFieldCache extends LuceneTestCase {
iw.shutdown();
AtomicReader ar = getOnlySegmentReader(ir);
BytesRef scratch = new BytesRef();
// Binary type: can be retrieved via getTerms()
try {
FieldCache.DEFAULT.getNumerics(ar, "binary", FieldCache.NUMERIC_UTILS_INT_PARSER, false);
@ -457,8 +447,8 @@ public class TestFieldCache extends LuceneTestCase {
} catch (IllegalStateException expected) {}
BinaryDocValues binary = FieldCache.DEFAULT.getTerms(ar, "binary", true);
binary.get(0, scratch);
assertEquals("binary value", scratch.utf8ToString());
final BytesRef term = binary.get(0);
assertEquals("binary value", term.utf8ToString());
try {
FieldCache.DEFAULT.getTermsIndex(ar, "binary");
@ -490,13 +480,13 @@ public class TestFieldCache extends LuceneTestCase {
} catch (IllegalStateException expected) {}
binary = FieldCache.DEFAULT.getTerms(ar, "sorted", true);
binary.get(0, scratch);
BytesRef scratch = binary.get(0);
assertEquals("sorted value", scratch.utf8ToString());
SortedDocValues sorted = FieldCache.DEFAULT.getTermsIndex(ar, "sorted");
assertEquals(0, sorted.getOrd(0));
assertEquals(1, sorted.getValueCount());
sorted.get(0, scratch);
scratch = sorted.get(0);
assertEquals("sorted value", scratch.utf8ToString());
SortedSetDocValues sortedSet = FieldCache.DEFAULT.getDocTermOrds(ar, "sorted", null);
@ -598,14 +588,13 @@ public class TestFieldCache extends LuceneTestCase {
NumericDocValues doubles = cache.getNumerics(ar, "bogusdoubles", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, true);
assertEquals(0, doubles.get(0));
BytesRef scratch = new BytesRef();
BinaryDocValues binaries = cache.getTerms(ar, "bogusterms", true);
binaries.get(0, scratch);
BytesRef scratch = binaries.get(0);
assertEquals(0, scratch.length);
SortedDocValues sorted = cache.getTermsIndex(ar, "bogustermsindex");
assertEquals(-1, sorted.getOrd(0));
sorted.get(0, scratch);
scratch = sorted.get(0);
assertEquals(0, scratch.length);
SortedSetDocValues sortedSet = cache.getDocTermOrds(ar, "bogusmultivalued", null);
@ -657,14 +646,13 @@ public class TestFieldCache extends LuceneTestCase {
NumericDocValues doubles = cache.getNumerics(ar, "bogusdoubles", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, true);
assertEquals(0, doubles.get(0));
BytesRef scratch = new BytesRef();
BinaryDocValues binaries = cache.getTerms(ar, "bogusterms", true);
binaries.get(0, scratch);
BytesRef scratch = binaries.get(0);
assertEquals(0, scratch.length);
SortedDocValues sorted = cache.getTermsIndex(ar, "bogustermsindex");
assertEquals(-1, sorted.getOrd(0));
sorted.get(0, scratch);
scratch = sorted.get(0);
assertEquals(0, scratch.length);
SortedSetDocValues sortedSet = cache.getDocTermOrds(ar, "bogusmultivalued", null);

5
lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSort.java
View File

@ -1054,7 +1054,10 @@ public class TestFieldCacheSort extends LuceneTestCase {
// this should not throw AIOOBE or RuntimeEx
IndexReader reader = UninvertingReader.wrap(DirectoryReader.open(indexStore),
Collections.singletonMap("string", Type.SORTED));
IndexSearcher searcher = newSearcher(reader);
// NOTE: we can't wrap this with newSearcher, because when the API is abused in this way,
// the number of ords can exceed the number of documents, and AssertingAtomicReader will get angry,
// rightfully so (its a broken dv)
IndexSearcher searcher = new IndexSearcher(reader);
searcher.search(new MatchAllDocsQuery(), null, 500, sort);
reader.close();
indexStore.close();

288
lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSortRandom.java Normal file
View File

@ -0,0 +1,288 @@
package org.apache.lucene.uninverting;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopFieldDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
/** random sorting tests with uninversion */
public class TestFieldCacheSortRandom extends LuceneTestCase {
public void testRandomStringSort() throws Exception {
Random random = new Random(random().nextLong());
final int NUM_DOCS = atLeast(100);
final Directory dir = newDirectory();
final RandomIndexWriter writer = new RandomIndexWriter(random, dir);
final boolean allowDups = random.nextBoolean();
final Set<String> seen = new HashSet<>();
final int maxLength = TestUtil.nextInt(random, 5, 100);
if (VERBOSE) {
System.out.println("TEST: NUM_DOCS=" + NUM_DOCS + " maxLength=" + maxLength + " allowDups=" + allowDups);
}
int numDocs = 0;
final List<BytesRef> docValues = new ArrayList<>();
// TODO: deletions
while (numDocs < NUM_DOCS) {
final Document doc = new Document();
// 10% of the time, the document is missing the value:
final BytesRef br;
if (random().nextInt(10) != 7) {
final String s;
if (random.nextBoolean()) {
s = TestUtil.randomSimpleString(random, maxLength);
} else {
s = TestUtil.randomUnicodeString(random, maxLength);
}
if (!allowDups) {
if (seen.contains(s)) {
continue;
}
seen.add(s);
}
if (VERBOSE) {
System.out.println(" " + numDocs + ": s=" + s);
}
doc.add(new StringField("stringdv", s, Field.Store.NO));
docValues.add(new BytesRef(s));
} else {
br = null;
if (VERBOSE) {
System.out.println(" " + numDocs + ": <missing>");
}
docValues.add(null);
}
doc.add(new IntField("id", numDocs, Field.Store.YES));
writer.addDocument(doc);
numDocs++;
if (random.nextInt(40) == 17) {
// force flush
writer.getReader().close();
}
}
Map<String,UninvertingReader.Type> mapping = new HashMap<>();
mapping.put("stringdv", Type.SORTED);
mapping.put("id", Type.INTEGER);
final IndexReader r = UninvertingReader.wrap(writer.getReader(), mapping);
writer.shutdown();
if (VERBOSE) {
System.out.println(" reader=" + r);
}
final IndexSearcher s = newSearcher(r, false);
final int ITERS = atLeast(100);
for(int iter=0;iter<ITERS;iter++) {
final boolean reverse = random.nextBoolean();
final TopFieldDocs hits;
final SortField sf;
final boolean sortMissingLast;
final boolean missingIsNull;
sf = new SortField("stringdv", SortField.Type.STRING, reverse);
sortMissingLast = random().nextBoolean();
missingIsNull = true;
if (sortMissingLast) {
sf.setMissingValue(SortField.STRING_LAST);
}
final Sort sort;
if (random.nextBoolean()) {
sort = new Sort(sf);
} else {
sort = new Sort(sf, SortField.FIELD_DOC);
}
final int hitCount = TestUtil.nextInt(random, 1, r.maxDoc() + 20);
final RandomFilter f = new RandomFilter(random, random.nextFloat(), docValues);
int queryType = random.nextInt(3);
if (queryType == 0) {
// force out of order
BooleanQuery bq = new BooleanQuery();
// Add a Query with SHOULD, since bw.scorer() returns BooleanScorer2
// which delegates to BS if there are no mandatory clauses.
bq.add(new MatchAllDocsQuery(), Occur.SHOULD);
// Set minNrShouldMatch to 1 so that BQ will not optimize rewrite to return
// the clause instead of BQ.
bq.setMinimumNumberShouldMatch(1);
hits = s.search(bq, f, hitCount, sort, random.nextBoolean(), random.nextBoolean());
} else if (queryType == 1) {
hits = s.search(new ConstantScoreQuery(f),
null, hitCount, sort, random.nextBoolean(), random.nextBoolean());
} else {
hits = s.search(new MatchAllDocsQuery(),
f, hitCount, sort, random.nextBoolean(), random.nextBoolean());
}
if (VERBOSE) {
System.out.println("\nTEST: iter=" + iter + " " + hits.totalHits + " hits; topN=" + hitCount + "; reverse=" + reverse + "; sortMissingLast=" + sortMissingLast + " sort=" + sort);
}
// Compute expected results:
Collections.sort(f.matchValues, new Comparator<BytesRef>() {
@Override
public int compare(BytesRef a, BytesRef b) {
if (a == null) {
if (b == null) {
return 0;
}
if (sortMissingLast) {
return 1;
} else {
return -1;
}
} else if (b == null) {
if (sortMissingLast) {
return -1;
} else {
return 1;
}
} else {
return a.compareTo(b);
}
}
});
if (reverse) {
Collections.reverse(f.matchValues);
}
final List<BytesRef> expected = f.matchValues;
if (VERBOSE) {
System.out.println(" expected:");
for(int idx=0;idx<expected.size();idx++) {
BytesRef br = expected.get(idx);
if (br == null && missingIsNull == false) {
br = new BytesRef();
}
System.out.println(" " + idx + ": " + (br == null ? "<missing>" : br.utf8ToString()));
if (idx == hitCount-1) {
break;
}
}
}
if (VERBOSE) {
System.out.println(" actual:");
for(int hitIDX=0;hitIDX<hits.scoreDocs.length;hitIDX++) {
final FieldDoc fd = (FieldDoc) hits.scoreDocs[hitIDX];
BytesRef br = (BytesRef) fd.fields[0];
System.out.println(" " + hitIDX + ": " + (br == null ? "<missing>" : br.utf8ToString()) + " id=" + s.doc(fd.doc).get("id"));
}
}
for(int hitIDX=0;hitIDX<hits.scoreDocs.length;hitIDX++) {
final FieldDoc fd = (FieldDoc) hits.scoreDocs[hitIDX];
BytesRef br = expected.get(hitIDX);
if (br == null && missingIsNull == false) {
br = new BytesRef();
}
// Normally, the old codecs (that don't support
// docsWithField via doc values) will always return
// an empty BytesRef for the missing case; however,
// if all docs in a given segment were missing, in
// that case it will return null! So we must map
// null here, too:
BytesRef br2 = (BytesRef) fd.fields[0];
if (br2 == null && missingIsNull == false) {
br2 = new BytesRef();
}
assertEquals(br, br2);
}
}
r.close();
dir.close();
}
private static class RandomFilter extends Filter {
private final Random random;
private float density;
private final List<BytesRef> docValues;
public final List<BytesRef> matchValues = Collections.synchronizedList(new ArrayList<BytesRef>());
// density should be 0.0 ... 1.0
public RandomFilter(Random random, float density, List<BytesRef> docValues) {
this.random = random;
this.density = density;
this.docValues = docValues;
}
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
final int maxDoc = context.reader().maxDoc();
final NumericDocValues idSource = DocValues.getNumeric(context.reader(), "id");
assertNotNull(idSource);
final FixedBitSet bits = new FixedBitSet(maxDoc);
for(int docID=0;docID<maxDoc;docID++) {
if (random.nextFloat() <= density && (acceptDocs == null || acceptDocs.get(docID))) {
bits.set(docID);
//System.out.println(" acc id=" + idSource.getInt(docID) + " docID=" + docID);
matchValues.add(docValues.get((int) idSource.get(docID)));
}
}
return bits;
}
}
}

14
lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheVsDocValues.java
View File

@ -200,8 +200,7 @@ public class TestFieldCacheVsDocValues extends LuceneTestCase {
BinaryDocValues s = FieldCache.DEFAULT.getTerms(ar, "field", false);
for(int docID=0;docID<docBytes.size();docID++) {
StoredDocument doc = ar.document(docID);
BytesRef bytes = new BytesRef();
s.get(docID, bytes);
BytesRef bytes = s.get(docID);
byte[] expected = docBytes.get(Integer.parseInt(doc.get("id")));
assertEquals(expected.length, bytes.length);
assertEquals(new BytesRef(expected), bytes);
@ -272,8 +271,7 @@ public class TestFieldCacheVsDocValues extends LuceneTestCase {
BinaryDocValues s = FieldCache.DEFAULT.getTerms(ar, "field", false);
for(int docID=0;docID<docBytes.size();docID++) {
StoredDocument doc = ar.document(docID);
BytesRef bytes = new BytesRef();
s.get(docID, bytes);
BytesRef bytes = s.get(docID);
byte[] expected = docBytes.get(Integer.parseInt(doc.get("id")));
assertEquals(expected.length, bytes.length);
assertEquals(new BytesRef(expected), bytes);
@ -495,7 +493,7 @@ public class TestFieldCacheVsDocValues extends LuceneTestCase {
// can be null for the segment if no docs actually had any SortedDocValues
// in this case FC.getDocTermsOrds returns EMPTY
if (actual == null) {
assertEquals(DocValues.EMPTY_SORTED_SET, expected);
assertEquals(expected.getValueCount(), 0);
return;
}
assertEquals(expected.getValueCount(), actual.getValueCount());
@ -511,11 +509,9 @@ public class TestFieldCacheVsDocValues extends LuceneTestCase {
}
// compare ord dictionary
BytesRef expectedBytes = new BytesRef();
BytesRef actualBytes = new BytesRef();
for (long i = 0; i < expected.getValueCount(); i++) {
expected.lookupOrd(i, expectedBytes);
actual.lookupOrd(i, actualBytes);
final BytesRef expectedBytes = BytesRef.deepCopyOf(expected.lookupOrd(i));
final BytesRef actualBytes = actual.lookupOrd(i);
assertEquals(expectedBytes, actualBytes);
}

16
lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheWithThreads.java
View File

@ -90,8 +90,6 @@ public class TestFieldCacheWithThreads extends LuceneTestCase {
SortedDocValues sdv = FieldCache.DEFAULT.getTermsIndex(ar, "sorted");
startingGun.await();
int iters = atLeast(1000);
BytesRef scratch = new BytesRef();
BytesRef scratch2 = new BytesRef();
for(int iter=0;iter<iters;iter++) {
int docID = threadRandom.nextInt(numDocs);
switch(threadRandom.nextInt(4)) {
@ -108,11 +106,10 @@ public class TestFieldCacheWithThreads extends LuceneTestCase {
assertEquals(numbers.get(docID).longValue(), FieldCache.DEFAULT.getNumerics(ar, "number", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, false).get(docID));
break;
}
bdv.get(docID, scratch);
assertEquals(binary.get(docID), scratch);
// Cannot share a single scratch against two "sources":
sdv.get(docID, scratch2);
assertEquals(sorted.get(docID), scratch2);
BytesRef term = bdv.get(docID);
assertEquals(binary.get(docID), term);
term = sdv.get(docID);
assertEquals(sorted.get(docID), term);
}
} catch (Exception e) {
throw new RuntimeException(e);
@ -207,12 +204,11 @@ public class TestFieldCacheWithThreads extends LuceneTestCase {
while(System.currentTimeMillis() < END_TIME) {
final SortedDocValues source;
source = stringDVDirect;
final BytesRef scratch = new BytesRef();
for(int iter=0;iter<100;iter++) {
final int docID = random.nextInt(sr.maxDoc());
source.get(docID, scratch);
assertEquals(docValues.get((int) docIDToID.get(docID)), scratch);
BytesRef term = source.get(docID);
assertEquals(docValues.get((int) docIDToID.get(docID)), term);
}
}
}

20
lucene/misc/src/test/org/apache/lucene/uninverting/TestUninvertingReader.java
View File

@ -72,11 +72,10 @@ public class TestUninvertingReader extends LuceneTestCase {
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
BytesRef value = new BytesRef();
v.lookupOrd(0, value);
BytesRef value = v.lookupOrd(0);
assertEquals(-3, NumericUtils.prefixCodedToInt(value));
v.lookupOrd(1, value);
value = v.lookupOrd(1);
assertEquals(5, NumericUtils.prefixCodedToInt(value));
ir.close();
@ -117,11 +116,10 @@ public class TestUninvertingReader extends LuceneTestCase {
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
BytesRef value = new BytesRef();
v.lookupOrd(0, value);
BytesRef value = v.lookupOrd(0);
assertEquals(Float.floatToRawIntBits(-3f), NumericUtils.prefixCodedToInt(value));
v.lookupOrd(1, value);
value = v.lookupOrd(1);
assertEquals(Float.floatToRawIntBits(5f), NumericUtils.prefixCodedToInt(value));
ir.close();
@ -161,11 +159,10 @@ public class TestUninvertingReader extends LuceneTestCase {
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
BytesRef value = new BytesRef();
v.lookupOrd(0, value);
BytesRef value = v.lookupOrd(0);
assertEquals(-3, NumericUtils.prefixCodedToLong(value));
v.lookupOrd(1, value);
value = v.lookupOrd(1);
assertEquals(5, NumericUtils.prefixCodedToLong(value));
ir.close();
@ -205,11 +202,10 @@ public class TestUninvertingReader extends LuceneTestCase {
assertEquals(1, v.nextOrd());
assertEquals(SortedSetDocValues.NO_MORE_ORDS, v.nextOrd());
BytesRef value = new BytesRef();
v.lookupOrd(0, value);
BytesRef value = v.lookupOrd(0);
assertEquals(Double.doubleToRawLongBits(-3d), NumericUtils.prefixCodedToLong(value));
v.lookupOrd(1, value);
value = v.lookupOrd(1);
assertEquals(Double.doubleToRawLongBits(5d), NumericUtils.prefixCodedToLong(value));
ir.close();

22
lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java
View File

@ -40,7 +40,6 @@ public abstract class DocTermsIndexDocValues extends FunctionValues {
protected final SortedDocValues termsIndex;
protected final ValueSource vs;
protected final MutableValueStr val = new MutableValueStr();
protected final BytesRef spare = new BytesRef();
protected final CharsRef spareChars = new CharsRef();
public DocTermsIndexDocValues(ValueSource vs, AtomicReaderContext context, String field) throws IOException {
@ -71,17 +70,18 @@ public abstract class DocTermsIndexDocValues extends FunctionValues {
@Override
public boolean bytesVal(int doc, BytesRef target) {
termsIndex.get(doc, target);
target.length = 0;
target.copyBytes(termsIndex.get(doc));
return target.length > 0;
}
@Override
public String strVal(int doc) {
termsIndex.get(doc, spare);
if (spare.length == 0) {
final BytesRef term = termsIndex.get(doc);
if (term.length == 0) {
return null;
}
UnicodeUtil.UTF8toUTF16(spare, spareChars);
UnicodeUtil.UTF8toUTF16(term, spareChars);
return spareChars.toString();
}
@ -149,14 +149,10 @@ public abstract class DocTermsIndexDocValues extends FunctionValues {
@Override
public void fillValue(int doc) {
int ord = termsIndex.getOrd(doc);
if (ord == -1) {
mval.value.bytes = BytesRef.EMPTY_BYTES;
mval.value.offset = 0;
mval.value.length = 0;
mval.exists = false;
} else {
termsIndex.lookupOrd(ord, mval.value);
mval.exists = true;
mval.value.length = 0;
mval.exists = ord >= 0;
if (mval.exists) {
mval.value.copyBytes(termsIndex.lookupOrd(ord));
}
}
};

11
lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java
View File

@ -59,7 +59,7 @@ public class BytesRefFieldSource extends FieldCacheSource {
@Override
public boolean bytesVal(int doc, BytesRef target) {
binaryValues.get(doc, target);
target.copyBytes(binaryValues.get(doc));
return target.length > 0;
}
@ -93,13 +93,8 @@ public class BytesRefFieldSource extends FieldCacheSource {
@Override
public void fillValue(int doc) {
mval.exists = docsWithField.get(doc);
if (mval.exists) {
binaryValues.get(doc, mval.value);
} else {
mval.value.bytes = BytesRef.EMPTY_BYTES;
mval.value.offset = 0;
mval.value.length = 0;
}
mval.value.length = 0;
mval.value.copyBytes(binaryValues.get(doc));
}
};
}

5
lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java
View File

@ -62,14 +62,13 @@ public class JoinDocFreqValueSource extends FieldCacheSource {
final TermsEnum termsEnum = t == null ? TermsEnum.EMPTY : t.iterator(null);
return new IntDocValues(this) {
final BytesRef ref = new BytesRef();
@Override
public int intVal(int doc)
{
try {
terms.get(doc, ref);
if (termsEnum.seekExact(ref)) {
final BytesRef term = terms.get(doc);
if (termsEnum.seekExact(term)) {
return termsEnum.docFreq();
} else {
return 0;

17
lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java
View File

@ -45,7 +45,6 @@ public final class SlowCollatedStringComparator extends FieldComparator<String>
final Collator collator;
private String bottom;
private String topValue;
private final BytesRef tempBR = new BytesRef();
public SlowCollatedStringComparator(int numHits, String field, Collator collator) {
values = new String[numHits];
@ -70,8 +69,8 @@ public final class SlowCollatedStringComparator extends FieldComparator<String>
@Override
public int compareBottom(int doc) {
currentDocTerms.get(doc, tempBR);
final String val2 = tempBR.length == 0 && docsWithField.get(doc) == false ? null : tempBR.utf8ToString();
final BytesRef term = currentDocTerms.get(doc);
final String val2 = term.length == 0 && docsWithField.get(doc) == false ? null : term.utf8ToString();
if (bottom == null) {
if (val2 == null) {
return 0;
@ -85,11 +84,11 @@ public final class SlowCollatedStringComparator extends FieldComparator<String>
@Override
public void copy(int slot, int doc) {
currentDocTerms.get(doc, tempBR);
if (tempBR.length == 0 && docsWithField.get(doc) == false) {
final BytesRef term = currentDocTerms.get(doc);
if (term.length == 0 && docsWithField.get(doc) == false) {
values[slot] = null;
} else {
values[slot] = tempBR.utf8ToString();
values[slot] = term.utf8ToString();
}
}
@ -131,12 +130,12 @@ public final class SlowCollatedStringComparator extends FieldComparator<String>
@Override
public int compareTop(int doc) {
currentDocTerms.get(doc, tempBR);
final BytesRef term = currentDocTerms.get(doc);
final String docValue;
if (tempBR.length == 0 && docsWithField.get(doc) == false) {
if (term.length == 0 && docsWithField.get(doc) == false) {
docValue = null;
} else {
docValue = tempBR.utf8ToString();
docValue = term.utf8ToString();
}
return compareValues(topValue, docValue);
}

2
lucene/spatial/src/java/org/apache/lucene/spatial/serialized/SerializedDVStrategy.java
View File

@ -215,7 +215,7 @@ public class SerializedDVStrategy extends SpatialStrategy {
boolean fillBytes(int doc) {
if (bytesRefDoc != doc) {
docValues.get(doc, bytesRef);
bytesRef.copyBytes(docValues.get(doc));
bytesRefDoc = doc;
}
return bytesRef.length != 0;

11
lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
View File

@ -488,17 +488,15 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
BinaryDocValues payloadsDV = MultiDocValues.getBinaryValues(searcher.getIndexReader(), "payloads");
List<AtomicReaderContext> leaves = searcher.getIndexReader().leaves();
List<LookupResult> results = new ArrayList<>();
BytesRef scratch = new BytesRef();
for (int i=0;i<hits.scoreDocs.length;i++) {
FieldDoc fd = (FieldDoc) hits.scoreDocs[i];
textDV.get(fd.doc, scratch);
String text = scratch.utf8ToString();
BytesRef term = textDV.get(fd.doc);
String text = term.utf8ToString();
long score = (Long) fd.fields[0];
BytesRef payload;
if (payloadsDV != null) {
payload = new BytesRef();
payloadsDV.get(fd.doc, payload);
payload = BytesRef.deepCopyOf(payloadsDV.get(fd.doc));
} else {
payload = null;
}
@ -512,8 +510,7 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
contextsDV.setDocument(fd.doc - leaves.get(segment).docBase);
long ord;
while ((ord = contextsDV.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
BytesRef context = new BytesRef();
contextsDV.lookupOrd(ord, context);
BytesRef context = BytesRef.deepCopyOf(contextsDV.lookupOrd(ord));
contexts.add(context);
}
} else {

7
lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java
View File

@ -155,18 +155,15 @@ public class BlendedInfixSuggester extends AnalyzingInfixSuggester {
// we reduce the num to the one initially requested
int actualNum = num / numFactor;
BytesRef scratch = new BytesRef();
for (int i = 0; i < hits.scoreDocs.length; i++) {
FieldDoc fd = (FieldDoc) hits.scoreDocs[i];
textDV.get(fd.doc, scratch);
String text = scratch.utf8ToString();
final String text = textDV.get(fd.doc).utf8ToString();
long weight = (Long) fd.fields[0];
BytesRef payload;
if (payloadsDV != null) {
payload = new BytesRef();
payloadsDV.get(fd.doc, payload);
payload = BytesRef.deepCopyOf(payloadsDV.get(fd.doc));
} else {
payload = null;
}

24
lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java
View File

@ -449,11 +449,11 @@ public class AssertingAtomicReader extends FilterAtomicReader {
}
@Override
public void get(int docID, BytesRef result) {
public BytesRef get(int docID) {
assert docID >= 0 && docID < maxDoc;
final BytesRef result = in.get(docID);
assert result.isValid();
in.get(docID, result);
assert result.isValid();
return result;
}
}
@ -479,11 +479,11 @@ public class AssertingAtomicReader extends FilterAtomicReader {
}
@Override
public void lookupOrd(int ord, BytesRef result) {
public BytesRef lookupOrd(int ord) {
assert ord >= 0 && ord < valueCount;
final BytesRef result = in.lookupOrd(ord);
assert result.isValid();
in.lookupOrd(ord, result);
assert result.isValid();
return result;
}
@Override
@ -494,11 +494,11 @@ public class AssertingAtomicReader extends FilterAtomicReader {
}
@Override
public void get(int docID, BytesRef result) {
public BytesRef get(int docID) {
assert docID >= 0 && docID < maxDoc;
final BytesRef result = in.get(docID);
assert result.isValid();
in.get(docID, result);
assert result.isValid();
return result;
}
@Override
@ -543,11 +543,11 @@ public class AssertingAtomicReader extends FilterAtomicReader {
}
@Override
public void lookupOrd(long ord, BytesRef result) {
public BytesRef lookupOrd(long ord) {
assert ord >= 0 && ord < valueCount;
final BytesRef result = in.lookupOrd(ord);
assert result.isValid();
in.lookupOrd(ord, result);
assert result.isValid();
return result;
}
@Override

226
lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java
View File

@ -206,11 +206,10 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(text, hitDoc.get("fieldname"));
assert ireader.leaves().size() == 1;
BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv1");
BytesRef scratch = new BytesRef();
dv.get(hits.scoreDocs[i].doc, scratch);
BytesRef scratch = dv.get(hits.scoreDocs[i].doc);
assertEquals(new BytesRef(longTerm), scratch);
dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv2");
dv.get(hits.scoreDocs[i].doc, scratch);
scratch = dv.get(hits.scoreDocs[i].doc);
assertEquals(new BytesRef(text), scratch);
}
@ -238,7 +237,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
Query query = new TermQuery(new Term("fieldname", "text"));
TopDocs hits = isearcher.search(query, null, 1);
assertEquals(1, hits.totalHits);
BytesRef scratch = new BytesRef();
// Iterate through the results:
for (int i = 0; i < hits.scoreDocs.length; i++) {
StoredDocument hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
@ -247,7 +245,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
NumericDocValues dv = ireader.leaves().get(0).reader().getNumericDocValues("dv1");
assertEquals(5, dv.get(hits.scoreDocs[i].doc));
BinaryDocValues dv2 = ireader.leaves().get(0).reader().getBinaryDocValues("dv2");
dv2.get(hits.scoreDocs[i].doc, scratch);
BytesRef scratch = dv2.get(hits.scoreDocs[i].doc);
assertEquals(new BytesRef("hello world"), scratch);
}
@ -276,7 +274,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
Query query = new TermQuery(new Term("fieldname", "text"));
TopDocs hits = isearcher.search(query, null, 1);
assertEquals(1, hits.totalHits);
BytesRef scratch = new BytesRef();
// Iterate through the results:
for (int i = 0; i < hits.scoreDocs.length; i++) {
StoredDocument hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
@ -284,12 +281,12 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assert ireader.leaves().size() == 1;
SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv1");
int ord = dv.getOrd(0);
dv.lookupOrd(ord, scratch);
BytesRef scratch = dv.lookupOrd(ord);
assertEquals(new BytesRef("hello hello"), scratch);
NumericDocValues dv2 = ireader.leaves().get(0).reader().getNumericDocValues("dv2");
assertEquals(5, dv2.get(hits.scoreDocs[i].doc));
BinaryDocValues dv3 = ireader.leaves().get(0).reader().getBinaryDocValues("dv3");
dv3.get(hits.scoreDocs[i].doc, scratch);
scratch = dv3.get(hits.scoreDocs[i].doc);
assertEquals(new BytesRef("hello world"), scratch);
}
@ -326,12 +323,12 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assert ireader.leaves().size() == 1;
SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv2");
int ord = dv.getOrd(0);
dv.lookupOrd(ord, scratch);
scratch = dv.lookupOrd(ord);
assertEquals(new BytesRef("hello hello"), scratch);
NumericDocValues dv2 = ireader.leaves().get(0).reader().getNumericDocValues("dv3");
assertEquals(5, dv2.get(hits.scoreDocs[i].doc));
BinaryDocValues dv3 = ireader.leaves().get(0).reader().getBinaryDocValues("dv1");
dv3.get(hits.scoreDocs[i].doc, scratch);
scratch = dv3.get(hits.scoreDocs[i].doc);
assertEquals(new BytesRef("hello world"), scratch);
}
@ -480,14 +477,13 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
Query query = new TermQuery(new Term("fieldname", "text"));
TopDocs hits = isearcher.search(query, null, 1);
assertEquals(1, hits.totalHits);
BytesRef scratch = new BytesRef();
// Iterate through the results:
for (int i = 0; i < hits.scoreDocs.length; i++) {
StoredDocument hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
assertEquals(text, hitDoc.get("fieldname"));
assert ireader.leaves().size() == 1;
BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv");
dv.get(hits.scoreDocs[i].doc, scratch);
BytesRef scratch = dv.get(hits.scoreDocs[i].doc);
assertEquals(new BytesRef("hello world"), scratch);
}
@ -527,7 +523,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
} else {
expected = "hello 2";
}
dv.get(i, scratch);
scratch = dv.get(i);
assertEquals(expected, scratch.utf8ToString());
}
@ -564,7 +560,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(text, hitDoc.get("fieldname"));
assert ireader.leaves().size() == 1;
SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
dv.lookupOrd(dv.getOrd(hits.scoreDocs[i].doc), scratch);
scratch = dv.lookupOrd(dv.getOrd(hits.scoreDocs[i].doc));
assertEquals(new BytesRef("hello world"), scratch);
}
@ -593,9 +589,9 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assert ireader.leaves().size() == 1;
SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
BytesRef scratch = new BytesRef();
dv.lookupOrd(dv.getOrd(0), scratch);
scratch = dv.lookupOrd(dv.getOrd(0));
assertEquals("hello world 1", scratch.utf8ToString());
dv.lookupOrd(dv.getOrd(1), scratch);
scratch = dv.lookupOrd(dv.getOrd(1));
assertEquals("hello world 2", scratch.utf8ToString());
ireader.close();
@ -626,12 +622,11 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assert ireader.leaves().size() == 1;
SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
assertEquals(2, dv.getValueCount());
BytesRef scratch = new BytesRef();
assertEquals(0, dv.getOrd(0));
dv.lookupOrd(0, scratch);
BytesRef scratch = dv.lookupOrd(0);
assertEquals("hello world 1", scratch.utf8ToString());
assertEquals(1, dv.getOrd(1));
dv.lookupOrd(1, scratch);
scratch = dv.lookupOrd(1);
assertEquals("hello world 2", scratch.utf8ToString());
assertEquals(0, dv.getOrd(2));
@ -663,10 +658,9 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assert ireader.leaves().size() == 1;
SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
assertEquals(2, dv.getValueCount()); // 2 ords
BytesRef scratch = new BytesRef();
dv.lookupOrd(0, scratch);
BytesRef scratch = dv.lookupOrd(0);
assertEquals(new BytesRef("hello world 1"), scratch);
dv.lookupOrd(1, scratch);
scratch = dv.lookupOrd(1);
assertEquals(new BytesRef("hello world 2"), scratch);
for(int i=0;i<2;i++) {
StoredDocument doc2 = ireader.leaves().get(0).reader().document(i);
@ -676,7 +670,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
} else {
expected = "hello world 2";
}
dv.lookupOrd(dv.getOrd(i), scratch);
scratch = dv.lookupOrd(dv.getOrd(i));
assertEquals(expected, scratch.utf8ToString());
}
@ -712,8 +706,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
} else {
assertEquals(0, dv.getOrd(0));
assertEquals(1, dv.getValueCount());
BytesRef ref = new BytesRef();
dv.lookupOrd(0, ref);
BytesRef ref = dv.lookupOrd(0);
assertEquals(new BytesRef(), ref);
}
@ -737,8 +730,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv");
BytesRef scratch = new BytesRef();
dv.get(0, scratch);
BytesRef scratch = dv.get(0);
assertEquals(new BytesRef("hello\nworld\r1"), scratch);
ireader.close();
@ -763,13 +755,12 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
BytesRef scratch = new BytesRef();
dv.lookupOrd(dv.getOrd(0), scratch);
BytesRef scratch = dv.lookupOrd(dv.getOrd(0));
assertEquals(new BytesRef("hello world 2"), scratch);
if (defaultCodecSupportsDocsWithField()) {
assertEquals(-1, dv.getOrd(1));
}
dv.get(1, scratch);
scratch = dv.get(1);
assertEquals(new BytesRef(""), scratch);
ireader.close();
directory.close();
@ -866,10 +857,9 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
SortedDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
BytesRef scratch = new BytesRef();
assertEquals(0, dv.getOrd(0));
assertEquals(0, dv.getOrd(1));
dv.lookupOrd(dv.getOrd(0), scratch);
BytesRef scratch = dv.lookupOrd(dv.getOrd(0));
assertEquals("", scratch.utf8ToString());
ireader.close();
@ -896,10 +886,9 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv");
BytesRef scratch = new BytesRef();
dv.get(0, scratch);
BytesRef scratch = dv.get(0);
assertEquals("", scratch.utf8ToString());
dv.get(1, scratch);
scratch = dv.get(1);
assertEquals("", scratch.utf8ToString());
ireader.close();
@ -925,8 +914,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv");
BytesRef scratch = new BytesRef();
dv.get(0, scratch);
BytesRef scratch = dv.get(0);
assertEquals(new BytesRef(bytes), scratch);
ireader.close();
@ -952,8 +940,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
BinaryDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
BytesRef scratch = new BytesRef();
dv.get(0, scratch);
BytesRef scratch = dv.get(0);
assertEquals(new BytesRef(bytes), scratch);
ireader.close();
directory.close();
@ -976,8 +963,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assert ireader.leaves().size() == 1;
BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv");
byte mybytes[] = new byte[20];
BytesRef scratch = new BytesRef(mybytes);
dv.get(0, scratch);
BytesRef scratch = dv.get(0);
assertEquals("boo!", scratch.utf8ToString());
assertFalse(scratch.bytes == mybytes);
@ -1002,8 +988,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assert ireader.leaves().size() == 1;
BinaryDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
byte mybytes[] = new byte[20];
BytesRef scratch = new BytesRef(mybytes);
dv.get(0, scratch);
BytesRef scratch = dv.get(0);
assertEquals("boo!", scratch.utf8ToString());
assertFalse(scratch.bytes == mybytes);
@ -1011,72 +996,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
directory.close();
}
public void testCodecUsesOwnBytesEachTime() throws IOException {
Analyzer analyzer = new MockAnalyzer(random());
Directory directory = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
conf.setMergePolicy(newLogMergePolicy());
RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, conf);
Document doc = new Document();
doc.add(new BinaryDocValuesField("dv", new BytesRef("foo!")));
iwriter.addDocument(doc);
doc = new Document();
doc.add(new BinaryDocValuesField("dv", new BytesRef("bar!")));
iwriter.addDocument(doc);
iwriter.shutdown();
// Now search the index:
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv");
BytesRef scratch = new BytesRef();
dv.get(0, scratch);
assertEquals("foo!", scratch.utf8ToString());
BytesRef scratch2 = new BytesRef();
dv.get(1, scratch2);
assertEquals("bar!", scratch2.utf8ToString());
// check scratch is still valid
assertEquals("foo!", scratch.utf8ToString());
ireader.close();
directory.close();
}
public void testCodecUsesOwnSortedBytesEachTime() throws IOException {
Analyzer analyzer = new MockAnalyzer(random());
Directory directory = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
conf.setMergePolicy(newLogMergePolicy());
RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, conf);
Document doc = new Document();
doc.add(new SortedDocValuesField("dv", new BytesRef("foo!")));
iwriter.addDocument(doc);
doc = new Document();
doc.add(new SortedDocValuesField("dv", new BytesRef("bar!")));
iwriter.addDocument(doc);
iwriter.shutdown();
// Now search the index:
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
BinaryDocValues dv = ireader.leaves().get(0).reader().getSortedDocValues("dv");
BytesRef scratch = new BytesRef();
dv.get(0, scratch);
assertEquals("foo!", scratch.utf8ToString());
BytesRef scratch2 = new BytesRef();
dv.get(1, scratch2);
assertEquals("bar!", scratch2.utf8ToString());
// check scratch is still valid
assertEquals("foo!", scratch.utf8ToString());
ireader.close();
directory.close();
}
/*
* Simple test case to show how to use the API
*/
@ -1181,11 +1100,10 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
SortedDocValues docValues = MultiDocValues.getSortedValues(reader, "field");
int[] sort = hash.sort(BytesRef.getUTF8SortedAsUnicodeComparator());
BytesRef expected = new BytesRef();
BytesRef actual = new BytesRef();
assertEquals(hash.size(), docValues.getValueCount());
for (int i = 0; i < hash.size(); i++) {
hash.get(sort[i], expected);
docValues.lookupOrd(i, actual);
final BytesRef actual = docValues.lookupOrd(i);
assertEquals(expected.utf8ToString(), actual.utf8ToString());
int ord = docValues.lookupTerm(expected);
assertEquals(i, ord);
@ -1198,7 +1116,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
DocsEnum termDocsEnum = slowR.termDocsEnum(new Term("id", entry.getKey()));
int docId = termDocsEnum.nextDoc();
expected = new BytesRef(entry.getValue());
docValues.get(docId, actual);
final BytesRef actual = docValues.get(docId);
assertEquals(expected, actual);
}
@ -1357,8 +1275,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
BinaryDocValues docValues = r.getBinaryDocValues("dv");
for (int i = 0; i < r.maxDoc(); i++) {
BytesRef binaryValue = r.document(i).getBinaryValue("stored");
BytesRef scratch = new BytesRef();
docValues.get(i, scratch);
BytesRef scratch = docValues.get(i);
assertEquals(binaryValue, scratch);
}
}
@ -1428,8 +1345,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
BinaryDocValues docValues = r.getSortedDocValues("dv");
for (int i = 0; i < r.maxDoc(); i++) {
BytesRef binaryValue = r.document(i).getBinaryValue("stored");
BytesRef scratch = new BytesRef();
docValues.get(i, scratch);
BytesRef scratch = docValues.get(i);
assertEquals(binaryValue, scratch);
}
}
@ -1470,8 +1386,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(0, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
BytesRef bytes = new BytesRef();
dv.lookupOrd(0, bytes);
BytesRef bytes = dv.lookupOrd(0);
assertEquals(new BytesRef("hello"), bytes);
ireader.close();
@ -1497,8 +1412,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(0, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
BytesRef bytes = new BytesRef();
dv.lookupOrd(0, bytes);
BytesRef bytes = dv.lookupOrd(0);
assertEquals(new BytesRef("hello"), bytes);
dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field2");
@ -1507,7 +1421,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(0, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
dv.lookupOrd(0, bytes);
bytes = dv.lookupOrd(0);
assertEquals(new BytesRef("world"), bytes);
ireader.close();
@ -1542,15 +1456,14 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(0, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
BytesRef bytes = new BytesRef();
dv.lookupOrd(0, bytes);
BytesRef bytes = dv.lookupOrd(0);
assertEquals(new BytesRef("hello"), bytes);
dv.setDocument(1);
assertEquals(1, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
dv.lookupOrd(1, bytes);
bytes = dv.lookupOrd(1);
assertEquals(new BytesRef("world"), bytes);
ireader.close();
@ -1577,11 +1490,10 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(1, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
BytesRef bytes = new BytesRef();
dv.lookupOrd(0, bytes);
BytesRef bytes = dv.lookupOrd(0);
assertEquals(new BytesRef("hello"), bytes);
dv.lookupOrd(1, bytes);
bytes = dv.lookupOrd(1);
assertEquals(new BytesRef("world"), bytes);
ireader.close();
@ -1608,11 +1520,10 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(1, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
BytesRef bytes = new BytesRef();
dv.lookupOrd(0, bytes);
BytesRef bytes = dv.lookupOrd(0);
assertEquals(new BytesRef("hello"), bytes);
dv.lookupOrd(1, bytes);
bytes = dv.lookupOrd(1);
assertEquals(new BytesRef("world"), bytes);
ireader.close();
@ -1655,14 +1566,13 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(1, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
BytesRef bytes = new BytesRef();
dv.lookupOrd(0, bytes);
BytesRef bytes = dv.lookupOrd(0);
assertEquals(new BytesRef("beer"), bytes);
dv.lookupOrd(1, bytes);
bytes = dv.lookupOrd(1);
assertEquals(new BytesRef("hello"), bytes);
dv.lookupOrd(2, bytes);
bytes = dv.lookupOrd(2);
assertEquals(new BytesRef("world"), bytes);
ireader.close();
@ -1694,8 +1604,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(0, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
BytesRef bytes = new BytesRef();
dv.lookupOrd(0, bytes);
BytesRef bytes = dv.lookupOrd(0);
assertEquals(new BytesRef("hello"), bytes);
ireader.close();
@ -1729,8 +1638,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(0, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
BytesRef bytes = new BytesRef();
dv.lookupOrd(0, bytes);
BytesRef bytes = dv.lookupOrd(0);
assertEquals(new BytesRef("hello"), bytes);
ireader.close();
@ -1763,8 +1671,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(0, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
BytesRef bytes = new BytesRef();
dv.lookupOrd(0, bytes);
BytesRef bytes = dv.lookupOrd(0);
assertEquals(new BytesRef("hello"), bytes);
ireader.close();
@ -1798,8 +1705,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(0, dv.nextOrd());
assertEquals(NO_MORE_ORDS, dv.nextOrd());
BytesRef bytes = new BytesRef();
dv.lookupOrd(0, bytes);
BytesRef bytes = dv.lookupOrd(0);
assertEquals(new BytesRef("hello"), bytes);
ireader.close();
@ -1955,7 +1861,6 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
for (AtomicReaderContext context : ir.leaves()) {
AtomicReader r = context.reader();
SortedSetDocValues docValues = r.getSortedSetDocValues("dv");
BytesRef scratch = new BytesRef();
for (int i = 0; i < r.maxDoc(); i++) {
String stringValues[] = r.document(i).getValues("stored");
if (docValues != null) {
@ -1965,7 +1870,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assert docValues != null;
long ord = docValues.nextOrd();
assert ord != NO_MORE_ORDS;
docValues.lookupOrd(ord, scratch);
BytesRef scratch = docValues.lookupOrd(ord);
assertEquals(stringValues[j], scratch.utf8ToString());
}
assert docValues == null || docValues.nextOrd() == NO_MORE_ORDS;
@ -2158,10 +2063,9 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(1, ir.leaves().size());
AtomicReader ar = ir.leaves().get(0).reader();
BinaryDocValues dv = ar.getBinaryDocValues("dv1");
BytesRef ref = new BytesRef();
dv.get(0, ref);
BytesRef ref = dv.get(0);
assertEquals(new BytesRef(), ref);
dv.get(1, ref);
ref = dv.get(1);
assertEquals(new BytesRef(), ref);
Bits docsWithField = ar.getDocsWithField("dv1");
assertTrue(docsWithField.get(0));
@ -2191,10 +2095,9 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(1, ir.leaves().size());
AtomicReader ar = ir.leaves().get(0).reader();
BinaryDocValues dv = ar.getBinaryDocValues("dv1");
BytesRef ref = new BytesRef();
dv.get(0, ref);
BytesRef ref = dv.get(0);
assertEquals(new BytesRef(), ref);
dv.get(1, ref);
ref = dv.get(1);
assertEquals(new BytesRef(), ref);
Bits docsWithField = ar.getDocsWithField("dv1");
assertTrue(docsWithField.get(0));
@ -2228,12 +2131,11 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
assertEquals(1, ir.leaves().size());
AtomicReader ar = ir.leaves().get(0).reader();
BinaryDocValues dv = ar.getBinaryDocValues("dv1");
BytesRef ref = new BytesRef();
dv.get(0, ref);
BytesRef ref = dv.get(0);
assertEquals(new BytesRef(), ref);
dv.get(1, ref);
ref = dv.get(1);
assertEquals(new BytesRef(), ref);
dv.get(2, ref);
ref = dv.get(2);
assertEquals(new BytesRef("boo"), ref);
Bits docsWithField = ar.getDocsWithField("dv1");
assertTrue(docsWithField.get(0));
@ -2308,10 +2210,9 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
NumericDocValues numerics = r.getNumericDocValues("dvNum");
for (int j = 0; j < r.maxDoc(); j++) {
BytesRef binaryValue = r.document(j).getBinaryValue("storedBin");
BytesRef scratch = new BytesRef();
binaries.get(j, scratch);
BytesRef scratch = binaries.get(j);
assertEquals(binaryValue, scratch);
sorted.get(j, scratch);
scratch = sorted.get(j);
assertEquals(binaryValue, scratch);
String expected = r.document(j).get("storedNum");
assertEquals(Long.parseLong(expected), numerics.get(j));
@ -2420,10 +2321,9 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
BytesRef binaryValue = r.document(j).getBinaryValue("storedBin");
if (binaryValue != null) {
if (binaries != null) {
BytesRef scratch = new BytesRef();
binaries.get(j, scratch);
BytesRef scratch = binaries.get(j);
assertEquals(binaryValue, scratch);
sorted.get(j, scratch);
scratch = sorted.get(j);
assertEquals(binaryValue, scratch);
assertTrue(binaryBits.get(j));
assertTrue(sortedBits.get(j));
@ -2451,8 +2351,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
for (int k = 0; k < values.length; k++) {
long ord = sortedSet.nextOrd();
assertTrue(ord != SortedSetDocValues.NO_MORE_ORDS);
BytesRef value = new BytesRef();
sortedSet.lookupOrd(ord, value);
BytesRef value = sortedSet.lookupOrd(ord);
assertEquals(values[k], value.utf8ToString());
}
assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
@ -2507,9 +2406,8 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTes
AtomicReader ar = SlowCompositeReaderWrapper.wrap(r);
BinaryDocValues values = ar.getBinaryDocValues("field");
BytesRef result = new BytesRef();
for(int j=0;j<5;j++) {
values.get(0, result);
BytesRef result = values.get(0);
assertTrue(result.length == 0 || result.length == 1<<i);
}
ar.close();

28
lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
View File

@ -2223,12 +2223,10 @@ public abstract class LuceneTestCase extends Assert {
BinaryDocValues leftValues = MultiDocValues.getBinaryValues(leftReader, field);
BinaryDocValues rightValues = MultiDocValues.getBinaryValues(rightReader, field);
if (leftValues != null && rightValues != null) {
BytesRef scratchLeft = new BytesRef();
BytesRef scratchRight = new BytesRef();
for(int docID=0;docID<leftReader.maxDoc();docID++) {
leftValues.get(docID, scratchLeft);
rightValues.get(docID, scratchRight);
assertEquals(info, scratchLeft, scratchRight);
final BytesRef left = BytesRef.deepCopyOf(leftValues.get(docID));
final BytesRef right = rightValues.get(docID);
assertEquals(info, left, right);
}
} else {
assertNull(info, leftValues);
@ -2246,15 +2244,15 @@ public abstract class LuceneTestCase extends Assert {
BytesRef scratchLeft = new BytesRef();
BytesRef scratchRight = new BytesRef();
for (int i = 0; i < leftValues.getValueCount(); i++) {
leftValues.lookupOrd(i, scratchLeft);
rightValues.lookupOrd(i, scratchRight);
assertEquals(info, scratchLeft, scratchRight);
final BytesRef left = BytesRef.deepCopyOf(leftValues.lookupOrd(i));
final BytesRef right = rightValues.lookupOrd(i);
assertEquals(info, left, right);
}
// bytes
for(int docID=0;docID<leftReader.maxDoc();docID++) {
leftValues.get(docID, scratchLeft);
rightValues.get(docID, scratchRight);
assertEquals(info, scratchLeft, scratchRight);
final BytesRef left = BytesRef.deepCopyOf(leftValues.get(docID));
final BytesRef right = rightValues.get(docID);
assertEquals(info, left, right);
}
} else {
assertNull(info, leftValues);
@ -2269,12 +2267,10 @@ public abstract class LuceneTestCase extends Assert {
// numOrds
assertEquals(info, leftValues.getValueCount(), rightValues.getValueCount());
// ords
BytesRef scratchLeft = new BytesRef();
BytesRef scratchRight = new BytesRef();
for (int i = 0; i < leftValues.getValueCount(); i++) {
leftValues.lookupOrd(i, scratchLeft);
rightValues.lookupOrd(i, scratchRight);
assertEquals(info, scratchLeft, scratchRight);
final BytesRef left = BytesRef.deepCopyOf(leftValues.lookupOrd(i));
final BytesRef right = rightValues.lookupOrd(i);
assertEquals(info, left, right);
}
// ord lists
for(int docID=0;docID<leftReader.maxDoc();docID++) {

10
solr/core/src/java/org/apache/solr/analytics/accumulator/facet/FieldFacetAccumulator.java
View File

@ -50,7 +50,6 @@ public class FieldFacetAccumulator extends ValueAccumulator {
protected final boolean multiValued;
protected final boolean numField;
protected final boolean dateField;
protected final BytesRef value;
protected SortedSetDocValues setValues;
protected SortedDocValues sortValues;
protected NumericDocValues numValues;
@ -70,7 +69,6 @@ public class FieldFacetAccumulator extends ValueAccumulator {
this.numField = schemaField.getType().getNumericType()!=null;
this.dateField = schemaField.getType().getClass().equals(TrieDateField.class);
this.parent = parent;
this.value = new BytesRef();
this.parser = AnalyticsParsers.getParser(schemaField.getType().getClass());
}
@ -108,7 +106,7 @@ public class FieldFacetAccumulator extends ValueAccumulator {
int term;
while ((term = (int)setValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
exists = true;
setValues.lookupOrd(term, value);
final BytesRef value = setValues.lookupOrd(term);
parent.collectField(doc, name, parser.parse(value) );
}
}
@ -129,11 +127,11 @@ public class FieldFacetAccumulator extends ValueAccumulator {
}
} else {
if(sortValues != null) {
sortValues.get(doc,value);
if( BytesRef.EMPTY_BYTES == value.bytes ){
final int ord = sortValues.getOrd(doc);
if (ord < 0) {
parent.collectField(doc, name, FacetingAccumulator.MISSING_VALUE );
} else {
parent.collectField(doc, name, parser.parse(value) );
parent.collectField(doc, name, parser.parse(sortValues.lookupOrd(ord)) );
}
} else {
parent.collectField(doc, name, FacetingAccumulator.MISSING_VALUE );

3
solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java
View File

@ -218,7 +218,6 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
searcher.search(query, pfilter.filter, collector);
IntObjectMap groups = groupExpandCollector.getGroups();
Map<String, DocSlice> outMap = new HashMap<>();
BytesRef bytesRef = new BytesRef();
CharsRef charsRef = new CharsRef();
FieldType fieldType = searcher.getSchema().getField(field).getType();
for (IntObjectCursor cursor : (Iterable<IntObjectCursor>) groups) {
@ -235,7 +234,7 @@ public class ExpandComponent extends SearchComponent implements PluginInfoInitia
scores[i] = scoreDoc.score;
}
DocSlice slice = new DocSlice(0, docs.length, docs, scores, topDocs.totalHits, topDocs.getMaxScore());
values.lookupOrd(ord, bytesRef);
final BytesRef bytesRef = values.lookupOrd(ord);
fieldType.indexedToReadable(bytesRef, charsRef);
String group = charsRef.toString();
outMap.put(group, slice);

10
solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java
View File

@ -62,8 +62,6 @@ public class FieldFacetStats {
SortedDocValues topLevelSortedValues = null;
private final BytesRef tempBR = new BytesRef();
public FieldFacetStats(SolrIndexSearcher searcher, String name, SchemaField field_sf, SchemaField facet_sf, boolean calcDistinct) {
this.name = name;
this.field_sf = field_sf;
@ -106,14 +104,12 @@ public class FieldFacetStats {
int term = topLevelSortedValues.getOrd(docID);
int arrIdx = term;
if (arrIdx >= 0 && arrIdx < topLevelSortedValues.getValueCount()) {
final BytesRef br;
final String key;
if (term == -1) {
br = null;
key = null;
} else {
br = tempBR;
topLevelSortedValues.lookupOrd(term, tempBR);
key = topLevelSortedValues.lookupOrd(term).utf8ToString();
}
String key = br == null ? null : br.utf8ToString();
while (facetStatsTerms.size() <= statsTermNum) {
facetStatsTerms.add(new HashMap<String, Integer>());
}

14
solr/core/src/java/org/apache/solr/request/DocValuesFacets.java
View File

@ -85,8 +85,6 @@ public class DocValuesFacets {
throw new UnsupportedOperationException("Currently this faceting method is limited to " + Integer.MAX_VALUE + " unique terms");
}
final BytesRef br = new BytesRef();
final BytesRef prefixRef;
if (prefix == null) {
prefixRef = null;
@ -132,7 +130,7 @@ public class DocValuesFacets {
if (multiValued) {
SortedSetDocValues sub = leaf.reader().getSortedSetDocValues(fieldName);
if (sub == null) {
sub = DocValues.EMPTY_SORTED_SET;
sub = DocValues.emptySortedSet();
}
final SortedDocValues singleton = DocValues.unwrapSingleton(sub);
if (singleton != null) {
@ -144,7 +142,7 @@ public class DocValuesFacets {
} else {
SortedDocValues sub = leaf.reader().getSortedDocValues(fieldName);
if (sub == null) {
sub = DocValues.EMPTY_SORTED;
sub = DocValues.emptySorted();
}
accumSingle(counts, startTermIndex, sub, disi, subIndex, ordinalMap);
}
@ -194,8 +192,8 @@ public class DocValuesFacets {
long pair = sorted[i];
int c = (int)(pair >>> 32);
int tnum = Integer.MAX_VALUE - (int)pair;
si.lookupOrd(startTermIndex+tnum, br);
ft.indexedToReadable(br, charsRef);
final BytesRef term = si.lookupOrd(startTermIndex+tnum);
ft.indexedToReadable(term, charsRef);
res.add(charsRef.toString(), c);
}
@ -213,8 +211,8 @@ public class DocValuesFacets {
int c = counts[i];
if (c<mincount || --off>=0) continue;
if (--lim<0) break;
si.lookupOrd(startTermIndex+i, br);
ft.indexedToReadable(br, charsRef);
final BytesRef term = si.lookupOrd(startTermIndex+i);
ft.indexedToReadable(term, charsRef);
res.add(charsRef.toString(), c);
}
}

9
solr/core/src/java/org/apache/solr/request/DocValuesStats.java
View File

@ -84,7 +84,7 @@ public class DocValuesStats {
}
}
if (si == null) {
si = DocValues.EMPTY_SORTED_SET;
si = DocValues.emptySortedSet();
}
if (si.getValueCount() >= Integer.MAX_VALUE) {
throw new UnsupportedOperationException("Currently this stats method is limited to " + Integer.MAX_VALUE + " unique terms");
@ -112,7 +112,7 @@ public class DocValuesStats {
if (multiValued) {
SortedSetDocValues sub = leaf.reader().getSortedSetDocValues(fieldName);
if (sub == null) {
sub = DocValues.EMPTY_SORTED_SET;
sub = DocValues.emptySortedSet();
}
final SortedDocValues singleton = DocValues.unwrapSingleton(sub);
if (singleton != null) {
@ -124,7 +124,7 @@ public class DocValuesStats {
} else {
SortedDocValues sub = leaf.reader().getSortedDocValues(fieldName);
if (sub == null) {
sub = DocValues.EMPTY_SORTED;
sub = DocValues.emptySorted();
}
accumSingle(counts, docBase, facetStats, sub, disi, subIndex, ordinalMap);
}
@ -132,11 +132,10 @@ public class DocValuesStats {
}
// add results in index order
BytesRef value = new BytesRef();
for (int ord = 0; ord < counts.length; ord++) {
int count = counts[ord];
if (count > 0) {
si.lookupOrd(ord, value);
final BytesRef value = si.lookupOrd(ord);
res.accumulate(value, count);
for (FieldFacetStats f : facetStats) {
f.accumulateTermNum(ord, value);

3
solr/core/src/java/org/apache/solr/schema/BoolField.java
View File

@ -193,11 +193,10 @@ class BoolFieldSource extends ValueSource {
// figure out what ord maps to true
int nord = sindex.getValueCount();
BytesRef br = new BytesRef();
// if no values in the segment, default trueOrd to something other then -1 (missing)
int tord = -2;
for (int i=0; i<nord; i++) {
sindex.lookupOrd(i, br);
final BytesRef br = sindex.lookupOrd(i);
if (br.length==1 && br.bytes[br.offset]=='T') {
tord = i;
break;

52
solr/core/src/test/org/apache/solr/request/TestFaceting.java
View File

@ -923,61 +923,9 @@ public class TestFaceting extends SolrTestCaseJ4 {
, "*[count(//lst[@name='facet_fields']/lst)=50]"
, "*[count(//lst[@name='facet_fields']/lst/int)=100]"
);
// Now, are all the UnInvertedFields still the same? Meaning they weren't re-fetched even when a bunch were
// requested at the same time?
assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
ui0, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f0_ws"));
assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
ui1, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f1_ws"));
assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
ui2, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f2_ws"));
assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
ui3, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f3_ws"));
assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
ui4, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f4_ws"));
assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
ui5, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f5_ws"));
assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
ui6, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f6_ws"));
assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
ui7, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f7_ws"));
assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
ui8, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f8_ws"));
assertEquals("UnInvertedField coming back from the seacher should not have changed! ",
ui9, DocValues.getSortedSet(currentSearcher.getAtomicReader(), "f9_ws"));
} finally {
currentSearcherRef.decref();
}
}
// assert same instance: either same object, or both wrapping same single-valued object
private void assertEquals(String msg, SortedSetDocValues dv1, SortedSetDocValues dv2) {
SortedDocValues singleton1 = DocValues.unwrapSingleton(dv1);
SortedDocValues singleton2 = DocValues.unwrapSingleton(dv2);
if (singleton1 == null || singleton2 == null) {
// actually a multi-valued field
if (dv1 instanceof MultiDocValues.MultiSortedSetDocValues) {
// if we produced more than one segment, ensure the core ordinal map is the same object
assertTrue(dv2 instanceof MultiDocValues.MultiSortedSetDocValues);
assertSame(((MultiDocValues.MultiSortedSetDocValues) dv1).mapping,
((MultiDocValues.MultiSortedSetDocValues) dv2).mapping);
} else {
// otherwise, same atomic instance
assertSame(dv1, dv2);
}
} else {
// just wrapping a field that is actually single-valued
if (singleton1 instanceof MultiDocValues.MultiSortedDocValues) {
// if we produced more than one segment, ensure the core ordinal map is the same object
assertTrue(singleton2 instanceof MultiDocValues.MultiSortedDocValues);
assertSame(((MultiDocValues.MultiSortedDocValues) singleton1).mapping,
((MultiDocValues.MultiSortedDocValues) singleton2).mapping);
} else {
// otherwise, same atomic instance
assertSame(singleton1, singleton2);
}
}
}
}