Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

merge DocTermsIndex into SortedDocValues 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4547@1413182 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2012-11-24 13:13:51 +00:00
parent 041ff1140f
commit 30eda3c474
34 changed files with 567 additions and 388 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
View File

@ -53,10 +53,10 @@ import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.index.SerialMergeScheduler;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldCache.DocTermsIndex;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
@ -341,12 +341,11 @@ public class TestPerfTasksLogic extends BenchmarkTestCase {
Benchmark benchmark = execBenchmark(algLines);
DirectoryReader r = DirectoryReader.open(benchmark.getRunData().getDirectory());
DocTermsIndex idx = FieldCache.DEFAULT.getTermsIndex(new SlowCompositeReaderWrapper(r), "country");
SortedDocValues idx = FieldCache.DEFAULT.getTermsIndex(new SlowCompositeReaderWrapper(r), "country");
final int maxDoc = r.maxDoc();
assertEquals(1000, maxDoc);
BytesRef br = new BytesRef();
for(int i=0;i<1000;i++) {
assertNotNull("doc " + i + " has null country", idx.getTerm(i, br));
assertTrue("doc " + i + " has null country", idx.getOrd(i) != -1);
}
r.close();
}

2
lucene/common-build.xml
View File

@ -438,7 +438,7 @@
description="Compiles core classes">
<compile
srcdir="${src.dir}"
destdir="${build.dir}/classes/java">
DESTDIR="${build.dir}/classes/java">
<classpath refid="classpath"/>
</compile>

140
lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java
View File

@ -17,6 +17,10 @@ package org.apache.lucene.index;
* limitations under the License.
*/
import java.io.IOException;
import java.util.Comparator;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
// nocommit need marker interface?
@ -28,14 +32,123 @@ public abstract class SortedDocValues extends BinaryDocValues {
public abstract void lookupOrd(int ord, BytesRef result);
// nocommit throws IOE or not?
// nocommit .getUniqueValueCount?
public abstract int getValueCount();
@Override
public void get(int docID, BytesRef result) {
int ord = getOrd(docID);
if (ord == -1) {
// nocommit what to do ... maybe we need to return
// BytesRef?
throw new IllegalArgumentException("doc has no value");
}
lookupOrd(ord, result);
}
public TermsEnum getTermsEnum() {
// nocommit who tests this base impl ...
// Default impl just uses the existing API; subclasses
// can specialize:
return new TermsEnum() {
private int currentOrd = -1;
private final BytesRef term = new BytesRef();
@Override
public SeekStatus seekCeil(BytesRef text, boolean useCache /* ignored */) throws IOException {
int low = 0;
int high = getValueCount()-1;
while (low <= high) {
int mid = (low + high) >>> 1;
seekExact(mid);
int cmp = term.compareTo(text);
if (cmp < 0)
low = mid + 1;
else if (cmp > 0)
high = mid - 1;
else {
return SeekStatus.FOUND; // key found
}
}
if (low == getValueCount()) {
return SeekStatus.END;
} else {
seekExact(low);
return SeekStatus.NOT_FOUND;
}
}
@Override
public void seekExact(long ord) throws IOException {
assert ord >= 0 && ord < getValueCount();
currentOrd = (int) ord;
lookupOrd(currentOrd, term);
}
@Override
public BytesRef next() throws IOException {
currentOrd++;
if (currentOrd >= getValueCount()) {
return null;
}
lookupOrd(currentOrd, term);
return term;
}
@Override
public BytesRef term() throws IOException {
return term;
}
@Override
public long ord() throws IOException {
return currentOrd;
}
@Override
public int docFreq() {
throw new UnsupportedOperationException();
}
@Override
public long totalTermFreq() {
return -1;
}
@Override
public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public Comparator<BytesRef> getComparator() {
return BytesRef.getUTF8SortedAsUnicodeComparator();
}
@Override
public void seekExact(BytesRef term, TermState state) throws IOException {
assert state != null && state instanceof OrdTermState;
this.seekExact(((OrdTermState)state).ord);
}
@Override
public TermState termState() throws IOException {
OrdTermState state = new OrdTermState();
state.ord = currentOrd;
return state;
}
};
}
@Override
public SortedDocValues newRAMInstance() {
// nocommit optimize this
@ -136,4 +249,31 @@ public abstract class SortedDocValues extends BinaryDocValues {
return 0;
}
}
// nocommit javadocs
public int lookupTerm(BytesRef key, BytesRef spare) {
// this special case is the reason that Arrays.binarySearch() isn't useful.
if (key == null) {
throw new IllegalArgumentException("key must not be null");
}
int low = 0;
int high = getValueCount()-1;
while (low <= high) {
int mid = (low + high) >>> 1;
lookupOrd(mid, spare);
int cmp = spare.compareTo(key);
if (cmp < 0) {
low = mid + 1;
} else if (cmp > 0) {
high = mid - 1;
} else {
return mid; // key found
}
}
return -(low + 1); // key not found.
}
}

79
lucene/core/src/java/org/apache/lucene/search/FieldCache.java
View File

@ -21,18 +21,17 @@ import java.io.IOException;
import java.io.PrintStream;
import org.apache.lucene.analysis.NumericTokenStream; // for javadocs
import org.apache.lucene.document.IntField; // for javadocs
import org.apache.lucene.document.FloatField; // for javadocs
import org.apache.lucene.document.LongField; // for javadocs
import org.apache.lucene.document.DoubleField; // for javadocs
import org.apache.lucene.index.DocTermOrds;
import org.apache.lucene.document.FloatField; // for javadocs
import org.apache.lucene.document.IntField; // for javadocs
import org.apache.lucene.document.LongField; // for javadocs
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.DocTermOrds;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.packed.PackedInts;
/**
* Expert: Maintains caches of term values.
@ -41,6 +40,8 @@ import org.apache.lucene.util.packed.PackedInts;
*
* @since lucene 1.4
* @see org.apache.lucene.util.FieldCacheSanityChecker
*
* @lucene.internal
*/
// nocommit abstract class...?
public interface FieldCache {
@ -480,7 +481,7 @@ public interface FieldCache {
public Doubles getDoubles(AtomicReader reader, String field, DoubleParser parser, boolean setDocsWithField) throws IOException;
/** Returned by {@link #getTerms} */
// nocommit: can we merge this api with the BinaryDocValues api?
// nocommit: merge this api with the BinaryDocValues api?
public abstract static class DocTerms {
/** The BytesRef argument must not be null; the method
* returns the same BytesRef, or an empty (length=0)
@ -515,64 +516,6 @@ public interface FieldCache {
* subsequent calls will share the same cache entry. */
public DocTerms getTerms (AtomicReader reader, String field, float acceptableOverheadRatio) throws IOException;
/** Returned by {@link #getTermsIndex} */
// nocommit: can we merge this api with the SortedDocValues api?
public abstract static class DocTermsIndex {
public int binarySearchLookup(BytesRef key, BytesRef spare) {
// this special case is the reason that Arrays.binarySearch() isn't useful.
if (key == null) {
throw new IllegalArgumentException("key must not be null");
}
int low = 0;
int high = numOrd()-1;
while (low <= high) {
int mid = (low + high) >>> 1;
int cmp = lookup(mid, spare).compareTo(key);
if (cmp < 0)
low = mid + 1;
else if (cmp > 0)
high = mid - 1;
else
return mid; // key found
}
return -(low + 1); // key not found.
}
/** The BytesRef argument must not be null; the method
* returns the same BytesRef, or an empty (length=0)
* BytesRef if this ord is the null ord (-1). */
public abstract BytesRef lookup(int ord, BytesRef reuse);
/** Convenience method, to lookup the Term for a doc.
* If this doc is deleted or did not have this field,
* this will return an empty (length=0) BytesRef. */
public BytesRef getTerm(int docID, BytesRef reuse) {
int ord = getOrd(docID);
if (ord == -1) {
return null;
}
return lookup(ord, reuse);
}
/** Returns sort ord for this document. Ord -1 is
* is returend for docs that are deleted or did not have
* this field. */
public abstract int getOrd(int docID);
/** Returns total unique ord count. */
public abstract int numOrd();
/** Number of documents */
public abstract int size();
/** Returns a TermsEnum that can iterate over the values in this index entry */
public abstract TermsEnum getTermsEnum();
}
/** Checks the internal cache for an appropriate entry, and if none
* is found, reads the term values in <code>field</code>
* and returns a {@link DocTerms} instance, providing a
@ -582,7 +525,7 @@ public interface FieldCache {
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
public DocTermsIndex getTermsIndex (AtomicReader reader, String field) throws IOException;
public SortedDocValues getTermsIndex (AtomicReader reader, String field) throws IOException;
/** Expert: just like {@link
* #getTermsIndex(AtomicReader,String)}, but you can specify
@ -590,7 +533,7 @@ public interface FieldCache {
* faster lookups (default is "true"). Note that the
* first call for a given reader and field "wins",
* subsequent calls will share the same cache entry. */
public DocTermsIndex getTermsIndex (AtomicReader reader, String field, float acceptableOverheadRatio) throws IOException;
public SortedDocValues getTermsIndex (AtomicReader reader, String field, float acceptableOverheadRatio) throws IOException;
/**
* Checks the internal cache for an appropriate entry, and if none is found, reads the term values
@ -662,7 +605,7 @@ public interface FieldCache {
* The most recently estimated size of the value, null unless
* estimateSize has been called.
*/
public final String getEstimatedSize() {
public String getEstimatedSize() {
return size;
}

81
lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
View File

@ -47,7 +47,6 @@ import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.packed.GrowableWriter;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PackedInts.Reader;
// nocommit rename to UninvertFieldCacheImpl or something ...
@ -73,7 +72,7 @@ class FieldCacheImpl implements FieldCache {
caches.put(Long.TYPE, new LongCache(this));
caches.put(Double.TYPE, new DoubleCache(this));
caches.put(DocTerms.class, new DocTermsCache(this));
caches.put(DocTermsIndex.class, new DocTermsIndexCache(this));
caches.put(SortedDocValues.class, new SortedDocValuesCache(this));
caches.put(DocTermOrds.class, new DocTermOrdsCache(this));
caches.put(DocsWithFieldCache.class, new DocsWithFieldCache(this));
}
@ -574,7 +573,6 @@ class FieldCacheImpl implements FieldCache {
// nocommit should we throw exc if parser isn't
// null? if setDocsWithField is true?
} else {
int maxDoc = reader.maxDoc();
final int[] values;
final IntParser parser = (IntParser) key.custom;
if (parser == null) {
@ -728,7 +726,6 @@ class FieldCacheImpl implements FieldCache {
// nocommit should we throw exc if parser isn't
// null? if setDocsWithField is true?
} else {
int maxDoc = reader.maxDoc();
final float[] values;
final FloatParser parser = (FloatParser) key.custom;
if (parser == null) {
@ -819,7 +816,6 @@ class FieldCacheImpl implements FieldCache {
// nocommit should we throw exc if parser isn't
// null? if setDocsWithField is true?
} else {
int maxDoc = reader.maxDoc();
final long[] values;
final LongParser parser = (LongParser) key.custom;
if (parser == null) {
@ -910,7 +906,6 @@ class FieldCacheImpl implements FieldCache {
// nocommit should we throw exc if parser isn't
// null? if setDocsWithField is true?
} else {
int maxDoc = reader.maxDoc();
final double[] values;
final DoubleParser parser = (DoubleParser) key.custom;
if (parser == null) {
@ -954,13 +949,13 @@ class FieldCacheImpl implements FieldCache {
}
}
public static class DocTermsIndexImpl extends DocTermsIndex {
public static class SortedDocValuesImpl extends SortedDocValues {
private final PagedBytes.Reader bytes;
private final PackedInts.Reader termOrdToBytesOffset;
private final PackedInts.Reader docToTermOrd;
private final int numOrd;
public DocTermsIndexImpl(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, PackedInts.Reader docToTermOrd, int numOrd) {
public SortedDocValuesImpl(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, PackedInts.Reader docToTermOrd, int numOrd) {
this.bytes = bytes;
this.docToTermOrd = docToTermOrd;
this.termOrdToBytesOffset = termOrdToBytesOffset;
@ -968,7 +963,7 @@ class FieldCacheImpl implements FieldCache {
}
@Override
public int numOrd() {
public int getValueCount() {
return numOrd;
}
@ -986,19 +981,31 @@ class FieldCacheImpl implements FieldCache {
}
@Override
public BytesRef lookup(int ord, BytesRef ret) {
public void lookupOrd(int ord, BytesRef ret) {
if (ord < 0) {
throw new IllegalArgumentException("ord must be >=0 (got ord=" + ord + ")");
}
return bytes.fill(ret, termOrdToBytesOffset.get(ord));
bytes.fill(ret, termOrdToBytesOffset.get(ord));
}
@Override
public int maxLength() {
// nocommit hmm
throw new UnsupportedOperationException();
}
@Override
public boolean isFixedLength() {
// nocommit hmm
throw new UnsupportedOperationException();
}
@Override
public TermsEnum getTermsEnum() {
return this.new DocTermsIndexEnum();
return this.new SortedDocValuesEnum();
}
class DocTermsIndexEnum extends TermsEnum {
class SortedDocValuesEnum extends TermsEnum {
int currentOrd;
int currentBlockNumber;
int end; // end position in the current block
@ -1007,7 +1014,7 @@ class FieldCacheImpl implements FieldCache {
final BytesRef term = new BytesRef();
public DocTermsIndexEnum() {
public SortedDocValuesEnum() {
currentOrd = -1;
currentBlockNumber = 0;
blocks = bytes.getBlocks();
@ -1043,8 +1050,9 @@ class FieldCacheImpl implements FieldCache {
}
}
@Override
public void seekExact(long ord) throws IOException {
assert ord >= 0 && ord <= numOrd;
assert ord >= 0 && ord < numOrd;
// TODO: if gap is small, could iterate from current position? Or let user decide that?
currentBlockNumber = bytes.fillAndGetIndex(term, termOrdToBytesOffset.get((int)ord));
end = blockEnds[currentBlockNumber];
@ -1140,16 +1148,16 @@ class FieldCacheImpl implements FieldCache {
// nocommit woudl be nice if .getTErms would return a
// DocTermsIndex if one already existed
public DocTermsIndex getTermsIndex(AtomicReader reader, String field) throws IOException {
public SortedDocValues getTermsIndex(AtomicReader reader, String field) throws IOException {
return getTermsIndex(reader, field, PackedInts.FAST);
}
public DocTermsIndex getTermsIndex(AtomicReader reader, String field, float acceptableOverheadRatio) throws IOException {
return (DocTermsIndex) caches.get(DocTermsIndex.class).get(reader, new CacheKey(field, acceptableOverheadRatio), false);
public SortedDocValues getTermsIndex(AtomicReader reader, String field, float acceptableOverheadRatio) throws IOException {
return (SortedDocValues) caches.get(SortedDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio), false);
}
static class DocTermsIndexCache extends Cache {
DocTermsIndexCache(FieldCacheImpl wrapper) {
static class SortedDocValuesCache extends Cache {
SortedDocValuesCache(FieldCacheImpl wrapper) {
super(wrapper);
}
@ -1160,36 +1168,7 @@ class FieldCacheImpl implements FieldCache {
final int maxDoc = reader.maxDoc();
SortedDocValues valuesIn = reader.getSortedDocValues(key.field);
if (valuesIn != null) {
final SortedDocValues ramInstance = valuesIn.newRAMInstance();
return new DocTermsIndex() {
@Override
public BytesRef lookup(int ord, BytesRef reuse) {
ramInstance.lookupOrd(ord, reuse);
return reuse;
}
@Override
public int getOrd(int docID) {
return ramInstance.getOrd(docID);
}
@Override
public int numOrd() {
return ramInstance.getValueCount();
}
@Override
public int size() {
return ramInstance.size();
}
@Override
public TermsEnum getTermsEnum() {
// nocommit: to the codec api? or can that termsenum just use this thing?
return null;
}
};
return valuesIn.newRAMInstance();
} else {
Terms terms = reader.terms(key.field);
@ -1283,7 +1262,7 @@ class FieldCacheImpl implements FieldCache {
}
// maybe an int-only impl?
return new DocTermsIndexImpl(bytes.freeze(true), termOrdToBytesOffset.getMutable(), docToTermOrd.getMutable(), termOrd);
return new SortedDocValuesImpl(bytes.freeze(true), termOrdToBytesOffset.getMutable(), docToTermOrd.getMutable(), termOrd);
}
}
}

17
lucene/core/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java
View File

@ -18,15 +18,16 @@ package org.apache.lucene.search;
import java.io.IOException;
import org.apache.lucene.document.DoubleField; // for javadocs
import org.apache.lucene.document.FloatField; // for javadocs
import org.apache.lucene.document.IntField; // for javadocs
import org.apache.lucene.document.LongField; // for javadocs
import org.apache.lucene.index.AtomicReader; // for javadocs
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.document.IntField; // for javadocs
import org.apache.lucene.document.FloatField; // for javadocs
import org.apache.lucene.document.LongField; // for javadocs
import org.apache.lucene.document.DoubleField; // for javadocs
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
/**
* A range filter built on top of a cached single term field (in {@link FieldCache}).
@ -89,10 +90,10 @@ public abstract class FieldCacheRangeFilter<T> extends Filter {
return new FieldCacheRangeFilter<String>(field, null, lowerVal, upperVal, includeLower, includeUpper) {
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
final FieldCache.DocTermsIndex fcsi = FieldCache.DEFAULT.getTermsIndex(context.reader(), field);
final SortedDocValues fcsi = FieldCache.DEFAULT.getTermsIndex(context.reader(), field);
final BytesRef spare = new BytesRef();
final int lowerPoint = lowerVal == null ? -1 : fcsi.binarySearchLookup(new BytesRef(lowerVal), spare);
final int upperPoint = upperVal == null ? -1 : fcsi.binarySearchLookup(new BytesRef(upperVal), spare);
final int lowerPoint = lowerVal == null ? -1 : fcsi.lookupTerm(new BytesRef(lowerVal), spare);
final int upperPoint = upperVal == null ? -1 : fcsi.lookupTerm(new BytesRef(upperVal), spare);
final int inclusiveLowerPoint, inclusiveUpperPoint;

9
lucene/core/src/java/org/apache/lucene/search/FieldCacheTermsFilter.java
View File

@ -22,9 +22,10 @@ import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DocsEnum; // javadoc @link
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
/**
* A {@link Filter} that only accepts documents whose single
@ -118,11 +119,11 @@ public class FieldCacheTermsFilter extends Filter {
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
final FieldCache.DocTermsIndex fcsi = getFieldCache().getTermsIndex(context.reader(), field);
final FixedBitSet bits = new FixedBitSet(fcsi.numOrd());
final SortedDocValues fcsi = getFieldCache().getTermsIndex(context.reader(), field);
final FixedBitSet bits = new FixedBitSet(fcsi.getValueCount());
final BytesRef spare = new BytesRef();
for (int i=0;i<terms.length;i++) {
int ord = fcsi.binarySearchLookup(terms[i], spare);
int ord = fcsi.lookupTerm(terms[i], spare);
if (ord >= 0) {
bits.set(ord);
}

49
lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
View File

@ -23,9 +23,9 @@ import java.util.Comparator;
import org.apache.lucene.index.AtomicReader; // javadocs
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.FieldCache.ByteParser;
import org.apache.lucene.search.FieldCache.DocTerms;
import org.apache.lucene.search.FieldCache.DocTermsIndex;
import org.apache.lucene.search.FieldCache.DoubleParser;
import org.apache.lucene.search.FieldCache.FloatParser;
import org.apache.lucene.search.FieldCache.IntParser;
@ -1096,7 +1096,7 @@ public abstract class FieldComparator<T> {
/* Current reader's doc ord/values.
@lucene.internal */
DocTermsIndex termsIndex;
SortedDocValues termsIndex;
private final String field;
@ -1159,8 +1159,8 @@ public abstract class FieldComparator<T> {
@Override
public int compareDocToValue(int doc, BytesRef value) {
BytesRef docValue = termsIndex.getTerm(doc, tempBR);
if (docValue == null) {
int ord = termsIndex.getOrd(doc);
if (ord == -1) {
if (value == null) {
return 0;
}
@ -1168,12 +1168,10 @@ public abstract class FieldComparator<T> {
} else if (value == null) {
return 1;
}
return docValue.compareTo(value);
termsIndex.lookupOrd(ord, tempBR);
return tempBR.compareTo(value);
}
// nocommit remove null from FC DocTerms/Index as an
// allowed value
/** Base class for specialized (per bit width of the
* ords) per-segment comparator. NOTE: this is messy;
* we do this only because hotspot can't reliably inline
@ -1223,10 +1221,10 @@ public abstract class FieldComparator<T> {
// Used per-segment when docToOrd is null:
private final class AnyOrdComparator extends PerSegmentComparator {
private final DocTermsIndex termsIndex;
private final SortedDocValues termsIndex;
private final int docBase;
public AnyOrdComparator(DocTermsIndex termsIndex, int docBase) {
public AnyOrdComparator(SortedDocValues termsIndex, int docBase) {
this.termsIndex = termsIndex;
this.docBase = docBase;
}
@ -1259,7 +1257,7 @@ public abstract class FieldComparator<T> {
if (values[slot] == null) {
values[slot] = new BytesRef();
}
termsIndex.lookup(ord, values[slot]);
termsIndex.lookupOrd(ord, values[slot]);
}
readerGen[slot] = currentReaderGen;
}
@ -1294,7 +1292,7 @@ public abstract class FieldComparator<T> {
bottomSameReader = true;
readerGen[bottomSlot] = currentReaderGen;
} else {
final int index = binarySearch(tempBR, termsIndex, bottomValue);
final int index = termsIndex.lookupTerm(bottomValue, tempBR);
if (index < 0) {
bottomOrd = -index - 2;
bottomSameReader = false;
@ -1912,31 +1910,4 @@ public abstract class FieldComparator<T> {
return docTerms.getBytes(doc, tempBR).compareTo(value);
}
}
// nocommit why do we have this AND DTI.binarySearch?
final protected static int binarySearch(BytesRef br, DocTermsIndex a, BytesRef key) {
return binarySearch(br, a, key, 0, a.numOrd()-1);
}
final protected static int binarySearch(BytesRef br, DocTermsIndex a, BytesRef key, int low, int high) {
while (low <= high) {
int mid = (low + high) >>> 1;
BytesRef midVal = a.lookup(mid, br);
int cmp;
if (midVal != null) {
cmp = midVal.compareTo(key);
} else {
cmp = -1;
}
if (cmp < 0)
low = mid + 1;
else if (cmp > 0)
high = mid - 1;
else
return mid;
}
return -(low + 1);
}
}

8
lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
View File

@ -1627,10 +1627,12 @@ public class TestIndexWriter extends LuceneTestCase {
w.close();
assertEquals(1, reader.docFreq(new Term("content", bigTerm)));
FieldCache.DocTermsIndex dti = FieldCache.DEFAULT.getTermsIndex(SlowCompositeReaderWrapper.wrap(reader), "content", random().nextFloat() * PackedInts.FAST);
assertEquals(4, dti.numOrd());
SortedDocValues dti = FieldCache.DEFAULT.getTermsIndex(SlowCompositeReaderWrapper.wrap(reader), "content", random().nextFloat() * PackedInts.FAST);
assertEquals(4, dti.getValueCount());
assertEquals(4, dti.size());
assertEquals(bigTermBytesRef, dti.lookup(2, new BytesRef()));
BytesRef br = new BytesRef();
dti.lookupOrd(2, br);
assertEquals(bigTermBytesRef, br);
reader.close();
dir.close();
}

5
lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java
View File

@ -22,6 +22,7 @@ import java.util.Comparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.Bits;
@ -88,9 +89,9 @@ public final class FieldCacheRewriteMethod extends MultiTermQuery.RewriteMethod
*/
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, final Bits acceptDocs) throws IOException {
final FieldCache.DocTermsIndex fcsi = FieldCache.DEFAULT.getTermsIndex(context.reader(), query.field);
final SortedDocValues fcsi = FieldCache.DEFAULT.getTermsIndex(context.reader(), query.field);
// Cannot use FixedBitSet because we require long index (ord):
final OpenBitSet termSet = new OpenBitSet(fcsi.numOrd());
final OpenBitSet termSet = new OpenBitSet(fcsi.getValueCount());
TermsEnum termsEnum = query.getTermsEnum(new Terms() {
@Override

6
lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java
View File

@ -142,7 +142,7 @@ class ElevationComparatorSource extends FieldComparatorSource {
public FieldComparator<Integer> newComparator(final String fieldname, final int numHits, int sortPos, boolean reversed) throws IOException {
return new FieldComparator<Integer>() {
FieldCache.DocTermsIndex idIndex;
SortedDocValues idIndex;
private final int[] values = new int[numHits];
private final BytesRef tempBR = new BytesRef();
int bottomVal;
@ -162,8 +162,8 @@ class ElevationComparatorSource extends FieldComparatorSource {
if (ord == -1) {
return 0;
} else {
BytesRef id = idIndex.lookup(ord, tempBR);
Integer prio = priority.get(id);
idIndex.lookupOrd(ord, tempBR);
Integer prio = priority.get(tempBR);
return prio == null ? 0 : prio.intValue();
}
}

23
lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
View File

@ -188,35 +188,42 @@ public class TestFieldCache extends LuceneTestCase {
}
// getTermsIndex
FieldCache.DocTermsIndex termsIndex = cache.getTermsIndex(reader, "theRandomUnicodeString");
SortedDocValues termsIndex = cache.getTermsIndex(reader, "theRandomUnicodeString");
assertSame("Second request to cache return same array", termsIndex, cache.getTermsIndex(reader, "theRandomUnicodeString"));
assertTrue("doubles Size: " + termsIndex.size() + " is not: " + NUM_DOCS, termsIndex.size() == NUM_DOCS);
final BytesRef br = new BytesRef();
for (int i = 0; i < NUM_DOCS; i++) {
final BytesRef term = termsIndex.getTerm(i, br);
final BytesRef term;
final int ord = termsIndex.getOrd(i);
if (ord == -1) {
term = null;
} else {
termsIndex.lookupOrd(ord, br);
term = br;
}
final String s = term == null ? null : term.utf8ToString();
assertTrue("for doc " + i + ": " + s + " does not equal: " + unicodeStrings[i], unicodeStrings[i] == null || unicodeStrings[i].equals(s));
}
int nTerms = termsIndex.numOrd();
int nTerms = termsIndex.getValueCount();
// System.out.println("nTerms="+nTerms);
TermsEnum tenum = termsIndex.getTermsEnum();
BytesRef val = new BytesRef();
for (int i=0; i<nTerms; i++) {
BytesRef val1 = tenum.next();
BytesRef val2 = termsIndex.lookup(i,val);
termsIndex.lookupOrd(i, val);
// System.out.println("i="+i);
assertEquals(val2, val1);
assertEquals(val, val1);
}
// seek the enum around (note this isn't a great test here)
int num = atLeast(100);
for (int i = 0; i < num; i++) {
int k = _TestUtil.nextInt(random(), 1, nTerms-1);
BytesRef val1 = termsIndex.lookup(k, val);
assertEquals(TermsEnum.SeekStatus.FOUND, tenum.seekCeil(val1));
assertEquals(val1, tenum.term());
termsIndex.lookupOrd(k, val);
assertEquals(TermsEnum.SeekStatus.FOUND, tenum.seekCeil(val));
assertEquals(val, tenum.term());
}
// test bad field

77
lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupHeadsCollector.java
View File

@ -17,15 +17,16 @@ package org.apache.lucene.search.grouping.term;
* limitations under the License.
*/
import java.io.IOException;
import java.util.*;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.*;
import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SentinelIntSet;
import java.io.IOException;
import java.util.*;
/**
* A base implementation of {@link org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector} for retrieving the most relevant groups when grouping
* on a string based group field. More specifically this all concrete implementations of this base implementation
@ -40,7 +41,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
final String groupField;
final BytesRef scratchBytesRef = new BytesRef();
FieldCache.DocTermsIndex groupIndex;
SortedDocValues groupIndex;
AtomicReaderContext readerContext;
protected TermAllGroupHeadsCollector(String groupField, int numberOfSorts) {
@ -124,7 +125,13 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
protected void retrieveGroupHeadAndAddIfNotExist(int doc) throws IOException {
final int ord = groupIndex.getOrd(doc);
final BytesRef groupValue = ord == -1 ? null : groupIndex.lookup(ord, scratchBytesRef);
final BytesRef groupValue;
if (ord == -1) {
groupValue = null;
} else {
groupIndex.lookupOrd(ord, scratchBytesRef);
groupValue = scratchBytesRef;
}
GroupHead groupHead = groups.get(groupValue);
if (groupHead == null) {
groupHead = new GroupHead(groupValue, sortWithinGroup, doc);
@ -199,7 +206,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
private final List<GroupHead> collectedGroups;
private final SortField[] fields;
private FieldCache.DocTermsIndex[] sortsIndex;
private SortedDocValues[] sortsIndex;
private Scorer scorer;
private GroupHead[] segmentGroupHeads;
@ -210,7 +217,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
final SortField[] sortFields = sortWithinGroup.getSort();
fields = new SortField[sortFields.length];
sortsIndex = new FieldCache.DocTermsIndex[sortFields.length];
sortsIndex = new SortedDocValues[sortFields.length];
for (int i = 0; i < sortFields.length; i++) {
reversed[i] = sortFields[i].getReverse() ? -1 : 1;
fields[i] = sortFields[i];
@ -230,7 +237,13 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
GroupHead groupHead;
if (!ordSet.exists(key)) {
ordSet.put(key);
BytesRef term = key == -1 ? null : groupIndex.getTerm(doc, new BytesRef());
BytesRef term;
if (key == -1) {
term = null;
} else {
term = new BytesRef();
groupIndex.lookupOrd(key, term);
}
groupHead = new GroupHead(doc, term);
collectedGroups.add(groupHead);
segmentGroupHeads[key+1] = groupHead;
@ -255,13 +268,13 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
// Clear ordSet and fill it with previous encountered groups that can occur in the current segment.
ordSet.clear();
segmentGroupHeads = new GroupHead[groupIndex.numOrd()+1];
segmentGroupHeads = new GroupHead[groupIndex.getValueCount()+1];
for (GroupHead collectedGroup : collectedGroups) {
int ord;
if (collectedGroup.groupValue == null) {
ord = -1;
} else {
ord = groupIndex.binarySearchLookup(collectedGroup.groupValue, scratchBytesRef);
ord = groupIndex.lookupTerm(collectedGroup.groupValue, scratchBytesRef);
}
if (collectedGroup.groupValue == null || ord >= 0) {
ordSet.put(ord);
@ -275,7 +288,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
if (collectedGroup.sortValues[i] == null) {
sortOrd = -1;
} else {
sortOrd = sortsIndex[i].binarySearchLookup(collectedGroup.sortValues[i], scratchBytesRef);
sortOrd = sortsIndex[i].lookupTerm(collectedGroup.sortValues[i], scratchBytesRef);
}
collectedGroup.sortOrds[i] = sortOrd;
}
@ -301,7 +314,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
sortOrds[i] = sortsIndex[i].getOrd(doc);
sortValues[i] = new BytesRef();
if (sortOrds[i] != -1) {
sortValues[i] = sortsIndex[i].getTerm(doc, sortValues[i]);
sortsIndex[i].get(doc, sortValues[i]);
}
}
}
@ -322,7 +335,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
if (sortsIndex[compIDX].getOrd(doc) == -1) {
scratchBytesRef.length = 0;
} else {
sortsIndex[compIDX].getTerm(doc, scratchBytesRef);
sortsIndex[compIDX].get(doc, scratchBytesRef);
}
return sortValues[compIDX].compareTo(scratchBytesRef);
} else {
@ -340,7 +353,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
if (sortOrds[i] == -1) {
sortValues[i].length = 0;
} else {
sortValues[i] = sortsIndex[i].getTerm(doc, sortValues[i]);
sortsIndex[i].get(doc, sortValues[i]);
}
}
}
@ -357,7 +370,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
private final List<GroupHead> collectedGroups;
private final SortField[] fields;
private FieldCache.DocTermsIndex[] sortsIndex;
private SortedDocValues[] sortsIndex;
private GroupHead[] segmentGroupHeads;
OrdAllGroupHeadsCollector(String groupField, Sort sortWithinGroup, int initialSize) {
@ -367,7 +380,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
final SortField[] sortFields = sortWithinGroup.getSort();
fields = new SortField[sortFields.length];
sortsIndex = new FieldCache.DocTermsIndex[sortFields.length];
sortsIndex = new SortedDocValues[sortFields.length];
for (int i = 0; i < sortFields.length; i++) {
reversed[i] = sortFields[i].getReverse() ? -1 : 1;
fields[i] = sortFields[i];
@ -386,7 +399,13 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
GroupHead groupHead;
if (!ordSet.exists(key)) {
ordSet.put(key);
BytesRef term = key == -1 ? null : groupIndex.getTerm(doc, new BytesRef());
BytesRef term;
if (key == -1) {
term = null;
} else {
term = new BytesRef();
groupIndex.lookupOrd(key, term);
}
groupHead = new GroupHead(doc, term);
collectedGroups.add(groupHead);
segmentGroupHeads[key+1] = groupHead;
@ -407,13 +426,13 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
// Clear ordSet and fill it with previous encountered groups that can occur in the current segment.
ordSet.clear();
segmentGroupHeads = new GroupHead[groupIndex.numOrd()+1];
segmentGroupHeads = new GroupHead[groupIndex.getValueCount()+1];
for (GroupHead collectedGroup : collectedGroups) {
int groupOrd;
if (collectedGroup.groupValue == null) {
groupOrd = -1;
} else {
groupOrd = groupIndex.binarySearchLookup(collectedGroup.groupValue, scratchBytesRef);
groupOrd = groupIndex.lookupTerm(collectedGroup.groupValue, scratchBytesRef);
}
if (collectedGroup.groupValue == null || groupOrd >= 0) {
ordSet.put(groupOrd);
@ -424,7 +443,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
if (collectedGroup.sortOrds[i] == -1) {
sortOrd = -1;
} else {
sortOrd = sortsIndex[i].binarySearchLookup(collectedGroup.sortValues[i], scratchBytesRef);
sortOrd = sortsIndex[i].lookupTerm(collectedGroup.sortValues[i], scratchBytesRef);
}
collectedGroup.sortOrds[i] = sortOrd;
}
@ -445,7 +464,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
sortOrds[i] = sortsIndex[i].getOrd(doc);
sortValues[i] = new BytesRef();
if (sortOrds[i] != -1) {
sortsIndex[i].getTerm(doc, sortValues[i]);
sortsIndex[i].get(doc, sortValues[i]);
}
}
}
@ -456,7 +475,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
if (sortsIndex[compIDX].getOrd(doc) == -1) {
scratchBytesRef.length = 0;
} else {
sortsIndex[compIDX].getTerm(doc, scratchBytesRef);
sortsIndex[compIDX].get(doc, scratchBytesRef);
}
return sortValues[compIDX].compareTo(scratchBytesRef);
} else {
@ -470,7 +489,7 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
if (sortOrds[i] == -1) {
sortValues[i].length = 0;
} else {
sortValues[i] = sortsIndex[i].getTerm(doc, sortValues[i]);
sortsIndex[i].lookupOrd(sortOrds[i], sortValues[i]);
}
}
this.doc = doc + readerContext.docBase;
@ -517,7 +536,13 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
GroupHead groupHead;
if (!ordSet.exists(key)) {
ordSet.put(key);
BytesRef term = key == -1 ? null : groupIndex.getTerm(doc, new BytesRef());
BytesRef term;
if (key == -1) {
term = null;
} else {
term = new BytesRef();
groupIndex.lookupOrd(key, term);
}
groupHead = new GroupHead(doc, term);
collectedGroups.add(groupHead);
segmentGroupHeads[key+1] = groupHead;
@ -535,13 +560,13 @@ public abstract class TermAllGroupHeadsCollector<GH extends AbstractAllGroupHead
// Clear ordSet and fill it with previous encountered groups that can occur in the current segment.
ordSet.clear();
segmentGroupHeads = new GroupHead[groupIndex.numOrd()+1];
segmentGroupHeads = new GroupHead[groupIndex.getValueCount()+1];
for (GroupHead collectedGroup : collectedGroups) {
int ord;
if (collectedGroup.groupValue == null) {
ord = -1;
} else {
ord = groupIndex.binarySearchLookup(collectedGroup.groupValue, scratchBytesRef);
ord = groupIndex.lookupTerm(collectedGroup.groupValue, scratchBytesRef);
}
if (collectedGroup.groupValue == null || ord >= 0) {
ordSet.put(ord);

26
lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermAllGroupsCollector.java
View File

@ -17,18 +17,18 @@ package org.apache.lucene.search.grouping.term;
* limitations under the License.
*/
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.grouping.AbstractAllGroupsCollector;
import org.apache.lucene.util.SentinelIntSet;
import org.apache.lucene.util.BytesRef;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.grouping.AbstractAllGroupsCollector;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SentinelIntSet;
/**
* A collector that collects all groups that match the
* query. Only the group value is collected, and the order
@ -52,7 +52,7 @@ public class TermAllGroupsCollector extends AbstractAllGroupsCollector<BytesRef>
private final SentinelIntSet ordSet;
private final List<BytesRef> groups;
private FieldCache.DocTermsIndex index;
private SortedDocValues index;
private final BytesRef spareBytesRef = new BytesRef();
/**
@ -87,7 +87,13 @@ public class TermAllGroupsCollector extends AbstractAllGroupsCollector<BytesRef>
int key = index.getOrd(doc);
if (!ordSet.exists(key)) {
ordSet.put(key);
BytesRef term = key == -1 ? null : index.lookup(key, new BytesRef());
BytesRef term;
if (key == -1) {
term = null;
} else {
term = new BytesRef();
index.lookupOrd(key, term);
}
groups.add(term);
}
}
@ -107,7 +113,7 @@ public class TermAllGroupsCollector extends AbstractAllGroupsCollector<BytesRef>
if (countedGroup == null) {
ordSet.put(-1);
} else {
int ord = index.binarySearchLookup(countedGroup, spareBytesRef);
int ord = index.lookupTerm(countedGroup, spareBytesRef);
if (ord >= 0) {
ordSet.put(ord);
}

23
lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermDistinctValuesCollector.java
View File

@ -17,20 +17,20 @@ package org.apache.lucene.search.grouping.term;
* limitations under the License.
*/
import java.io.IOException;
import java.util.*;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.FieldCache.DocTermsIndex; // javadocs
import org.apache.lucene.search.grouping.AbstractDistinctValuesCollector;
import org.apache.lucene.search.grouping.SearchGroup;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SentinelIntSet;
import java.io.IOException;
import java.util.*;
/**
* A term based implementation of {@link org.apache.lucene.search.grouping.AbstractDistinctValuesCollector} that relies
* on {@link DocTermsIndex} to count the distinct values per group.
* on {@link SortedDocValues} to count the distinct values per group.
*
* @lucene.experimental
*/
@ -43,8 +43,8 @@ public class TermDistinctValuesCollector extends AbstractDistinctValuesCollector
private final GroupCount groupCounts[];
private final BytesRef spare = new BytesRef();
private FieldCache.DocTermsIndex groupFieldTermIndex;
private FieldCache.DocTermsIndex countFieldTermIndex;
private SortedDocValues groupFieldTermIndex;
private SortedDocValues countFieldTermIndex;
/**
* Constructs {@link TermDistinctValuesCollector} instance.
@ -69,7 +69,6 @@ public class TermDistinctValuesCollector extends AbstractDistinctValuesCollector
if (slot < 0) {
return;
}
int groupOrd = groupFieldTermIndex.getOrd(doc);
GroupCount gc = groupCounts[slot];
int countOrd = countFieldTermIndex.getOrd(doc);
@ -77,7 +76,9 @@ public class TermDistinctValuesCollector extends AbstractDistinctValuesCollector
if (countOrd == -1) {
gc.uniqueValues.add(null);
} else {
gc.uniqueValues.add(countFieldTermIndex.lookup(countOrd, new BytesRef()));
BytesRef br = new BytesRef();
countFieldTermIndex.lookupOrd(countOrd, br);
gc.uniqueValues.add(br);
}
gc.ords = Arrays.copyOf(gc.ords, gc.ords.length + 1);
@ -107,7 +108,7 @@ public class TermDistinctValuesCollector extends AbstractDistinctValuesCollector
ordSet.clear();
BytesRef scratch = new BytesRef();
for (GroupCount group : groups) {
int groupOrd = group.groupValue == null ? -1 : groupFieldTermIndex.binarySearchLookup(group.groupValue, spare);
int groupOrd = group.groupValue == null ? -1 : groupFieldTermIndex.lookupTerm(group.groupValue, spare);
if (group.groupValue != null && groupOrd < 0) {
continue;
}
@ -117,7 +118,7 @@ public class TermDistinctValuesCollector extends AbstractDistinctValuesCollector
Arrays.fill(group.ords, -2);
int i = 0;
for (BytesRef value : group.uniqueValues) {
int countOrd = value == null ? -1 : countFieldTermIndex.binarySearchLookup(value, scratch);
int countOrd = value == null ? -1 : countFieldTermIndex.lookupTerm(value, scratch);
if (value == null || countOrd >= 0) {
group.ords[i++] = countOrd;
}

14
lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java
View File

@ -17,14 +17,15 @@ package org.apache.lucene.search.grouping.term;
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.grouping.AbstractFirstPassGroupingCollector;
import org.apache.lucene.util.BytesRef;
import java.io.IOException;
/**
* Concrete implementation of {@link org.apache.lucene.search.grouping.AbstractFirstPassGroupingCollector} that groups based on
* field values and more specifically uses {@link org.apache.lucene.search.FieldCache.DocTermsIndex}
@ -35,7 +36,7 @@ import java.io.IOException;
public class TermFirstPassGroupingCollector extends AbstractFirstPassGroupingCollector<BytesRef> {
private final BytesRef scratchBytesRef = new BytesRef();
private FieldCache.DocTermsIndex index;
private SortedDocValues index;
private String groupField;
@ -63,7 +64,12 @@ public class TermFirstPassGroupingCollector extends AbstractFirstPassGroupingCol
@Override
protected BytesRef getDocGroupValue(int doc) {
final int ord = index.getOrd(doc);
return ord == -1 ? null : index.lookup(ord, scratchBytesRef);
if (ord == -1) {
return null;
} else {
index.lookupOrd(ord, scratchBytesRef);
return scratchBytesRef;
}
}
@Override

78
lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java
View File

@ -17,17 +17,18 @@ package org.apache.lucene.search.grouping.term;
* limitations under the License.
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DocTermOrds;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.grouping.AbstractGroupFacetCollector;
import org.apache.lucene.util.*;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* An implementation of {@link AbstractGroupFacetCollector} that computes grouped facets based on the indexed terms
* from the {@link FieldCache}.
@ -40,7 +41,7 @@ public abstract class TermGroupFacetCollector extends AbstractGroupFacetCollecto
final SentinelIntSet segmentGroupedFacetHits;
final BytesRef spare = new BytesRef();
FieldCache.DocTermsIndex groupFieldTermsIndex;
SortedDocValues groupFieldTermsIndex;
/**
* Factory method for creating the right implementation based on the fact whether the facet field contains
@ -76,7 +77,7 @@ public abstract class TermGroupFacetCollector extends AbstractGroupFacetCollecto
// Implementation for single valued facet fields.
static class SV extends TermGroupFacetCollector {
private FieldCache.DocTermsIndex facetFieldTermsIndex;
private SortedDocValues facetFieldTermsIndex;
SV(String groupField, String facetField, BytesRef facetPrefix, int initialSize) {
super(groupField, facetField, facetPrefix, initialSize);
@ -89,7 +90,7 @@ public abstract class TermGroupFacetCollector extends AbstractGroupFacetCollecto
}
int groupOrd = groupFieldTermsIndex.getOrd(doc);
int segmentGroupedFacetsIndex = groupOrd * (facetFieldTermsIndex.numOrd()+1) + facetOrd;
int segmentGroupedFacetsIndex = groupOrd * (facetFieldTermsIndex.getValueCount()+1) + facetOrd;
if (segmentGroupedFacetHits.exists(segmentGroupedFacetsIndex)) {
return;
}
@ -98,12 +99,24 @@ public abstract class TermGroupFacetCollector extends AbstractGroupFacetCollecto
segmentFacetCounts[facetOrd+1]++;
segmentGroupedFacetHits.put(segmentGroupedFacetsIndex);
groupedFacetHits.add(
new GroupedFacetHit(
groupOrd == -1 ? null : groupFieldTermsIndex.lookup(groupOrd, new BytesRef()),
facetOrd == -1 ? null : facetFieldTermsIndex.lookup(facetOrd, new BytesRef())
)
);
BytesRef groupKey;
if (groupOrd == -1) {
groupKey = null;
} else {
groupKey = new BytesRef();
groupFieldTermsIndex.lookupOrd(groupOrd, groupKey);
}
BytesRef facetKey;
if (facetOrd == -1) {
facetKey = null;
} else {
facetKey = new BytesRef();
facetFieldTermsIndex.lookupOrd(facetOrd, facetKey);
}
groupedFacetHits.add(new GroupedFacetHit(groupKey, facetKey));
}
public void setNextReader(AtomicReaderContext context) throws IOException {
@ -115,39 +128,39 @@ public abstract class TermGroupFacetCollector extends AbstractGroupFacetCollecto
facetFieldTermsIndex = FieldCache.DEFAULT.getTermsIndex(context.reader(), facetField);
// 1+ to allow for the -1 "not set":
segmentFacetCounts = new int[facetFieldTermsIndex.numOrd()+1];
segmentFacetCounts = new int[facetFieldTermsIndex.getValueCount()+1];
segmentTotalCount = 0;
segmentGroupedFacetHits.clear();
for (GroupedFacetHit groupedFacetHit : groupedFacetHits) {
int facetOrd = groupedFacetHit.facetValue == null ? -1 : facetFieldTermsIndex.binarySearchLookup(groupedFacetHit.facetValue, spare);
int facetOrd = groupedFacetHit.facetValue == null ? -1 : facetFieldTermsIndex.lookupTerm(groupedFacetHit.facetValue, spare);
if (groupedFacetHit.facetValue != null && facetOrd < 0) {
continue;
}
int groupOrd = groupedFacetHit.groupValue == null ? -1 : groupFieldTermsIndex.binarySearchLookup(groupedFacetHit.groupValue, spare);
int groupOrd = groupedFacetHit.groupValue == null ? -1 : groupFieldTermsIndex.lookupTerm(groupedFacetHit.groupValue, spare);
if (groupedFacetHit.groupValue != null && groupOrd < 0) {
continue;
}
int segmentGroupedFacetsIndex = groupOrd * (facetFieldTermsIndex.numOrd()+1) + facetOrd;
int segmentGroupedFacetsIndex = groupOrd * (facetFieldTermsIndex.getValueCount()+1) + facetOrd;
segmentGroupedFacetHits.put(segmentGroupedFacetsIndex);
}
if (facetPrefix != null) {
startFacetOrd = facetFieldTermsIndex.binarySearchLookup(facetPrefix, spare);
startFacetOrd = facetFieldTermsIndex.lookupTerm(facetPrefix, spare);
if (startFacetOrd < 0) {
// Points to the ord one higher than facetPrefix
startFacetOrd = -startFacetOrd - 1;
}
BytesRef facetEndPrefix = BytesRef.deepCopyOf(facetPrefix);
facetEndPrefix.append(UnicodeUtil.BIG_TERM);
endFacetOrd = facetFieldTermsIndex.binarySearchLookup(facetEndPrefix, spare);
endFacetOrd = facetFieldTermsIndex.lookupTerm(facetEndPrefix, spare);
assert endFacetOrd < 0;
endFacetOrd = -endFacetOrd - 1; // Points to the ord one higher than facetEndPrefix
} else {
startFacetOrd = -1;
endFacetOrd = facetFieldTermsIndex.numOrd();
endFacetOrd = facetFieldTermsIndex.getValueCount();
}
}
@ -199,9 +212,14 @@ public abstract class TermGroupFacetCollector extends AbstractGroupFacetCollecto
segmentFacetCounts[facetFieldDocTermOrds.numTerms()]++;
segmentGroupedFacetHits.put(segmentGroupedFacetsIndex);
groupedFacetHits.add(
new GroupedFacetHit(groupOrd == -1 ? null : groupFieldTermsIndex.lookup(groupOrd, new BytesRef()), null)
);
BytesRef groupKey;
if (groupOrd == -1) {
groupKey = null;
} else {
groupKey = new BytesRef();
groupFieldTermsIndex.lookupOrd(groupOrd, groupKey);
}
groupedFacetHits.add(new GroupedFacetHit(groupKey, null));
return;
}
@ -234,9 +252,17 @@ public abstract class TermGroupFacetCollector extends AbstractGroupFacetCollecto
segmentFacetCounts[facetOrd]++;
segmentGroupedFacetHits.put(segmentGroupedFacetsIndex);
BytesRef groupKey;
if (groupOrd == -1) {
groupKey = null;
} else {
groupKey = new BytesRef();
groupFieldTermsIndex.lookupOrd(groupOrd, groupKey);
}
groupedFacetHits.add(
new GroupedFacetHit(
groupOrd == -1 ? null : groupFieldTermsIndex.lookup(groupOrd, new BytesRef()),
new GroupedFacetHit(groupKey,
facetOrd == facetFieldDocTermOrds.numTerms() ? null : BytesRef.deepCopyOf(facetFieldDocTermOrds.lookupTerm(facetOrdTermsEnum, facetOrd))
)
);
@ -259,7 +285,7 @@ public abstract class TermGroupFacetCollector extends AbstractGroupFacetCollecto
segmentGroupedFacetHits.clear();
for (GroupedFacetHit groupedFacetHit : groupedFacetHits) {
int groupOrd = groupedFacetHit.groupValue == null ? -1 : groupFieldTermsIndex.binarySearchLookup(groupedFacetHit.groupValue, spare);
int groupOrd = groupedFacetHit.groupValue == null ? -1 : groupFieldTermsIndex.lookupTerm(groupedFacetHit.groupValue, spare);
if (groupedFacetHit.groupValue != null && groupOrd < 0) {
continue;
}

13
lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermSecondPassGroupingCollector.java
View File

@ -17,16 +17,17 @@ package org.apache.lucene.search.grouping.term;
* limitations under the License.
*/
import java.io.IOException;
import java.util.Collection;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.grouping.AbstractSecondPassGroupingCollector;
import org.apache.lucene.search.grouping.SearchGroup;
import org.apache.lucene.util.SentinelIntSet;
import org.apache.lucene.util.BytesRef;
import java.io.IOException;
import java.util.Collection;
import org.apache.lucene.util.SentinelIntSet;
/**
* Concrete implementation of {@link org.apache.lucene.search.grouping.AbstractSecondPassGroupingCollector} that groups based on
@ -38,7 +39,7 @@ import java.util.Collection;
public class TermSecondPassGroupingCollector extends AbstractSecondPassGroupingCollector<BytesRef> {
private final SentinelIntSet ordSet;
private FieldCache.DocTermsIndex index;
private SortedDocValues index;
private final BytesRef spareBytesRef = new BytesRef();
private final String groupField;
@ -61,7 +62,7 @@ public class TermSecondPassGroupingCollector extends AbstractSecondPassGroupingC
ordSet.clear();
for (SearchGroupDocs<BytesRef> group : groupMap.values()) {
// System.out.println(" group=" + (group.groupValue == null ? "null" : group.groupValue.utf8ToString()));
int ord = group.groupValue == null ? -1 : index.binarySearchLookup(group.groupValue, spareBytesRef);
int ord = group.groupValue == null ? -1 : index.lookupTerm(group.groupValue, spareBytesRef);
if (group.groupValue == null || ord >= 0) {
groupDocs[ordSet.put(ord)] = group;
}

23
lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java
View File

@ -17,26 +17,27 @@
package org.apache.lucene.queries.function.docvalues;
import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.ValueSourceScorer;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueStr;
import java.io.IOException;
/**
* Internal class, subject to change.
* Serves as base class for FunctionValues based on DocTermsIndex.
*/
public abstract class DocTermsIndexDocValues extends FunctionValues {
protected final FieldCache.DocTermsIndex termsIndex;
protected final SortedDocValues termsIndex;
protected final ValueSource vs;
protected final MutableValueStr val = new MutableValueStr();
protected final BytesRef spare = new BytesRef();
@ -51,7 +52,7 @@ public abstract class DocTermsIndexDocValues extends FunctionValues {
this.vs = vs;
}
public FieldCache.DocTermsIndex getDocTermsIndex() {
public SortedDocValues getSortedDocValues() {
return termsIndex;
}
@ -70,7 +71,7 @@ public abstract class DocTermsIndexDocValues extends FunctionValues {
target.length = 0;
return false;
}
termsIndex.lookup(ord, target);
termsIndex.lookupOrd(ord, target);
return true;
}
@ -78,7 +79,7 @@ public abstract class DocTermsIndexDocValues extends FunctionValues {
public String strVal(int doc) {
int ord=termsIndex.getOrd(doc);
if (ord==-1) return null;
termsIndex.lookup(ord, spare);
termsIndex.lookupOrd(ord, spare);
UnicodeUtil.UTF8toUTF16(spare, spareChars);
return spareChars.toString();
}
@ -101,7 +102,7 @@ public abstract class DocTermsIndexDocValues extends FunctionValues {
int lower = Integer.MIN_VALUE;
if (lowerVal != null) {
lower = termsIndex.binarySearchLookup(new BytesRef(lowerVal), spare);
lower = termsIndex.lookupTerm(new BytesRef(lowerVal), spare);
if (lower < 0) {
lower = -lower-1;
} else if (!includeLower) {
@ -111,7 +112,7 @@ public abstract class DocTermsIndexDocValues extends FunctionValues {
int upper = Integer.MAX_VALUE;
if (upperVal != null) {
upper = termsIndex.binarySearchLookup(new BytesRef(upperVal), spare);
upper = termsIndex.lookupTerm(new BytesRef(upperVal), spare);
if (upper < 0) {
upper = -upper-2;
} else if (!includeUpper) {
@ -153,7 +154,7 @@ public abstract class DocTermsIndexDocValues extends FunctionValues {
if (!mval.exists) {
mval.value.length = 0;
} else {
mval.value = termsIndex.lookup(ord, mval.value);
termsIndex.lookupOrd(ord, mval.value);
}
}
};

11
lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/OrdFieldSource.java
View File

@ -17,12 +17,16 @@
package org.apache.lucene.queries.function.valuesource;
import java.io.IOException;
import java.util.Map;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.CompositeReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.IntDocValues;
@ -30,9 +34,6 @@ import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueInt;
import java.io.IOException;
import java.util.Map;
/**
* Obtains the ordinal of the field value from the default Lucene {@link org.apache.lucene.search.FieldCache} using getStringIndex().
* <br>
@ -72,7 +73,7 @@ public class OrdFieldSource extends ValueSource {
final AtomicReader r = topReader instanceof CompositeReader
? new SlowCompositeReaderWrapper((CompositeReader)topReader)
: (AtomicReader) topReader;
final FieldCache.DocTermsIndex sindex = FieldCache.DEFAULT.getTermsIndex(r, field);
final SortedDocValues sindex = FieldCache.DEFAULT.getTermsIndex(r, field);
return new IntDocValues(this) {
protected String toTerm(String readableValue) {
return readableValue;
@ -87,7 +88,7 @@ public class OrdFieldSource extends ValueSource {
}
@Override
public int numOrd() {
return sindex.numOrd();
return sindex.getValueCount();
}
@Override

11
lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ReverseOrdFieldSource.java
View File

@ -17,20 +17,21 @@
package org.apache.lucene.queries.function.valuesource;
import java.io.IOException;
import java.util.Map;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.CompositeReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.IntDocValues;
import org.apache.lucene.search.FieldCache;
import java.io.IOException;
import java.util.Map;
/**
* Obtains the ordinal of the field value from the default Lucene {@link org.apache.lucene.search.FieldCache} using getTermsIndex()
* and reverses the order.
@ -73,8 +74,8 @@ public class ReverseOrdFieldSource extends ValueSource {
: (AtomicReader) topReader;
final int off = readerContext.docBase;
final FieldCache.DocTermsIndex sindex = FieldCache.DEFAULT.getTermsIndex(r, field);
final int end = sindex.numOrd();
final SortedDocValues sindex = FieldCache.DEFAULT.getTermsIndex(r, field);
final int end = sindex.getValueCount();
return new IntDocValues(this) {
@Override

37
solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java
View File

@ -16,16 +16,17 @@ package org.apache.solr.handler.component;
* limitations under the License.
*/
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
/**
* FieldFacetStats is a utility to accumulate statistics on a set of values in one field,
@ -39,7 +40,7 @@ import java.util.Map;
public class FieldFacetStats {
public final String name;
final FieldCache.DocTermsIndex si;
final SortedDocValues si;
final SchemaField facet_sf;
final SchemaField field_sf;
@ -55,7 +56,7 @@ public class FieldFacetStats {
private final BytesRef tempBR = new BytesRef();
public FieldFacetStats(String name, FieldCache.DocTermsIndex si, SchemaField field_sf, SchemaField facet_sf, int numStatsTerms) {
public FieldFacetStats(String name, SortedDocValues si, SchemaField field_sf, SchemaField facet_sf, int numStatsTerms) {
this.name = name;
this.si = si;
this.field_sf = field_sf;
@ -63,7 +64,7 @@ public class FieldFacetStats {
this.numStatsTerms = numStatsTerms;
startTermIndex = 0;
endTermIndex = si.numOrd();
endTermIndex = si.getValueCount();
nTerms = endTermIndex - startTermIndex;
facetStatsValues = new HashMap<String, StatsValues>();
@ -82,7 +83,8 @@ public class FieldFacetStats {
if (ord == -1) {
return null;
} else {
return si.lookup(ord, ret);
si.lookupOrd(ord, ret);
return ret;
}
}
@ -90,7 +92,14 @@ public class FieldFacetStats {
int term = si.getOrd(docID);
int arrIdx = term - startTermIndex;
if (arrIdx >= 0 && arrIdx < nTerms) {
final BytesRef br = si.lookup(term, tempBR);
final BytesRef br;
if (term == -1) {
br = null;
} else {
br = tempBR;
si.lookupOrd(term, tempBR);
}
String key = (br == null)?null:facet_sf.getType().indexedToReadable(br.utf8ToString());
StatsValues stats = facetStatsValues.get(key);
if (stats == null) {
@ -117,7 +126,13 @@ public class FieldFacetStats {
int term = si.getOrd(docID);
int arrIdx = term - startTermIndex;
if (arrIdx >= 0 && arrIdx < nTerms) {
final BytesRef br = si.lookup(term, tempBR);
final BytesRef br;
if (term == -1) {
br = null;
} else {
br = tempBR;
si.lookupOrd(term, tempBR);
}
String key = br == null ? null : br.utf8ToString();
HashMap<String, Integer> statsTermCounts = facetStatsTerms.get(statsTermNum);
Integer statsTermCount = statsTermCounts.get(key);

18
solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java
View File

@ -23,22 +23,23 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.StatsParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.UnInvertedField;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.TrieField;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.request.UnInvertedField;
/**
* Stats component calculates simple statistics on numeric field values
@ -240,7 +241,7 @@ class SimpleStats {
public NamedList<?> getFieldCacheStats(String fieldName, String[] facet ) {
SchemaField sf = searcher.getSchema().getField(fieldName);
FieldCache.DocTermsIndex si;
SortedDocValues si;
try {
si = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), fieldName);
}
@ -248,12 +249,12 @@ class SimpleStats {
throw new RuntimeException( "failed to open field cache for: "+fieldName, e );
}
StatsValues allstats = StatsValuesFactory.createStatsValues(sf);
final int nTerms = si.numOrd();
final int nTerms = si.getValueCount();
if ( nTerms <= 0 || docs.size() <= 0 ) return allstats.getStatsValues();
// don't worry about faceting if no documents match...
List<FieldFacetStats> facetStats = new ArrayList<FieldFacetStats>();
FieldCache.DocTermsIndex facetTermsIndex;
SortedDocValues facetTermsIndex;
for( String facetField : facet ) {
SchemaField fsf = searcher.getSchema().getField(facetField);
@ -283,9 +284,10 @@ class SimpleStats {
tempBR.length = 0;
raw = tempBR;
} else {
raw = si.lookup(docOrd, tempBR);
if( raw.length > 0 ) {
allstats.accumulate(raw);
raw = tempBR;
si.lookupOrd(docOrd, tempBR);
if( tempBR.length > 0 ) {
allstats.accumulate(tempBR);
} else {
allstats.missing();
}

19
solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java
View File

@ -17,15 +17,20 @@
package org.apache.solr.request;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.*;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.solr.common.SolrException;
@ -36,10 +41,6 @@ import org.apache.solr.search.DocSet;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.BoundedTreeSet;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.*;
class PerSegmentSingleValuedFaceting {
@ -223,7 +224,7 @@ class PerSegmentSingleValuedFaceting {
this.context = context;
}
FieldCache.DocTermsIndex si;
SortedDocValues si;
int startTermIndex;
int endTermIndex;
int[] counts;
@ -239,16 +240,16 @@ class PerSegmentSingleValuedFaceting {
if (prefix!=null) {
BytesRef prefixRef = new BytesRef(prefix);
startTermIndex = si.binarySearchLookup(prefixRef, tempBR);
startTermIndex = si.lookupTerm(prefixRef, tempBR);
if (startTermIndex<0) startTermIndex=-startTermIndex-1;
prefixRef.append(UnicodeUtil.BIG_TERM);
// TODO: we could constrain the lower endpoint if we had a binarySearch method that allowed passing start/end
endTermIndex = si.binarySearchLookup(prefixRef, tempBR);
endTermIndex = si.lookupTerm(prefixRef, tempBR);
assert endTermIndex < 0;
endTermIndex = -endTermIndex-1;
} else {
startTermIndex=-1;
endTermIndex=si.numOrd();
endTermIndex=si.getValueCount();
}
final int nTerms=endTermIndex-startTermIndex;

14
solr/core/src/java/org/apache/solr/request/SimpleFacets.java
View File

@ -477,7 +477,7 @@ public class SimpleFacets {
FieldType ft = searcher.getSchema().getFieldType(fieldName);
NamedList<Integer> res = new NamedList<Integer>();
FieldCache.DocTermsIndex si = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), fieldName);
SortedDocValues si = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), fieldName);
final BytesRef prefixRef;
if (prefix == null) {
@ -493,15 +493,15 @@ public class SimpleFacets {
int startTermIndex, endTermIndex;
if (prefix!=null) {
startTermIndex = si.binarySearchLookup(prefixRef, br);
startTermIndex = si.lookupTerm(prefixRef, br);
if (startTermIndex<0) startTermIndex=-startTermIndex-1;
prefixRef.append(UnicodeUtil.BIG_TERM);
endTermIndex = si.binarySearchLookup(prefixRef, br);
endTermIndex = si.lookupTerm(prefixRef, br);
assert endTermIndex < 0;
endTermIndex = -endTermIndex-1;
} else {
startTermIndex=-1;
endTermIndex=si.numOrd();
endTermIndex=si.getValueCount();
}
final int nTerms=endTermIndex-startTermIndex;
@ -564,7 +564,8 @@ public class SimpleFacets {
long pair = sorted[i];
int c = (int)(pair >>> 32);
int tnum = Integer.MAX_VALUE - (int)pair;
ft.indexedToReadable(si.lookup(startTermIndex+tnum, br), charsRef);
si.lookupOrd(startTermIndex+tnum, br);
ft.indexedToReadable(br, charsRef);
res.add(charsRef.toString(), c);
}
@ -582,7 +583,8 @@ public class SimpleFacets {
int c = counts[i];
if (c<mincount || --off>=0) continue;
if (--lim<0) break;
ft.indexedToReadable(si.lookup(startTermIndex+i, br), charsRef);
si.lookupOrd(startTermIndex+i, br);
ft.indexedToReadable(br, charsRef);
res.add(charsRef.toString(), c);
}
}

35
solr/core/src/java/org/apache/solr/request/UnInvertedField.java
View File

@ -17,37 +17,36 @@
package org.apache.solr.request;
import org.apache.lucene.search.FieldCache;
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.index.DocTermOrds;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.SolrException;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.component.FieldFacetStats;
import org.apache.solr.handler.component.StatsValues;
import org.apache.solr.handler.component.StatsValuesFactory;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.TrieField;
import org.apache.solr.search.*;
import org.apache.solr.util.LongPriorityQueue;
import org.apache.solr.util.PrimUtils;
import org.apache.solr.handler.component.StatsValues;
import org.apache.solr.handler.component.StatsValuesFactory;
import org.apache.solr.handler.component.FieldFacetStats;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.UnicodeUtil;
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
/**
*
@ -481,7 +480,7 @@ public class UnInvertedField extends DocTermOrds {
int i = 0;
final FieldFacetStats[] finfo = new FieldFacetStats[facet.length];
//Initialize facetstats, if facets have been passed in
FieldCache.DocTermsIndex si;
SortedDocValues si;
for (String f : facet) {
SchemaField facet_sf = searcher.getSchema().getField(f);
try {

31
solr/core/src/java/org/apache/solr/schema/BoolField.java
View File

@ -17,29 +17,30 @@
package org.apache.solr.schema;
import java.io.IOException;
import java.io.Reader;
import java.util.Map;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.GeneralField;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.BoolDocValues;
import org.apache.lucene.queries.function.valuesource.OrdFieldSource;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueBool;
import org.apache.solr.search.QParser;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.analysis.SolrAnalyzer;
import java.util.Map;
import java.io.Reader;
import java.io.IOException;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
/**
*
*/
@ -167,14 +168,14 @@ class BoolFieldSource extends ValueSource {
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
final FieldCache.DocTermsIndex sindex = FieldCache.DEFAULT.getTermsIndex(readerContext.reader(), field);
final SortedDocValues sindex = FieldCache.DEFAULT.getTermsIndex(readerContext.reader(), field);
// figure out what ord maps to true
int nord = sindex.numOrd();
int nord = sindex.getValueCount();
BytesRef br = new BytesRef();
int tord = -1;
for (int i=0; i<nord; i++) {
sindex.lookup(i, br);
sindex.lookupOrd(i, br);
if (br.length==1 && br.bytes[br.offset]=='T') {
tord = i;
break;

7
solr/core/src/java/org/apache/solr/schema/DateField.java
View File

@ -500,8 +500,8 @@ class DateFieldSource extends FieldCacheSource {
if (ord == -1) {
return null;
} else {
final BytesRef br = termsIndex.lookup(ord, spare);
return ft.indexedToReadable(br, spareChars).toString();
termsIndex.lookupOrd(ord, spare);
return ft.indexedToReadable(spare, spareChars).toString();
}
}
@ -511,7 +511,8 @@ class DateFieldSource extends FieldCacheSource {
if (ord == -1) {
return null;
} else {
final BytesRef br = termsIndex.lookup(ord, new BytesRef());
final BytesRef br = new BytesRef();
termsIndex.lookupOrd(ord, br);
return ft.toObject(null, br);
}
}

17
solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java
View File

@ -149,7 +149,12 @@ class SortableDoubleFieldSource extends FieldCacheSource {
@Override
public double doubleVal(int doc) {
int ord=termsIndex.getOrd(doc);
return ord==-1 ? def : NumberUtils.SortableStr2double(termsIndex.lookup(ord, spare));
if (ord == -1) {
return def;
} else {
termsIndex.lookupOrd(ord, spare);
return NumberUtils.SortableStr2double(spare);
}
}
@Override
@ -160,7 +165,12 @@ class SortableDoubleFieldSource extends FieldCacheSource {
@Override
public Object objectVal(int doc) {
int ord=termsIndex.getOrd(doc);
return ord==-1 ? null : NumberUtils.SortableStr2double(termsIndex.lookup(ord, spare));
if (ord==-1) {
return null;
} else {
termsIndex.lookupOrd(ord, spare);
return NumberUtils.SortableStr2double(spare);
}
}
@Override
@ -185,7 +195,8 @@ class SortableDoubleFieldSource extends FieldCacheSource {
mval.value = def;
mval.exists = false;
} else {
mval.value = NumberUtils.SortableStr2double(termsIndex.lookup(ord, spare));
termsIndex.lookupOrd(ord, spare);
mval.value = NumberUtils.SortableStr2double(spare);
mval.exists = true;
}
}

17
solr/core/src/java/org/apache/solr/schema/SortableFloatField.java
View File

@ -137,7 +137,12 @@ class SortableFloatFieldSource extends FieldCacheSource {
@Override
public float floatVal(int doc) {
int ord=termsIndex.getOrd(doc);
return ord==-1 ? def : NumberUtils.SortableStr2float(termsIndex.lookup(ord, spare));
if (ord==-1) {
return def;
} else {
termsIndex.lookupOrd(ord, spare);
return NumberUtils.SortableStr2float(spare);
}
}
@Override
@ -168,7 +173,12 @@ class SortableFloatFieldSource extends FieldCacheSource {
@Override
public Object objectVal(int doc) {
int ord=termsIndex.getOrd(doc);
return ord==-1 ? null : NumberUtils.SortableStr2float(termsIndex.lookup(ord, spare));
if (ord==-1) {
return null;
} else {
termsIndex.lookupOrd(ord, spare);
return NumberUtils.SortableStr2float(spare);
}
}
@Override
@ -188,7 +198,8 @@ class SortableFloatFieldSource extends FieldCacheSource {
mval.value = def;
mval.exists = false;
} else {
mval.value = NumberUtils.SortableStr2float(termsIndex.lookup(ord, spare));
termsIndex.lookupOrd(ord, spare);
mval.value = NumberUtils.SortableStr2float(spare);
mval.exists = true;
}
}

17
solr/core/src/java/org/apache/solr/schema/SortableIntField.java
View File

@ -144,7 +144,12 @@ class SortableIntFieldSource extends FieldCacheSource {
@Override
public int intVal(int doc) {
int ord=termsIndex.getOrd(doc);
return ord==-1 ? def : NumberUtils.SortableStr2int(termsIndex.lookup(ord, spare),0,3);
if (ord==-1) {
return def;
} else {
termsIndex.lookupOrd(ord, spare);
return NumberUtils.SortableStr2int(spare,0,3);
}
}
@Override
@ -170,7 +175,12 @@ class SortableIntFieldSource extends FieldCacheSource {
@Override
public Object objectVal(int doc) {
int ord=termsIndex.getOrd(doc);
return ord==-1 ? null : NumberUtils.SortableStr2int(termsIndex.lookup(ord, spare));
if (ord==-1) {
return null;
} else {
termsIndex.lookupOrd(ord, spare);
return NumberUtils.SortableStr2int(spare);
}
}
@Override
@ -190,7 +200,8 @@ class SortableIntFieldSource extends FieldCacheSource {
mval.value = def;
mval.exists = false;
} else {
mval.value = NumberUtils.SortableStr2int(termsIndex.lookup(ord, spare),0,3);
termsIndex.lookupOrd(ord, spare);
mval.value = NumberUtils.SortableStr2int(spare,0,3);
mval.exists = true;
}
}

17
solr/core/src/java/org/apache/solr/schema/SortableLongField.java
View File

@ -147,7 +147,12 @@ class SortableLongFieldSource extends FieldCacheSource {
@Override
public long longVal(int doc) {
int ord=termsIndex.getOrd(doc);
return ord==-1 ? def : NumberUtils.SortableStr2long(termsIndex.lookup(ord, spare),0,5);
if (ord==-1) {
return def;
} else {
termsIndex.lookupOrd(ord, spare);
return NumberUtils.SortableStr2long(spare,0,5);
}
}
@Override
@ -163,7 +168,12 @@ class SortableLongFieldSource extends FieldCacheSource {
@Override
public Object objectVal(int doc) {
int ord=termsIndex.getOrd(doc);
return ord==-1 ? null : NumberUtils.SortableStr2long(termsIndex.lookup(ord, spare));
if (ord==-1) {
return null;
} else {
termsIndex.lookupOrd(ord, spare);
return NumberUtils.SortableStr2long(spare);
}
}
@Override
@ -188,7 +198,8 @@ class SortableLongFieldSource extends FieldCacheSource {
mval.value = def;
mval.exists = false;
} else {
mval.value = NumberUtils.SortableStr2long(termsIndex.lookup(ord, spare),0,5);
termsIndex.lookupOrd(ord, spare);
mval.value = NumberUtils.SortableStr2long(spare,0,5);
mval.exists = true;
}
}

2
solr/core/src/java/org/apache/solr/schema/StrFieldSource.java
View File

@ -52,7 +52,7 @@ public class StrFieldSource extends FieldCacheSource {
@Override
public int numOrd() {
return termsIndex.numOrd();
return termsIndex.getValueCount();
}
@Override

22
solr/core/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java
View File

@ -17,8 +17,11 @@
package org.apache.solr.search;
import java.io.IOException;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.FieldComparatorSource;
@ -26,8 +29,6 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.packed.PackedInts;
import java.io.IOException;
public class MissingStringLastComparatorSource extends FieldComparatorSource {
private final BytesRef missingValueProxy;
@ -62,7 +63,7 @@ class TermOrdValComparator_SML extends FieldComparator<Comparable> {
private final BytesRef[] values;
private final int[] readerGen;
private FieldCache.DocTermsIndex termsIndex;
private SortedDocValues termsIndex;
private final String field;
private final BytesRef NULL_VAL;
@ -137,7 +138,7 @@ class TermOrdValComparator_SML extends FieldComparator<Comparable> {
protected final int[] readerGen;
protected int currentReaderGen = -1;
protected FieldCache.DocTermsIndex termsIndex;
protected SortedDocValues termsIndex;
protected int bottomSlot = -1;
protected int bottomOrd;
@ -202,7 +203,7 @@ class TermOrdValComparator_SML extends FieldComparator<Comparable> {
bottomSameReader = true;
readerGen[bottomSlot] = currentReaderGen;
} else {
final int index = binarySearch(tempBR, termsIndex, bottomValue);
final int index = termsIndex.lookupTerm(bottomValue, tempBR);
if (index < 0) {
bottomOrd = -index - 2;
bottomSameReader = false;
@ -224,8 +225,8 @@ class TermOrdValComparator_SML extends FieldComparator<Comparable> {
@Override
public int compareDocToValue(int doc, BytesRef value) {
final BytesRef docValue = termsIndex.getTerm(doc, tempBR);
if (docValue == null) {
int docOrd = termsIndex.getOrd(doc);
if (docOrd == -1) {
if (value == null) {
return 0;
}
@ -233,7 +234,8 @@ class TermOrdValComparator_SML extends FieldComparator<Comparable> {
} else if (value == null) {
return -1;
}
return docValue.compareTo(value);
termsIndex.lookupOrd(docOrd, tempBR);
return tempBR.compareTo(value);
}
}
@ -270,7 +272,7 @@ class TermOrdValComparator_SML extends FieldComparator<Comparable> {
if (order == NULL_ORD) {
return bottomValue.compareTo(parent.NULL_VAL);
} else {
termsIndex.lookup(order, tempBR);
termsIndex.lookupOrd(order, tempBR);
return bottomValue.compareTo(tempBR);
}
}
@ -288,7 +290,7 @@ class TermOrdValComparator_SML extends FieldComparator<Comparable> {
if (values[slot] == null) {
values[slot] = new BytesRef();
}
termsIndex.lookup(ord, values[slot]);
termsIndex.lookupOrd(ord, values[slot]);
}
readerGen[slot] = currentReaderGen;
}