Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-4819: revert 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1454996 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2013-03-11 04:40:47 +00:00
parent 5ea281a9d6
commit 1600711a61
13 changed files with 21 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lucene/CHANGES.txt
View File

@ -41,11 +41,6 @@ New Features
once as a keyword and once as an ordinary token allow stemmers to emit
a stemmed version along with the un-stemmed version. (Simon Willnauer)
Optimizations
* LUCENE-4819: Added Sorted[Set]DocValues.termsEnum(), and optimized the
default codec for improved enumeration performance. (Robert Muir)
======================= Lucene 4.2.0 =======================
Changes in backwards compatibility policy

10
lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
View File

@ -32,7 +32,9 @@ import org.apache.lucene.index.MultiDocValues.OrdinalMap;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedDocValuesTermsEnum;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.SortedSetDocValuesTermsEnum;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
@ -267,7 +269,7 @@ public abstract class DocValuesConsumer implements Closeable {
SortedDocValues dv = dvs[sub];
Bits liveDocs = reader.getLiveDocs();
if (liveDocs == null) {
liveTerms[sub] = dv.termsEnum();
liveTerms[sub] = new SortedDocValuesTermsEnum(dv);
} else {
OpenBitSet bitset = new OpenBitSet(dv.getValueCount());
for (int i = 0; i < reader.maxDoc(); i++) {
@ -275,7 +277,7 @@ public abstract class DocValuesConsumer implements Closeable {
bitset.set(dv.getOrd(i));
}
}
liveTerms[sub] = new BitsFilteredTermsEnum(dv.termsEnum(), bitset);
liveTerms[sub] = new BitsFilteredTermsEnum(new SortedDocValuesTermsEnum(dv), bitset);
}
}
@ -399,7 +401,7 @@ public abstract class DocValuesConsumer implements Closeable {
SortedSetDocValues dv = dvs[sub];
Bits liveDocs = reader.getLiveDocs();
if (liveDocs == null) {
liveTerms[sub] = dv.termsEnum();
liveTerms[sub] = new SortedSetDocValuesTermsEnum(dv);
} else {
OpenBitSet bitset = new OpenBitSet(dv.getValueCount());
for (int i = 0; i < reader.maxDoc(); i++) {
@ -411,7 +413,7 @@ public abstract class DocValuesConsumer implements Closeable {
}
}
}
liveTerms[sub] = new BitsFilteredTermsEnum(dv.termsEnum(), bitset);
liveTerms[sub] = new BitsFilteredTermsEnum(new SortedSetDocValuesTermsEnum(dv), bitset);
}
}

117
lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java
View File

@ -18,7 +18,6 @@ package org.apache.lucene.codecs.lucene42;
*/
import java.io.IOException;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
@ -26,8 +25,6 @@ import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
@ -35,10 +32,8 @@ import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
@ -290,11 +285,6 @@ class Lucene42DocValuesProducer extends DocValuesProducer {
public int getValueCount() {
return (int)entry.numOrds;
}
@Override
public TermsEnum termsEnum() {
return new FSTTermsEnum(fst);
}
};
}
@ -379,11 +369,6 @@ class Lucene42DocValuesProducer extends DocValuesProducer {
public long getValueCount() {
return entry.numOrds;
}
@Override
public TermsEnum termsEnum() {
return new FSTTermsEnum(fst);
}
};
}
@ -411,106 +396,4 @@ class Lucene42DocValuesProducer extends DocValuesProducer {
long offset;
long numOrds;
}
// exposes FSTEnum directly as a TermsEnum: avoids binary-search next()
static class FSTTermsEnum extends TermsEnum {
final BytesRefFSTEnum<Long> in;
// this is all for the complicated seek(ord)...
// maybe we should add a FSTEnum that supports this operation?
final FST<Long> fst;
final FST.BytesReader bytesReader;
final Arc<Long> firstArc = new Arc<Long>();
final Arc<Long> scratchArc = new Arc<Long>();
final IntsRef scratchInts = new IntsRef();
final BytesRef scratchBytes = new BytesRef();
FSTTermsEnum(FST<Long> fst) {
this.fst = fst;
in = new BytesRefFSTEnum<Long>(fst);
bytesReader = fst.getBytesReader();
}
@Override
public BytesRef next() throws IOException {
InputOutput<Long> io = in.next();
if (io == null) {
return null;
} else {
return io.input;
}
}
@Override
public Comparator<BytesRef> getComparator() {
return BytesRef.getUTF8SortedAsUnicodeComparator();
}
@Override
public SeekStatus seekCeil(BytesRef text, boolean useCache) throws IOException {
if (in.seekCeil(text) == null) {
return SeekStatus.NOT_FOUND;
} else if (term().equals(text)) {
// TODO: add SeekStatus to FSTEnum like in https://issues.apache.org/jira/browse/LUCENE-3729
// to remove this comparision?
return SeekStatus.FOUND;
} else {
return SeekStatus.NOT_FOUND;
}
}
@Override
public boolean seekExact(BytesRef text, boolean useCache) throws IOException {
if (in.seekExact(text) == null) {
return false;
} else {
return true;
}
}
@Override
public void seekExact(long ord) throws IOException {
// TODO: would be better to make this simpler and faster.
// but we dont want to introduce a bug that corrupts our enum state!
bytesReader.setPosition(0);
fst.getFirstArc(firstArc);
IntsRef output = Util.getByOutput(fst, ord, bytesReader, firstArc, scratchArc, scratchInts);
scratchBytes.bytes = new byte[output.length];
scratchBytes.offset = 0;
scratchBytes.length = 0;
Util.toBytesRef(output, scratchBytes);
// TODO: we could do this lazily, better to try to push into FSTEnum though?
in.seekExact(scratchBytes);
}
@Override
public BytesRef term() throws IOException {
return in.current().input;
}
@Override
public long ord() throws IOException {
return in.current().output;
}
@Override
public int docFreq() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public long totalTermFreq() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
throw new UnsupportedOperationException();
}
}
}

4
lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
View File

@ -217,7 +217,7 @@ public class MultiDocValues {
} else {
TermsEnum enums[] = new TermsEnum[values.length];
for (int i = 0; i < values.length; i++) {
enums[i] = values[i].termsEnum();
enums[i] = new SortedDocValuesTermsEnum(values[i]);
}
OrdinalMap mapping = new OrdinalMap(r.getCoreCacheKey(), enums);
return new MultiSortedDocValues(values, starts, mapping);
@ -261,7 +261,7 @@ public class MultiDocValues {
} else {
TermsEnum enums[] = new TermsEnum[values.length];
for (int i = 0; i < values.length; i++) {
enums[i] = values[i].termsEnum();
enums[i] = new SortedSetDocValuesTermsEnum(values[i]);
}
OrdinalMap mapping = new OrdinalMap(r.getCoreCacheKey(), enums);
return new MultiSortedSetDocValues(values, starts, mapping);

8
lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java
View File

@ -114,12 +114,4 @@ public abstract class SortedDocValues extends BinaryDocValues {
return -(low + 1); // key not found.
}
/**
* Returns a {@link TermsEnum} over the values.
* The enum supports {@link TermsEnum#ord()} and {@link TermsEnum#seekExact(long)}.
*/
public TermsEnum termsEnum() {
return new SortedDocValuesTermsEnum(this);
}
}

2
lucene/core/src/java/org/apache/lucene/index/SortedDocValuesTermsEnum.java
View File

@ -26,7 +26,7 @@ import org.apache.lucene.util.BytesRef;
/** Implements a {@link TermsEnum} wrapping a provided
* {@link SortedDocValues}. */
class SortedDocValuesTermsEnum extends TermsEnum {
public class SortedDocValuesTermsEnum extends TermsEnum {
private final SortedDocValues values;
private int currentOrd = -1;
private final BytesRef term = new BytesRef();

8
lucene/core/src/java/org/apache/lucene/index/SortedSetDocValues.java
View File

@ -117,12 +117,4 @@ public abstract class SortedSetDocValues {
return -(low + 1); // key not found.
}
/**
* Returns a {@link TermsEnum} over the values.
* The enum supports {@link TermsEnum#ord()} and {@link TermsEnum#seekExact(long)}.
*/
public TermsEnum termsEnum() {
return new SortedSetDocValuesTermsEnum(this);
}
}

2
lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesTermsEnum.java
View File

@ -26,7 +26,7 @@ import org.apache.lucene.util.BytesRef;
/** Implements a {@link TermsEnum} wrapping a provided
* {@link SortedSetDocValues}. */
class SortedSetDocValuesTermsEnum extends TermsEnum {
public class SortedSetDocValuesTermsEnum extends TermsEnum {
private final SortedSetDocValues values;
private long currentOrd = -1;
private final BytesRef term = new BytesRef();

3
lucene/core/src/java/org/apache/lucene/search/DocTermOrdsRewriteMethod.java
View File

@ -23,6 +23,7 @@ import java.util.Comparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.SortedSetDocValuesTermsEnum;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.Bits;
@ -97,7 +98,7 @@ public final class DocTermOrdsRewriteMethod extends MultiTermQuery.RewriteMethod
@Override
public TermsEnum iterator(TermsEnum reuse) {
return docTermOrds.termsEnum();
return new SortedSetDocValuesTermsEnum(docTermOrds);
}
@Override

3
lucene/core/src/java/org/apache/lucene/search/FieldCacheRewriteMethod.java
View File

@ -23,6 +23,7 @@ import java.util.Comparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedDocValuesTermsEnum;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.Bits;
@ -97,7 +98,7 @@ public final class FieldCacheRewriteMethod extends MultiTermQuery.RewriteMethod
@Override
public TermsEnum iterator(TermsEnum reuse) {
return fcsi.termsEnum();
return new SortedDocValuesTermsEnum(fcsi);
}
@Override

2
lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
View File

@ -219,7 +219,7 @@ public class TestFieldCache extends LuceneTestCase {
int nTerms = termsIndex.getValueCount();
TermsEnum tenum = termsIndex.termsEnum();
TermsEnum tenum = new SortedDocValuesTermsEnum(termsIndex);
BytesRef val = new BytesRef();
for (int i=0; i<nTerms; i++) {
BytesRef val1 = tenum.next();

6
lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermGroupFacetCollector.java
View File

@ -24,7 +24,9 @@ import java.util.List;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DocTermOrds;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedDocValuesTermsEnum;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.SortedSetDocValuesTermsEnum;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.grouping.AbstractGroupFacetCollector;
@ -168,7 +170,7 @@ public abstract class TermGroupFacetCollector extends AbstractGroupFacetCollecto
@Override
protected SegmentResult createSegmentResult() throws IOException {
return new SegmentResult(segmentFacetCounts, segmentTotalCount, facetFieldTermsIndex.termsEnum(), startFacetOrd, endFacetOrd);
return new SegmentResult(segmentFacetCounts, segmentTotalCount, new SortedDocValuesTermsEnum(facetFieldTermsIndex), startFacetOrd, endFacetOrd);
}
private static class SegmentResult extends AbstractGroupFacetCollector.SegmentResult {
@ -287,7 +289,7 @@ public abstract class TermGroupFacetCollector extends AbstractGroupFacetCollecto
if (facetFieldNumTerms == 0) {
facetOrdTermsEnum = null;
} else {
facetOrdTermsEnum = facetFieldDocTermOrds.termsEnum();
facetOrdTermsEnum = new SortedSetDocValuesTermsEnum(facetFieldDocTermOrds);
}
// [facetFieldNumTerms() + 1] for all possible facet values and docs not containing facet field
segmentFacetCounts = new int[facetFieldNumTerms + 1];

3
solr/core/src/java/org/apache/solr/request/PerSegmentSingleValuedFaceting.java
View File

@ -23,6 +23,7 @@ import java.util.concurrent.*;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedDocValuesTermsEnum;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
@ -154,7 +155,7 @@ class PerSegmentSingleValuedFaceting {
seg.pos = seg.startTermIndex;
}
if (seg.pos < seg.endTermIndex) {
seg.tenum = seg.si.termsEnum();
seg.tenum = new SortedDocValuesTermsEnum(seg.si);
seg.tenum.seekExact(seg.pos);
seg.tempBR = seg.tenum.term();
queue.add(seg);