Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Re-enable test. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4547@1437183 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Adrien Grand 2013-01-22 21:24:49 +00:00
parent 94e1ff6217
commit ab9fdf8cce
4 changed files with 99 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
View File

@ -20,7 +20,6 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -29,14 +28,12 @@ import java.util.WeakHashMap;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocTermOrds;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.OrdTermState;
import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.ArrayUtil;
@ -628,10 +625,21 @@ class FieldCacheImpl implements FieldCache {
@Override
protected Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField /* ignored */)
throws IOException {
final String field = key.field;
final String field = key.field;
final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
final int maxDoc = reader.maxDoc();
if (fieldInfo == null) {
// field does not exist or has no value
return new Bits.MatchNoBits(maxDoc);
} else if (fieldInfo.hasDocValues()) {
// doc values are dense
return new Bits.MatchAllBits(maxDoc);
}
// Visit all docs that have terms for this field
FixedBitSet res = null;
Terms terms = reader.terms(field);
final int maxDoc = reader.maxDoc();
if (terms != null) {
final int termsDocCount = terms.getDocCount();
assert termsDocCount <= maxDoc;

44
lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java
View File

@ -26,6 +26,7 @@ import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.ValueSourceScorer;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.UnicodeUtil;
@ -33,11 +34,12 @@ import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueStr;
/**
* Internal class, subject to change.
* Serves as base class for FunctionValues based on DocTermsIndex.
* @lucene.internal
*/
public abstract class DocTermsIndexDocValues extends FunctionValues {
protected final SortedDocValues termsIndex;
protected final Bits valid;
protected final ValueSource vs;
protected final MutableValueStr val = new MutableValueStr();
protected final BytesRef spare = new BytesRef();
@ -46,40 +48,42 @@ public abstract class DocTermsIndexDocValues extends FunctionValues {
public DocTermsIndexDocValues(ValueSource vs, AtomicReaderContext context, String field) throws IOException {
try {
termsIndex = FieldCache.DEFAULT.getTermsIndex(context.reader(), field);
valid = FieldCache.DEFAULT.getDocsWithField(context.reader(), field);
} catch (RuntimeException e) {
throw new DocTermsIndexException(field, e);
}
this.vs = vs;
}
public SortedDocValues getSortedDocValues() {
return termsIndex;
}
protected abstract String toTerm(String readableValue);
@Override
public boolean exists(int doc) {
return termsIndex.getOrd(doc) != -1;
return valid.get(doc);
}
@Override
public int ordVal(int doc) {
return termsIndex.getOrd(doc);
}
@Override
public int numOrd() {
return termsIndex.getValueCount();
}
@Override
public boolean bytesVal(int doc, BytesRef target) {
int ord=termsIndex.getOrd(doc);
if (ord==-1) {
target.length = 0;
return false;
}
termsIndex.lookupOrd(ord, target);
return true;
termsIndex.get(doc, target);
return target.length > 0;
}
@Override
public String strVal(int doc) {
int ord=termsIndex.getOrd(doc);
if (ord==-1) return null;
termsIndex.lookupOrd(ord, spare);
termsIndex.get(doc, spare);
if (spare.length == 0) {
return null;
}
UnicodeUtil.UTF8toUTF16(spare, spareChars);
return spareChars.toString();
}
@ -149,13 +153,7 @@ public abstract class DocTermsIndexDocValues extends FunctionValues {
@Override
public void fillValue(int doc) {
int ord = termsIndex.getOrd(doc);
mval.exists = ord != -1;
if (!mval.exists) {
mval.value.length = 0;
} else {
termsIndex.lookupOrd(ord, mval.value);
}
termsIndex.get(doc, mval.value);
}
};
}

78
lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java
View File

@ -17,13 +17,19 @@ package org.apache.lucene.queries.function.valuesource;
* limitations under the License.
*/
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.docvalues.DocTermsIndexDocValues;
import java.io.IOException;
import java.util.Map;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.docvalues.DocTermsIndexDocValues;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
/**
* An implementation for retrieving {@link FunctionValues} instances for string based fields.
*/
@ -35,23 +41,59 @@ public class BytesRefFieldSource extends FieldCacheSource {
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
return new DocTermsIndexDocValues(this, readerContext, field) {
final FieldInfo fieldInfo = readerContext.reader().getFieldInfos().fieldInfo(field);
// To be sorted or not to be sorted, that is the question
// nocommit: do it cleaner?
if (fieldInfo != null && fieldInfo.hasDocValues() && fieldInfo.getDocValuesType() == DocValuesType.BINARY) {
final BinaryDocValues binaryValues = FieldCache.DEFAULT.getTerms(readerContext.reader(), field);
return new FunctionValues() {
@Override
protected String toTerm(String readableValue) {
return readableValue;
}
@Override
public boolean exists(int doc) {
return true; // doc values are dense
}
@Override
public Object objectVal(int doc) {
return strVal(doc);
}
@Override
public boolean bytesVal(int doc, BytesRef target) {
binaryValues.get(doc, target);
return target.length > 0;
}
@Override
public String toString(int doc) {
return description() + '=' + strVal(doc);
}
public String strVal(int doc) {
final BytesRef bytes = new BytesRef();
return bytesVal(doc, bytes)
? bytes.utf8ToString()
: null;
}
};
@Override
public Object objectVal(int doc) {
return strVal(doc);
}
@Override
public String toString(int doc) {
return description() + '=' + strVal(doc);
}
};
} else {
return new DocTermsIndexDocValues(this, readerContext, field) {
@Override
protected String toTerm(String readableValue) {
return readableValue;
}
@Override
public Object objectVal(int doc) {
return strVal(doc);
}
@Override
public String toString(int doc) {
return description() + '=' + strVal(doc);
}
};
}
}
}

10
lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java
View File

@ -37,7 +37,6 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.packed.PackedInts;
import org.junit.Ignore;
import com.carrotsearch.randomizedtesting.generators.RandomInts;
@ -71,11 +70,13 @@ public class TestDocValuesFieldSources extends LuceneTestCase {
RandomIndexWriter iw = new RandomIndexWriter(random(), d, iwConfig);
for (int i = 0; i < nDocs; ++i) {
id.setIntValue(i);
id.setLongValue(i);
switch (type) {
case SORTED:
case BINARY:
vals[i] = _TestUtil.randomSimpleString(random(), 20);
do {
vals[i] = _TestUtil.randomSimpleString(random(), 20);
} while (((String) vals[i]).isEmpty());
f.setBytesValue(new BytesRef((String) vals[i]));
break;
case NUMERIC:
@ -115,7 +116,6 @@ public class TestDocValuesFieldSources extends LuceneTestCase {
} else if (vs instanceof LongFieldSource) {
assertTrue(values.objectVal(i) instanceof Long);
assertTrue(values.bytesVal(i, bytes));
assertEquals(8, bytes.length);
} else {
throw new AssertionError();
}
@ -143,8 +143,6 @@ public class TestDocValuesFieldSources extends LuceneTestCase {
d.close();
}
// nocommit
@Ignore("fix this test")
public void test() throws IOException {
for (DocValuesType type : DocValuesType.values()) {
test(type);