From ab9fdf8ccee785b7155452bf9185f04902d6e36b Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 22 Jan 2013 21:24:49 +0000 Subject: [PATCH] Re-enable test. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4547@1437183 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/lucene/search/FieldCacheImpl.java | 20 +++-- .../docvalues/DocTermsIndexDocValues.java | 44 +++++------ .../valuesource/BytesRefFieldSource.java | 78 ++++++++++++++----- .../function/TestDocValuesFieldSources.java | 10 +-- 4 files changed, 99 insertions(+), 53 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java b/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java index f556c02fcca..866b946787b 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java +++ b/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java @@ -20,7 +20,6 @@ package org.apache.lucene.search; import java.io.IOException; import java.io.PrintStream; import java.util.ArrayList; -import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -29,14 +28,12 @@ import java.util.WeakHashMap; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocTermOrds; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.OrdTermState; import org.apache.lucene.index.SegmentReader; import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.ArrayUtil; @@ -628,10 +625,21 @@ class FieldCacheImpl implements FieldCache { @Override protected Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField /* ignored */) throws IOException { - final String field = key.field; + final String field = key.field; + final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field); + final int maxDoc = reader.maxDoc(); + + if (fieldInfo == null) { + // field does not exist or has no value + return new Bits.MatchNoBits(maxDoc); + } else if (fieldInfo.hasDocValues()) { + // doc values are dense + return new Bits.MatchAllBits(maxDoc); + } + + // Visit all docs that have terms for this field FixedBitSet res = null; Terms terms = reader.terms(field); - final int maxDoc = reader.maxDoc(); if (terms != null) { final int termsDocCount = terms.getDocCount(); assert termsDocCount <= maxDoc; diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java index 12055771588..ed3f13ed6c4 100755 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java @@ -26,6 +26,7 @@ import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.ValueSourceScorer; import org.apache.lucene.search.FieldCache; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.UnicodeUtil; @@ -33,11 +34,12 @@ import org.apache.lucene.util.mutable.MutableValue; import org.apache.lucene.util.mutable.MutableValueStr; /** - * Internal class, subject to change. * Serves as base class for FunctionValues based on DocTermsIndex. + * @lucene.internal */ public abstract class DocTermsIndexDocValues extends FunctionValues { protected final SortedDocValues termsIndex; + protected final Bits valid; protected final ValueSource vs; protected final MutableValueStr val = new MutableValueStr(); protected final BytesRef spare = new BytesRef(); @@ -46,40 +48,42 @@ public abstract class DocTermsIndexDocValues extends FunctionValues { public DocTermsIndexDocValues(ValueSource vs, AtomicReaderContext context, String field) throws IOException { try { termsIndex = FieldCache.DEFAULT.getTermsIndex(context.reader(), field); + valid = FieldCache.DEFAULT.getDocsWithField(context.reader(), field); } catch (RuntimeException e) { throw new DocTermsIndexException(field, e); } this.vs = vs; } - public SortedDocValues getSortedDocValues() { - return termsIndex; - } - protected abstract String toTerm(String readableValue); @Override public boolean exists(int doc) { - return termsIndex.getOrd(doc) != -1; + return valid.get(doc); } + @Override + public int ordVal(int doc) { + return termsIndex.getOrd(doc); + } + + @Override + public int numOrd() { + return termsIndex.getValueCount(); + } @Override public boolean bytesVal(int doc, BytesRef target) { - int ord=termsIndex.getOrd(doc); - if (ord==-1) { - target.length = 0; - return false; - } - termsIndex.lookupOrd(ord, target); - return true; + termsIndex.get(doc, target); + return target.length > 0; } @Override public String strVal(int doc) { - int ord=termsIndex.getOrd(doc); - if (ord==-1) return null; - termsIndex.lookupOrd(ord, spare); + termsIndex.get(doc, spare); + if (spare.length == 0) { + return null; + } UnicodeUtil.UTF8toUTF16(spare, spareChars); return spareChars.toString(); } @@ -149,13 +153,7 @@ public abstract class DocTermsIndexDocValues extends FunctionValues { @Override public void fillValue(int doc) { - int ord = termsIndex.getOrd(doc); - mval.exists = ord != -1; - if (!mval.exists) { - mval.value.length = 0; - } else { - termsIndex.lookupOrd(ord, mval.value); - } + termsIndex.get(doc, mval.value); } }; } diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java index a33c38d1394..48283e2bb2d 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java @@ -17,13 +17,19 @@ package org.apache.lucene.queries.function.valuesource; * limitations under the License. */ -import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.queries.function.FunctionValues; -import org.apache.lucene.queries.function.docvalues.DocTermsIndexDocValues; - import java.io.IOException; import java.util.Map; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfo.DocValuesType; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.docvalues.DocTermsIndexDocValues; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; + /** * An implementation for retrieving {@link FunctionValues} instances for string based fields. */ @@ -35,23 +41,59 @@ public class BytesRefFieldSource extends FieldCacheSource { @Override public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException { - return new DocTermsIndexDocValues(this, readerContext, field) { + final FieldInfo fieldInfo = readerContext.reader().getFieldInfos().fieldInfo(field); + // To be sorted or not to be sorted, that is the question + // nocommit: do it cleaner? + if (fieldInfo != null && fieldInfo.hasDocValues() && fieldInfo.getDocValuesType() == DocValuesType.BINARY) { + final BinaryDocValues binaryValues = FieldCache.DEFAULT.getTerms(readerContext.reader(), field); + return new FunctionValues() { - @Override - protected String toTerm(String readableValue) { - return readableValue; - } + @Override + public boolean exists(int doc) { + return true; // doc values are dense + } - @Override - public Object objectVal(int doc) { - return strVal(doc); - } + @Override + public boolean bytesVal(int doc, BytesRef target) { + binaryValues.get(doc, target); + return target.length > 0; + } - @Override - public String toString(int doc) { - return description() + '=' + strVal(doc); - } + public String strVal(int doc) { + final BytesRef bytes = new BytesRef(); + return bytesVal(doc, bytes) + ? bytes.utf8ToString() + : null; + } - }; + @Override + public Object objectVal(int doc) { + return strVal(doc); + } + + @Override + public String toString(int doc) { + return description() + '=' + strVal(doc); + } + }; + } else { + return new DocTermsIndexDocValues(this, readerContext, field) { + + @Override + protected String toTerm(String readableValue) { + return readableValue; + } + + @Override + public Object objectVal(int doc) { + return strVal(doc); + } + + @Override + public String toString(int doc) { + return description() + '=' + strVal(doc); + } + }; + } } } diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java index 398c959da72..2abf2c3fcb0 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java @@ -37,7 +37,6 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; import org.apache.lucene.util.packed.PackedInts; -import org.junit.Ignore; import com.carrotsearch.randomizedtesting.generators.RandomInts; @@ -71,11 +70,13 @@ public class TestDocValuesFieldSources extends LuceneTestCase { RandomIndexWriter iw = new RandomIndexWriter(random(), d, iwConfig); for (int i = 0; i < nDocs; ++i) { - id.setIntValue(i); + id.setLongValue(i); switch (type) { case SORTED: case BINARY: - vals[i] = _TestUtil.randomSimpleString(random(), 20); + do { + vals[i] = _TestUtil.randomSimpleString(random(), 20); + } while (((String) vals[i]).isEmpty()); f.setBytesValue(new BytesRef((String) vals[i])); break; case NUMERIC: @@ -115,7 +116,6 @@ public class TestDocValuesFieldSources extends LuceneTestCase { } else if (vs instanceof LongFieldSource) { assertTrue(values.objectVal(i) instanceof Long); assertTrue(values.bytesVal(i, bytes)); - assertEquals(8, bytes.length); } else { throw new AssertionError(); } @@ -143,8 +143,6 @@ public class TestDocValuesFieldSources extends LuceneTestCase { d.close(); } - // nocommit - @Ignore("fix this test") public void test() throws IOException { for (DocValuesType type : DocValuesType.values()) { test(type);