Apache/lucene
1
0
Fork 0
mirror of https://github.com/apache/lucene.git synced 2025-02-27 13:06:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

move away from MultiDocValues

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4547@1422070 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2012-12-14 20:37:34 +00:00
parent 9503269122
commit 493f8e718d
19 changed files with 366 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java
View File

@ -34,7 +34,6 @@ import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40NormsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoFormat;
import org.apache.lucene.codecs.lucene40.Lucene40TermVectorsFormat;
import org.apache.lucene.codecs.lucene41.values.Lucene41DocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
@ -145,4 +144,6 @@ public class Lucene41Codec extends Codec {
private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene41");
// nocommit
private final SimpleDocValuesFormat defaultDVFormat = SimpleDocValuesFormat.forName("Memory");
// nocommit need simpleNormsFormat
}

2
lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
View File

@ -257,7 +257,7 @@ final class SegmentCoreReaders {
// Field does not exist
return null;
}
if (fi.omitsNorms()) {
if (!fi.isIndexed() || fi.omitsNorms()) {
return null;
}
// nocommit change to assert != null!!

1
lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
View File

@ -556,6 +556,7 @@ class FieldCacheImpl implements FieldCache {
throws IOException {
final NumericDocValues valuesIn = reader.getNumericDocValues(key.field);
System.out.println("ints values=" + valuesIn);
if (valuesIn != null) {
return new Ints() {
@Override

6
lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
View File

@ -47,7 +47,6 @@ import org.apache.lucene.document.SortedBytesDocValuesField;
import org.apache.lucene.document.StraightBytesDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.DocIdSetIterator;
@ -365,7 +364,9 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
assertEquals(7, i);
}
}
// nocommit re-enable if/when we backport DV 2.0 to 4.0
/*
if (is40Index) {
// check docvalues fields
Source dvByte = MultiDocValues.getDocValues(reader, "dvByte").getSource();
@ -407,6 +408,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
assertEquals(id, dvShort.getInt(i));
}
}
*/
ScoreDoc[] hits = searcher.search(new TermQuery(new Term(new String("content"), "aaa")), null, 1000).scoreDocs;

25
lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
View File

@ -546,11 +546,12 @@ public void testFilesOpenClose() throws IOException {
// TODO: maybe this can reuse the logic of test dueling codecs?
public static void assertIndexEquals(DirectoryReader index1, DirectoryReader index2) throws IOException {
Assume.assumeTrue(_TestUtil.canUseSimpleNorms());
assertEquals("IndexReaders have different values for numDocs.", index1.numDocs(), index2.numDocs());
assertEquals("IndexReaders have different values for maxDoc.", index1.maxDoc(), index2.maxDoc());
assertEquals("Only one IndexReader has deletions.", index1.hasDeletions(), index2.hasDeletions());
assertEquals("Single segment test differs.", index1.leaves().size() == 1, index2.leaves().size() == 1);
// check field names
FieldInfos fieldInfos1 = MultiFields.getMergedFieldInfos(index1);
FieldInfos fieldInfos2 = MultiFields.getMergedFieldInfos(index2);
@ -565,21 +566,17 @@ public void testFilesOpenClose() throws IOException {
// check norms
for(FieldInfo fieldInfo : fieldInfos1) {
String curField = fieldInfo.name;
DocValues norms1 = MultiDocValues.getNormDocValues(index1, curField);
DocValues norms2 = MultiDocValues.getNormDocValues(index2, curField);
if (norms1 != null && norms2 != null)
{
NumericDocValues norms1 = MultiSimpleDocValues.simpleNormValues(index1, curField);
NumericDocValues norms2 = MultiSimpleDocValues.simpleNormValues(index2, curField);
if (norms1 != null && norms2 != null) {
// todo: generalize this (like TestDuelingCodecs assert)
byte[] b1 = (byte[]) norms1.getSource().getArray();
byte[] b2 = (byte[]) norms2.getSource().getArray();
assertEquals(b1.length, b2.length);
for (int i = 0; i < b1.length; i++) {
assertEquals("Norm different for doc " + i + " and field '" + curField + "'.", b1[i], b2[i]);
assertEquals(norms1.size(), norms2.size());
for (int i = 0; i < norms1.size(); i++) {
assertEquals("Norm different for doc " + i + " and field '" + curField + "'.", norms1.get(i), norms2.get(i));
}
}
else
{
assertSame(norms1, norms2);
} else {
assertNull(norms1);
assertNull(norms2);
}
}

5
lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
View File

@ -16,7 +16,6 @@ package org.apache.lucene.index;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
@ -38,13 +37,14 @@ import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.junit.Assume;
public class TestDirectoryReaderReopen extends LuceneTestCase {
public void testReopen() throws Exception {
Assume.assumeTrue(_TestUtil.canUseSimpleNorms());
final Directory dir1 = newDirectory();
createIndex(random(), dir1, false);
@ -196,6 +196,7 @@ public class TestDirectoryReaderReopen extends LuceneTestCase {
}
public void testThreadSafety() throws Exception {
Assume.assumeTrue(_TestUtil.canUseSimpleNorms());
final Directory dir = newDirectory();
// NOTE: this also controls the number of threads!
final int n = _TestUtil.nextInt(random(), 20, 40);

20
lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
View File

@ -55,6 +55,7 @@ import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
@ -65,6 +66,7 @@ import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.junit.Assume;
/**
*
@ -595,7 +597,7 @@ public class TestDocValuesIndexing extends LuceneTestCase {
}
}
return docValues;
}
}
@SuppressWarnings("fallthrough")
private Source getSource(DocValues values) throws IOException {
@ -761,6 +763,7 @@ public class TestDocValuesIndexing extends LuceneTestCase {
}
public void testMultiValuedDocValuesField() throws Exception {
Assume.assumeTrue(_TestUtil.canUseSimpleDV());
Directory d = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), d);
Document doc = new Document();
@ -782,12 +785,13 @@ public class TestDocValuesIndexing extends LuceneTestCase {
w.forceMerge(1);
DirectoryReader r = w.getReader();
w.close();
assertEquals(17, getOnlySegmentReader(r).docValues("field").loadSource().getInt(0));
assertEquals(17, FieldCache.DEFAULT.getInts(getOnlySegmentReader(r), "field", false).get(0));
r.close();
d.close();
}
public void testDifferentTypedDocValuesField() throws Exception {
assumeTrue("requires simple dv", _TestUtil.canUseSimpleDV());
Directory d = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), d);
Document doc = new Document();
@ -809,7 +813,7 @@ public class TestDocValuesIndexing extends LuceneTestCase {
w.forceMerge(1);
DirectoryReader r = w.getReader();
w.close();
assertEquals(17, getOnlySegmentReader(r).docValues("field").loadSource().getInt(0));
assertEquals(17, FieldCache.DEFAULT.getInts(getOnlySegmentReader(r), "field", false).get(0));
r.close();
d.close();
}
@ -1031,6 +1035,7 @@ public class TestDocValuesIndexing extends LuceneTestCase {
// LUCENE-3870
public void testLengthPrefixAcrossTwoPages() throws Exception {
assumeTrue("requires simple dv", _TestUtil.canUseSimpleDV());
Directory d = newDirectory();
IndexWriter w = new IndexWriter(d, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
Document doc = new Document();
@ -1044,14 +1049,15 @@ public class TestDocValuesIndexing extends LuceneTestCase {
w.addDocument(doc);
w.forceMerge(1);
DirectoryReader r = w.getReader();
Source s = getOnlySegmentReader(r).docValues("field").getSource();
BinaryDocValues s = FieldCache.DEFAULT.getTerms(getOnlySegmentReader(r), "field");
BytesRef bytes1 = s.getBytes(0, new BytesRef());
BytesRef bytes1 = new BytesRef();
s.get(0, bytes1);
assertEquals(bytes.length, bytes1.length);
bytes[0] = 0;
assertEquals(b, bytes1);
bytes1 = s.getBytes(1, new BytesRef());
s.get(1, bytes1);
assertEquals(bytes.length, bytes1.length);
bytes[0] = 1;
assertEquals(b, bytes1);
@ -1076,7 +1082,7 @@ public class TestDocValuesIndexing extends LuceneTestCase {
w.close();
d.close();
}
public void testDocValuesUnstored() throws IOException {
//nocommit convert!
Directory dir = newDirectory();

121
lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
View File

@ -38,9 +38,11 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.automaton.AutomatonTestUtil;
import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.automaton.RegExp;
import org.junit.Assume;
/**
* Compares one codec against another
@ -66,6 +68,8 @@ public class TestDuelingCodecs extends LuceneTestCase {
leftCodec = Codec.forName("SimpleText");
rightCodec = new RandomCodec(random());
Assume.assumeTrue(rightCodec.simpleNormsFormat() != null);
leftDir = newDirectory();
rightDir = newDirectory();
@ -105,10 +109,19 @@ public class TestDuelingCodecs extends LuceneTestCase {
@Override
public void tearDown() throws Exception {
leftReader.close();
rightReader.close();
leftDir.close();
rightDir.close();
if (leftReader != null) {
leftReader.close();
}
if (rightReader != null) {
rightReader.close();
}
if (leftDir != null) {
leftDir.close();
}
if (rightDir != null) {
rightDir.close();
}
super.tearDown();
}
@ -506,6 +519,7 @@ public class TestDuelingCodecs extends LuceneTestCase {
* checks that norms are the same across all fields
*/
public void assertNorms(IndexReader leftReader, IndexReader rightReader) throws Exception {
Assume.assumeTrue(_TestUtil.canUseSimpleNorms());
Fields leftFields = MultiFields.getFields(leftReader);
Fields rightFields = MultiFields.getFields(rightReader);
// Fields could be null if there are no postings,
@ -518,8 +532,8 @@ public class TestDuelingCodecs extends LuceneTestCase {
for (String field : leftFields) {
// nocommit cutover to per-segment comparison
DocValues leftNorms = MultiDocValues.getNormDocValues(leftReader, field);
DocValues rightNorms = MultiDocValues.getNormDocValues(rightReader, field);
NumericDocValues leftNorms = MultiSimpleDocValues.simpleNormValues(leftReader, field);
NumericDocValues rightNorms = MultiSimpleDocValues.simpleNormValues(rightReader, field);
if (leftNorms != null && rightNorms != null) {
assertDocValues(leftNorms, rightNorms);
} else {
@ -600,69 +614,52 @@ public class TestDuelingCodecs extends LuceneTestCase {
* checks that docvalues across all fields are equivalent
*/
public void assertDocValues(IndexReader leftReader, IndexReader rightReader) throws Exception {
Set<String> leftValues = getDVFields(leftReader);
Set<String> rightValues = getDVFields(rightReader);
assertEquals(info, leftValues, rightValues);
Set<String> leftFields = getDVFields(leftReader);
Set<String> rightFields = getDVFields(rightReader);
assertEquals(info, leftFields, rightFields);
for (String field : leftValues) {
// nocommit cutover to per-segment comparison
DocValues leftDocValues = MultiDocValues.getDocValues(leftReader, field);
DocValues rightDocValues = MultiDocValues.getDocValues(rightReader, field);
if (leftDocValues != null && rightDocValues != null) {
assertDocValues(leftDocValues, rightDocValues);
} else {
assertNull(leftDocValues);
assertNull(rightDocValues);
for (String field : leftFields) {
{
NumericDocValues leftValues = MultiSimpleDocValues.simpleNumericValues(leftReader, field);
NumericDocValues rightValues = MultiSimpleDocValues.simpleNumericValues(rightReader, field);
if (leftValues != null && rightValues != null) {
assertDocValues(leftValues, rightValues);
} else {
assertNull(leftValues);
assertNull(rightValues);
}
}
{
BinaryDocValues leftValues = MultiSimpleDocValues.simpleBinaryValues(leftReader, field);
BinaryDocValues rightValues = MultiSimpleDocValues.simpleBinaryValues(rightReader, field);
if (leftValues != null && rightValues != null) {
assertEquals(leftValues.size(), rightValues.size());
assertEquals(leftValues.maxLength(), rightValues.maxLength());
assertEquals(leftValues.isFixedLength(), rightValues.isFixedLength());
BytesRef scratchLeft = new BytesRef();
BytesRef scratchRight = new BytesRef();
for(int docID=0;docID<leftValues.size();docID++) {
leftValues.get(docID, scratchLeft);
rightValues.get(docID, scratchRight);
assertEquals(scratchLeft, scratchRight);
}
} else {
assertNull(leftValues);
assertNull(rightValues);
}
}
}
}
public void assertDocValues(DocValues leftDocValues, DocValues rightDocValues) throws Exception {
public void assertDocValues(NumericDocValues leftDocValues, NumericDocValues rightDocValues) throws Exception {
assertNotNull(info, leftDocValues);
assertNotNull(info, rightDocValues);
assertEquals(info, leftDocValues.getType(), rightDocValues.getType());
assertEquals(info, leftDocValues.getValueSize(), rightDocValues.getValueSize());
assertDocValuesSource(leftDocValues.getDirectSource(), rightDocValues.getDirectSource());
assertDocValuesSource(leftDocValues.getSource(), rightDocValues.getSource());
}
/**
* checks source API
*/
public void assertDocValuesSource(DocValues.Source left, DocValues.Source right) throws Exception {
DocValues.Type leftType = left.getType();
assertEquals(info, leftType, right.getType());
switch(leftType) {
case VAR_INTS:
case FIXED_INTS_8:
case FIXED_INTS_16:
case FIXED_INTS_32:
case FIXED_INTS_64:
for (int i = 0; i < leftReader.maxDoc(); i++) {
assertEquals(info, left.getInt(i), right.getInt(i));
}
break;
case FLOAT_32:
case FLOAT_64:
for (int i = 0; i < leftReader.maxDoc(); i++) {
assertEquals(info, left.getFloat(i), right.getFloat(i), 0F);
}
break;
case BYTES_FIXED_STRAIGHT:
case BYTES_FIXED_DEREF:
case BYTES_VAR_STRAIGHT:
case BYTES_VAR_DEREF:
BytesRef b1 = new BytesRef();
BytesRef b2 = new BytesRef();
for (int i = 0; i < leftReader.maxDoc(); i++) {
left.getBytes(i, b1);
right.getBytes(i, b2);
assertEquals(info, b1, b2);
}
break;
// TODO: can we test these?
case BYTES_VAR_SORTED:
case BYTES_FIXED_SORTED:
assertEquals(info, leftDocValues.size(), rightDocValues.size());
for(int docID=0;docID<leftDocValues.size();docID++) {
assertEquals(leftDocValues.get(docID),
rightDocValues.get(docID));
}
}

10
lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
View File

@ -30,6 +30,7 @@ import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.junit.Assume;
/**
* Tests the maxTermFrequency statistic in FieldInvertState
@ -67,9 +68,12 @@ public class TestMaxTermFrequency extends LuceneTestCase {
}
public void test() throws Exception {
byte fooNorms[] = (byte[])MultiDocValues.getNormDocValues(reader, "foo").getSource().getArray();
for (int i = 0; i < reader.maxDoc(); i++)
assertEquals(expected.get(i).intValue(), fooNorms[i] & 0xff);
// nocommit remove
Assume.assumeTrue(_TestUtil.canUseSimpleNorms());
NumericDocValues fooNorms = MultiSimpleDocValues.simpleNormValues(reader, "foo");
for (int i = 0; i < reader.maxDoc(); i++) {
assertEquals(expected.get(i).intValue(), fooNorms.get(i) & 0xff);
}
}
/**

20
lucene/core/src/test/org/apache/lucene/index/TestNorms.java
View File

@ -37,12 +37,14 @@ import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.junit.Assume;
/**
* Test that norms info is preserved during index life - including
* separate norms, addDocument, addIndexes, forceMerge.
*/
@SuppressCodecs({ "SimpleText", "Memory", "Direct" })
// nocommit put SimpleText back in suppress list:
@SuppressCodecs({ "Memory", "Direct" })
@Slow
public class TestNorms extends LuceneTestCase {
final String byteTestField = "normsTestByte";
@ -66,6 +68,8 @@ public class TestNorms extends LuceneTestCase {
// LUCENE-1260
public void testCustomEncoder() throws Exception {
// nocommit remove:
Assume.assumeTrue(_TestUtil.canUseSimpleNorms());
Directory dir = newDirectory();
IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
config.setSimilarity(new CustomNormEncodingSimilarity());
@ -84,13 +88,15 @@ public class TestNorms extends LuceneTestCase {
IndexReader reader = writer.getReader();
writer.close();
byte fooNorms[] = (byte[]) MultiDocValues.getNormDocValues(reader, "foo").getSource().getArray();
for (int i = 0; i < reader.maxDoc(); i++)
assertEquals(0, fooNorms[i]);
NumericDocValues fooNorms = MultiSimpleDocValues.simpleNormValues(reader, "foo");
for (int i = 0; i < reader.maxDoc(); i++) {
assertEquals(0, fooNorms.get(i));
}
byte barNorms[] = (byte[]) MultiDocValues.getNormDocValues(reader, "bar").getSource().getArray();
for (int i = 0; i < reader.maxDoc(); i++)
assertEquals(1, barNorms[i]);
NumericDocValues barNorms = MultiSimpleDocValues.simpleNormValues(reader, "bar");
for (int i = 0; i < reader.maxDoc(); i++) {
assertEquals(1, barNorms.get(i));
}
reader.close();
dir.close();

26
lucene/core/src/test/org/apache/lucene/index/TestOmitNorms.java
View File

@ -19,8 +19,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@ -28,6 +26,9 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.junit.Assume;
public class TestOmitNorms extends LuceneTestCase {
// Tests whether the DocumentWriter correctly enable the
@ -265,7 +266,9 @@ public class TestOmitNorms extends LuceneTestCase {
* Indexes at least 1 document with f1, and at least 1 document with f2.
* returns the norms for "field".
*/
byte[] getNorms(String field, Field f1, Field f2) throws IOException {
NumericDocValues getNorms(String field, Field f1, Field f2) throws IOException {
// nocommit remove
Assume.assumeTrue(_TestUtil.canUseSimpleNorms());
Directory dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy());
RandomIndexWriter riw = new RandomIndexWriter(random(), dir, iwc);
@ -290,16 +293,21 @@ public class TestOmitNorms extends LuceneTestCase {
IndexReader ir1 = riw.getReader();
// todo: generalize
DocValues dv1 = MultiDocValues.getNormDocValues(ir1, field);
byte[] norms1 = dv1 == null ? null : (byte[]) dv1.getSource().getArray();
NumericDocValues norms1 = MultiSimpleDocValues.simpleNormValues(ir1, field);
// fully merge and validate MultiNorms against single segment.
riw.forceMerge(1);
DirectoryReader ir2 = riw.getReader();
DocValues dv2 = getOnlySegmentReader(ir2).normValues(field);
byte[] norms2 = dv2 == null ? null : (byte[]) dv2.getSource().getArray();
assertArrayEquals(norms1, norms2);
NumericDocValues norms2 = getOnlySegmentReader(ir2).simpleNormValues(field);
if (norms1 == null) {
assertNull(norms2);
} else {
assertEquals(norms1.size(), norms2.size());
for(int docID=0;docID<norms1.size();docID++) {
assertEquals(norms1.get(docID), norms2.get(docID));
}
}
ir1.close();
ir2.close();
riw.close();

2
lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
View File

@ -30,6 +30,7 @@ import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.packed.PackedInts;
import org.junit.Assume;
public class TestSegmentMerger extends LuceneTestCase {
@ -78,6 +79,7 @@ public class TestSegmentMerger extends LuceneTestCase {
}
public void testMerge() throws IOException {
Assume.assumeTrue(_TestUtil.canUseSimpleNorms());
final Codec codec = Codec.getDefault();
final SegmentInfo si = new SegmentInfo(mergedDir, Constants.LUCENE_MAIN_VERSION, mergedSegment, -1, false, codec, null, null);

6
lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java
View File

@ -29,6 +29,7 @@ import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.junit.Assume;
public class TestSegmentReader extends LuceneTestCase {
private Directory dir;
@ -173,7 +174,8 @@ public class TestSegmentReader extends LuceneTestCase {
}
public static void checkNorms(AtomicReader reader) throws IOException {
// test omit norms
Assume.assumeTrue(_TestUtil.canUseSimpleNorms());
// test omit norms
for (int i=0; i<DocHelper.fields.length; i++) {
IndexableField f = DocHelper.fields[i];
if (f.fieldType().indexed()) {
@ -181,7 +183,7 @@ public class TestSegmentReader extends LuceneTestCase {
assertEquals(reader.normValues(f.name()) != null, !DocHelper.noNorms.containsKey(f.name()));
if (reader.normValues(f.name()) == null) {
// test for norms of null
DocValues norms = MultiDocValues.getNormDocValues(reader, f.name());
NumericDocValues norms = MultiSimpleDocValues.simpleNormValues(reader, f.name());
assertNull(norms);
}
}

11
lucene/core/src/test/org/apache/lucene/index/TestSimpleDocValuesIndexing.java
View File

@ -25,8 +25,8 @@ import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Map;
import java.util.Random;
import java.util.Set;
@ -47,9 +47,9 @@ import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DocValues.SortedSource;
import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.index.DocValues.SourceCache.DirectSourceCache;
import org.apache.lucene.index.DocValues.SourceCache;
import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.index.DocValues.SourceCache.DirectSourceCache;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.DocIdSetIterator;
@ -63,6 +63,7 @@ import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.junit.Assume;
/**
*
@ -79,6 +80,7 @@ public class TestSimpleDocValuesIndexing extends LuceneTestCase {
* Simple test case to show how to use the API
*/
public void testDocValuesSimple() throws IOException {
Assume.assumeTrue(_TestUtil.canUseSimpleDV());
Directory dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, writerConfig(false));
for (int i = 0; i < 5; i++) {
@ -107,11 +109,10 @@ public class TestSimpleDocValuesIndexing extends LuceneTestCase {
TopDocs search = searcher.search(query, 10);
assertEquals(5, search.totalHits);
ScoreDoc[] scoreDocs = search.scoreDocs;
DocValues docValues = MultiDocValues.getDocValues(reader, "docId");
Source source = docValues.getSource();
NumericDocValues docValues = getOnlySegmentReader(reader).getNumericDocValues("docId");
for (int i = 0; i < scoreDocs.length; i++) {
assertEquals(i, scoreDocs[i].doc);
assertEquals(i, source.getInt(scoreDocs[i].doc));
assertEquals(i, docValues.get(scoreDocs[i].doc));
}
reader.close();
dir.close();

10
lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java
View File

@ -27,11 +27,11 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.junit.Assume;
/**
* Tests the uniqueTermCount statistic in FieldInvertState
@ -69,9 +69,11 @@ public class TestUniqueTermCount extends LuceneTestCase {
}
public void test() throws Exception {
byte fooNorms[] = (byte[])MultiDocValues.getNormDocValues(reader, "foo").getSource().getArray();
for (int i = 0; i < reader.maxDoc(); i++)
assertEquals(expected.get(i).intValue(), fooNorms[i] & 0xff);
Assume.assumeTrue(_TestUtil.canUseSimpleNorms());
NumericDocValues fooNorms = MultiSimpleDocValues.simpleNormValues(reader, "foo");
for (int i = 0; i < reader.maxDoc(); i++) {
assertEquals(expected.get(i).intValue(), fooNorms.get(i) & 0xff);
}
}
/**

17
lucene/core/src/test/org/apache/lucene/search/TestDocValuesScoring.java
View File

@ -23,7 +23,6 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FloatDocValuesField;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Norm;
@ -67,10 +66,10 @@ public class TestDocValuesScoring extends LuceneTestCase {
iw.close();
// no boosting
IndexSearcher searcher1 = newSearcher(ir);
IndexSearcher searcher1 = newSearcher(ir, false);
final Similarity base = searcher1.getSimilarity();
// boosting
IndexSearcher searcher2 = newSearcher(ir);
IndexSearcher searcher2 = newSearcher(ir, false);
searcher2.setSimilarity(new PerFieldSimilarityWrapper() {
final Similarity fooSim = new BoostingSimilarity(base, "foo_boost");
@ -160,17 +159,17 @@ public class TestDocValuesScoring extends LuceneTestCase {
@Override
public ExactSimScorer exactSimScorer(SimWeight stats, AtomicReaderContext context) throws IOException {
final ExactSimScorer sub = sim.exactSimScorer(stats, context);
final Source values = context.reader().docValues(boostField).getSource();
final FieldCache.Floats values = FieldCache.DEFAULT.getFloats(context.reader(), boostField, false);
return new ExactSimScorer() {
@Override
public float score(int doc, int freq) {
return (float) values.getFloat(doc) * sub.score(doc, freq);
return values.get(doc) * sub.score(doc, freq);
}
@Override
public Explanation explain(int doc, Explanation freq) {
Explanation boostExplanation = new Explanation((float) values.getFloat(doc), "indexDocValue(" + boostField + ")");
Explanation boostExplanation = new Explanation(values.get(doc), "indexDocValue(" + boostField + ")");
Explanation simExplanation = sub.explain(doc, freq);
Explanation expl = new Explanation(boostExplanation.getValue() * simExplanation.getValue(), "product of:");
expl.addDetail(boostExplanation);
@ -183,12 +182,12 @@ public class TestDocValuesScoring extends LuceneTestCase {
@Override
public SloppySimScorer sloppySimScorer(SimWeight stats, AtomicReaderContext context) throws IOException {
final SloppySimScorer sub = sim.sloppySimScorer(stats, context);
final Source values = context.reader().docValues(boostField).getSource();
final FieldCache.Floats values = FieldCache.DEFAULT.getFloats(context.reader(), boostField, false);
return new SloppySimScorer() {
@Override
public float score(int doc, float freq) {
return (float) values.getFloat(doc) * sub.score(doc, freq);
return values.get(doc) * sub.score(doc, freq);
}
@Override
@ -203,7 +202,7 @@ public class TestDocValuesScoring extends LuceneTestCase {
@Override
public Explanation explain(int doc, Explanation freq) {
Explanation boostExplanation = new Explanation((float) values.getFloat(doc), "indexDocValue(" + boostField + ")");
Explanation boostExplanation = new Explanation(values.get(doc), "indexDocValue(" + boostField + ")");
Explanation simExplanation = sub.explain(doc, freq);
Explanation expl = new Explanation(boostExplanation.getValue() * simExplanation.getValue(), "product of:");
expl.addDetail(boostExplanation);

4
lucene/core/src/test/org/apache/lucene/search/TestSort.java
View File

@ -1388,14 +1388,14 @@ public class TestSort extends LuceneTestCase {
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
final int maxDoc = context.reader().maxDoc();
final DocValues.Source idSource = context.reader().docValues("id").getSource();
final FieldCache.Ints idSource = FieldCache.DEFAULT.getInts(context.reader(), "id", false);
assertNotNull(idSource);
final FixedBitSet bits = new FixedBitSet(maxDoc);
for(int docID=0;docID<maxDoc;docID++) {
if (random.nextFloat() <= density && (acceptDocs == null || acceptDocs.get(docID))) {
bits.set(docID);
//System.out.println(" acc id=" + idSource.getInt(docID) + " docID=" + docID);
matchValues.add(docValues.get((int) idSource.getInt(docID)));
matchValues.add(docValues.get(idSource.get(docID)));
}
}

202
lucene/test-framework/src/java/org/apache/lucene/index/MultiSimpleDocValues.java Normal file
View File

@ -0,0 +1,202 @@
package org.apache.lucene.index;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.List;
import org.apache.lucene.util.BytesRef;
public class MultiSimpleDocValues {
public static NumericDocValues simpleNormValues(final IndexReader r, final String field) throws IOException {
final List<AtomicReaderContext> leaves = r.leaves();
boolean anyReal = false;
long minValue = Long.MAX_VALUE;
long maxValue = Long.MIN_VALUE;
for(AtomicReaderContext ctx : leaves) {
NumericDocValues norms = ctx.reader().simpleNormValues(field);
if (norms == null) {
norms = new NumericDocValues.EMPTY(ctx.reader().maxDoc());
} else {
anyReal = true;
}
maxValue = Math.max(norms.maxValue(), maxValue);
minValue = Math.min(norms.minValue(), minValue);
}
if (!anyReal) {
return null;
} else {
final long finalMaxValue = maxValue;
final long finalMinValue = minValue;
return new NumericDocValues() {
@Override
public long get(int docID) {
int subIndex = ReaderUtil.subIndex(docID, leaves);
NumericDocValues norms;
try {
norms = leaves.get(subIndex).reader().simpleNormValues(field);
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
if (norms == null) {
return 0;
} else {
return norms.get(docID - leaves.get(subIndex).docBase);
}
}
@Override
public int size() {
return r.maxDoc();
}
@Override
public long minValue() {
return finalMinValue;
}
@Override
public long maxValue() {
return finalMaxValue;
}
};
}
}
public static NumericDocValues simpleNumericValues(final IndexReader r, final String field) throws IOException {
final List<AtomicReaderContext> leaves = r.leaves();
boolean anyReal = false;
long minValue = Long.MAX_VALUE;
long maxValue = Long.MIN_VALUE;
for(AtomicReaderContext ctx : leaves) {
NumericDocValues values = ctx.reader().getNumericDocValues(field);
if (values == null) {
values = new NumericDocValues.EMPTY(ctx.reader().maxDoc());
} else {
anyReal = true;
}
maxValue = Math.max(values.maxValue(), maxValue);
minValue = Math.min(values.minValue(), minValue);
}
if (!anyReal) {
return null;
} else {
final long finalMaxValue = maxValue;
final long finalMinValue = minValue;
return new NumericDocValues() {
@Override
public long get(int docID) {
int subIndex = ReaderUtil.subIndex(docID, leaves);
NumericDocValues values;
try {
values = leaves.get(subIndex).reader().getNumericDocValues(field);
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
if (values == null) {
return 0;
} else {
return values.get(docID - leaves.get(subIndex).docBase);
}
}
@Override
public int size() {
return r.maxDoc();
}
@Override
public long minValue() {
return finalMinValue;
}
@Override
public long maxValue() {
return finalMaxValue;
}
};
}
}
public static BinaryDocValues simpleBinaryValues(final IndexReader r, final String field) throws IOException {
final List<AtomicReaderContext> leaves = r.leaves();
boolean anyReal = false;
int maxLength = -1;
for(AtomicReaderContext ctx : leaves) {
BinaryDocValues values = ctx.reader().getBinaryDocValues(field);
if (values == null) {
values = new BinaryDocValues.EMPTY(ctx.reader().maxDoc());
} else {
anyReal = true;
}
maxLength = Math.max(maxLength, values.maxLength());
}
if (!anyReal) {
return null;
} else {
final int finalMaxLength = maxLength;
return new BinaryDocValues() {
@Override
public void get(int docID, BytesRef result) {
int subIndex = ReaderUtil.subIndex(docID, leaves);
BinaryDocValues values;
try {
values = leaves.get(subIndex).reader().getBinaryDocValues(field);
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
if (values != null) {
values.get(docID - leaves.get(subIndex).docBase, result);
} else {
result.length = 0;
result.bytes = BinaryDocValues.MISSING;
}
}
@Override
public int size() {
return r.maxDoc();
}
@Override
public boolean isFixedLength() {
// Harmless lie?
return false;
}
@Override
public int maxLength() {
return finalMaxLength;
}
};
}
}
}

5
lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java
View File

@ -715,6 +715,11 @@ public class _TestUtil {
return Codec.getDefault().simpleDocValuesFormat() != null;
}
// nocommit remove this once all codecs support simple norms!!
public static boolean canUseSimpleNorms() {
return Codec.getDefault().simpleNormsFormat() != null;
}
public static boolean anyFilesExceptWriteLock(Directory dir) throws IOException {
String[] files = dir.listAll();
if (files.length > 1 || (files.length == 1 && !files[0].equals("write.lock"))) {