mirror of
https://github.com/apache/lucene.git
synced 2025-02-08 02:58:58 +00:00
dump 4.0-internal specific tests (tested in BaseDocValuesFormatTestCase)
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4547@1439725 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir
parent
726659bf9a
commit
348ed1d255
@ -1,469 +0,0 @@
|
||||
package org.apache.lucene.codecs.lucene40.values;
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
import java.util.Comparator;
|
||||
import java.util.Random;
|
||||
|
||||
import org.apache.lucene.analysis.Analyzer;
|
||||
import org.apache.lucene.analysis.TokenStream;
|
||||
import org.apache.lucene.document.FieldType;
|
||||
import org.apache.lucene.index.IndexableField;
|
||||
import org.apache.lucene.index.IndexableFieldType;
|
||||
import org.apache.lucene.index.StorableField;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.Counter;
|
||||
import org.apache.lucene.util.LuceneTestCase;
|
||||
import org.apache.lucene.util.UnicodeUtil;
|
||||
import org.apache.lucene.util._TestUtil;
|
||||
import org.apache.lucene.util.packed.PackedInts;
|
||||
|
||||
// TODO: some of this should be under lucene40 codec tests? is talking to codec directly?f
|
||||
public class TestDocValues extends LuceneTestCase {
|
||||
/* nocommit
|
||||
private static final Comparator<BytesRef> COMP = BytesRef.getUTF8SortedAsUnicodeComparator();
|
||||
// TODO -- for sorted test, do our own Sort of the
|
||||
// values and verify it's identical
|
||||
|
||||
public void testBytesStraight() throws IOException {
|
||||
runTestBytes(Bytes.Mode.STRAIGHT, true);
|
||||
runTestBytes(Bytes.Mode.STRAIGHT, false);
|
||||
}
|
||||
|
||||
public void testBytesDeref() throws IOException {
|
||||
runTestBytes(Bytes.Mode.DEREF, true);
|
||||
runTestBytes(Bytes.Mode.DEREF, false);
|
||||
}
|
||||
|
||||
public void testBytesSorted() throws IOException {
|
||||
runTestBytes(Bytes.Mode.SORTED, true);
|
||||
runTestBytes(Bytes.Mode.SORTED, false);
|
||||
}
|
||||
|
||||
public void runTestBytes(final Bytes.Mode mode, final boolean fixedSize)
|
||||
throws IOException {
|
||||
DocValueHolder valueHolder = new DocValueHolder();
|
||||
valueHolder.comp = COMP;
|
||||
final BytesRef bytesRef = new BytesRef();
|
||||
|
||||
Directory dir = newDirectory();
|
||||
final Counter trackBytes = Counter.newCounter();
|
||||
DocValuesConsumer w = Bytes.getWriter(dir, "test", mode, fixedSize, COMP, trackBytes, newIOContext(random()),
|
||||
random().nextFloat() * PackedInts.FAST);
|
||||
int maxDoc = 220;
|
||||
final String[] values = new String[maxDoc];
|
||||
final int fixedLength = 1 + atLeast(50);
|
||||
for (int i = 0; i < 100; i++) {
|
||||
final String s;
|
||||
if (i > 0 && random().nextInt(5) <= 2) {
|
||||
// use prior value
|
||||
s = values[2 * random().nextInt(i)];
|
||||
} else {
|
||||
s = _TestUtil.randomFixedByteLengthUnicodeString(random(), fixedSize? fixedLength : 1 + random().nextInt(39));
|
||||
}
|
||||
values[2 * i] = s;
|
||||
|
||||
UnicodeUtil.UTF16toUTF8(s, 0, s.length(), bytesRef);
|
||||
valueHolder.bytes = bytesRef;
|
||||
w.add(2 * i, valueHolder);
|
||||
}
|
||||
w.finish(maxDoc);
|
||||
assertEquals(0, trackBytes.get());
|
||||
|
||||
DocValues r = Bytes.getValues(dir, "test", mode, fixedSize, maxDoc, COMP, newIOContext(random()));
|
||||
|
||||
// Verify we can load source twice:
|
||||
for (int iter = 0; iter < 2; iter++) {
|
||||
Source s;
|
||||
DocValues.SortedSource ss;
|
||||
if (mode == Bytes.Mode.SORTED) {
|
||||
// default is unicode so we can simply pass null here
|
||||
s = ss = getSortedSource(r);
|
||||
} else {
|
||||
s = getSource(r);
|
||||
ss = null;
|
||||
}
|
||||
for (int i = 0; i < 100; i++) {
|
||||
final int idx = 2 * i;
|
||||
assertNotNull("doc " + idx + "; value=" + values[idx], s.getBytes(idx,
|
||||
bytesRef));
|
||||
assertEquals("doc " + idx, values[idx], s.getBytes(idx, bytesRef)
|
||||
.utf8ToString());
|
||||
if (ss != null) {
|
||||
assertEquals("doc " + idx, values[idx], ss.getByOrd(ss.ord(idx),
|
||||
bytesRef).utf8ToString());
|
||||
int ord = ss
|
||||
.getOrdByValue(new BytesRef(values[idx]), new BytesRef());
|
||||
assertTrue(ord >= 0);
|
||||
assertEquals(ss.ord(idx), ord);
|
||||
}
|
||||
}
|
||||
|
||||
// Lookup random strings:
|
||||
if (mode == Bytes.Mode.SORTED) {
|
||||
final int valueCount = ss.getValueCount();
|
||||
Random random = random();
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
BytesRef bytesValue = new BytesRef(_TestUtil.randomFixedByteLengthUnicodeString(random, fixedSize? fixedLength : 1 + random.nextInt(39)));
|
||||
int ord = ss.getOrdByValue(bytesValue, new BytesRef());
|
||||
if (ord >= 0) {
|
||||
assertTrue(bytesValue
|
||||
.bytesEquals(ss.getByOrd(ord, bytesRef)));
|
||||
int count = 0;
|
||||
for (int k = 0; k < 100; k++) {
|
||||
if (bytesValue.utf8ToString().equals(values[2 * k])) {
|
||||
assertEquals(ss.ord(2 * k), ord);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
assertTrue(count > 0);
|
||||
} else {
|
||||
assert ord < 0;
|
||||
int insertIndex = (-ord)-1;
|
||||
if (insertIndex == 0) {
|
||||
final BytesRef firstRef = ss.getByOrd(1, bytesRef);
|
||||
// random string was before our first
|
||||
assertTrue(firstRef.compareTo(bytesValue) > 0);
|
||||
} else if (insertIndex == valueCount) {
|
||||
final BytesRef lastRef = ss.getByOrd(valueCount-1, bytesRef);
|
||||
// random string was after our last
|
||||
assertTrue(lastRef.compareTo(bytesValue) < 0);
|
||||
} else {
|
||||
// TODO: I don't think this actually needs a deep copy?
|
||||
final BytesRef before = BytesRef.deepCopyOf(ss.getByOrd(insertIndex-1, bytesRef));
|
||||
BytesRef after = ss.getByOrd(insertIndex, bytesRef);
|
||||
assertTrue(COMP.compare(before, bytesValue) < 0);
|
||||
assertTrue(COMP.compare(bytesValue, after) < 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
r.close();
|
||||
dir.close();
|
||||
}
|
||||
|
||||
public void testVariableIntsLimits() throws IOException {
|
||||
long[][] minMax = new long[][] { { Long.MIN_VALUE, Long.MAX_VALUE },
|
||||
{ Long.MIN_VALUE + 1, 1 }, { -1, Long.MAX_VALUE },
|
||||
{ Long.MIN_VALUE, -1 }, { 1, Long.MAX_VALUE },
|
||||
{ -1, Long.MAX_VALUE - 1 }, { Long.MIN_VALUE + 2, 1 }, };
|
||||
Type[] expectedTypes = new Type[] { Type.FIXED_INTS_64,
|
||||
Type.FIXED_INTS_64, Type.FIXED_INTS_64,
|
||||
Type.FIXED_INTS_64, Type.VAR_INTS, Type.VAR_INTS,
|
||||
Type.VAR_INTS, };
|
||||
DocValueHolder valueHolder = new DocValueHolder();
|
||||
for (int i = 0; i < minMax.length; i++) {
|
||||
Directory dir = newDirectory();
|
||||
final Counter trackBytes = Counter.newCounter();
|
||||
DocValuesConsumer w = Ints.getWriter(dir, "test", trackBytes, Type.VAR_INTS, newIOContext(random()));
|
||||
valueHolder.numberValue = minMax[i][0];
|
||||
w.add(0, valueHolder);
|
||||
valueHolder.numberValue = minMax[i][1];
|
||||
w.add(1, valueHolder);
|
||||
w.finish(2);
|
||||
assertEquals(0, trackBytes.get());
|
||||
DocValues r = Ints.getValues(dir, "test", 2, Type.VAR_INTS, newIOContext(random()));
|
||||
Source source = getSource(r);
|
||||
assertEquals(i + " with min: " + minMax[i][0] + " max: " + minMax[i][1],
|
||||
expectedTypes[i], source.getType());
|
||||
assertEquals(minMax[i][0], source.getInt(0));
|
||||
assertEquals(minMax[i][1], source.getInt(1));
|
||||
|
||||
r.close();
|
||||
dir.close();
|
||||
}
|
||||
}
|
||||
|
||||
public void testVInts() throws IOException {
|
||||
testInts(Type.VAR_INTS, 63);
|
||||
}
|
||||
|
||||
public void testFixedInts() throws IOException {
|
||||
testInts(Type.FIXED_INTS_64, 63);
|
||||
testInts(Type.FIXED_INTS_32, 31);
|
||||
testInts(Type.FIXED_INTS_16, 15);
|
||||
testInts(Type.FIXED_INTS_8, 7);
|
||||
|
||||
}
|
||||
|
||||
public void testGetInt8Array() throws IOException {
|
||||
DocValueHolder valueHolder = new DocValueHolder();
|
||||
byte[] sourceArray = new byte[] {1,2,3};
|
||||
Directory dir = newDirectory();
|
||||
final Counter trackBytes = Counter.newCounter();
|
||||
DocValuesConsumer w = Ints.getWriter(dir, "test", trackBytes, Type.FIXED_INTS_8, newIOContext(random()));
|
||||
for (int i = 0; i < sourceArray.length; i++) {
|
||||
valueHolder.numberValue = (long) sourceArray[i];
|
||||
w.add(i, valueHolder);
|
||||
}
|
||||
w.finish(sourceArray.length);
|
||||
DocValues r = Ints.getValues(dir, "test", sourceArray.length, Type.FIXED_INTS_8, newIOContext(random()));
|
||||
Source source = r.getSource();
|
||||
assertTrue(source.hasArray());
|
||||
byte[] loaded = ((byte[])source.getArray());
|
||||
assertEquals(loaded.length, sourceArray.length);
|
||||
for (int i = 0; i < loaded.length; i++) {
|
||||
assertEquals("value didn't match at index " + i, sourceArray[i], loaded[i]);
|
||||
}
|
||||
r.close();
|
||||
dir.close();
|
||||
}
|
||||
|
||||
public void testGetInt16Array() throws IOException {
|
||||
DocValueHolder valueHolder = new DocValueHolder();
|
||||
short[] sourceArray = new short[] {1,2,3};
|
||||
Directory dir = newDirectory();
|
||||
final Counter trackBytes = Counter.newCounter();
|
||||
DocValuesConsumer w = Ints.getWriter(dir, "test", trackBytes, Type.FIXED_INTS_16, newIOContext(random()));
|
||||
for (int i = 0; i < sourceArray.length; i++) {
|
||||
valueHolder.numberValue = (long) sourceArray[i];
|
||||
w.add(i, valueHolder);
|
||||
}
|
||||
w.finish(sourceArray.length);
|
||||
DocValues r = Ints.getValues(dir, "test", sourceArray.length, Type.FIXED_INTS_16, newIOContext(random()));
|
||||
Source source = r.getSource();
|
||||
assertTrue(source.hasArray());
|
||||
short[] loaded = ((short[])source.getArray());
|
||||
assertEquals(loaded.length, sourceArray.length);
|
||||
for (int i = 0; i < loaded.length; i++) {
|
||||
assertEquals("value didn't match at index " + i, sourceArray[i], loaded[i]);
|
||||
}
|
||||
r.close();
|
||||
dir.close();
|
||||
}
|
||||
|
||||
public void testGetInt64Array() throws IOException {
|
||||
DocValueHolder valueHolder = new DocValueHolder();
|
||||
long[] sourceArray = new long[] {1,2,3};
|
||||
Directory dir = newDirectory();
|
||||
final Counter trackBytes = Counter.newCounter();
|
||||
DocValuesConsumer w = Ints.getWriter(dir, "test", trackBytes, Type.FIXED_INTS_64, newIOContext(random()));
|
||||
for (int i = 0; i < sourceArray.length; i++) {
|
||||
valueHolder.numberValue = sourceArray[i];
|
||||
w.add(i, valueHolder);
|
||||
}
|
||||
w.finish(sourceArray.length);
|
||||
DocValues r = Ints.getValues(dir, "test", sourceArray.length, Type.FIXED_INTS_64, newIOContext(random()));
|
||||
Source source = r.getSource();
|
||||
assertTrue(source.hasArray());
|
||||
long[] loaded = ((long[])source.getArray());
|
||||
assertEquals(loaded.length, sourceArray.length);
|
||||
for (int i = 0; i < loaded.length; i++) {
|
||||
assertEquals("value didn't match at index " + i, sourceArray[i], loaded[i]);
|
||||
}
|
||||
r.close();
|
||||
dir.close();
|
||||
}
|
||||
|
||||
public void testGetInt32Array() throws IOException {
|
||||
DocValueHolder valueHolder = new DocValueHolder();
|
||||
int[] sourceArray = new int[] {1,2,3};
|
||||
Directory dir = newDirectory();
|
||||
final Counter trackBytes = Counter.newCounter();
|
||||
DocValuesConsumer w = Ints.getWriter(dir, "test", trackBytes, Type.FIXED_INTS_32, newIOContext(random()));
|
||||
for (int i = 0; i < sourceArray.length; i++) {
|
||||
valueHolder.numberValue = (long) sourceArray[i];
|
||||
w.add(i, valueHolder);
|
||||
}
|
||||
w.finish(sourceArray.length);
|
||||
DocValues r = Ints.getValues(dir, "test", sourceArray.length, Type.FIXED_INTS_32, newIOContext(random()));
|
||||
Source source = r.getSource();
|
||||
assertTrue(source.hasArray());
|
||||
int[] loaded = ((int[])source.getArray());
|
||||
assertEquals(loaded.length, sourceArray.length);
|
||||
for (int i = 0; i < loaded.length; i++) {
|
||||
assertEquals("value didn't match at index " + i, sourceArray[i], loaded[i]);
|
||||
}
|
||||
r.close();
|
||||
dir.close();
|
||||
}
|
||||
|
||||
public void testGetFloat32Array() throws IOException {
|
||||
DocValueHolder valueHolder = new DocValueHolder();
|
||||
float[] sourceArray = new float[] {1,2,3};
|
||||
Directory dir = newDirectory();
|
||||
final Counter trackBytes = Counter.newCounter();
|
||||
DocValuesConsumer w = Floats.getWriter(dir, "test", trackBytes, newIOContext(random()), Type.FLOAT_32);
|
||||
for (int i = 0; i < sourceArray.length; i++) {
|
||||
valueHolder.numberValue = sourceArray[i];
|
||||
w.add(i, valueHolder);
|
||||
}
|
||||
w.finish(sourceArray.length);
|
||||
DocValues r = Floats.getValues(dir, "test", 3, newIOContext(random()), Type.FLOAT_32);
|
||||
Source source = r.getSource();
|
||||
assertTrue(source.hasArray());
|
||||
float[] loaded = ((float[])source.getArray());
|
||||
assertEquals(loaded.length, sourceArray.length);
|
||||
for (int i = 0; i < loaded.length; i++) {
|
||||
assertEquals("value didn't match at index " + i, sourceArray[i], loaded[i], 0.0f);
|
||||
}
|
||||
r.close();
|
||||
dir.close();
|
||||
}
|
||||
|
||||
public void testGetFloat64Array() throws IOException {
|
||||
DocValueHolder valueHolder = new DocValueHolder();
|
||||
double[] sourceArray = new double[] {1,2,3};
|
||||
Directory dir = newDirectory();
|
||||
final Counter trackBytes = Counter.newCounter();
|
||||
DocValuesConsumer w = Floats.getWriter(dir, "test", trackBytes, newIOContext(random()), Type.FLOAT_64);
|
||||
for (int i = 0; i < sourceArray.length; i++) {
|
||||
valueHolder.numberValue = sourceArray[i];
|
||||
w.add(i, valueHolder);
|
||||
}
|
||||
w.finish(sourceArray.length);
|
||||
DocValues r = Floats.getValues(dir, "test", 3, newIOContext(random()), Type.FLOAT_64);
|
||||
Source source = r.getSource();
|
||||
assertTrue(source.hasArray());
|
||||
double[] loaded = ((double[])source.getArray());
|
||||
assertEquals(loaded.length, sourceArray.length);
|
||||
for (int i = 0; i < loaded.length; i++) {
|
||||
assertEquals("value didn't match at index " + i, sourceArray[i], loaded[i], 0.0d);
|
||||
}
|
||||
r.close();
|
||||
dir.close();
|
||||
}
|
||||
|
||||
private void testInts(Type type, int maxBit) throws IOException {
|
||||
DocValueHolder valueHolder = new DocValueHolder();
|
||||
long maxV = 1;
|
||||
final int NUM_VALUES = 333 + random().nextInt(333);
|
||||
final long[] values = new long[NUM_VALUES];
|
||||
for (int rx = 1; rx < maxBit; rx++, maxV *= 2) {
|
||||
Directory dir = newDirectory();
|
||||
final Counter trackBytes = Counter.newCounter();
|
||||
DocValuesConsumer w = Ints.getWriter(dir, "test", trackBytes, type, newIOContext(random()));
|
||||
for (int i = 0; i < NUM_VALUES; i++) {
|
||||
final long v = _TestUtil.nextLong(random(), -maxV, maxV);
|
||||
valueHolder.numberValue = values[i] = v;
|
||||
w.add(i, valueHolder);
|
||||
}
|
||||
final int additionalDocs = 1 + random().nextInt(9);
|
||||
w.finish(NUM_VALUES + additionalDocs);
|
||||
assertEquals(0, trackBytes.get());
|
||||
|
||||
DocValues r = Ints.getValues(dir, "test", NUM_VALUES + additionalDocs, type, newIOContext(random()));
|
||||
for (int iter = 0; iter < 2; iter++) {
|
||||
Source s = getSource(r);
|
||||
assertEquals(type, s.getType());
|
||||
for (int i = 0; i < NUM_VALUES; i++) {
|
||||
final long v = s.getInt(i);
|
||||
assertEquals("index " + i, values[i], v);
|
||||
}
|
||||
}
|
||||
|
||||
r.close();
|
||||
dir.close();
|
||||
}
|
||||
}
|
||||
|
||||
public void testFloats4() throws IOException {
|
||||
runTestFloats(Type.FLOAT_32);
|
||||
}
|
||||
|
||||
private void runTestFloats(Type type) throws IOException {
|
||||
DocValueHolder valueHolder = new DocValueHolder();
|
||||
Directory dir = newDirectory();
|
||||
final Counter trackBytes = Counter.newCounter();
|
||||
DocValuesConsumer w = Floats.getWriter(dir, "test", trackBytes, newIOContext(random()), type);
|
||||
final int NUM_VALUES = 777 + random().nextInt(777);
|
||||
final double[] values = new double[NUM_VALUES];
|
||||
for (int i = 0; i < NUM_VALUES; i++) {
|
||||
final double v = type == Type.FLOAT_32 ? random().nextFloat() : random()
|
||||
.nextDouble();
|
||||
valueHolder.numberValue = values[i] = v;
|
||||
w.add(i, valueHolder);
|
||||
}
|
||||
final int additionalValues = 1 + random().nextInt(10);
|
||||
w.finish(NUM_VALUES + additionalValues);
|
||||
assertEquals(0, trackBytes.get());
|
||||
|
||||
DocValues r = Floats.getValues(dir, "test", NUM_VALUES + additionalValues, newIOContext(random()), type);
|
||||
for (int iter = 0; iter < 2; iter++) {
|
||||
Source s = getSource(r);
|
||||
for (int i = 0; i < NUM_VALUES; i++) {
|
||||
assertEquals("" + i, values[i], s.getFloat(i), 0.0f);
|
||||
}
|
||||
}
|
||||
r.close();
|
||||
dir.close();
|
||||
}
|
||||
|
||||
public void testFloats8() throws IOException {
|
||||
runTestFloats(Type.FLOAT_64);
|
||||
}
|
||||
|
||||
|
||||
private Source getSource(DocValues values) throws IOException {
|
||||
// getSource uses cache internally
|
||||
switch(random().nextInt(5)) {
|
||||
case 2:
|
||||
return values.getDirectSource();
|
||||
case 1:
|
||||
return values.getSource();
|
||||
default:
|
||||
return values.getSource();
|
||||
}
|
||||
}
|
||||
|
||||
private SortedSource getSortedSource(DocValues values) throws IOException {
|
||||
return getSource(values).asSortedSource();
|
||||
}
|
||||
|
||||
public static class DocValueHolder implements StorableField {
|
||||
BytesRef bytes;
|
||||
Number numberValue;
|
||||
Comparator<BytesRef> comp;
|
||||
|
||||
@Override
|
||||
public String name() {
|
||||
return "test";
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef binaryValue() {
|
||||
return bytes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Number numericValue() {
|
||||
return numberValue;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String stringValue() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Reader readerValue() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FieldType fieldType() {
|
||||
return null;
|
||||
}
|
||||
}*/
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user