Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5063: Compress integer and long field caches and remove FieldCache.get(Byte|Short)s, default parsers and related class/methods (merged from r1494753 and r1495146). 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1495156 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Adrien Grand 2013-06-20 20:07:57 +00:00
parent 09cc436a74
commit 543d33f941
47 changed files with 611 additions and 2127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
lucene/CHANGES.txt
View File

@ -103,6 +103,12 @@ Changes in backwards compatibility policy
on segment size and noCFSRatio. The default implemantion was pulled up from
TieredMergePolicy. (Simon Willnauer)
* LUCENE-5063: FieldCache.get(Bytes|Shorts), SortField.Type.(BYTE|SHORT) and
FieldCache.DEFAULT_(BYTE|SHORT|INT|LONG|FLOAT|DOUBLE)_PARSER are now
deprecated. These methods/types assume that data is stored as strings although
Lucene has much better support for numeric data through (Int|Long)Field,
NumericRangeQuery and FieldCache.get(Int|Long)s. (Adrien Grand)
Bug Fixes
* LUCENE-4997: Internal test framework's tests are sensitive to previous
@ -233,6 +239,9 @@ New Features
* LUCENE-5025: FST's Builder can now handle more than 2.1 billion
"tail nodes" while building a minimal FST. (Aaron Binns, Adrien
Grand, Mike McCandless)
* LUCENE-5063: FieldCache.DEFAULT.get(Ints|Longs) now uses bit-packing to save
memory. (Adrien Grand)
Build

227
lucene/core/src/java/org/apache/lucene/search/FieldCache.java
View File

@ -49,34 +49,6 @@ import org.apache.lucene.util.RamUsageEstimator;
*/
public interface FieldCache {
/** Field values as 8-bit signed bytes */
public static abstract class Bytes {
/** Return a single Byte representation of this field's value. */
public abstract byte get(int docID);
/** Zero value for every document */
public static final Bytes EMPTY = new Bytes() {
@Override
public byte get(int docID) {
return 0;
}
};
}
/** Field values as 16-bit signed shorts */
public static abstract class Shorts {
/** Return a short representation of this field's value. */
public abstract short get(int docID);
/** Zero value for every document */
public static final Shorts EMPTY = new Shorts() {
@Override
public short get(int docID) {
return 0;
}
};
}
/** Field values as 32-bit signed integers */
public static abstract class Ints {
/** Return an integer representation of this field's value. */
@ -178,22 +150,6 @@ public interface FieldCache {
public TermsEnum termsEnum(Terms terms) throws IOException;
}
/** Interface to parse bytes from document fields.
* @see FieldCache#getBytes(AtomicReader, String, FieldCache.ByteParser, boolean)
*/
public interface ByteParser extends Parser {
/** Return a single Byte representation of this field's value. */
public byte parseByte(BytesRef term);
}
/** Interface to parse shorts from document fields.
* @see FieldCache#getShorts(AtomicReader, String, FieldCache.ShortParser, boolean)
*/
public interface ShortParser extends Parser {
/** Return a short representation of this field's value. */
public short parseShort(BytesRef term);
}
/** Interface to parse ints from document fields.
* @see FieldCache#getInts(AtomicReader, String, FieldCache.IntParser, boolean)
*/
@ -229,135 +185,6 @@ public interface FieldCache {
/** Expert: The cache used internally by sorting and range query classes. */
public static FieldCache DEFAULT = new FieldCacheImpl();
/** The default parser for byte values, which are encoded by {@link Byte#toString(byte)} */
public static final ByteParser DEFAULT_BYTE_PARSER = new ByteParser() {
@Override
public byte parseByte(BytesRef term) {
// TODO: would be far better to directly parse from
// UTF8 bytes... but really users should use
// IntField, instead, which already decodes
// directly from byte[]
return Byte.parseByte(term.utf8ToString());
}
@Override
public String toString() {
return FieldCache.class.getName()+".DEFAULT_BYTE_PARSER";
}
@Override
public TermsEnum termsEnum(Terms terms) throws IOException {
return terms.iterator(null);
}
};
/** The default parser for short values, which are encoded by {@link Short#toString(short)} */
public static final ShortParser DEFAULT_SHORT_PARSER = new ShortParser() {
@Override
public short parseShort(BytesRef term) {
// TODO: would be far better to directly parse from
// UTF8 bytes... but really users should use
// IntField, instead, which already decodes
// directly from byte[]
return Short.parseShort(term.utf8ToString());
}
@Override
public String toString() {
return FieldCache.class.getName()+".DEFAULT_SHORT_PARSER";
}
@Override
public TermsEnum termsEnum(Terms terms) throws IOException {
return terms.iterator(null);
}
};
/** The default parser for int values, which are encoded by {@link Integer#toString(int)} */
public static final IntParser DEFAULT_INT_PARSER = new IntParser() {
@Override
public int parseInt(BytesRef term) {
// TODO: would be far better to directly parse from
// UTF8 bytes... but really users should use
// IntField, instead, which already decodes
// directly from byte[]
return Integer.parseInt(term.utf8ToString());
}
@Override
public TermsEnum termsEnum(Terms terms) throws IOException {
return terms.iterator(null);
}
@Override
public String toString() {
return FieldCache.class.getName()+".DEFAULT_INT_PARSER";
}
};
/** The default parser for float values, which are encoded by {@link Float#toString(float)} */
public static final FloatParser DEFAULT_FLOAT_PARSER = new FloatParser() {
@Override
public float parseFloat(BytesRef term) {
// TODO: would be far better to directly parse from
// UTF8 bytes... but really users should use
// FloatField, instead, which already decodes
// directly from byte[]
return Float.parseFloat(term.utf8ToString());
}
@Override
public TermsEnum termsEnum(Terms terms) throws IOException {
return terms.iterator(null);
}
@Override
public String toString() {
return FieldCache.class.getName()+".DEFAULT_FLOAT_PARSER";
}
};
/** The default parser for long values, which are encoded by {@link Long#toString(long)} */
public static final LongParser DEFAULT_LONG_PARSER = new LongParser() {
@Override
public long parseLong(BytesRef term) {
// TODO: would be far better to directly parse from
// UTF8 bytes... but really users should use
// LongField, instead, which already decodes
// directly from byte[]
return Long.parseLong(term.utf8ToString());
}
@Override
public TermsEnum termsEnum(Terms terms) throws IOException {
return terms.iterator(null);
}
@Override
public String toString() {
return FieldCache.class.getName()+".DEFAULT_LONG_PARSER";
}
};
/** The default parser for double values, which are encoded by {@link Double#toString(double)} */
public static final DoubleParser DEFAULT_DOUBLE_PARSER = new DoubleParser() {
@Override
public double parseDouble(BytesRef term) {
// TODO: would be far better to directly parse from
// UTF8 bytes... but really users should use
// DoubleField, instead, which already decodes
// directly from byte[]
return Double.parseDouble(term.utf8ToString());
}
@Override
public TermsEnum termsEnum(Terms terms) throws IOException {
return terms.iterator(null);
}
@Override
public String toString() {
return FieldCache.class.getName()+".DEFAULT_DOUBLE_PARSER";
}
};
/**
* A parser instance for int values encoded by {@link NumericUtils}, e.g. when indexed
* via {@link IntField}/{@link NumericTokenStream}.
@ -449,60 +276,6 @@ public interface FieldCache {
*/
public Bits getDocsWithField(AtomicReader reader, String field) throws IOException;
/** Checks the internal cache for an appropriate entry, and if none is
* found, reads the terms in <code>field</code> as a single byte and returns an array
* of size <code>reader.maxDoc()</code> of the value each document
* has in the given field.
* @param reader Used to get field values.
* @param field Which field contains the single byte values.
* @param setDocsWithField If true then {@link #getDocsWithField} will
* also be computed and stored in the FieldCache.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
public Bytes getBytes(AtomicReader reader, String field, boolean setDocsWithField) throws IOException;
/** Checks the internal cache for an appropriate entry, and if none is found,
* reads the terms in <code>field</code> as bytes and returns an array of
* size <code>reader.maxDoc()</code> of the value each document has in the
* given field.
* @param reader Used to get field values.
* @param field Which field contains the bytes.
* @param parser Computes byte for string values.
* @param setDocsWithField If true then {@link #getDocsWithField} will
* also be computed and stored in the FieldCache.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
public Bytes getBytes(AtomicReader reader, String field, ByteParser parser, boolean setDocsWithField) throws IOException;
/** Checks the internal cache for an appropriate entry, and if none is
* found, reads the terms in <code>field</code> as shorts and returns an array
* of size <code>reader.maxDoc()</code> of the value each document
* has in the given field.
* @param reader Used to get field values.
* @param field Which field contains the shorts.
* @param setDocsWithField If true then {@link #getDocsWithField} will
* also be computed and stored in the FieldCache.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
public Shorts getShorts (AtomicReader reader, String field, boolean setDocsWithField) throws IOException;
/** Checks the internal cache for an appropriate entry, and if none is found,
* reads the terms in <code>field</code> as shorts and returns an array of
* size <code>reader.maxDoc()</code> of the value each document has in the
* given field.
* @param reader Used to get field values.
* @param field Which field contains the shorts.
* @param parser Computes short for string values.
* @param setDocsWithField If true then {@link #getDocsWithField} will
* also be computed and stored in the FieldCache.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
public Shorts getShorts (AtomicReader reader, String field, ShortParser parser, boolean setDocsWithField) throws IOException;
/** Checks the internal cache for an appropriate entry, and if none is
* found, reads the terms in <code>field</code> as integers and returns an array
* of size <code>reader.maxDoc()</code> of the value each document

329
lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
View File

@ -38,7 +38,6 @@ import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FieldCacheSanityChecker;
@ -63,8 +62,6 @@ class FieldCacheImpl implements FieldCache {
private synchronized void init() {
caches = new HashMap<Class<?>,Cache>(9);
caches.put(Byte.TYPE, new ByteCache(this));
caches.put(Short.TYPE, new ShortCache(this));
caches.put(Integer.TYPE, new IntCache(this));
caches.put(Float.TYPE, new FloatCache(this));
caches.put(Long.TYPE, new LongCache(this));
@ -355,192 +352,6 @@ class FieldCacheImpl implements FieldCache {
}
caches.get(DocsWithFieldCache.class).put(reader, new CacheKey(field, null), bits);
}
// inherit javadocs
public Bytes getBytes (AtomicReader reader, String field, boolean setDocsWithField) throws IOException {
return getBytes(reader, field, null, setDocsWithField);
}
// inherit javadocs
public Bytes getBytes(AtomicReader reader, String field, ByteParser parser, boolean setDocsWithField)
throws IOException {
final NumericDocValues valuesIn = reader.getNumericDocValues(field);
if (valuesIn != null) {
// Not cached here by FieldCacheImpl (cached instead
// per-thread by SegmentReader):
return new Bytes() {
@Override
public byte get(int docID) {
return (byte) valuesIn.get(docID);
}
};
} else {
final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
if (info == null) {
return Bytes.EMPTY;
} else if (info.hasDocValues()) {
throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
} else if (!info.isIndexed()) {
return Bytes.EMPTY;
}
return (Bytes) caches.get(Byte.TYPE).get(reader, new CacheKey(field, parser), setDocsWithField);
}
}
static class BytesFromArray extends Bytes {
private final byte[] values;
public BytesFromArray(byte[] values) {
this.values = values;
}
@Override
public byte get(int docID) {
return values[docID];
}
}
static final class ByteCache extends Cache {
ByteCache(FieldCacheImpl wrapper) {
super(wrapper);
}
@Override
protected Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField)
throws IOException {
int maxDoc = reader.maxDoc();
final byte[] values;
final ByteParser parser = (ByteParser) key.custom;
if (parser == null) {
// Confusing: must delegate to wrapper (vs simply
// setting parser = DEFAULT_SHORT_PARSER) so cache
// key includes DEFAULT_SHORT_PARSER:
return wrapper.getBytes(reader, key.field, DEFAULT_BYTE_PARSER, setDocsWithField);
}
values = new byte[maxDoc];
Uninvert u = new Uninvert() {
private byte currentValue;
@Override
public void visitTerm(BytesRef term) {
currentValue = parser.parseByte(term);
}
@Override
public void visitDoc(int docID) {
values[docID] = currentValue;
}
@Override
protected TermsEnum termsEnum(Terms terms) throws IOException {
return parser.termsEnum(terms);
}
};
u.uninvert(reader, key.field, setDocsWithField);
if (setDocsWithField) {
wrapper.setDocsWithField(reader, key.field, u.docsWithField);
}
return new BytesFromArray(values);
}
}
// inherit javadocs
public Shorts getShorts (AtomicReader reader, String field, boolean setDocsWithField) throws IOException {
return getShorts(reader, field, null, setDocsWithField);
}
// inherit javadocs
public Shorts getShorts(AtomicReader reader, String field, ShortParser parser, boolean setDocsWithField)
throws IOException {
final NumericDocValues valuesIn = reader.getNumericDocValues(field);
if (valuesIn != null) {
// Not cached here by FieldCacheImpl (cached instead
// per-thread by SegmentReader):
return new Shorts() {
@Override
public short get(int docID) {
return (short) valuesIn.get(docID);
}
};
} else {
final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
if (info == null) {
return Shorts.EMPTY;
} else if (info.hasDocValues()) {
throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
} else if (!info.isIndexed()) {
return Shorts.EMPTY;
}
return (Shorts) caches.get(Short.TYPE).get(reader, new CacheKey(field, parser), setDocsWithField);
}
}
static class ShortsFromArray extends Shorts {
private final short[] values;
public ShortsFromArray(short[] values) {
this.values = values;
}
@Override
public short get(int docID) {
return values[docID];
}
}
static final class ShortCache extends Cache {
ShortCache(FieldCacheImpl wrapper) {
super(wrapper);
}
@Override
protected Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField)
throws IOException {
int maxDoc = reader.maxDoc();
final short[] values;
final ShortParser parser = (ShortParser) key.custom;
if (parser == null) {
// Confusing: must delegate to wrapper (vs simply
// setting parser = DEFAULT_SHORT_PARSER) so cache
// key includes DEFAULT_SHORT_PARSER:
return wrapper.getShorts(reader, key.field, DEFAULT_SHORT_PARSER, setDocsWithField);
}
values = new short[maxDoc];
Uninvert u = new Uninvert() {
private short currentValue;
@Override
public void visitTerm(BytesRef term) {
currentValue = parser.parseShort(term);
}
@Override
public void visitDoc(int docID) {
values[docID] = currentValue;
}
@Override
protected TermsEnum termsEnum(Terms terms) throws IOException {
return parser.termsEnum(terms);
}
};
u.uninvert(reader, key.field, setDocsWithField);
if (setDocsWithField) {
wrapper.setDocsWithField(reader, key.field, u.docsWithField);
}
return new ShortsFromArray(values);
}
}
// inherit javadocs
public Ints getInts (AtomicReader reader, String field, boolean setDocsWithField) throws IOException {
@ -574,15 +385,19 @@ class FieldCacheImpl implements FieldCache {
}
static class IntsFromArray extends Ints {
private final int[] values;
private final PackedInts.Reader values;
private final int minValue;
public IntsFromArray(int[] values) {
public IntsFromArray(PackedInts.Reader values, int minValue) {
assert values.getBitsPerValue() <= 32;
this.values = values;
this.minValue = minValue;
}
@Override
public int get(int docID) {
return values[docID];
final long delta = values.get(docID);
return minValue + (int) delta;
}
}
@ -598,6 +413,15 @@ class FieldCacheImpl implements FieldCache {
}
}
private static class GrowableWriterAndMinValue {
GrowableWriterAndMinValue(GrowableWriter array, long minValue) {
this.writer = array;
this.minValue = minValue;
}
public GrowableWriter writer;
public long minValue;
}
static final class IntCache extends Cache {
IntCache(FieldCacheImpl wrapper) {
super(wrapper);
@ -610,22 +434,17 @@ class FieldCacheImpl implements FieldCache {
final IntParser parser = (IntParser) key.custom;
if (parser == null) {
// Confusing: must delegate to wrapper (vs simply
// setting parser =
// DEFAULT_INT_PARSER/NUMERIC_UTILS_INT_PARSER) so
// cache key includes
// DEFAULT_INT_PARSER/NUMERIC_UTILS_INT_PARSER:
try {
return wrapper.getInts(reader, key.field, DEFAULT_INT_PARSER, setDocsWithField);
} catch (NumberFormatException ne) {
return wrapper.getInts(reader, key.field, NUMERIC_UTILS_INT_PARSER, setDocsWithField);
}
// setting parser = NUMERIC_UTILS_INT_PARSER) so
// cache key includes NUMERIC_UTILS_INT_PARSER:
return wrapper.getInts(reader, key.field, NUMERIC_UTILS_INT_PARSER, setDocsWithField);
}
final HoldsOneThing<int[]> valuesRef = new HoldsOneThing<int[]>();
final HoldsOneThing<GrowableWriterAndMinValue> valuesRef = new HoldsOneThing<GrowableWriterAndMinValue>();
Uninvert u = new Uninvert() {
private int minValue;
private int currentValue;
private int[] values;
private GrowableWriter values;
@Override
public void visitTerm(BytesRef term) {
@ -635,16 +454,28 @@ class FieldCacheImpl implements FieldCache {
// (which will hit a NumberFormatException
// when we first try the DEFAULT_INT_PARSER),
// we don't double-alloc:
values = new int[reader.maxDoc()];
valuesRef.set(values);
int startBitsPerValue;
// Make sure than missing values (0) can be stored without resizing
if (currentValue < 0) {
minValue = currentValue;
startBitsPerValue = PackedInts.bitsRequired((-minValue) & 0xFFFFFFFFL);
} else {
minValue = 0;
startBitsPerValue = PackedInts.bitsRequired(currentValue);
}
values = new GrowableWriter(startBitsPerValue, reader.maxDoc(), PackedInts.FAST);
if (minValue != 0) {
values.fill(0, values.size(), (-minValue) & 0xFFFFFFFFL); // default value must be 0
}
valuesRef.set(new GrowableWriterAndMinValue(values, minValue));
}
}
@Override
public void visitDoc(int docID) {
values[docID] = currentValue;
values.set(docID, (currentValue - minValue) & 0xFFFFFFFFL);
}
@Override
protected TermsEnum termsEnum(Terms terms) throws IOException {
return parser.termsEnum(terms);
@ -656,11 +487,11 @@ class FieldCacheImpl implements FieldCache {
if (setDocsWithField) {
wrapper.setDocsWithField(reader, key.field, u.docsWithField);
}
int[] values = valuesRef.get();
GrowableWriterAndMinValue values = valuesRef.get();
if (values == null) {
values = new int[reader.maxDoc()];
return new IntsFromArray(new PackedInts.NullReader(reader.maxDoc()), 0);
}
return new IntsFromArray(values);
return new IntsFromArray(values.writer.getMutable(), (int) values.minValue);
}
}
@ -792,15 +623,9 @@ class FieldCacheImpl implements FieldCache {
final FloatParser parser = (FloatParser) key.custom;
if (parser == null) {
// Confusing: must delegate to wrapper (vs simply
// setting parser =
// DEFAULT_FLOAT_PARSER/NUMERIC_UTILS_FLOAT_PARSER) so
// cache key includes
// DEFAULT_FLOAT_PARSER/NUMERIC_UTILS_FLOAT_PARSER:
try {
return wrapper.getFloats(reader, key.field, DEFAULT_FLOAT_PARSER, setDocsWithField);
} catch (NumberFormatException ne) {
return wrapper.getFloats(reader, key.field, NUMERIC_UTILS_FLOAT_PARSER, setDocsWithField);
}
// setting parser = NUMERIC_UTILS_FLOAT_PARSER) so
// cache key includes NUMERIC_UTILS_FLOAT_PARSER:
return wrapper.getFloats(reader, key.field, NUMERIC_UTILS_FLOAT_PARSER, setDocsWithField);
}
final HoldsOneThing<float[]> valuesRef = new HoldsOneThing<float[]>();
@ -879,15 +704,17 @@ class FieldCacheImpl implements FieldCache {
}
static class LongsFromArray extends Longs {
private final long[] values;
private final PackedInts.Reader values;
private final long minValue;
public LongsFromArray(long[] values) {
public LongsFromArray(PackedInts.Reader values, long minValue) {
this.values = values;
this.minValue = minValue;
}
@Override
public long get(int docID) {
return values[docID];
return minValue + values.get(docID);
}
}
@ -903,22 +730,17 @@ class FieldCacheImpl implements FieldCache {
final LongParser parser = (LongParser) key.custom;
if (parser == null) {
// Confusing: must delegate to wrapper (vs simply
// setting parser =
// DEFAULT_LONG_PARSER/NUMERIC_UTILS_LONG_PARSER) so
// cache key includes
// DEFAULT_LONG_PARSER/NUMERIC_UTILS_LONG_PARSER:
try {
return wrapper.getLongs(reader, key.field, DEFAULT_LONG_PARSER, setDocsWithField);
} catch (NumberFormatException ne) {
return wrapper.getLongs(reader, key.field, NUMERIC_UTILS_LONG_PARSER, setDocsWithField);
}
// setting parser = NUMERIC_UTILS_LONG_PARSER) so
// cache key includes NUMERIC_UTILS_LONG_PARSER:
return wrapper.getLongs(reader, key.field, NUMERIC_UTILS_LONG_PARSER, setDocsWithField);
}
final HoldsOneThing<long[]> valuesRef = new HoldsOneThing<long[]>();
final HoldsOneThing<GrowableWriterAndMinValue> valuesRef = new HoldsOneThing<GrowableWriterAndMinValue>();
Uninvert u = new Uninvert() {
private long minValue;
private long currentValue;
private long[] values;
private GrowableWriter values;
@Override
public void visitTerm(BytesRef term) {
@ -928,14 +750,26 @@ class FieldCacheImpl implements FieldCache {
// (which will hit a NumberFormatException
// when we first try the DEFAULT_INT_PARSER),
// we don't double-alloc:
values = new long[reader.maxDoc()];
valuesRef.set(values);
int startBitsPerValue;
// Make sure than missing values (0) can be stored without resizing
if (currentValue < 0) {
minValue = currentValue;
startBitsPerValue = minValue == Long.MIN_VALUE ? 64 : PackedInts.bitsRequired(-minValue);
} else {
minValue = 0;
startBitsPerValue = PackedInts.bitsRequired(currentValue);
}
values = new GrowableWriter(startBitsPerValue, reader.maxDoc(), PackedInts.FAST);
if (minValue != 0) {
values.fill(0, values.size(), -minValue); // default value must be 0
}
valuesRef.set(new GrowableWriterAndMinValue(values, minValue));
}
}
@Override
public void visitDoc(int docID) {
values[docID] = currentValue;
values.set(docID, currentValue - minValue);
}
@Override
@ -949,11 +783,11 @@ class FieldCacheImpl implements FieldCache {
if (setDocsWithField) {
wrapper.setDocsWithField(reader, key.field, u.docsWithField);
}
long[] values = valuesRef.get();
GrowableWriterAndMinValue values = valuesRef.get();
if (values == null) {
values = new long[reader.maxDoc()];
return new LongsFromArray(new PackedInts.NullReader(reader.maxDoc()), 0L);
}
return new LongsFromArray(values);
return new LongsFromArray(values.writer.getMutable(), values.minValue);
}
}
@ -1014,15 +848,9 @@ class FieldCacheImpl implements FieldCache {
final DoubleParser parser = (DoubleParser) key.custom;
if (parser == null) {
// Confusing: must delegate to wrapper (vs simply
// setting parser =
// DEFAULT_DOUBLE_PARSER/NUMERIC_UTILS_DOUBLE_PARSER) so
// cache key includes
// DEFAULT_DOUBLE_PARSER/NUMERIC_UTILS_DOUBLE_PARSER:
try {
return wrapper.getDoubles(reader, key.field, DEFAULT_DOUBLE_PARSER, setDocsWithField);
} catch (NumberFormatException ne) {
return wrapper.getDoubles(reader, key.field, NUMERIC_UTILS_DOUBLE_PARSER, setDocsWithField);
}
// setting parser = NUMERIC_UTILS_DOUBLE_PARSER) so
// cache key includes NUMERIC_UTILS_DOUBLE_PARSER:
return wrapper.getDoubles(reader, key.field, NUMERIC_UTILS_DOUBLE_PARSER, setDocsWithField);
}
final HoldsOneThing<double[]> valuesRef = new HoldsOneThing<double[]>();
@ -1146,7 +974,6 @@ class FieldCacheImpl implements FieldCache {
final PagedBytes bytes = new PagedBytes(15);
int startTermsBPV;
int startNumUniqueTerms;
final int termCountHardLimit;
if (maxDoc == Integer.MAX_VALUE) {
@ -1170,15 +997,11 @@ class FieldCacheImpl implements FieldCache {
}
startTermsBPV = PackedInts.bitsRequired(numUniqueTerms);
startNumUniqueTerms = (int) numUniqueTerms;
} else {
startTermsBPV = 1;
startNumUniqueTerms = 1;
}
} else {
startTermsBPV = 1;
startNumUniqueTerms = 1;
}
MonotonicAppendingLongBuffer termOrdToBytesOffset = new MonotonicAppendingLongBuffer();

106
lucene/core/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java
View File

@ -193,109 +193,7 @@ public abstract class FieldCacheRangeFilter<T> extends Filter {
}
};
}
/**
* Creates a numeric range filter using {@link FieldCache#getBytes(AtomicReader,String,boolean)}. This works with all
* byte fields containing exactly one numeric term in the field. The range can be half-open by setting one
* of the values to <code>null</code>.
*/
public static FieldCacheRangeFilter<Byte> newByteRange(String field, Byte lowerVal, Byte upperVal, boolean includeLower, boolean includeUpper) {
return newByteRange(field, null, lowerVal, upperVal, includeLower, includeUpper);
}
/**
* Creates a numeric range filter using {@link FieldCache#getBytes(AtomicReader,String,FieldCache.ByteParser,boolean)}. This works with all
* byte fields containing exactly one numeric term in the field. The range can be half-open by setting one
* of the values to <code>null</code>.
*/
public static FieldCacheRangeFilter<Byte> newByteRange(String field, FieldCache.ByteParser parser, Byte lowerVal, Byte upperVal, boolean includeLower, boolean includeUpper) {
return new FieldCacheRangeFilter<Byte>(field, parser, lowerVal, upperVal, includeLower, includeUpper) {
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
final byte inclusiveLowerPoint, inclusiveUpperPoint;
if (lowerVal != null) {
final byte i = lowerVal.byteValue();
if (!includeLower && i == Byte.MAX_VALUE)
return null;
inclusiveLowerPoint = (byte) (includeLower ? i : (i + 1));
} else {
inclusiveLowerPoint = Byte.MIN_VALUE;
}
if (upperVal != null) {
final byte i = upperVal.byteValue();
if (!includeUpper && i == Byte.MIN_VALUE)
return null;
inclusiveUpperPoint = (byte) (includeUpper ? i : (i - 1));
} else {
inclusiveUpperPoint = Byte.MAX_VALUE;
}
if (inclusiveLowerPoint > inclusiveUpperPoint)
return null;
final FieldCache.Bytes values = FieldCache.DEFAULT.getBytes(context.reader(), field, (FieldCache.ByteParser) parser, false);
return new FieldCacheDocIdSet(context.reader().maxDoc(), acceptDocs) {
@Override
protected boolean matchDoc(int doc) {
final byte value = values.get(doc);
return value >= inclusiveLowerPoint && value <= inclusiveUpperPoint;
}
};
}
};
}
/**
* Creates a numeric range filter using {@link FieldCache#getShorts(AtomicReader,String,boolean)}. This works with all
* short fields containing exactly one numeric term in the field. The range can be half-open by setting one
* of the values to <code>null</code>.
*/
public static FieldCacheRangeFilter<Short> newShortRange(String field, Short lowerVal, Short upperVal, boolean includeLower, boolean includeUpper) {
return newShortRange(field, null, lowerVal, upperVal, includeLower, includeUpper);
}
/**
* Creates a numeric range filter using {@link FieldCache#getShorts(AtomicReader,String,FieldCache.ShortParser,boolean)}. This works with all
* short fields containing exactly one numeric term in the field. The range can be half-open by setting one
* of the values to <code>null</code>.
*/
public static FieldCacheRangeFilter<Short> newShortRange(String field, FieldCache.ShortParser parser, Short lowerVal, Short upperVal, boolean includeLower, boolean includeUpper) {
return new FieldCacheRangeFilter<Short>(field, parser, lowerVal, upperVal, includeLower, includeUpper) {
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
final short inclusiveLowerPoint, inclusiveUpperPoint;
if (lowerVal != null) {
short i = lowerVal.shortValue();
if (!includeLower && i == Short.MAX_VALUE)
return null;
inclusiveLowerPoint = (short) (includeLower ? i : (i + 1));
} else {
inclusiveLowerPoint = Short.MIN_VALUE;
}
if (upperVal != null) {
short i = upperVal.shortValue();
if (!includeUpper && i == Short.MIN_VALUE)
return null;
inclusiveUpperPoint = (short) (includeUpper ? i : (i - 1));
} else {
inclusiveUpperPoint = Short.MAX_VALUE;
}
if (inclusiveLowerPoint > inclusiveUpperPoint)
return null;
final FieldCache.Shorts values = FieldCache.DEFAULT.getShorts(context.reader(), field, (FieldCache.ShortParser) parser, false);
return new FieldCacheDocIdSet(context.reader().maxDoc(), acceptDocs) {
@Override
protected boolean matchDoc(int doc) {
final short value = values.get(doc);
return value >= inclusiveLowerPoint && value <= inclusiveUpperPoint;
}
};
}
};
}
/**
* Creates a numeric range filter using {@link FieldCache#getInts(AtomicReader,String,boolean)}. This works with all
* int fields containing exactly one numeric term in the field. The range can be half-open by setting one
@ -521,7 +419,7 @@ public abstract class FieldCacheRangeFilter<T> extends Filter {
}
@Override
@SuppressWarnings({"unchecked","rawtypes"})
@SuppressWarnings({"rawtypes"})
public final boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof FieldCacheRangeFilter)) return false;

148
lucene/core/src/java/org/apache/lucene/search/FieldComparator.java
View File

@ -22,12 +22,10 @@ import java.io.IOException;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.FieldCache.ByteParser;
import org.apache.lucene.search.FieldCache.DoubleParser;
import org.apache.lucene.search.FieldCache.FloatParser;
import org.apache.lucene.search.FieldCache.IntParser;
import org.apache.lucene.search.FieldCache.LongParser;
import org.apache.lucene.search.FieldCache.ShortParser;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@ -218,78 +216,6 @@ public abstract class FieldComparator<T> {
}
}
/** Parses field's values as byte (using {@link
* FieldCache#getBytes} and sorts by ascending value */
public static final class ByteComparator extends NumericComparator<Byte> {
private final byte[] values;
private final ByteParser parser;
private FieldCache.Bytes currentReaderValues;
private byte bottom;
ByteComparator(int numHits, String field, FieldCache.Parser parser, Byte missingValue) {
super(field, missingValue);
values = new byte[numHits];
this.parser = (ByteParser) parser;
}
@Override
public int compare(int slot1, int slot2) {
return Byte.compare(values[slot1], values[slot2]);
}
@Override
public int compareBottom(int doc) {
byte v2 = currentReaderValues.get(doc);
// Test for v2 == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
v2 = missingValue;
}
return Byte.compare(bottom, v2);
}
@Override
public void copy(int slot, int doc) {
byte v2 = currentReaderValues.get(doc);
// Test for v2 == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
v2 = missingValue;
}
values[slot] = v2;
}
@Override
public FieldComparator<Byte> setNextReader(AtomicReaderContext context) throws IOException {
// NOTE: must do this before calling super otherwise
// we compute the docsWithField Bits twice!
currentReaderValues = FieldCache.DEFAULT.getBytes(context.reader(), field, parser, missingValue != null);
return super.setNextReader(context);
}
@Override
public void setBottom(final int bottom) {
this.bottom = values[bottom];
}
@Override
public Byte value(int slot) {
return Byte.valueOf(values[slot]);
}
@Override
public int compareDocToValue(int doc, Byte value) {
byte docValue = currentReaderValues.get(doc);
// Test for docValue == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) {
docValue = missingValue;
}
return Byte.compare(docValue, value.byteValue());
}
}
/** Parses field's values as double (using {@link
* FieldCache#getDoubles} and sorts by ascending value */
public static final class DoubleComparator extends NumericComparator<Double> {
@ -439,80 +365,6 @@ public abstract class FieldComparator<T> {
}
}
/** Parses field's values as short (using {@link
* FieldCache#getShorts} and sorts by ascending value */
public static final class ShortComparator extends NumericComparator<Short> {
private final short[] values;
private final ShortParser parser;
private FieldCache.Shorts currentReaderValues;
private short bottom;
ShortComparator(int numHits, String field, FieldCache.Parser parser, Short missingValue) {
super(field, missingValue);
values = new short[numHits];
this.parser = (ShortParser) parser;
}
@Override
public int compare(int slot1, int slot2) {
return Short.compare(values[slot1], values[slot2]);
}
@Override
public int compareBottom(int doc) {
short v2 = currentReaderValues.get(doc);
// Test for v2 == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
v2 = missingValue;
}
return Short.compare(bottom, v2);
}
@Override
public void copy(int slot, int doc) {
short v2 = currentReaderValues.get(doc);
// Test for v2 == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) {
v2 = missingValue;
}
values[slot] = v2;
}
@Override
public FieldComparator<Short> setNextReader(AtomicReaderContext context) throws IOException {
// NOTE: must do this before calling super otherwise
// we compute the docsWithField Bits twice!
currentReaderValues = FieldCache.DEFAULT.getShorts(context.reader(), field, parser, missingValue != null);
return super.setNextReader(context);
}
@Override
public void setBottom(final int bottom) {
this.bottom = values[bottom];
}
@Override
public Short value(int slot) {
return Short.valueOf(values[slot]);
}
@Override
public int compareDocToValue(int doc, Short valueObj) {
final short value = valueObj.shortValue();
short docValue = currentReaderValues.get(doc);
// Test for docValue == 0 to save Bits.get method call for
// the common case (doc has value and value is non-zero):
if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) {
docValue = missingValue;
}
return Short.compare(docValue, value);
}
}
/** Parses field's values as int (using {@link
* FieldCache#getInts} and sorts by ascending value */
public static final class IntComparator extends NumericComparator<Integer> {

26
lucene/core/src/java/org/apache/lucene/search/SortField.java
View File

@ -71,18 +71,10 @@ public class SortField {
* lower values are at the front. */
DOUBLE,
/** Sort using term values as encoded Shorts. Sort values are Short and
* lower values are at the front. */
SHORT,
/** Sort using a custom Comparator. Sort values are any Comparable and
* sorting is done according to natural order. */
CUSTOM,
/** Sort using term values as encoded Bytes. Sort values are Byte and
* lower values are at the front. */
BYTE,
/** Sort using term values as Strings, but comparing by
* value (using String.compareTo) for all comparisons.
* This is typically slower than {@link #STRING}, which
@ -164,8 +156,6 @@ public class SortField {
public SortField(String field, FieldCache.Parser parser, boolean reverse) {
if (parser instanceof FieldCache.IntParser) initFieldType(field, Type.INT);
else if (parser instanceof FieldCache.FloatParser) initFieldType(field, Type.FLOAT);
else if (parser instanceof FieldCache.ShortParser) initFieldType(field, Type.SHORT);
else if (parser instanceof FieldCache.ByteParser) initFieldType(field, Type.BYTE);
else if (parser instanceof FieldCache.LongParser) initFieldType(field, Type.LONG);
else if (parser instanceof FieldCache.DoubleParser) initFieldType(field, Type.DOUBLE);
else {
@ -177,7 +167,7 @@ public class SortField {
}
public SortField setMissingValue(Object missingValue) {
if (type != Type.BYTE && type != Type.SHORT && type != Type.INT && type != Type.FLOAT && type != Type.LONG && type != Type.DOUBLE) {
if (type != Type.INT && type != Type.FLOAT && type != Type.LONG && type != Type.DOUBLE) {
throw new IllegalArgumentException( "Missing value only works for numeric types" );
}
this.missingValue = missingValue;
@ -274,14 +264,6 @@ public class SortField {
buffer.append("<string_val" + ": \"").append(field).append("\">");
break;
case BYTE:
buffer.append("<byte: \"").append(field).append("\">");
break;
case SHORT:
buffer.append("<short: \"").append(field).append("\">");
break;
case INT:
buffer.append("<int" + ": \"").append(field).append("\">");
break;
@ -389,12 +371,6 @@ public class SortField {
case DOUBLE:
return new FieldComparator.DoubleComparator(numHits, field, parser, (Double) missingValue);
case BYTE:
return new FieldComparator.ByteComparator(numHits, field, parser, (Byte) missingValue);
case SHORT:
return new FieldComparator.ShortComparator(numHits, field, parser, (Short) missingValue);
case CUSTOM:
assert comparatorSource != null;
return comparatorSource.newComparator(field, numHits, sortPos, reverse);

28
lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java
View File

@ -25,20 +25,30 @@ import org.apache.lucene.util.RamUsageEstimator;
/**
* Implements {@link PackedInts.Mutable}, but grows the
* bit count of the underlying packed ints on-demand.
* <p>Beware that this class will accept to set negative values but in order
* to do this, it will grow the number of bits per value to 64.
*
* <p>@lucene.internal</p>
*/
public class GrowableWriter implements PackedInts.Mutable {
private long currentMaxValue;
private long currentMask;
private PackedInts.Mutable current;
private final float acceptableOverheadRatio;
/**
* @param startBitsPerValue the initial number of bits per value, may grow depending on the data
* @param valueCount the number of values
* @param acceptableOverheadRatio an acceptable overhead ratio
*/
public GrowableWriter(int startBitsPerValue, int valueCount, float acceptableOverheadRatio) {
this.acceptableOverheadRatio = acceptableOverheadRatio;
current = PackedInts.getMutable(valueCount, startBitsPerValue, this.acceptableOverheadRatio);
currentMaxValue = PackedInts.maxValue(current.getBitsPerValue());
currentMask = mask(current.getBitsPerValue());
}
private static long mask(int bitsPerValue) {
return bitsPerValue == 64 ? ~0L : PackedInts.maxValue(bitsPerValue);
}
@Override
@ -71,16 +81,16 @@ public class GrowableWriter implements PackedInts.Mutable {
}
private void ensureCapacity(long value) {
assert value >= 0;
if (value <= currentMaxValue) {
if ((value & currentMask) == value) {
return;
}
final int bitsRequired = PackedInts.bitsRequired(value);
final int bitsRequired = value < 0 ? 64 : PackedInts.bitsRequired(value);
assert bitsRequired > current.getBitsPerValue();
final int valueCount = size();
PackedInts.Mutable next = PackedInts.getMutable(valueCount, bitsRequired, acceptableOverheadRatio);
PackedInts.copy(current, 0, next, 0, valueCount, PackedInts.DEFAULT_BUFFER_SIZE);
current = next;
currentMaxValue = PackedInts.maxValue(current.getBitsPerValue());
currentMask = mask(current.getBitsPerValue());
}
@Override
@ -110,6 +120,10 @@ public class GrowableWriter implements PackedInts.Mutable {
public int set(int index, long[] arr, int off, int len) {
long max = 0;
for (int i = off, end = off + len; i < end; ++i) {
// bitwise or is nice because either all values are positive and the
// or-ed result will require as many bits per value as the max of the
// values, or one of them is negative and the result will be negative,
// forcing GrowableWriter to use 64 bits per value
max |= arr[i];
}
ensureCapacity(max);

2
lucene/core/src/test/org/apache/lucene/TestSearch.java
View File

@ -127,7 +127,7 @@ public class TestSearch extends LuceneTestCase {
for (int j = 0; j < docs.length; j++) {
Document d = new Document();
d.add(newTextField("contents", docs[j], Field.Store.YES));
d.add(newStringField("id", ""+j, Field.Store.NO));
d.add(new IntField("id", j, Field.Store.NO));
writer.addDocument(d);
}
writer.close();

2
lucene/core/src/test/org/apache/lucene/TestSearchForDuplicates.java
View File

@ -81,7 +81,7 @@ public class TestSearchForDuplicates extends LuceneTestCase {
for (int j = 0; j < MAX_DOCS; j++) {
Document d = new Document();
d.add(newTextField(PRIORITY_FIELD, HIGH_PRIORITY, Field.Store.YES));
d.add(newTextField(ID_FIELD, Integer.toString(j), Field.Store.YES));
d.add(new IntField(ID_FIELD, j, Field.Store.YES));
writer.addDocument(d);
}
writer.close();

3
lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
View File

@ -33,6 +33,7 @@ import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
@ -764,7 +765,7 @@ public void testFilesOpenClose() throws IOException {
setMergePolicy(newLogMergePolicy(10))
);
Document doc = new Document();
doc.add(newStringField("number", "17", Field.Store.NO));
doc.add(new IntField("number", 17, Field.Store.NO));
writer.addDocument(doc);
writer.commit();

14
lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java
View File

@ -87,23 +87,17 @@ public class TestDocValuesWithThreads extends LuceneTestCase {
BytesRef scratch2 = new BytesRef();
for(int iter=0;iter<iters;iter++) {
int docID = threadRandom.nextInt(numDocs);
switch(threadRandom.nextInt(6)) {
switch(threadRandom.nextInt(4)) {
case 0:
assertEquals((byte) numbers.get(docID).longValue(), FieldCache.DEFAULT.getBytes(ar, "number", false).get(docID));
break;
case 1:
assertEquals((short) numbers.get(docID).longValue(), FieldCache.DEFAULT.getShorts(ar, "number", false).get(docID));
break;
case 2:
assertEquals((int) numbers.get(docID).longValue(), FieldCache.DEFAULT.getInts(ar, "number", false).get(docID));
break;
case 3:
case 1:
assertEquals(numbers.get(docID).longValue(), FieldCache.DEFAULT.getLongs(ar, "number", false).get(docID));
break;
case 4:
case 2:
assertEquals(Float.intBitsToFloat((int) numbers.get(docID).longValue()), FieldCache.DEFAULT.getFloats(ar, "number", false).get(docID), 0.0f);
break;
case 5:
case 3:
assertEquals(Double.longBitsToDouble(numbers.get(docID).longValue()), FieldCache.DEFAULT.getDoubles(ar, "number", false).get(docID), 0.0);
break;
}

2
lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
View File

@ -59,7 +59,7 @@ public class TestRollingUpdates extends LuceneTestCase {
// TODO: sometimes update ids not in order...
for(int docIter=0;docIter<numUpdates;docIter++) {
final Document doc = docs.nextDoc();
final String myID = ""+id;
final String myID = Integer.toString(id);
if (id == SIZE-1) {
id = 0;
} else {

17
lucene/core/src/test/org/apache/lucene/search/BaseTestRangeFilter.java
View File

@ -22,7 +22,12 @@ import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.FloatField;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.RandomIndexWriter;
@ -115,9 +120,17 @@ public class BaseTestRangeFilter extends LuceneTestCase {
Document doc = new Document();
Field idField = newStringField(random, "id", "", Field.Store.YES);
Field intIdField = new IntField("id_int", 0, Store.YES);
Field floatIdField = new FloatField("id_float", 0, Store.YES);
Field longIdField = new LongField("id_long", 0, Store.YES);
Field doubleIdField = new DoubleField("id_double", 0, Store.YES);
Field randField = newStringField(random, "rand", "", Field.Store.YES);
Field bodyField = newStringField(random, "body", "", Field.Store.NO);
doc.add(idField);
doc.add(intIdField);
doc.add(floatIdField);
doc.add(longIdField);
doc.add(doubleIdField);
doc.add(randField);
doc.add(bodyField);
@ -133,6 +146,10 @@ public class BaseTestRangeFilter extends LuceneTestCase {
for (int d = minId; d <= maxId; d++) {
idField.setStringValue(pad(d));
intIdField.setIntValue(d);
floatIdField.setFloatValue(d);
longIdField.setLongValue(d);
doubleIdField.setDoubleValue(d);
int r = index.allowNegativeRandomInts ? random.nextInt() : random
.nextInt(Integer.MAX_VALUE);
if (index.maxR < r) {

178
lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
View File

@ -23,33 +23,45 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.FloatField;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.*;
import org.apache.lucene.search.FieldCache.Bytes;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocTermOrds;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.FieldCache.Doubles;
import org.apache.lucene.search.FieldCache.Floats;
import org.apache.lucene.search.FieldCache.Ints;
import org.apache.lucene.search.FieldCache.Longs;
import org.apache.lucene.search.FieldCache.Shorts;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util._TestUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@ -70,8 +82,6 @@ public class TestFieldCache extends LuceneTestCase {
RandomIndexWriter writer= new RandomIndexWriter(random(), directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
long theLong = Long.MAX_VALUE;
double theDouble = Double.MAX_VALUE;
byte theByte = Byte.MAX_VALUE;
short theShort = Short.MAX_VALUE;
int theInt = Integer.MAX_VALUE;
float theFloat = Float.MAX_VALUE;
unicodeStrings = new String[NUM_DOCS];
@ -81,14 +91,12 @@ public class TestFieldCache extends LuceneTestCase {
}
for (int i = 0; i < NUM_DOCS; i++){
Document doc = new Document();
doc.add(newStringField("theLong", String.valueOf(theLong--), Field.Store.NO));
doc.add(newStringField("theDouble", String.valueOf(theDouble--), Field.Store.NO));
doc.add(newStringField("theByte", String.valueOf(theByte--), Field.Store.NO));
doc.add(newStringField("theShort", String.valueOf(theShort--), Field.Store.NO));
doc.add(newStringField("theInt", String.valueOf(theInt--), Field.Store.NO));
doc.add(newStringField("theFloat", String.valueOf(theFloat--), Field.Store.NO));
doc.add(new LongField("theLong", theLong--, Field.Store.NO));
doc.add(new DoubleField("theDouble", theDouble--, Field.Store.NO));
doc.add(new IntField("theInt", theInt--, Field.Store.NO));
doc.add(new FloatField("theFloat", theFloat--, Field.Store.NO));
if (i%2 == 0) {
doc.add(newStringField("sparse", String.valueOf(i), Field.Store.NO));
doc.add(new IntField("sparse", i, Field.Store.NO));
}
if (i%2 == 0) {
@ -133,7 +141,16 @@ public class TestFieldCache extends LuceneTestCase {
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
cache.setInfoStream(new PrintStream(bos, false, "UTF-8"));
cache.getDoubles(reader, "theDouble", false);
cache.getFloats(reader, "theDouble", false);
cache.getFloats(reader, "theDouble", new FieldCache.FloatParser() {
@Override
public TermsEnum termsEnum(Terms terms) throws IOException {
return NumericUtils.filterPrefixCodedLongs(terms.iterator(null));
}
@Override
public float parseFloat(BytesRef term) {
return NumericUtils.sortableIntToFloat((int) NumericUtils.prefixCodedToLong(term));
}
}, false);
assertTrue(bos.toString("UTF-8").indexOf("WARNING") != -1);
} finally {
FieldCache.DEFAULT.purgeAllCaches();
@ -144,42 +161,28 @@ public class TestFieldCache extends LuceneTestCase {
FieldCache cache = FieldCache.DEFAULT;
FieldCache.Doubles doubles = cache.getDoubles(reader, "theDouble", random().nextBoolean());
assertSame("Second request to cache return same array", doubles, cache.getDoubles(reader, "theDouble", random().nextBoolean()));
assertSame("Second request with explicit parser return same array", doubles, cache.getDoubles(reader, "theDouble", FieldCache.DEFAULT_DOUBLE_PARSER, random().nextBoolean()));
assertSame("Second request with explicit parser return same array", doubles, cache.getDoubles(reader, "theDouble", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, random().nextBoolean()));
for (int i = 0; i < NUM_DOCS; i++) {
assertTrue(doubles.get(i) + " does not equal: " + (Double.MAX_VALUE - i), doubles.get(i) == (Double.MAX_VALUE - i));
}
FieldCache.Longs longs = cache.getLongs(reader, "theLong", random().nextBoolean());
assertSame("Second request to cache return same array", longs, cache.getLongs(reader, "theLong", random().nextBoolean()));
assertSame("Second request with explicit parser return same array", longs, cache.getLongs(reader, "theLong", FieldCache.DEFAULT_LONG_PARSER, random().nextBoolean()));
assertSame("Second request with explicit parser return same array", longs, cache.getLongs(reader, "theLong", FieldCache.NUMERIC_UTILS_LONG_PARSER, random().nextBoolean()));
for (int i = 0; i < NUM_DOCS; i++) {
assertTrue(longs.get(i) + " does not equal: " + (Long.MAX_VALUE - i) + " i=" + i, longs.get(i) == (Long.MAX_VALUE - i));
}
FieldCache.Bytes bytes = cache.getBytes(reader, "theByte", random().nextBoolean());
assertSame("Second request to cache return same array", bytes, cache.getBytes(reader, "theByte", random().nextBoolean()));
assertSame("Second request with explicit parser return same array", bytes, cache.getBytes(reader, "theByte", FieldCache.DEFAULT_BYTE_PARSER, random().nextBoolean()));
for (int i = 0; i < NUM_DOCS; i++) {
assertTrue(bytes.get(i) + " does not equal: " + (Byte.MAX_VALUE - i), bytes.get(i) == (byte) (Byte.MAX_VALUE - i));
}
FieldCache.Shorts shorts = cache.getShorts(reader, "theShort", random().nextBoolean());
assertSame("Second request to cache return same array", shorts, cache.getShorts(reader, "theShort", random().nextBoolean()));
assertSame("Second request with explicit parser return same array", shorts, cache.getShorts(reader, "theShort", FieldCache.DEFAULT_SHORT_PARSER, random().nextBoolean()));
for (int i = 0; i < NUM_DOCS; i++) {
assertTrue(shorts.get(i) + " does not equal: " + (Short.MAX_VALUE - i), shorts.get(i) == (short) (Short.MAX_VALUE - i));
}
FieldCache.Ints ints = cache.getInts(reader, "theInt", random().nextBoolean());
assertSame("Second request to cache return same array", ints, cache.getInts(reader, "theInt", random().nextBoolean()));
assertSame("Second request with explicit parser return same array", ints, cache.getInts(reader, "theInt", FieldCache.DEFAULT_INT_PARSER, random().nextBoolean()));
assertSame("Second request with explicit parser return same array", ints, cache.getInts(reader, "theInt", FieldCache.NUMERIC_UTILS_INT_PARSER, random().nextBoolean()));
for (int i = 0; i < NUM_DOCS; i++) {
assertTrue(ints.get(i) + " does not equal: " + (Integer.MAX_VALUE - i), ints.get(i) == (Integer.MAX_VALUE - i));
}
FieldCache.Floats floats = cache.getFloats(reader, "theFloat", random().nextBoolean());
assertSame("Second request to cache return same array", floats, cache.getFloats(reader, "theFloat", random().nextBoolean()));
assertSame("Second request with explicit parser return same array", floats, cache.getFloats(reader, "theFloat", FieldCache.DEFAULT_FLOAT_PARSER, random().nextBoolean()));
assertSame("Second request with explicit parser return same array", floats, cache.getFloats(reader, "theFloat", FieldCache.NUMERIC_UTILS_FLOAT_PARSER, random().nextBoolean()));
for (int i = 0; i < NUM_DOCS; i++) {
assertTrue(floats.get(i) + " does not equal: " + (Float.MAX_VALUE - i), floats.get(i) == (Float.MAX_VALUE - i));
}
@ -587,12 +590,6 @@ public class TestFieldCache extends LuceneTestCase {
cache.purgeAllCaches();
assertEquals(0, cache.getCacheEntries().length);
Bytes bytes = cache.getBytes(ar, "bogusbytes", true);
assertEquals(0, bytes.get(0));
Shorts shorts = cache.getShorts(ar, "bogusshorts", true);
assertEquals(0, shorts.get(0));
Ints ints = cache.getInts(ar, "bogusints", true);
assertEquals(0, ints.get(0));
@ -652,12 +649,6 @@ public class TestFieldCache extends LuceneTestCase {
cache.purgeAllCaches();
assertEquals(0, cache.getCacheEntries().length);
Bytes bytes = cache.getBytes(ar, "bogusbytes", true);
assertEquals(0, bytes.get(0));
Shorts shorts = cache.getShorts(ar, "bogusshorts", true);
assertEquals(0, shorts.get(0));
Ints ints = cache.getInts(ar, "bogusints", true);
assertEquals(0, ints.get(0));
@ -692,4 +683,97 @@ public class TestFieldCache extends LuceneTestCase {
ir.close();
dir.close();
}
// Make sure that the use of GrowableWriter doesn't prevent from using the full long range
public void testLongFieldCache() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
cfg.setMergePolicy(newLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
Document doc = new Document();
LongField field = new LongField("f", 0L, Store.YES);
doc.add(field);
final long[] values = new long[_TestUtil.nextInt(random(), 1, 10)];
for (int i = 0; i < values.length; ++i) {
final long v;
switch (random().nextInt(10)) {
case 0:
v = Long.MIN_VALUE;
break;
case 1:
v = 0;
break;
case 2:
v = Long.MAX_VALUE;
break;
default:
v = _TestUtil.nextLong(random(), -10, 10);
break;
}
values[i] = v;
if (v == 0 && random().nextBoolean()) {
// missing
iw.addDocument(new Document());
} else {
field.setLongValue(v);
iw.addDocument(doc);
}
}
iw.forceMerge(1);
final DirectoryReader reader = iw.getReader();
final FieldCache.Longs longs = FieldCache.DEFAULT.getLongs(getOnlySegmentReader(reader), "f", false);
for (int i = 0; i < values.length; ++i) {
assertEquals(values[i], longs.get(i));
}
reader.close();
iw.close();
dir.close();
}
// Make sure that the use of GrowableWriter doesn't prevent from using the full int range
public void testIntFieldCache() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
cfg.setMergePolicy(newLogMergePolicy());
RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
Document doc = new Document();
IntField field = new IntField("f", 0, Store.YES);
doc.add(field);
final int[] values = new int[_TestUtil.nextInt(random(), 1, 10)];
for (int i = 0; i < values.length; ++i) {
final int v;
switch (random().nextInt(10)) {
case 0:
v = Integer.MIN_VALUE;
break;
case 1:
v = 0;
break;
case 2:
v = Integer.MAX_VALUE;
break;
default:
v = _TestUtil.nextInt(random(), -10, 10);
break;
}
values[i] = v;
if (v == 0 && random().nextBoolean()) {
// missing
iw.addDocument(new Document());
} else {
field.setIntValue(v);
iw.addDocument(doc);
}
}
iw.forceMerge(1);
final DirectoryReader reader = iw.getReader();
final FieldCache.Ints ints = FieldCache.DEFAULT.getInts(getOnlySegmentReader(reader), "f", false);
for (int i = 0; i < values.length; ++i) {
assertEquals(values[i], ints.get(i));
}
reader.close();
iw.close();
dir.close();
}
}

242
lucene/core/src/test/org/apache/lucene/search/TestFieldCacheRangeFilter.java
View File

@ -19,15 +19,17 @@ package org.apache.lucene.search;
import java.io.IOException;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.IntField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
import org.junit.Test;
/**
@ -187,98 +189,6 @@ public class TestFieldCacheRangeFilter extends BaseTestRangeFilter {
assertEquals("max,nul,T,T", 1, result.length);
}
// byte-ranges cannot be tested, because all ranges are too big for bytes, need an extra range for that
@Test
public void testFieldCacheRangeFilterShorts() throws IOException {
IndexReader reader = signedIndexReader;
IndexSearcher search = newSearcher(reader);
int numDocs = reader.numDocs();
int medId = ((maxId - minId) / 2);
Short minIdO = Short.valueOf((short) minId);
Short maxIdO = Short.valueOf((short) maxId);
Short medIdO = Short.valueOf((short) medId);
assertEquals("num of docs", numDocs, 1+ maxId - minId);
ScoreDoc[] result;
Query q = new TermQuery(new Term("body","body"));
// test id, bounded on both ends
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",minIdO,maxIdO,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",minIdO,maxIdO,T,F), numDocs).scoreDocs;
assertEquals("all but last", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",minIdO,maxIdO,F,T), numDocs).scoreDocs;
assertEquals("all but first", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",minIdO,maxIdO,F,F), numDocs).scoreDocs;
assertEquals("all but ends", numDocs-2, result.length);
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",medIdO,maxIdO,T,T), numDocs).scoreDocs;
assertEquals("med and up", 1+ maxId-medId, result.length);
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",minIdO,medIdO,T,T), numDocs).scoreDocs;
assertEquals("up to med", 1+ medId-minId, result.length);
// unbounded id
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",null,null,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",minIdO,null,T,F), numDocs).scoreDocs;
assertEquals("min and up", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",null,maxIdO,F,T), numDocs).scoreDocs;
assertEquals("max and down", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",minIdO,null,F,F), numDocs).scoreDocs;
assertEquals("not min, but up", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",null,maxIdO,F,F), numDocs).scoreDocs;
assertEquals("not max, but down", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",medIdO,maxIdO,T,F), numDocs).scoreDocs;
assertEquals("med and up, not max", maxId-medId, result.length);
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",minIdO,medIdO,F,T), numDocs).scoreDocs;
assertEquals("not min, up to med", medId-minId, result.length);
// very small sets
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",minIdO,minIdO,F,F), numDocs).scoreDocs;
assertEquals("min,min,F,F", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",medIdO,medIdO,F,F), numDocs).scoreDocs;
assertEquals("med,med,F,F", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",maxIdO,maxIdO,F,F), numDocs).scoreDocs;
assertEquals("max,max,F,F", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",minIdO,minIdO,T,T), numDocs).scoreDocs;
assertEquals("min,min,T,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",null,minIdO,F,T), numDocs).scoreDocs;
assertEquals("nul,min,F,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",maxIdO,maxIdO,T,T), numDocs).scoreDocs;
assertEquals("max,max,T,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",maxIdO,null,T,F), numDocs).scoreDocs;
assertEquals("max,nul,T,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",medIdO,medIdO,T,T), numDocs).scoreDocs;
assertEquals("med,med,T,T", 1, result.length);
// special cases
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",Short.valueOf(Short.MAX_VALUE),null,F,F), numDocs).scoreDocs;
assertEquals("overflow special case", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",null,Short.valueOf(Short.MIN_VALUE),F,F), numDocs).scoreDocs;
assertEquals("overflow special case", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newShortRange("id",maxIdO,minIdO,T,T), numDocs).scoreDocs;
assertEquals("inverse range", 0, result.length);
}
@Test
public void testFieldCacheRangeFilterInts() throws IOException {
@ -298,75 +208,75 @@ public class TestFieldCacheRangeFilter extends BaseTestRangeFilter {
// test id, bounded on both ends
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",minIdO,maxIdO,T,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",minIdO,maxIdO,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",minIdO,maxIdO,T,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",minIdO,maxIdO,T,F), numDocs).scoreDocs;
assertEquals("all but last", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",minIdO,maxIdO,F,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",minIdO,maxIdO,F,T), numDocs).scoreDocs;
assertEquals("all but first", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",minIdO,maxIdO,F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",minIdO,maxIdO,F,F), numDocs).scoreDocs;
assertEquals("all but ends", numDocs-2, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",medIdO,maxIdO,T,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",medIdO,maxIdO,T,T), numDocs).scoreDocs;
assertEquals("med and up", 1+ maxId-medId, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",minIdO,medIdO,T,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",minIdO,medIdO,T,T), numDocs).scoreDocs;
assertEquals("up to med", 1+ medId-minId, result.length);
// unbounded id
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",null,null,T,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",null,null,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",minIdO,null,T,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",minIdO,null,T,F), numDocs).scoreDocs;
assertEquals("min and up", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",null,maxIdO,F,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",null,maxIdO,F,T), numDocs).scoreDocs;
assertEquals("max and down", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",minIdO,null,F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",minIdO,null,F,F), numDocs).scoreDocs;
assertEquals("not min, but up", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",null,maxIdO,F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",null,maxIdO,F,F), numDocs).scoreDocs;
assertEquals("not max, but down", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",medIdO,maxIdO,T,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",medIdO,maxIdO,T,F), numDocs).scoreDocs;
assertEquals("med and up, not max", maxId-medId, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",minIdO,medIdO,F,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",minIdO,medIdO,F,T), numDocs).scoreDocs;
assertEquals("not min, up to med", medId-minId, result.length);
// very small sets
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",minIdO,minIdO,F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",minIdO,minIdO,F,F), numDocs).scoreDocs;
assertEquals("min,min,F,F", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",medIdO,medIdO,F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",medIdO,medIdO,F,F), numDocs).scoreDocs;
assertEquals("med,med,F,F", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",maxIdO,maxIdO,F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",maxIdO,maxIdO,F,F), numDocs).scoreDocs;
assertEquals("max,max,F,F", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",minIdO,minIdO,T,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",minIdO,minIdO,T,T), numDocs).scoreDocs;
assertEquals("min,min,T,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",null,minIdO,F,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",null,minIdO,F,T), numDocs).scoreDocs;
assertEquals("nul,min,F,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",maxIdO,maxIdO,T,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",maxIdO,maxIdO,T,T), numDocs).scoreDocs;
assertEquals("max,max,T,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",maxIdO,null,T,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",maxIdO,null,T,F), numDocs).scoreDocs;
assertEquals("max,nul,T,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",medIdO,medIdO,T,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",medIdO,medIdO,T,T), numDocs).scoreDocs;
assertEquals("med,med,T,T", 1, result.length);
// special cases
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",Integer.valueOf(Integer.MAX_VALUE),null,F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",Integer.valueOf(Integer.MAX_VALUE),null,F,F), numDocs).scoreDocs;
assertEquals("overflow special case", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",null,Integer.valueOf(Integer.MIN_VALUE),F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",null,Integer.valueOf(Integer.MIN_VALUE),F,F), numDocs).scoreDocs;
assertEquals("overflow special case", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newIntRange("id",maxIdO,minIdO,T,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",maxIdO,minIdO,T,T), numDocs).scoreDocs;
assertEquals("inverse range", 0, result.length);
}
@ -389,75 +299,75 @@ public class TestFieldCacheRangeFilter extends BaseTestRangeFilter {
// test id, bounded on both ends
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",minIdO,maxIdO,T,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",minIdO,maxIdO,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",minIdO,maxIdO,T,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",minIdO,maxIdO,T,F), numDocs).scoreDocs;
assertEquals("all but last", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",minIdO,maxIdO,F,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",minIdO,maxIdO,F,T), numDocs).scoreDocs;
assertEquals("all but first", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",minIdO,maxIdO,F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",minIdO,maxIdO,F,F), numDocs).scoreDocs;
assertEquals("all but ends", numDocs-2, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",medIdO,maxIdO,T,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",medIdO,maxIdO,T,T), numDocs).scoreDocs;
assertEquals("med and up", 1+ maxId-medId, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",minIdO,medIdO,T,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",minIdO,medIdO,T,T), numDocs).scoreDocs;
assertEquals("up to med", 1+ medId-minId, result.length);
// unbounded id
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",null,null,T,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",null,null,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",minIdO,null,T,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",minIdO,null,T,F), numDocs).scoreDocs;
assertEquals("min and up", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",null,maxIdO,F,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",null,maxIdO,F,T), numDocs).scoreDocs;
assertEquals("max and down", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",minIdO,null,F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",minIdO,null,F,F), numDocs).scoreDocs;
assertEquals("not min, but up", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",null,maxIdO,F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",null,maxIdO,F,F), numDocs).scoreDocs;
assertEquals("not max, but down", numDocs-1, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",medIdO,maxIdO,T,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",medIdO,maxIdO,T,F), numDocs).scoreDocs;
assertEquals("med and up, not max", maxId-medId, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",minIdO,medIdO,F,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",minIdO,medIdO,F,T), numDocs).scoreDocs;
assertEquals("not min, up to med", medId-minId, result.length);
// very small sets
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",minIdO,minIdO,F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",minIdO,minIdO,F,F), numDocs).scoreDocs;
assertEquals("min,min,F,F", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",medIdO,medIdO,F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",medIdO,medIdO,F,F), numDocs).scoreDocs;
assertEquals("med,med,F,F", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",maxIdO,maxIdO,F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",maxIdO,maxIdO,F,F), numDocs).scoreDocs;
assertEquals("max,max,F,F", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",minIdO,minIdO,T,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",minIdO,minIdO,T,T), numDocs).scoreDocs;
assertEquals("min,min,T,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",null,minIdO,F,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",null,minIdO,F,T), numDocs).scoreDocs;
assertEquals("nul,min,F,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",maxIdO,maxIdO,T,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",maxIdO,maxIdO,T,T), numDocs).scoreDocs;
assertEquals("max,max,T,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",maxIdO,null,T,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",maxIdO,null,T,F), numDocs).scoreDocs;
assertEquals("max,nul,T,T", 1, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",medIdO,medIdO,T,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",medIdO,medIdO,T,T), numDocs).scoreDocs;
assertEquals("med,med,T,T", 1, result.length);
// special cases
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",Long.valueOf(Long.MAX_VALUE),null,F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",Long.valueOf(Long.MAX_VALUE),null,F,F), numDocs).scoreDocs;
assertEquals("overflow special case", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",null,Long.valueOf(Long.MIN_VALUE),F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",null,Long.valueOf(Long.MIN_VALUE),F,F), numDocs).scoreDocs;
assertEquals("overflow special case", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newLongRange("id",maxIdO,minIdO,T,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newLongRange("id_long",maxIdO,minIdO,T,T), numDocs).scoreDocs;
assertEquals("inverse range", 0, result.length);
}
@ -476,19 +386,19 @@ public class TestFieldCacheRangeFilter extends BaseTestRangeFilter {
ScoreDoc[] result;
Query q = new TermQuery(new Term("body","body"));
result = search.search(q,FieldCacheRangeFilter.newFloatRange("id",minIdO,medIdO,T,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newFloatRange("id_float",minIdO,medIdO,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs/2, result.length);
int count = 0;
result = search.search(q,FieldCacheRangeFilter.newFloatRange("id",null,medIdO,F,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newFloatRange("id_float",null,medIdO,F,T), numDocs).scoreDocs;
count += result.length;
result = search.search(q,FieldCacheRangeFilter.newFloatRange("id",medIdO,null,F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newFloatRange("id_float",medIdO,null,F,F), numDocs).scoreDocs;
count += result.length;
assertEquals("sum of two concenatted ranges", numDocs, count);
result = search.search(q,FieldCacheRangeFilter.newFloatRange("id",null,null,T,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newFloatRange("id_float",null,null,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newFloatRange("id",Float.valueOf(Float.POSITIVE_INFINITY),null,F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newFloatRange("id_float",Float.valueOf(Float.POSITIVE_INFINITY),null,F,F), numDocs).scoreDocs;
assertEquals("infinity special case", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newFloatRange("id",null,Float.valueOf(Float.NEGATIVE_INFINITY),F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newFloatRange("id_float",null,Float.valueOf(Float.NEGATIVE_INFINITY),F,F), numDocs).scoreDocs;
assertEquals("infinity special case", 0, result.length);
}
@ -505,19 +415,19 @@ public class TestFieldCacheRangeFilter extends BaseTestRangeFilter {
ScoreDoc[] result;
Query q = new TermQuery(new Term("body","body"));
result = search.search(q,FieldCacheRangeFilter.newDoubleRange("id",minIdO,medIdO,T,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newDoubleRange("id_double",minIdO,medIdO,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs/2, result.length);
int count = 0;
result = search.search(q,FieldCacheRangeFilter.newDoubleRange("id",null,medIdO,F,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newDoubleRange("id_double",null,medIdO,F,T), numDocs).scoreDocs;
count += result.length;
result = search.search(q,FieldCacheRangeFilter.newDoubleRange("id",medIdO,null,F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newDoubleRange("id_double",medIdO,null,F,F), numDocs).scoreDocs;
count += result.length;
assertEquals("sum of two concenatted ranges", numDocs, count);
result = search.search(q,FieldCacheRangeFilter.newDoubleRange("id",null,null,T,T), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newDoubleRange("id_double",null,null,T,T), numDocs).scoreDocs;
assertEquals("find all", numDocs, result.length);
result = search.search(q,FieldCacheRangeFilter.newDoubleRange("id",Double.valueOf(Double.POSITIVE_INFINITY),null,F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newDoubleRange("id_double",Double.valueOf(Double.POSITIVE_INFINITY),null,F,F), numDocs).scoreDocs;
assertEquals("infinity special case", 0, result.length);
result = search.search(q,FieldCacheRangeFilter.newDoubleRange("id",null, Double.valueOf(Double.NEGATIVE_INFINITY),F,F), numDocs).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newDoubleRange("id_double",null, Double.valueOf(Double.NEGATIVE_INFINITY),F,F), numDocs).scoreDocs;
assertEquals("infinity special case", 0, result.length);
}
@ -529,13 +439,15 @@ public class TestFieldCacheRangeFilter extends BaseTestRangeFilter {
for (int d = -20; d <= 20; d++) {
Document doc = new Document();
doc.add(newStringField("id", Integer.toString(d), Field.Store.NO));
doc.add(new IntField("id_int", d, Field.Store.NO));
doc.add(newStringField("body", "body", Field.Store.NO));
writer.addDocument(doc);
}
writer.forceMerge(1);
writer.deleteDocuments(new Term("id","0"));
BytesRef term0 = new BytesRef();
NumericUtils.intToPrefixCoded(0, 0, term0);
writer.deleteDocuments(new Term("id_int", term0));
writer.close();
IndexReader reader = DirectoryReader.open(dir);
@ -545,19 +457,19 @@ public class TestFieldCacheRangeFilter extends BaseTestRangeFilter {
ScoreDoc[] result;
Query q = new TermQuery(new Term("body","body"));
result = search.search(q,FieldCacheRangeFilter.newByteRange("id",Byte.valueOf((byte) -20),Byte.valueOf((byte) 20),T,T), 100).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",-20,20,T,T), 100).scoreDocs;
assertEquals("find all", 40, result.length);
result = search.search(q,FieldCacheRangeFilter.newByteRange("id",Byte.valueOf((byte) 0),Byte.valueOf((byte) 20),T,T), 100).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",0,20,T,T), 100).scoreDocs;
assertEquals("find all", 20, result.length);
result = search.search(q,FieldCacheRangeFilter.newByteRange("id",Byte.valueOf((byte) -20),Byte.valueOf((byte) 0),T,T), 100).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",-20,0,T,T), 100).scoreDocs;
assertEquals("find all", 20, result.length);
result = search.search(q,FieldCacheRangeFilter.newByteRange("id",Byte.valueOf((byte) 10),Byte.valueOf((byte) 20),T,T), 100).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",10,20,T,T), 100).scoreDocs;
assertEquals("find all", 11, result.length);
result = search.search(q,FieldCacheRangeFilter.newByteRange("id",Byte.valueOf((byte) -20),Byte.valueOf((byte) -10),T,T), 100).scoreDocs;
result = search.search(q,FieldCacheRangeFilter.newIntRange("id_int",-20,-10,T,T), 100).scoreDocs;
assertEquals("find all", 11, result.length);
reader.close();
dir.close();

2
lucene/core/src/test/org/apache/lucene/search/TestSearchAfter.java
View File

@ -111,8 +111,6 @@ public class TestSearchAfter extends LuceneTestCase {
assertQuery(query, filter, Sort.INDEXORDER);
for(int rev=0;rev<2;rev++) {
boolean reversed = rev == 1;
assertQuery(query, filter, new Sort(new SortField[] {new SortField("byte", SortField.Type.BYTE, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("short", SortField.Type.SHORT, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("int", SortField.Type.INT, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("long", SortField.Type.LONG, reversed)}));
assertQuery(query, filter, new Sort(new SortField[] {new SortField("float", SortField.Type.FLOAT, reversed)}));

2
lucene/core/src/test/org/apache/lucene/search/TestShardSearching.java
View File

@ -219,7 +219,7 @@ public class TestShardSearching extends ShardSearchingTestBase {
//sort = new Sort(SortField.FIELD_DOC);
sort = null;
} else if (what == 2) {
sort = new Sort(new SortField[] {new SortField("docid", SortField.Type.INT, random().nextBoolean())});
sort = new Sort(new SortField[] {new SortField("docid_int", SortField.Type.INT, random().nextBoolean())});
} else {
sort = new Sort(new SortField[] {new SortField("title", SortField.Type.STRING, random().nextBoolean())});
}

435
lucene/core/src/test/org/apache/lucene/search/TestSort.java
View File

@ -24,7 +24,11 @@ import java.util.List;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FloatField;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
@ -329,259 +333,19 @@ public class TestSort extends LuceneTestCase {
ir.close();
dir.close();
}
/** Tests sorting on type byte */
public void testByte() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(newStringField("value", "23", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(ir);
Sort sort = new Sort(new SortField("value", SortField.Type.BYTE));
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(3, td.totalHits);
// numeric order
assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value"));
assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value"));
assertEquals("23", searcher.doc(td.scoreDocs[2].doc).get("value"));
ir.close();
dir.close();
}
/** Tests sorting on type byte with a missing value */
public void testByteMissing() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(ir);
Sort sort = new Sort(new SortField("value", SortField.Type.BYTE));
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(3, td.totalHits);
// null value is treated as a 0
assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value"));
assertNull(searcher.doc(td.scoreDocs[1].doc).get("value"));
assertEquals("4", searcher.doc(td.scoreDocs[2].doc).get("value"));
ir.close();
dir.close();
}
/** Tests sorting on type byte, specifying the missing value should be treated as Byte.MAX_VALUE */
public void testByteMissingLast() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(ir);
SortField sortField = new SortField("value", SortField.Type.BYTE);
sortField.setMissingValue(Byte.MAX_VALUE);
Sort sort = new Sort(sortField);
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(3, td.totalHits);
// null value is treated Byte.MAX_VALUE
assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value"));
assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value"));
assertNull(searcher.doc(td.scoreDocs[2].doc).get("value"));
ir.close();
dir.close();
}
/** Tests sorting on type byte in reverse */
public void testByteReverse() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(newStringField("value", "23", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(ir);
Sort sort = new Sort(new SortField("value", SortField.Type.BYTE, true));
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(3, td.totalHits);
// reverse numeric order
assertEquals("23", searcher.doc(td.scoreDocs[0].doc).get("value"));
assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value"));
assertEquals("-1", searcher.doc(td.scoreDocs[2].doc).get("value"));
ir.close();
dir.close();
}
/** Tests sorting on type short */
public void testShort() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(newStringField("value", "300", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(ir);
Sort sort = new Sort(new SortField("value", SortField.Type.SHORT));
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(3, td.totalHits);
// numeric order
assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value"));
assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value"));
assertEquals("300", searcher.doc(td.scoreDocs[2].doc).get("value"));
ir.close();
dir.close();
}
/** Tests sorting on type short with a missing value */
public void testShortMissing() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(ir);
Sort sort = new Sort(new SortField("value", SortField.Type.SHORT));
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(3, td.totalHits);
// null is treated as a 0
assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value"));
assertNull(searcher.doc(td.scoreDocs[1].doc).get("value"));
assertEquals("4", searcher.doc(td.scoreDocs[2].doc).get("value"));
ir.close();
dir.close();
}
/** Tests sorting on type short, specifying the missing value should be treated as Short.MAX_VALUE */
public void testShortMissingLast() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(ir);
SortField sortField = new SortField("value", SortField.Type.SHORT);
sortField.setMissingValue(Short.MAX_VALUE);
Sort sort = new Sort(sortField);
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(3, td.totalHits);
// null is treated as Short.MAX_VALUE
assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value"));
assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value"));
assertNull(searcher.doc(td.scoreDocs[2].doc).get("value"));
ir.close();
dir.close();
}
/** Tests sorting on type short in reverse */
public void testShortReverse() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(newStringField("value", "300", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(ir);
Sort sort = new Sort(new SortField("value", SortField.Type.SHORT, true));
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(3, td.totalHits);
// reverse numeric order
assertEquals("300", searcher.doc(td.scoreDocs[0].doc).get("value"));
assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value"));
assertEquals("-1", searcher.doc(td.scoreDocs[2].doc).get("value"));
ir.close();
dir.close();
}
/** Tests sorting on type int */
public void testInt() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(newStringField("value", "300000", Field.Store.YES));
doc.add(new IntField("value", 300000, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1", Field.Store.YES));
doc.add(new IntField("value", -1, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4", Field.Store.YES));
doc.add(new IntField("value", 4, Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
@ -607,10 +371,10 @@ public class TestSort extends LuceneTestCase {
Document doc = new Document();
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1", Field.Store.YES));
doc.add(new IntField("value", -1, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4", Field.Store.YES));
doc.add(new IntField("value", 4, Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
@ -636,10 +400,10 @@ public class TestSort extends LuceneTestCase {
Document doc = new Document();
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1", Field.Store.YES));
doc.add(new IntField("value", -1, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4", Field.Store.YES));
doc.add(new IntField("value", 4, Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
@ -665,13 +429,13 @@ public class TestSort extends LuceneTestCase {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(newStringField("value", "300000", Field.Store.YES));
doc.add(new IntField("value", 300000, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1", Field.Store.YES));
doc.add(new IntField("value", -1, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4", Field.Store.YES));
doc.add(new IntField("value", 4, Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
@ -695,13 +459,13 @@ public class TestSort extends LuceneTestCase {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(newStringField("value", "3000000000", Field.Store.YES));
doc.add(new LongField("value", 3000000000L, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1", Field.Store.YES));
doc.add(new LongField("value", -1, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4", Field.Store.YES));
doc.add(new LongField("value", 4, Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
@ -727,10 +491,10 @@ public class TestSort extends LuceneTestCase {
Document doc = new Document();
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1", Field.Store.YES));
doc.add(new LongField("value", -1, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4", Field.Store.YES));
doc.add(new LongField("value", 4, Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
@ -756,10 +520,10 @@ public class TestSort extends LuceneTestCase {
Document doc = new Document();
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1", Field.Store.YES));
doc.add(new LongField("value", -1, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4", Field.Store.YES));
doc.add(new LongField("value", 4, Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
@ -785,13 +549,13 @@ public class TestSort extends LuceneTestCase {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(newStringField("value", "3000000000", Field.Store.YES));
doc.add(new LongField("value", 3000000000L, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1", Field.Store.YES));
doc.add(new LongField("value", -1, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4", Field.Store.YES));
doc.add(new LongField("value", 4, Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
@ -815,13 +579,13 @@ public class TestSort extends LuceneTestCase {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(newStringField("value", "30.1", Field.Store.YES));
doc.add(new FloatField("value", 30.1f, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1.3", Field.Store.YES));
doc.add(new FloatField("value", -1.3f, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4.2", Field.Store.YES));
doc.add(new FloatField("value", 4.2f, Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
@ -847,10 +611,10 @@ public class TestSort extends LuceneTestCase {
Document doc = new Document();
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1.3", Field.Store.YES));
doc.add(new FloatField("value", -1.3f, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4.2", Field.Store.YES));
doc.add(new FloatField("value", 4.2f, Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
@ -876,10 +640,10 @@ public class TestSort extends LuceneTestCase {
Document doc = new Document();
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1.3", Field.Store.YES));
doc.add(new FloatField("value", -1.3f, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4.2", Field.Store.YES));
doc.add(new FloatField("value", 4.2f, Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
@ -905,13 +669,13 @@ public class TestSort extends LuceneTestCase {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(newStringField("value", "30.1", Field.Store.YES));
doc.add(new FloatField("value", 30.1f, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1.3", Field.Store.YES));
doc.add(new FloatField("value", -1.3f, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4.2", Field.Store.YES));
doc.add(new FloatField("value", 4.2f, Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
@ -935,16 +699,16 @@ public class TestSort extends LuceneTestCase {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(newStringField("value", "30.1", Field.Store.YES));
doc.add(new DoubleField("value", 30.1, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1.3", Field.Store.YES));
doc.add(new DoubleField("value", -1.3, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4.2333333333333", Field.Store.YES));
doc.add(new DoubleField("value", 4.2333333333333, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4.2333333333332", Field.Store.YES));
doc.add(new DoubleField("value", 4.2333333333332, Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
@ -969,10 +733,10 @@ public class TestSort extends LuceneTestCase {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(newStringField("value", "+0", Field.Store.YES));
doc.add(new DoubleField("value", +0d, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-0", Field.Store.YES));
doc.add(new DoubleField("value", -0d, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
IndexReader ir = writer.getReader();
@ -984,8 +748,13 @@ public class TestSort extends LuceneTestCase {
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(2, td.totalHits);
// numeric order
assertEquals("-0", searcher.doc(td.scoreDocs[0].doc).get("value"));
assertEquals("+0", searcher.doc(td.scoreDocs[1].doc).get("value"));
double v0 = searcher.doc(td.scoreDocs[0].doc).getField("value").numericValue().doubleValue();
double v1 = searcher.doc(td.scoreDocs[1].doc).getField("value").numericValue().doubleValue();
assertEquals(0, v0, 0d);
assertEquals(0, v1, 0d);
// check sign bits
assertEquals(1, Double.doubleToLongBits(v0) >>> 63);
assertEquals(0, Double.doubleToLongBits(v1) >>> 63);
ir.close();
dir.close();
@ -998,13 +767,13 @@ public class TestSort extends LuceneTestCase {
Document doc = new Document();
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1.3", Field.Store.YES));
doc.add(new DoubleField("value", -1.3, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4.2333333333333", Field.Store.YES));
doc.add(new DoubleField("value", 4.2333333333333, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4.2333333333332", Field.Store.YES));
doc.add(new DoubleField("value", 4.2333333333332, Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
@ -1031,13 +800,13 @@ public class TestSort extends LuceneTestCase {
Document doc = new Document();
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1.3", Field.Store.YES));
doc.add(new DoubleField("value", -1.3, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4.2333333333333", Field.Store.YES));
doc.add(new DoubleField("value", 4.2333333333333, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4.2333333333332", Field.Store.YES));
doc.add(new DoubleField("value", 4.2333333333332, Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
@ -1064,16 +833,16 @@ public class TestSort extends LuceneTestCase {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(newStringField("value", "30.1", Field.Store.YES));
doc.add(new DoubleField("value", 30.1, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "-1.3", Field.Store.YES));
doc.add(new DoubleField("value", -1.3, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4.2333333333333", Field.Store.YES));
doc.add(new DoubleField("value", 4.2333333333333, Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(newStringField("value", "4.2333333333332", Field.Store.YES));
doc.add(new DoubleField("value", 4.2333333333332, Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
@ -1150,7 +919,7 @@ public class TestSort extends LuceneTestCase {
for(int seg=0;seg<2;seg++) {
for(int docIDX=0;docIDX<10;docIDX++) {
Document doc = new Document();
doc.add(newStringField("id", ""+docIDX, Field.Store.YES));
doc.add(new IntField("id", docIDX, Field.Store.YES));
StringBuilder sb = new StringBuilder();
for(int i=0;i<id;i++) {
sb.append(' ');
@ -1250,94 +1019,6 @@ public class TestSort extends LuceneTestCase {
dir.close();
}
/**
* test sorts for a custom byte parser that uses a simple char encoding
*/
public void testCustomByteParser() throws Exception {
List<String> letters = Arrays.asList(new String[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J" });
Collections.shuffle(letters, random());
Directory dir = newDirectory();
RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
for (String letter : letters) {
Document doc = new Document();
doc.add(newStringField("parser", letter, Field.Store.YES));
iw.addDocument(doc);
}
IndexReader ir = iw.getReader();
iw.close();
IndexSearcher searcher = newSearcher(ir);
Sort sort = new Sort(new SortField("parser", new FieldCache.ByteParser() {
@Override
public byte parseByte(BytesRef term) {
return (byte) (term.bytes[term.offset]-'A');
}
@Override
public TermsEnum termsEnum(Terms terms) throws IOException {
return terms.iterator(null);
}
}), SortField.FIELD_DOC );
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
// results should be in alphabetical order
assertEquals(10, td.totalHits);
Collections.sort(letters);
for (int i = 0; i < letters.size(); i++) {
assertEquals(letters.get(i), searcher.doc(td.scoreDocs[i].doc).get("parser"));
}
ir.close();
dir.close();
}
/**
* test sorts for a custom short parser that uses a simple char encoding
*/
public void testCustomShortParser() throws Exception {
List<String> letters = Arrays.asList(new String[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J" });
Collections.shuffle(letters, random());
Directory dir = newDirectory();
RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
for (String letter : letters) {
Document doc = new Document();
doc.add(newStringField("parser", letter, Field.Store.YES));
iw.addDocument(doc);
}
IndexReader ir = iw.getReader();
iw.close();
IndexSearcher searcher = newSearcher(ir);
Sort sort = new Sort(new SortField("parser", new FieldCache.ShortParser() {
@Override
public short parseShort(BytesRef term) {
return (short) (term.bytes[term.offset]-'A');
}
@Override
public TermsEnum termsEnum(Terms terms) throws IOException {
return terms.iterator(null);
}
}), SortField.FIELD_DOC );
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
// results should be in alphabetical order
assertEquals(10, td.totalHits);
Collections.sort(letters);
for (int i = 0; i < letters.size(); i++) {
assertEquals(letters.get(i), searcher.doc(td.scoreDocs[i].doc).get("parser"));
}
ir.close();
dir.close();
}
/**
* test sorts for a custom long parser that uses a simple char encoding
*/

136
lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java
View File

@ -223,142 +223,6 @@ public class TestSortDocValues extends LuceneTestCase {
dir.close();
}
/** Tests sorting on type byte */
public void testByte() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(new NumericDocValuesField("value", 23));
doc.add(newStringField("value", "23", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(new NumericDocValuesField("value", -1));
doc.add(newStringField("value", "-1", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(new NumericDocValuesField("value", 4));
doc.add(newStringField("value", "4", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(ir);
Sort sort = new Sort(new SortField("value", SortField.Type.BYTE));
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(3, td.totalHits);
// numeric order
assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value"));
assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value"));
assertEquals("23", searcher.doc(td.scoreDocs[2].doc).get("value"));
assertNoFieldCaches();
ir.close();
dir.close();
}
/** Tests sorting on type byte in reverse */
public void testByteReverse() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(new NumericDocValuesField("value", 23));
doc.add(newStringField("value", "23", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(new NumericDocValuesField("value", -1));
doc.add(newStringField("value", "-1", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(new NumericDocValuesField("value", 4));
doc.add(newStringField("value", "4", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(ir);
Sort sort = new Sort(new SortField("value", SortField.Type.BYTE, true));
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(3, td.totalHits);
// reverse numeric order
assertEquals("23", searcher.doc(td.scoreDocs[0].doc).get("value"));
assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value"));
assertEquals("-1", searcher.doc(td.scoreDocs[2].doc).get("value"));
assertNoFieldCaches();
ir.close();
dir.close();
}
/** Tests sorting on type short */
public void testShort() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(new NumericDocValuesField("value", 300));
doc.add(newStringField("value", "300", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(new NumericDocValuesField("value", -1));
doc.add(newStringField("value", "-1", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(new NumericDocValuesField("value", 4));
doc.add(newStringField("value", "4", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(ir);
Sort sort = new Sort(new SortField("value", SortField.Type.SHORT));
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(3, td.totalHits);
// numeric order
assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value"));
assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value"));
assertEquals("300", searcher.doc(td.scoreDocs[2].doc).get("value"));
assertNoFieldCaches();
ir.close();
dir.close();
}
/** Tests sorting on type short in reverse */
public void testShortReverse() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
doc.add(new NumericDocValuesField("value", 300));
doc.add(newStringField("value", "300", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(new NumericDocValuesField("value", -1));
doc.add(newStringField("value", "-1", Field.Store.YES));
writer.addDocument(doc);
doc = new Document();
doc.add(new NumericDocValuesField("value", 4));
doc.add(newStringField("value", "4", Field.Store.YES));
writer.addDocument(doc);
IndexReader ir = writer.getReader();
writer.close();
IndexSearcher searcher = newSearcher(ir);
Sort sort = new Sort(new SortField("value", SortField.Type.SHORT, true));
TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
assertEquals(3, td.totalHits);
// reverse numeric order
assertEquals("300", searcher.doc(td.scoreDocs[0].doc).get("value"));
assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value"));
assertEquals("-1", searcher.doc(td.scoreDocs[2].doc).get("value"));
assertNoFieldCaches();
ir.close();
dir.close();
}
/** Tests sorting on type int */
public void testInt() throws IOException {
Directory dir = newDirectory();

37
lucene/core/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java
View File

@ -16,21 +16,25 @@ package org.apache.lucene.util;
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.document.FloatField;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.FieldCacheSanityChecker.Insanity;
import org.apache.lucene.util.FieldCacheSanityChecker.InsanityType;
import java.io.IOException;
public class TestFieldCacheSanityChecker extends LuceneTestCase {
protected AtomicReader readerA;
@ -51,18 +55,14 @@ public class TestFieldCacheSanityChecker extends LuceneTestCase {
long theLong = Long.MAX_VALUE;
double theDouble = Double.MAX_VALUE;
byte theByte = Byte.MAX_VALUE;
short theShort = Short.MAX_VALUE;
int theInt = Integer.MAX_VALUE;
float theFloat = Float.MAX_VALUE;
for (int i = 0; i < NUM_DOCS; i++){
Document doc = new Document();
doc.add(newStringField("theLong", String.valueOf(theLong--), Field.Store.NO));
doc.add(newStringField("theDouble", String.valueOf(theDouble--), Field.Store.NO));
doc.add(newStringField("theByte", String.valueOf(theByte--), Field.Store.NO));
doc.add(newStringField("theShort", String.valueOf(theShort--), Field.Store.NO));
doc.add(newStringField("theInt", String.valueOf(theInt--), Field.Store.NO));
doc.add(newStringField("theFloat", String.valueOf(theFloat--), Field.Store.NO));
doc.add(new LongField("theLong", theLong--, Field.Store.NO));
doc.add(new DoubleField("theDouble", theDouble--, Field.Store.NO));
doc.add(new IntField("theInt", theInt--, Field.Store.NO));
doc.add(new FloatField("theFloat", theFloat--, Field.Store.NO));
if (0 == i % 3) {
wA.addDocument(doc);
} else {
@ -95,12 +95,12 @@ public class TestFieldCacheSanityChecker extends LuceneTestCase {
cache.purgeAllCaches();
cache.getDoubles(readerA, "theDouble", false);
cache.getDoubles(readerA, "theDouble", FieldCache.DEFAULT_DOUBLE_PARSER, false);
cache.getDoubles(readerAclone, "theDouble", FieldCache.DEFAULT_DOUBLE_PARSER, false);
cache.getDoubles(readerB, "theDouble", FieldCache.DEFAULT_DOUBLE_PARSER, false);
cache.getDoubles(readerA, "theDouble", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, false);
cache.getDoubles(readerAclone, "theDouble", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, false);
cache.getDoubles(readerB, "theDouble", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, false);
cache.getInts(readerX, "theInt", false);
cache.getInts(readerX, "theInt", FieldCache.DEFAULT_INT_PARSER, false);
cache.getInts(readerX, "theInt", FieldCache.NUMERIC_UTILS_INT_PARSER, false);
// // //
@ -119,9 +119,8 @@ public class TestFieldCacheSanityChecker extends LuceneTestCase {
FieldCache cache = FieldCache.DEFAULT;
cache.purgeAllCaches();
cache.getInts(readerX, "theInt", FieldCache.DEFAULT_INT_PARSER, false);
cache.getInts(readerX, "theInt", FieldCache.NUMERIC_UTILS_INT_PARSER, false);
cache.getTerms(readerX, "theInt");
cache.getBytes(readerX, "theByte", false);
// // //
@ -147,8 +146,6 @@ public class TestFieldCacheSanityChecker extends LuceneTestCase {
cache.getTerms(readerB, "theInt");
cache.getTerms(readerX, "theInt");
cache.getBytes(readerX, "theByte", false);
// // //

3
lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
View File

@ -650,8 +650,11 @@ public class TestPackedInts extends LuceneTestCase {
wrt.set(99, (1 << 23) - 1);
assertEquals(1 << 10, wrt.get(valueCount - 1));
wrt.set(1, Long.MAX_VALUE);
wrt.set(2, -3);
assertEquals(64, wrt.getBitsPerValue());
assertEquals(1 << 10, wrt.get(valueCount - 1));
assertEquals(Long.MAX_VALUE, wrt.get(1));
assertEquals(-3L, wrt.get(2));
assertEquals(2, wrt.get(4));
assertEquals((1 << 23) - 1, wrt.get(99));
assertEquals(10, wrt.get(7));

16
lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
View File

@ -78,7 +78,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
Document doc = new Document();
addGroupField(doc, groupField, "author1", valueType);
doc.add(newTextField("content", "random text", Field.Store.NO));
doc.add(newStringField("id_1", "1", Field.Store.NO));
doc.add(new IntField("id_1", 1, Field.Store.NO));
doc.add(newStringField("id_2", "1", Field.Store.NO));
w.addDocument(doc);
@ -86,7 +86,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
doc = new Document();
addGroupField(doc, groupField, "author1", valueType);
doc.add(newTextField("content", "some more random text blob", Field.Store.NO));
doc.add(newStringField("id_1", "2", Field.Store.NO));
doc.add(new IntField("id_1", 2, Field.Store.NO));
doc.add(newStringField("id_2", "2", Field.Store.NO));
w.addDocument(doc);
@ -94,7 +94,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
doc = new Document();
addGroupField(doc, groupField, "author1", valueType);
doc.add(newTextField("content", "some more random textual data", Field.Store.NO));
doc.add(newStringField("id_1", "3", Field.Store.NO));
doc.add(new IntField("id_1", 3, Field.Store.NO));
doc.add(newStringField("id_2", "3", Field.Store.NO));
w.addDocument(doc);
w.commit(); // To ensure a second segment
@ -103,7 +103,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
doc = new Document();
addGroupField(doc, groupField, "author2", valueType);
doc.add(newTextField("content", "some random text", Field.Store.NO));
doc.add(newStringField("id_1", "4", Field.Store.NO));
doc.add(new IntField("id_1", 4, Field.Store.NO));
doc.add(newStringField("id_2", "4", Field.Store.NO));
w.addDocument(doc);
@ -111,7 +111,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
doc = new Document();
addGroupField(doc, groupField, "author3", valueType);
doc.add(newTextField("content", "some more random text", Field.Store.NO));
doc.add(newStringField("id_1", "5", Field.Store.NO));
doc.add(new IntField("id_1", 5, Field.Store.NO));
doc.add(newStringField("id_2", "5", Field.Store.NO));
w.addDocument(doc);
@ -119,21 +119,21 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase {
doc = new Document();
addGroupField(doc, groupField, "author3", valueType);
doc.add(newTextField("content", "random blob", Field.Store.NO));
doc.add(newStringField("id_1", "6", Field.Store.NO));
doc.add(new IntField("id_1", 6, Field.Store.NO));
doc.add(newStringField("id_2", "6", Field.Store.NO));
w.addDocument(doc);
// 6 -- no author field
doc = new Document();
doc.add(newTextField("content", "random word stuck in alot of other text", Field.Store.NO));
doc.add(newStringField("id_1", "6", Field.Store.NO));
doc.add(new IntField("id_1", 6, Field.Store.NO));
doc.add(newStringField("id_2", "6", Field.Store.NO));
w.addDocument(doc);
// 7 -- no author field
doc = new Document();
doc.add(newTextField("content", "random word stuck in alot of other text", Field.Store.NO));
doc.add(newStringField("id_1", "7", Field.Store.NO));
doc.add(new IntField("id_1", 7, Field.Store.NO));
doc.add(newStringField("id_2", "7", Field.Store.NO));
w.addDocument(doc);

16
lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java
View File

@ -459,7 +459,7 @@ public class TestBlockJoin extends LuceneTestCase {
for(int parentDocID=0;parentDocID<numParentDocs;parentDocID++) {
Document parentDoc = new Document();
Document parentJoinDoc = new Document();
Field id = newStringField("parentID", ""+parentDocID, Field.Store.YES);
Field id = new IntField("parentID", parentDocID, Field.Store.YES);
parentDoc.add(id);
parentJoinDoc.add(id);
parentJoinDoc.add(newStringField("isParent", "x", Field.Store.NO));
@ -472,8 +472,8 @@ public class TestBlockJoin extends LuceneTestCase {
}
if (doDeletes) {
parentDoc.add(newStringField("blockID", ""+parentDocID, Field.Store.NO));
parentJoinDoc.add(newStringField("blockID", ""+parentDocID, Field.Store.NO));
parentDoc.add(new IntField("blockID", parentDocID, Field.Store.NO));
parentJoinDoc.add(new IntField("blockID", parentDocID, Field.Store.NO));
}
final List<Document> joinDocs = new ArrayList<>();
@ -497,7 +497,7 @@ public class TestBlockJoin extends LuceneTestCase {
Document joinChildDoc = new Document();
joinDocs.add(joinChildDoc);
Field childID = newStringField("childID", ""+childDocID, Field.Store.YES);
Field childID = new IntField("childID", childDocID, Field.Store.YES);
childDoc.add(childID);
joinChildDoc.add(childID);
@ -522,7 +522,7 @@ public class TestBlockJoin extends LuceneTestCase {
}
if (doDeletes) {
joinChildDoc.add(newStringField("blockID", ""+parentDocID, Field.Store.NO));
joinChildDoc.add(new IntField("blockID", parentDocID, Field.Store.NO));
}
w.addDocument(childDoc);
@ -541,8 +541,10 @@ public class TestBlockJoin extends LuceneTestCase {
if (VERBOSE) {
System.out.println("DELETE parentID=" + deleteID);
}
w.deleteDocuments(new Term("blockID", ""+deleteID));
joinW.deleteDocuments(new Term("blockID", ""+deleteID));
BytesRef term = new BytesRef();
NumericUtils.intToPrefixCodedBytes(deleteID, 0, term);
w.deleteDocuments(new Term("blockID", term));
joinW.deleteDocuments(new Term("blockID", term));
}
final IndexReader r = w.getReader();

121
lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ByteFieldSource.java
View File

@ -1,121 +0,0 @@
package org.apache.lucene.queries.function.valuesource;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.Map;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.search.FieldCache;
/**
* Obtains int field values from the {@link org.apache.lucene.search.FieldCache}
* using <code>getInts()</code>
* and makes those values available as other numeric types, casting as needed. *
*
*
*/
public class ByteFieldSource extends FieldCacheSource {
private final FieldCache.ByteParser parser;
public ByteFieldSource(String field) {
this(field, null);
}
public ByteFieldSource(String field, FieldCache.ByteParser parser) {
super(field);
this.parser = parser;
}
@Override
public String description() {
return "byte(" + field + ')';
}
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
final FieldCache.Bytes arr = cache.getBytes(readerContext.reader(), field, parser, false);
return new FunctionValues() {
@Override
public byte byteVal(int doc) {
return arr.get(doc);
}
@Override
public short shortVal(int doc) {
return (short) arr.get(doc);
}
@Override
public float floatVal(int doc) {
return (float) arr.get(doc);
}
@Override
public int intVal(int doc) {
return (int) arr.get(doc);
}
@Override
public long longVal(int doc) {
return (long) arr.get(doc);
}
@Override
public double doubleVal(int doc) {
return (double) arr.get(doc);
}
@Override
public String strVal(int doc) {
return Byte.toString(arr.get(doc));
}
@Override
public String toString(int doc) {
return description() + '=' + byteVal(doc);
}
@Override
public Object objectVal(int doc) {
return arr.get(doc); // TODO: valid?
}
};
}
@Override
public boolean equals(Object o) {
if (o.getClass() != ByteFieldSource.class) return false;
ByteFieldSource
other = (ByteFieldSource) o;
return super.equals(other)
&& (this.parser == null ? other.parser == null :
this.parser.getClass() == other.parser.getClass());
}
@Override
public int hashCode() {
int h = parser == null ? Byte.class.hashCode() : parser.getClass().hashCode();
h += super.hashCode();
return h;
}
}

114
lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ShortFieldSource.java
View File

@ -1,114 +0,0 @@
package org.apache.lucene.queries.function.valuesource;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.Map;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.search.FieldCache;
/**
* Obtains short field values from the {@link org.apache.lucene.search.FieldCache}
* using <code>getShorts()</code>
* and makes those values available as other numeric types, casting as needed.
**/
public class ShortFieldSource extends FieldCacheSource {
final FieldCache.ShortParser parser;
public ShortFieldSource(String field) {
this(field, null);
}
public ShortFieldSource(String field, FieldCache.ShortParser parser) {
super(field);
this.parser = parser;
}
@Override
public String description() {
return "short(" + field + ')';
}
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
final FieldCache.Shorts arr = cache.getShorts(readerContext.reader(), field, parser, false);
return new FunctionValues() {
@Override
public byte byteVal(int doc) {
return (byte) arr.get(doc);
}
@Override
public short shortVal(int doc) {
return arr.get(doc);
}
@Override
public float floatVal(int doc) {
return (float) arr.get(doc);
}
@Override
public int intVal(int doc) {
return (int) arr.get(doc);
}
@Override
public long longVal(int doc) {
return (long) arr.get(doc);
}
@Override
public double doubleVal(int doc) {
return (double) arr.get(doc);
}
@Override
public String strVal(int doc) {
return Short.toString(arr.get(doc));
}
@Override
public String toString(int doc) {
return description() + '=' + shortVal(doc);
}
};
}
@Override
public boolean equals(Object o) {
if (o.getClass() != ShortFieldSource.class) return false;
ShortFieldSource
other = (ShortFieldSource) o;
return super.equals(other)
&& (this.parser == null ? other.parser == null :
this.parser.getClass() == other.parser.getClass());
}
@Override
public int hashCode() {
int h = parser == null ? Short.class.hashCode() : parser.getClass().hashCode();
h += super.hashCode();
return h;
}
}

20
lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java
View File

@ -52,26 +52,6 @@ public class TestCustomScoreQuery extends FunctionTestSetup {
createIndex(true);
}
/**
* Test that CustomScoreQuery of Type.BYTE returns the expected scores.
*/
@Test
public void testCustomScoreByte() throws Exception {
// INT field values are small enough to be parsed as byte
doTestCustomScore(BYTE_VALUESOURCE, 1.0);
doTestCustomScore(BYTE_VALUESOURCE, 2.0);
}
/**
* Test that CustomScoreQuery of Type.SHORT returns the expected scores.
*/
@Test
public void testCustomScoreShort() throws Exception {
// INT field values are small enough to be parsed as short
doTestCustomScore(SHORT_VALUESOURCE, 1.0);
doTestCustomScore(SHORT_VALUESOURCE, 3.0);
}
/**
* Test that CustomScoreQuery of Type.INT returns the expected scores.
*/

34
lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java
View File

@ -1,18 +1,25 @@
package org.apache.lucene.queries.function;
import java.io.IOException;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.FloatField;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.queries.function.valuesource.ByteFieldSource;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
import org.apache.lucene.queries.function.valuesource.IntFieldSource;
import org.apache.lucene.queries.function.valuesource.ShortFieldSource;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.junit.AfterClass;
@ -53,10 +60,21 @@ public abstract class FunctionTestSetup extends LuceneTestCase {
protected static final String INT_FIELD = "iii";
protected static final String FLOAT_FIELD = "fff";
protected ValueSource BYTE_VALUESOURCE = new ByteFieldSource(INT_FIELD);
protected ValueSource SHORT_VALUESOURCE = new ShortFieldSource(INT_FIELD);
private static final FieldCache.FloatParser CUSTOM_FLOAT_PARSER = new FieldCache.FloatParser() {
@Override
public TermsEnum termsEnum(Terms terms) throws IOException {
return FieldCache.NUMERIC_UTILS_INT_PARSER.termsEnum(terms);
}
@Override
public float parseFloat(BytesRef term) {
return (float) FieldCache.NUMERIC_UTILS_INT_PARSER.parseInt(term);
}
};
protected ValueSource INT_VALUESOURCE = new IntFieldSource(INT_FIELD);
protected ValueSource INT_AS_FLOAT_VALUESOURCE = new FloatFieldSource(INT_FIELD);
protected ValueSource INT_AS_FLOAT_VALUESOURCE = new FloatFieldSource(INT_FIELD, CUSTOM_FLOAT_PARSER);
protected ValueSource FLOAT_VALUESOURCE = new FloatFieldSource(FLOAT_FIELD);
private static final String DOC_TEXT_LINES[] = {
@ -140,10 +158,10 @@ public abstract class FunctionTestSetup extends LuceneTestCase {
f = newField(TEXT_FIELD, "text of doc" + scoreAndID + textLine(i), customType2); // for regular search
d.add(f);
f = newField(INT_FIELD, "" + scoreAndID, customType); // for function scoring
f = new IntField(INT_FIELD, scoreAndID, Store.YES); // for function scoring
d.add(f);
f = newField(FLOAT_FIELD, scoreAndID + ".000", customType); // for function scoring
f = new FloatField(FLOAT_FIELD, scoreAndID, Store.YES); // for function scoring
d.add(f);
iw.addDocument(d);

34
lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java
View File

@ -19,12 +19,6 @@ package org.apache.lucene.queries.function;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queries.function.FunctionQuery;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.ByteFieldSource;
import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
import org.apache.lucene.queries.function.valuesource.IntFieldSource;
import org.apache.lucene.queries.function.valuesource.ShortFieldSource;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.QueryUtils;
import org.apache.lucene.search.ScoreDoc;
@ -50,20 +44,6 @@ public class TestFieldScoreQuery extends FunctionTestSetup {
createIndex(true);
}
/** Test that FieldScoreQuery of Type.BYTE returns docs in expected order. */
@Test
public void testRankByte () throws Exception {
// INT field values are small enough to be parsed as byte
doTestRank(BYTE_VALUESOURCE);
}
/** Test that FieldScoreQuery of Type.SHORT returns docs in expected order. */
@Test
public void testRankShort () throws Exception {
// INT field values are small enough to be parsed as short
doTestRank(SHORT_VALUESOURCE);
}
/** Test that FieldScoreQuery of Type.INT returns docs in expected order. */
@Test
public void testRankInt () throws Exception {
@ -99,20 +79,6 @@ public class TestFieldScoreQuery extends FunctionTestSetup {
r.close();
}
/** Test that FieldScoreQuery of Type.BYTE returns the expected scores. */
@Test
public void testExactScoreByte () throws Exception {
// INT field values are small enough to be parsed as byte
doTestExactScore(BYTE_VALUESOURCE);
}
/** Test that FieldScoreQuery of Type.SHORT returns the expected scores. */
@Test
public void testExactScoreShort () throws Exception {
// INT field values are small enough to be parsed as short
doTestExactScore(SHORT_VALUESOURCE);
}
/** Test that FieldScoreQuery of Type.INT returns the expected scores. */
@Test
public void testExactScoreInt () throws Exception {

7
lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java
View File

@ -18,9 +18,10 @@ package org.apache.lucene.queries.function;
*/
import java.io.IOException;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.IntField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
@ -46,13 +47,13 @@ public class TestFunctionQuerySort extends LuceneTestCase {
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
Document doc = new Document();
Field field = new StringField("value", "", Field.Store.YES);
Field field = new IntField("value", 0, Field.Store.YES);
doc.add(field);
// Save docs unsorted (decreasing value n, n-1, ...)
final int NUM_VALS = 5;
for (int val = NUM_VALS; val > 0; val--) {
field.setStringValue(Integer.toString(val));
field.setIntValue(val);
writer.addDocument(doc);
}

48
lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java
View File

@ -22,13 +22,16 @@ import java.util.List;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FloatField;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.queries.function.valuesource.ByteFieldSource;
import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource;
import org.apache.lucene.queries.function.valuesource.ConstValueSource;
import org.apache.lucene.queries.function.valuesource.DivFloatFunction;
@ -54,7 +57,6 @@ import org.apache.lucene.queries.function.valuesource.QueryValueSource;
import org.apache.lucene.queries.function.valuesource.RangeMapFloatFunction;
import org.apache.lucene.queries.function.valuesource.ReciprocalFloatFunction;
import org.apache.lucene.queries.function.valuesource.ScaleFloatFunction;
import org.apache.lucene.queries.function.valuesource.ShortFieldSource;
import org.apache.lucene.queries.function.valuesource.SumFloatFunction;
import org.apache.lucene.queries.function.valuesource.SumTotalTermFreqValueSource;
import org.apache.lucene.queries.function.valuesource.TFValueSource;
@ -85,9 +87,9 @@ public class TestValueSources extends LuceneTestCase {
static IndexSearcher searcher;
static final List<String[]> documents = Arrays.asList(new String[][] {
/* id, byte, double, float, int, long, short, string, text */
new String[] { "0", "5", "3.63", "5.2", "35", "4343", "945", "test", "this is a test test test" },
new String[] { "1", "12", "5.65", "9.3", "54", "1954", "123", "bar", "second test" },
/* id, double, float, int, long, string, text */
new String[] { "0", "3.63", "5.2", "35", "4343", "test", "this is a test test test" },
new String[] { "1", "5.65", "9.3", "54", "1954", "bar", "second test" },
});
@BeforeClass
@ -99,18 +101,14 @@ public class TestValueSources extends LuceneTestCase {
Document document = new Document();
Field idField = new StringField("id", "", Field.Store.NO);
document.add(idField);
Field byteField = new StringField("byte", "", Field.Store.NO);
document.add(byteField);
Field doubleField = new StringField("double", "", Field.Store.NO);
Field doubleField = new DoubleField("double", 0d, Field.Store.NO);
document.add(doubleField);
Field floatField = new StringField("float", "", Field.Store.NO);
Field floatField = new FloatField("float", 0f, Field.Store.NO);
document.add(floatField);
Field intField = new StringField("int", "", Field.Store.NO);
Field intField = new IntField("int", 0, Field.Store.NO);
document.add(intField);
Field longField = new StringField("long", "", Field.Store.NO);
Field longField = new LongField("long", 0L, Field.Store.NO);
document.add(longField);
Field shortField = new StringField("short", "", Field.Store.NO);
document.add(shortField);
Field stringField = new StringField("string", "", Field.Store.NO);
document.add(stringField);
Field textField = new TextField("text", "", Field.Store.NO);
@ -118,14 +116,12 @@ public class TestValueSources extends LuceneTestCase {
for (String [] doc : documents) {
idField.setStringValue(doc[0]);
byteField.setStringValue(doc[1]);
doubleField.setStringValue(doc[2]);
floatField.setStringValue(doc[3]);
intField.setStringValue(doc[4]);
longField.setStringValue(doc[5]);
shortField.setStringValue(doc[6]);
stringField.setStringValue(doc[7]);
textField.setStringValue(doc[8]);
doubleField.setDoubleValue(Double.valueOf(doc[1]));
floatField.setFloatValue(Float.valueOf(doc[2]));
intField.setIntValue(Integer.valueOf(doc[3]));
longField.setLongValue(Long.valueOf(doc[4]));
stringField.setStringValue(doc[5]);
textField.setStringValue(doc[6]);
iw.addDocument(document);
}
@ -143,11 +139,6 @@ public class TestValueSources extends LuceneTestCase {
dir = null;
}
public void testByte() throws Exception {
assertHits(new FunctionQuery(new ByteFieldSource("byte")),
new float[] { 5f, 12f });
}
public void testConst() throws Exception {
assertHits(new FunctionQuery(new ConstValueSource(0.3f)),
new float[] { 0.3f, 0.3f });
@ -298,11 +289,6 @@ public class TestValueSources extends LuceneTestCase {
new float[] { 0.0f, 1.0f });
}
public void testShort() throws Exception {
assertHits(new FunctionQuery(new ShortFieldSource("short")),
new float[] { 945f, 123f });
}
public void testSumFloat() throws Exception {
assertHits(new FunctionQuery(new SumFloatFunction(new ValueSource[] {
new ConstValueSource(1f), new ConstValueSource(2f)})),

9
lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java
View File

@ -36,6 +36,7 @@ import java.util.zip.GZIPInputStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
@ -158,6 +159,7 @@ public class LineFileDocs implements Closeable {
final Field titleDV;
final Field body;
final Field id;
final Field idNum;
final Field date;
public DocState(boolean useDocValues) {
@ -180,6 +182,9 @@ public class LineFileDocs implements Closeable {
id = new StringField("docid", "", Field.Store.YES);
doc.add(id);
idNum = new IntField("docid_int", 0, Field.Store.NO);
doc.add(idNum);
date = new StringField("date", "", Field.Store.YES);
doc.add(date);
@ -233,7 +238,9 @@ public class LineFileDocs implements Closeable {
}
docState.titleTokenized.setStringValue(title);
docState.date.setStringValue(line.substring(1+spot, spot2));
docState.id.setStringValue(Integer.toString(id.getAndIncrement()));
final int i = id.getAndIncrement();
docState.id.setStringValue(Integer.toString(i));
docState.idNum.setIntValue(i);
return docState.doc;
}
}

26
lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java
View File

@ -49,7 +49,12 @@ import org.apache.lucene.codecs.lucene42.Lucene42Codec;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType.NumericType;
import org.apache.lucene.document.FloatField;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.index.AtomicReader;
@ -78,14 +83,15 @@ import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.FilteredQuery.FilterStrategy;
import org.apache.lucene.search.FilteredQuery;
import org.apache.lucene.search.FilteredQuery.FilterStrategy;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.junit.Assert;
import com.carrotsearch.randomizedtesting.RandomizedContext;
import com.carrotsearch.randomizedtesting.generators.RandomInts;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
@ -872,6 +878,7 @@ public class _TestUtil {
final Field field1 = (Field) f;
final Field field2;
final DocValuesType dvType = field1.fieldType().docValueType();
final NumericType numType = field1.fieldType().numericType();
if (dvType != null) {
switch(dvType) {
case NUMERIC:
@ -886,6 +893,23 @@ public class _TestUtil {
default:
throw new IllegalStateException("unknown Type: " + dvType);
}
} else if (numType != null) {
switch (numType) {
case INT:
field2 = new IntField(field1.name(), field1.numericValue().intValue(), field1.fieldType());
break;
case FLOAT:
field2 = new FloatField(field1.name(), field1.numericValue().intValue(), field1.fieldType());
break;
case LONG:
field2 = new LongField(field1.name(), field1.numericValue().intValue(), field1.fieldType());
break;
case DOUBLE:
field2 = new DoubleField(field1.name(), field1.numericValue().intValue(), field1.fieldType());
break;
default:
throw new IllegalStateException("unknown Type: " + numType);
}
} else {
field2 = new Field(field1.name(), field1.stringValue(), field1.fieldType());
}

4
solr/CHANGES.txt
View File

@ -79,6 +79,10 @@ Upgrading from Solr 4.3.0
* SOLR-4778: The signature of LogWatcher.registerListener has changed, from
(ListenerConfig, CoreContainer) to (ListenerConfig). Users implementing their
own LogWatcher classes will need to change their code accordingly.
* LUCENE-5063: ByteField and ShortField have been deprecated and will be removed
in 5.0. If you are still using these field types, you should migrate your
fields to TrieIntField.
Detailed Change List
----------------------

2
solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
View File

@ -49,9 +49,7 @@ public class StatsValuesFactory {
if (DoubleField.class.isInstance(fieldType) ||
IntField.class.isInstance(fieldType) ||
LongField.class.isInstance(fieldType) ||
ShortField.class.isInstance(fieldType) ||
FloatField.class.isInstance(fieldType) ||
ByteField.class.isInstance(fieldType) ||
TrieField.class.isInstance(fieldType) ||
SortableDoubleField.class.isInstance(fieldType) ||
SortableIntField.class.isInstance(fieldType) ||

2
solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java
View File

@ -260,11 +260,9 @@ public class BinaryResponseWriter implements BinaryQueryResponseWriter {
KNOWN_TYPES.add(BCDIntField.class);
KNOWN_TYPES.add(BCDLongField.class);
KNOWN_TYPES.add(BCDStrField.class);
KNOWN_TYPES.add(ByteField.class);
KNOWN_TYPES.add(DateField.class);
KNOWN_TYPES.add(DoubleField.class);
KNOWN_TYPES.add(FloatField.class);
KNOWN_TYPES.add(ShortField.class);
KNOWN_TYPES.add(IntField.class);
KNOWN_TYPES.add(LongField.class);
KNOWN_TYPES.add(SortableLongField.class);

98
solr/core/src/java/org/apache/solr/schema/ByteField.java
View File

@ -1,98 +0,0 @@
package org.apache.solr.schema;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.ByteFieldSource;
import org.apache.lucene.index.GeneralField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.search.SortField;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import java.io.IOException;
import java.util.Map;
/**
* A numeric field that can contain 8-bit signed two's complement integer
* values, encoded as simple Strings.
*
* <p>
* Field values will sort numerically, but Range Queries (and other features
* that rely on numeric ranges) will not work as expected: values will be
* evaluated in unicode String order, not numeric order.
* </p>
*
* <ul>
* <li>Min Value Allowed: -128</li>
* <li>Max Value Allowed: 127</li>
* </ul>
*
* @see Byte
*/
public class ByteField extends PrimitiveFieldType {
@Override
protected void init(IndexSchema schema, Map<String, String> args) {
super.init(schema, args);
restrictProps(SORT_MISSING_FIRST | SORT_MISSING_LAST);
}
/////////////////////////////////////////////////////////////
@Override
public SortField getSortField(SchemaField field, boolean reverse) {
field.checkSortability();
return new SortField(field.name, SortField.Type.BYTE, reverse);
}
@Override
public ValueSource getValueSource(SchemaField field, QParser qparser) {
field.checkFieldCacheSource(qparser);
return new ByteFieldSource(field.name);
}
@Override
public void write(TextResponseWriter writer, String name, StorableField f) throws IOException {
String s = f.stringValue();
// these values may be from a legacy lucene index, which may
// not be properly formatted in some output formats, or may
// incorrectly have a zero length.
if (s.length()==0) {
// zero length value means someone mistakenly indexed the value
// instead of simply leaving it out. Write a null value instead of a numeric.
writer.writeNull(name);
return;
}
try {
byte val = Byte.parseByte(s);
writer.writeInt(name, val);
} catch (NumberFormatException e){
// can't parse - write out the contents as a string so nothing is lost and
// clients don't get a parse error.
writer.writeStr(name, s, true);
}
}
@Override
public Byte toObject(StorableField f) {
return Byte.valueOf(toExternal(f));
}
}

38
solr/core/src/java/org/apache/solr/schema/DoubleField.java
View File

@ -17,18 +17,20 @@
package org.apache.solr.schema;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.DoubleFieldSource;
import org.apache.lucene.index.GeneralField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.search.SortField;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import java.io.IOException;
import java.util.Map;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.DoubleFieldSource;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
/**
* A legacy numeric field type that encodes "Double" values as simple Strings.
* This class should not be used except by people with existing indexes that
@ -44,6 +46,20 @@ import java.util.Map;
* @see TrieDoubleField
*/
public class DoubleField extends PrimitiveFieldType {
private static final FieldCache.DoubleParser PARSER = new FieldCache.DoubleParser() {
@Override
public TermsEnum termsEnum(Terms terms) throws IOException {
return terms.iterator(null);
}
@Override
public double parseDouble(BytesRef term) {
return Double.parseDouble(term.utf8ToString());
}
};
@Override
protected void init(IndexSchema schema, Map<String, String> args) {
super.init(schema, args);
@ -54,13 +70,13 @@ public class DoubleField extends PrimitiveFieldType {
@Override
public SortField getSortField(SchemaField field, boolean reverse) {
field.checkSortability();
return new SortField(field.name, SortField.Type.DOUBLE, reverse);
return new SortField(field.name, PARSER, reverse);
}
@Override
public ValueSource getValueSource(SchemaField field, QParser qparser) {
field.checkFieldCacheSource(qparser);
return new DoubleFieldSource(field.name);
return new DoubleFieldSource(field.name, PARSER);
}
@Override

20
solr/core/src/java/org/apache/solr/schema/FloatField.java
View File

@ -19,11 +19,15 @@ package org.apache.solr.schema;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.search.QParser;
import org.apache.lucene.index.GeneralField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.solr.response.TextResponseWriter;
import java.util.Map;
@ -43,6 +47,20 @@ import java.io.IOException;
* @see TrieFloatField
*/
public class FloatField extends PrimitiveFieldType {
private static final FieldCache.FloatParser PARSER = new FieldCache.FloatParser() {
@Override
public TermsEnum termsEnum(Terms terms) throws IOException {
return terms.iterator(null);
}
@Override
public float parseFloat(BytesRef term) {
return Float.parseFloat(term.utf8ToString());
}
};
@Override
protected void init(IndexSchema schema, Map<String,String> args) {
super.init(schema, args);
@ -58,7 +76,7 @@ public class FloatField extends PrimitiveFieldType {
@Override
public ValueSource getValueSource(SchemaField field, QParser qparser) {
field.checkFieldCacheSource(qparser);
return new FloatFieldSource(field.name);
return new FloatFieldSource(field.name, PARSER);
}
@Override

22
solr/core/src/java/org/apache/solr/schema/IntField.java
View File

@ -19,11 +19,15 @@ package org.apache.solr.schema;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.IntFieldSource;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.search.QParser;
import org.apache.lucene.index.GeneralField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.solr.response.TextResponseWriter;
import java.util.Map;
@ -43,6 +47,20 @@ import java.io.IOException;
* @see TrieIntField
*/
public class IntField extends PrimitiveFieldType {
private static final FieldCache.IntParser PARSER = new FieldCache.IntParser() {
@Override
public TermsEnum termsEnum(Terms terms) throws IOException {
return terms.iterator(null);
}
@Override
public int parseInt(BytesRef term) {
return Integer.parseInt(term.utf8ToString());
}
};
@Override
protected void init(IndexSchema schema, Map<String,String> args) {
super.init(schema, args);
@ -52,13 +70,13 @@ public class IntField extends PrimitiveFieldType {
@Override
public SortField getSortField(SchemaField field,boolean reverse) {
field.checkSortability();
return new SortField(field.name,SortField.Type.INT, reverse);
return new SortField(field.name, PARSER, reverse);
}
@Override
public ValueSource getValueSource(SchemaField field, QParser qparser) {
field.checkFieldCacheSource(qparser);
return new IntFieldSource(field.name);
return new IntFieldSource(field.name, PARSER);
}
@Override

22
solr/core/src/java/org/apache/solr/schema/LongField.java
View File

@ -22,7 +22,11 @@ import org.apache.lucene.queries.function.valuesource.LongFieldSource;
import org.apache.lucene.index.GeneralField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
@ -43,6 +47,20 @@ import java.util.Map;
* @see TrieLongField
*/
public class LongField extends PrimitiveFieldType {
private static final FieldCache.LongParser PARSER = new FieldCache.LongParser() {
@Override
public TermsEnum termsEnum(Terms terms) throws IOException {
return terms.iterator(null);
}
@Override
public long parseLong(BytesRef term) {
return Long.parseLong(term.utf8ToString());
}
};
@Override
protected void init(IndexSchema schema, Map<String,String> args) {
super.init(schema, args);
@ -54,13 +72,13 @@ public class LongField extends PrimitiveFieldType {
@Override
public SortField getSortField(SchemaField field,boolean reverse) {
field.checkSortability();
return new SortField(field.name,SortField.Type.LONG, reverse);
return new SortField(field.name, PARSER, reverse);
}
@Override
public ValueSource getValueSource(SchemaField field, QParser qparser) {
field.checkFieldCacheSource(qparser);
return new LongFieldSource(field.name);
return new LongFieldSource(field.name, PARSER);
}
@Override

101
solr/core/src/java/org/apache/solr/schema/ShortField.java
View File

@ -1,101 +0,0 @@
package org.apache.solr.schema;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.ShortFieldSource;
import org.apache.lucene.index.GeneralField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.search.SortField;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import java.io.IOException;
import java.util.Map;
/**
* A numeric field that can contain 16-bit signed two's complement integer
* values, encoded as simple Strings.
*
* <p>
* Field values will sort numerically, but Range Queries (and other features
* that rely on numeric ranges) will not work as expected: values will be
* evaluated in unicode String order, not numeric order.
* </p>
*
* <ul>
* <li>Min Value Allowed: -32768</li>
* <li>Max Value Allowed: 32767</li>
* </ul>
*
* @see Short
**/
public class ShortField extends PrimitiveFieldType {
@Override
protected void init(IndexSchema schema, Map<String, String> args) {
super.init(schema, args);
restrictProps(SORT_MISSING_FIRST | SORT_MISSING_LAST);
}
/////////////////////////////////////////////////////////////
@Override
public SortField getSortField(SchemaField field, boolean reverse) {
field.checkSortability();
return new SortField(field.name, SortField.Type.SHORT, reverse);
}
@Override
public ValueSource getValueSource(SchemaField field, QParser qparser) {
field.checkFieldCacheSource(qparser);
return new ShortFieldSource(field.name);
}
@Override
public void write(TextResponseWriter writer, String name, StorableField f) throws IOException {
String s = f.stringValue();
// these values may be from a legacy lucene index, which may
// not be properly formatted in some output formats, or may
// incorrectly have a zero length.
if (s.length()==0) {
// zero length value means someone mistakenly indexed the value
// instead of simply leaving it out. Write a null value instead of a numeric.
writer.writeNull(name);
return;
}
try {
short val = Short.parseShort(s);
writer.writeInt(name, val);
} catch (NumberFormatException e){
// can't parse - write out the contents as a string so nothing is lost and
// clients don't get a parse error.
writer.writeStr(name, s, true);
}
}
@Override
public Short toObject(StorableField f) {
return Short.valueOf(toExternal(f));
}
}

4
solr/core/src/test-files/solr/collection1/conf/schema-folding.xml
View File

@ -221,8 +221,6 @@
<fieldType name="float" class="solr.TrieFloatField" precisionStep="4" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="long" class="solr.TrieLongField" precisionStep="4" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="double" class="solr.TrieDoubleField" precisionStep="4" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="byte" class="solr.ByteField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="short" class="solr.ShortField" omitNorms="true" positionIncrementGap="0"/>
<fieldtype name="boolean" class="solr.BoolField" sortMissingLast="true"/>
<fieldtype name="date" class="solr.TrieDateField" precisionStep="0"/>
</types>
@ -233,8 +231,6 @@
<field name="float_f" type="float"/>
<field name="long_f" type="long"/>
<field name="double_f" type="double"/>
<field name="byte_f" type="byte"/>
<field name="short_f" type="short"/>
<field name="bool_f" type="boolean"/>
<field name="date_f" type="date"/>

6
solr/core/src/test-files/solr/collection1/conf/schema.xml
View File

@ -48,8 +48,6 @@
<fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="byte" class="solr.ByteField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="short" class="solr.ShortField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tint" class="solr.TrieIntField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
@ -609,10 +607,6 @@
<dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
<dynamicField name="*_dt1" type="date" indexed="true" stored="true" multiValued="false"/>
<dynamicField name="*_bcd" type="bcdstr" indexed="true" stored="true"/>
<dynamicField name="*_by" type="byte" indexed="true" stored="true"/>
<dynamicField name="*_by1" type="byte" indexed="true" stored="true" multiValued="false"/>
<dynamicField name="*_sh" type="short" indexed="true" stored="true"/>
<dynamicField name="*_sh1" type="short" indexed="true" stored="true" multiValued="false"/>
<!-- some trie-coded dynamic fields for faster range queries -->

2
solr/core/src/test/org/apache/solr/core/QueryResultKeyTest.java
View File

@ -38,7 +38,7 @@ public class QueryResultKeyTest extends SolrTestCaseJ4 {
// the hashcode should be the same even when the list
// of filters is in a different order
Sort sort = new Sort(new SortField("test", SortField.Type.BYTE));
Sort sort = new Sort(new SortField("test", SortField.Type.INT));
List<Query> filters = new ArrayList<Query>();
filters.add(new TermQuery(new Term("test", "field")));
filters.add(new TermQuery(new Term("test2", "field2")));

18
solr/core/src/test/org/apache/solr/schema/PrimitiveFieldTypeTest.java
View File

@ -59,11 +59,9 @@ public class PrimitiveFieldTypeTest extends SolrTestCaseJ4 {
TrieIntField ti;
SortableIntField si;
LongField l;
ShortField sf;
FloatField f;
DoubleField d;
BoolField b;
ByteField bf;
// ***********************
@ -95,10 +93,6 @@ public class PrimitiveFieldTypeTest extends SolrTestCaseJ4 {
l.init(schema, initMap);
assertFalse(l.hasProperty(FieldType.OMIT_NORMS));
sf = new ShortField();
sf.init(schema, initMap);
assertFalse(sf.hasProperty(FieldType.OMIT_NORMS));
f = new FloatField();
f.init(schema, initMap);
assertFalse(f.hasProperty(FieldType.OMIT_NORMS));
@ -115,10 +109,6 @@ public class PrimitiveFieldTypeTest extends SolrTestCaseJ4 {
b.init(schema, initMap);
assertFalse(b.hasProperty(FieldType.OMIT_NORMS));
bf = new ByteField();
bf.init(schema, initMap);
assertFalse(bf.hasProperty(FieldType.OMIT_NORMS));
// Non-primitive fields
t = new TextField();
t.init(schema, initMap);
@ -157,10 +147,6 @@ public class PrimitiveFieldTypeTest extends SolrTestCaseJ4 {
l.init(schema, initMap);
assertTrue(l.hasProperty(FieldType.OMIT_NORMS));
sf = new ShortField();
sf.init(schema, initMap);
assertTrue(sf.hasProperty(FieldType.OMIT_NORMS));
f = new FloatField();
f.init(schema, initMap);
assertTrue(f.hasProperty(FieldType.OMIT_NORMS));
@ -177,10 +163,6 @@ public class PrimitiveFieldTypeTest extends SolrTestCaseJ4 {
b.init(schema, initMap);
assertTrue(b.hasProperty(FieldType.OMIT_NORMS));
bf = new ByteField();
bf.init(schema, initMap);
assertTrue(bf.hasProperty(FieldType.OMIT_NORMS));
// Non-primitive fields
t = new TextField();
t.init(schema, initMap);

4
solr/core/src/test/org/apache/solr/search/TestFoldingMultitermQuery.java
View File

@ -46,8 +46,6 @@ public class TestFoldingMultitermQuery extends SolrTestCaseJ4 {
"float_f", num,
"long_f", num,
"double_f", num,
"byte_f", num,
"short_f", num,
"bool_f", boolVal,
"date_f", "200" + Integer.toString(i % 10) + "-01-01T00:00:00Z",
"content", docs[i],
@ -269,7 +267,7 @@ public class TestFoldingMultitermQuery extends SolrTestCaseJ4 {
@Test
public void testNonTextTypes() {
String[] intTypes = {"int_f", "float_f", "long_f", "double_f", "byte_f", "short_f"};
String[] intTypes = {"int_f", "float_f", "long_f", "double_f"};
for (String str : intTypes) {
assertQ(req("q", str + ":" + "0"),

2
solr/core/src/test/org/apache/solr/search/TestValueSourceCache.java
View File

@ -78,8 +78,6 @@ public class TestValueSourceCache extends SolrTestCaseJ4 {
"val1_i1",
"val1_l1",
"val1_b1",
"val1_by1",
"val1_sh1"
};
for (String template : templates) {
for (String nums : numbers) {