diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 5671a0d9784..9c22802ec37 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -103,6 +103,12 @@ Changes in backwards compatibility policy on segment size and noCFSRatio. The default implemantion was pulled up from TieredMergePolicy. (Simon Willnauer) +* LUCENE-5063: FieldCache.get(Bytes|Shorts), SortField.Type.(BYTE|SHORT) and + FieldCache.DEFAULT_(BYTE|SHORT|INT|LONG|FLOAT|DOUBLE)_PARSER are now + deprecated. These methods/types assume that data is stored as strings although + Lucene has much better support for numeric data through (Int|Long)Field, + NumericRangeQuery and FieldCache.get(Int|Long)s. (Adrien Grand) + Bug Fixes * LUCENE-4997: Internal test framework's tests are sensitive to previous @@ -233,6 +239,9 @@ New Features * LUCENE-5025: FST's Builder can now handle more than 2.1 billion "tail nodes" while building a minimal FST. (Aaron Binns, Adrien Grand, Mike McCandless) + +* LUCENE-5063: FieldCache.DEFAULT.get(Ints|Longs) now uses bit-packing to save + memory. (Adrien Grand) Build diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldCache.java b/lucene/core/src/java/org/apache/lucene/search/FieldCache.java index 4a3100a5732..f87fb51acf3 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FieldCache.java +++ b/lucene/core/src/java/org/apache/lucene/search/FieldCache.java @@ -49,34 +49,6 @@ import org.apache.lucene.util.RamUsageEstimator; */ public interface FieldCache { - /** Field values as 8-bit signed bytes */ - public static abstract class Bytes { - /** Return a single Byte representation of this field's value. */ - public abstract byte get(int docID); - - /** Zero value for every document */ - public static final Bytes EMPTY = new Bytes() { - @Override - public byte get(int docID) { - return 0; - } - }; - } - - /** Field values as 16-bit signed shorts */ - public static abstract class Shorts { - /** Return a short representation of this field's value. */ - public abstract short get(int docID); - - /** Zero value for every document */ - public static final Shorts EMPTY = new Shorts() { - @Override - public short get(int docID) { - return 0; - } - }; - } - /** Field values as 32-bit signed integers */ public static abstract class Ints { /** Return an integer representation of this field's value. */ @@ -178,22 +150,6 @@ public interface FieldCache { public TermsEnum termsEnum(Terms terms) throws IOException; } - /** Interface to parse bytes from document fields. - * @see FieldCache#getBytes(AtomicReader, String, FieldCache.ByteParser, boolean) - */ - public interface ByteParser extends Parser { - /** Return a single Byte representation of this field's value. */ - public byte parseByte(BytesRef term); - } - - /** Interface to parse shorts from document fields. - * @see FieldCache#getShorts(AtomicReader, String, FieldCache.ShortParser, boolean) - */ - public interface ShortParser extends Parser { - /** Return a short representation of this field's value. */ - public short parseShort(BytesRef term); - } - /** Interface to parse ints from document fields. * @see FieldCache#getInts(AtomicReader, String, FieldCache.IntParser, boolean) */ @@ -229,135 +185,6 @@ public interface FieldCache { /** Expert: The cache used internally by sorting and range query classes. */ public static FieldCache DEFAULT = new FieldCacheImpl(); - /** The default parser for byte values, which are encoded by {@link Byte#toString(byte)} */ - public static final ByteParser DEFAULT_BYTE_PARSER = new ByteParser() { - @Override - public byte parseByte(BytesRef term) { - // TODO: would be far better to directly parse from - // UTF8 bytes... but really users should use - // IntField, instead, which already decodes - // directly from byte[] - return Byte.parseByte(term.utf8ToString()); - } - @Override - public String toString() { - return FieldCache.class.getName()+".DEFAULT_BYTE_PARSER"; - } - @Override - public TermsEnum termsEnum(Terms terms) throws IOException { - return terms.iterator(null); - } - }; - - /** The default parser for short values, which are encoded by {@link Short#toString(short)} */ - public static final ShortParser DEFAULT_SHORT_PARSER = new ShortParser() { - @Override - public short parseShort(BytesRef term) { - // TODO: would be far better to directly parse from - // UTF8 bytes... but really users should use - // IntField, instead, which already decodes - // directly from byte[] - return Short.parseShort(term.utf8ToString()); - } - @Override - public String toString() { - return FieldCache.class.getName()+".DEFAULT_SHORT_PARSER"; - } - - @Override - public TermsEnum termsEnum(Terms terms) throws IOException { - return terms.iterator(null); - } - }; - - /** The default parser for int values, which are encoded by {@link Integer#toString(int)} */ - public static final IntParser DEFAULT_INT_PARSER = new IntParser() { - @Override - public int parseInt(BytesRef term) { - // TODO: would be far better to directly parse from - // UTF8 bytes... but really users should use - // IntField, instead, which already decodes - // directly from byte[] - return Integer.parseInt(term.utf8ToString()); - } - - @Override - public TermsEnum termsEnum(Terms terms) throws IOException { - return terms.iterator(null); - } - - @Override - public String toString() { - return FieldCache.class.getName()+".DEFAULT_INT_PARSER"; - } - }; - - /** The default parser for float values, which are encoded by {@link Float#toString(float)} */ - public static final FloatParser DEFAULT_FLOAT_PARSER = new FloatParser() { - @Override - public float parseFloat(BytesRef term) { - // TODO: would be far better to directly parse from - // UTF8 bytes... but really users should use - // FloatField, instead, which already decodes - // directly from byte[] - return Float.parseFloat(term.utf8ToString()); - } - - @Override - public TermsEnum termsEnum(Terms terms) throws IOException { - return terms.iterator(null); - } - - @Override - public String toString() { - return FieldCache.class.getName()+".DEFAULT_FLOAT_PARSER"; - } - }; - - /** The default parser for long values, which are encoded by {@link Long#toString(long)} */ - public static final LongParser DEFAULT_LONG_PARSER = new LongParser() { - @Override - public long parseLong(BytesRef term) { - // TODO: would be far better to directly parse from - // UTF8 bytes... but really users should use - // LongField, instead, which already decodes - // directly from byte[] - return Long.parseLong(term.utf8ToString()); - } - - @Override - public TermsEnum termsEnum(Terms terms) throws IOException { - return terms.iterator(null); - } - - @Override - public String toString() { - return FieldCache.class.getName()+".DEFAULT_LONG_PARSER"; - } - }; - - /** The default parser for double values, which are encoded by {@link Double#toString(double)} */ - public static final DoubleParser DEFAULT_DOUBLE_PARSER = new DoubleParser() { - @Override - public double parseDouble(BytesRef term) { - // TODO: would be far better to directly parse from - // UTF8 bytes... but really users should use - // DoubleField, instead, which already decodes - // directly from byte[] - return Double.parseDouble(term.utf8ToString()); - } - - @Override - public TermsEnum termsEnum(Terms terms) throws IOException { - return terms.iterator(null); - } - - @Override - public String toString() { - return FieldCache.class.getName()+".DEFAULT_DOUBLE_PARSER"; - } - }; - /** * A parser instance for int values encoded by {@link NumericUtils}, e.g. when indexed * via {@link IntField}/{@link NumericTokenStream}. @@ -449,60 +276,6 @@ public interface FieldCache { */ public Bits getDocsWithField(AtomicReader reader, String field) throws IOException; - /** Checks the internal cache for an appropriate entry, and if none is - * found, reads the terms in field as a single byte and returns an array - * of size reader.maxDoc() of the value each document - * has in the given field. - * @param reader Used to get field values. - * @param field Which field contains the single byte values. - * @param setDocsWithField If true then {@link #getDocsWithField} will - * also be computed and stored in the FieldCache. - * @return The values in the given field for each document. - * @throws IOException If any error occurs. - */ - public Bytes getBytes(AtomicReader reader, String field, boolean setDocsWithField) throws IOException; - - /** Checks the internal cache for an appropriate entry, and if none is found, - * reads the terms in field as bytes and returns an array of - * size reader.maxDoc() of the value each document has in the - * given field. - * @param reader Used to get field values. - * @param field Which field contains the bytes. - * @param parser Computes byte for string values. - * @param setDocsWithField If true then {@link #getDocsWithField} will - * also be computed and stored in the FieldCache. - * @return The values in the given field for each document. - * @throws IOException If any error occurs. - */ - public Bytes getBytes(AtomicReader reader, String field, ByteParser parser, boolean setDocsWithField) throws IOException; - - /** Checks the internal cache for an appropriate entry, and if none is - * found, reads the terms in field as shorts and returns an array - * of size reader.maxDoc() of the value each document - * has in the given field. - * @param reader Used to get field values. - * @param field Which field contains the shorts. - * @param setDocsWithField If true then {@link #getDocsWithField} will - * also be computed and stored in the FieldCache. - * @return The values in the given field for each document. - * @throws IOException If any error occurs. - */ - public Shorts getShorts (AtomicReader reader, String field, boolean setDocsWithField) throws IOException; - - /** Checks the internal cache for an appropriate entry, and if none is found, - * reads the terms in field as shorts and returns an array of - * size reader.maxDoc() of the value each document has in the - * given field. - * @param reader Used to get field values. - * @param field Which field contains the shorts. - * @param parser Computes short for string values. - * @param setDocsWithField If true then {@link #getDocsWithField} will - * also be computed and stored in the FieldCache. - * @return The values in the given field for each document. - * @throws IOException If any error occurs. - */ - public Shorts getShorts (AtomicReader reader, String field, ShortParser parser, boolean setDocsWithField) throws IOException; - /** Checks the internal cache for an appropriate entry, and if none is * found, reads the terms in field as integers and returns an array * of size reader.maxDoc() of the value each document diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java b/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java index 89ab8558b1f..9b3d3a1bd4d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java +++ b/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java @@ -38,7 +38,6 @@ import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FieldCacheSanityChecker; @@ -63,8 +62,6 @@ class FieldCacheImpl implements FieldCache { private synchronized void init() { caches = new HashMap,Cache>(9); - caches.put(Byte.TYPE, new ByteCache(this)); - caches.put(Short.TYPE, new ShortCache(this)); caches.put(Integer.TYPE, new IntCache(this)); caches.put(Float.TYPE, new FloatCache(this)); caches.put(Long.TYPE, new LongCache(this)); @@ -355,192 +352,6 @@ class FieldCacheImpl implements FieldCache { } caches.get(DocsWithFieldCache.class).put(reader, new CacheKey(field, null), bits); } - - // inherit javadocs - public Bytes getBytes (AtomicReader reader, String field, boolean setDocsWithField) throws IOException { - return getBytes(reader, field, null, setDocsWithField); - } - - // inherit javadocs - public Bytes getBytes(AtomicReader reader, String field, ByteParser parser, boolean setDocsWithField) - throws IOException { - final NumericDocValues valuesIn = reader.getNumericDocValues(field); - if (valuesIn != null) { - // Not cached here by FieldCacheImpl (cached instead - // per-thread by SegmentReader): - return new Bytes() { - @Override - public byte get(int docID) { - return (byte) valuesIn.get(docID); - } - }; - } else { - final FieldInfo info = reader.getFieldInfos().fieldInfo(field); - if (info == null) { - return Bytes.EMPTY; - } else if (info.hasDocValues()) { - throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType()); - } else if (!info.isIndexed()) { - return Bytes.EMPTY; - } - return (Bytes) caches.get(Byte.TYPE).get(reader, new CacheKey(field, parser), setDocsWithField); - } - } - - static class BytesFromArray extends Bytes { - private final byte[] values; - - public BytesFromArray(byte[] values) { - this.values = values; - } - - @Override - public byte get(int docID) { - return values[docID]; - } - } - - static final class ByteCache extends Cache { - ByteCache(FieldCacheImpl wrapper) { - super(wrapper); - } - - @Override - protected Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField) - throws IOException { - - int maxDoc = reader.maxDoc(); - final byte[] values; - final ByteParser parser = (ByteParser) key.custom; - if (parser == null) { - // Confusing: must delegate to wrapper (vs simply - // setting parser = DEFAULT_SHORT_PARSER) so cache - // key includes DEFAULT_SHORT_PARSER: - return wrapper.getBytes(reader, key.field, DEFAULT_BYTE_PARSER, setDocsWithField); - } - - values = new byte[maxDoc]; - - Uninvert u = new Uninvert() { - private byte currentValue; - - @Override - public void visitTerm(BytesRef term) { - currentValue = parser.parseByte(term); - } - - @Override - public void visitDoc(int docID) { - values[docID] = currentValue; - } - - @Override - protected TermsEnum termsEnum(Terms terms) throws IOException { - return parser.termsEnum(terms); - } - }; - - u.uninvert(reader, key.field, setDocsWithField); - - if (setDocsWithField) { - wrapper.setDocsWithField(reader, key.field, u.docsWithField); - } - - return new BytesFromArray(values); - } - } - - // inherit javadocs - public Shorts getShorts (AtomicReader reader, String field, boolean setDocsWithField) throws IOException { - return getShorts(reader, field, null, setDocsWithField); - } - - // inherit javadocs - public Shorts getShorts(AtomicReader reader, String field, ShortParser parser, boolean setDocsWithField) - throws IOException { - final NumericDocValues valuesIn = reader.getNumericDocValues(field); - if (valuesIn != null) { - // Not cached here by FieldCacheImpl (cached instead - // per-thread by SegmentReader): - return new Shorts() { - @Override - public short get(int docID) { - return (short) valuesIn.get(docID); - } - }; - } else { - final FieldInfo info = reader.getFieldInfos().fieldInfo(field); - if (info == null) { - return Shorts.EMPTY; - } else if (info.hasDocValues()) { - throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType()); - } else if (!info.isIndexed()) { - return Shorts.EMPTY; - } - return (Shorts) caches.get(Short.TYPE).get(reader, new CacheKey(field, parser), setDocsWithField); - } - } - - static class ShortsFromArray extends Shorts { - private final short[] values; - - public ShortsFromArray(short[] values) { - this.values = values; - } - - @Override - public short get(int docID) { - return values[docID]; - } - } - - static final class ShortCache extends Cache { - ShortCache(FieldCacheImpl wrapper) { - super(wrapper); - } - - @Override - protected Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField) - throws IOException { - - int maxDoc = reader.maxDoc(); - final short[] values; - final ShortParser parser = (ShortParser) key.custom; - if (parser == null) { - // Confusing: must delegate to wrapper (vs simply - // setting parser = DEFAULT_SHORT_PARSER) so cache - // key includes DEFAULT_SHORT_PARSER: - return wrapper.getShorts(reader, key.field, DEFAULT_SHORT_PARSER, setDocsWithField); - } - - values = new short[maxDoc]; - Uninvert u = new Uninvert() { - private short currentValue; - - @Override - public void visitTerm(BytesRef term) { - currentValue = parser.parseShort(term); - } - - @Override - public void visitDoc(int docID) { - values[docID] = currentValue; - } - - @Override - protected TermsEnum termsEnum(Terms terms) throws IOException { - return parser.termsEnum(terms); - } - }; - - u.uninvert(reader, key.field, setDocsWithField); - - if (setDocsWithField) { - wrapper.setDocsWithField(reader, key.field, u.docsWithField); - } - return new ShortsFromArray(values); - } - } // inherit javadocs public Ints getInts (AtomicReader reader, String field, boolean setDocsWithField) throws IOException { @@ -574,15 +385,19 @@ class FieldCacheImpl implements FieldCache { } static class IntsFromArray extends Ints { - private final int[] values; + private final PackedInts.Reader values; + private final int minValue; - public IntsFromArray(int[] values) { + public IntsFromArray(PackedInts.Reader values, int minValue) { + assert values.getBitsPerValue() <= 32; this.values = values; + this.minValue = minValue; } @Override public int get(int docID) { - return values[docID]; + final long delta = values.get(docID); + return minValue + (int) delta; } } @@ -598,6 +413,15 @@ class FieldCacheImpl implements FieldCache { } } + private static class GrowableWriterAndMinValue { + GrowableWriterAndMinValue(GrowableWriter array, long minValue) { + this.writer = array; + this.minValue = minValue; + } + public GrowableWriter writer; + public long minValue; + } + static final class IntCache extends Cache { IntCache(FieldCacheImpl wrapper) { super(wrapper); @@ -610,22 +434,17 @@ class FieldCacheImpl implements FieldCache { final IntParser parser = (IntParser) key.custom; if (parser == null) { // Confusing: must delegate to wrapper (vs simply - // setting parser = - // DEFAULT_INT_PARSER/NUMERIC_UTILS_INT_PARSER) so - // cache key includes - // DEFAULT_INT_PARSER/NUMERIC_UTILS_INT_PARSER: - try { - return wrapper.getInts(reader, key.field, DEFAULT_INT_PARSER, setDocsWithField); - } catch (NumberFormatException ne) { - return wrapper.getInts(reader, key.field, NUMERIC_UTILS_INT_PARSER, setDocsWithField); - } + // setting parser = NUMERIC_UTILS_INT_PARSER) so + // cache key includes NUMERIC_UTILS_INT_PARSER: + return wrapper.getInts(reader, key.field, NUMERIC_UTILS_INT_PARSER, setDocsWithField); } - final HoldsOneThing valuesRef = new HoldsOneThing(); + final HoldsOneThing valuesRef = new HoldsOneThing(); Uninvert u = new Uninvert() { + private int minValue; private int currentValue; - private int[] values; + private GrowableWriter values; @Override public void visitTerm(BytesRef term) { @@ -635,16 +454,28 @@ class FieldCacheImpl implements FieldCache { // (which will hit a NumberFormatException // when we first try the DEFAULT_INT_PARSER), // we don't double-alloc: - values = new int[reader.maxDoc()]; - valuesRef.set(values); + int startBitsPerValue; + // Make sure than missing values (0) can be stored without resizing + if (currentValue < 0) { + minValue = currentValue; + startBitsPerValue = PackedInts.bitsRequired((-minValue) & 0xFFFFFFFFL); + } else { + minValue = 0; + startBitsPerValue = PackedInts.bitsRequired(currentValue); + } + values = new GrowableWriter(startBitsPerValue, reader.maxDoc(), PackedInts.FAST); + if (minValue != 0) { + values.fill(0, values.size(), (-minValue) & 0xFFFFFFFFL); // default value must be 0 + } + valuesRef.set(new GrowableWriterAndMinValue(values, minValue)); } } @Override public void visitDoc(int docID) { - values[docID] = currentValue; + values.set(docID, (currentValue - minValue) & 0xFFFFFFFFL); } - + @Override protected TermsEnum termsEnum(Terms terms) throws IOException { return parser.termsEnum(terms); @@ -656,11 +487,11 @@ class FieldCacheImpl implements FieldCache { if (setDocsWithField) { wrapper.setDocsWithField(reader, key.field, u.docsWithField); } - int[] values = valuesRef.get(); + GrowableWriterAndMinValue values = valuesRef.get(); if (values == null) { - values = new int[reader.maxDoc()]; + return new IntsFromArray(new PackedInts.NullReader(reader.maxDoc()), 0); } - return new IntsFromArray(values); + return new IntsFromArray(values.writer.getMutable(), (int) values.minValue); } } @@ -792,15 +623,9 @@ class FieldCacheImpl implements FieldCache { final FloatParser parser = (FloatParser) key.custom; if (parser == null) { // Confusing: must delegate to wrapper (vs simply - // setting parser = - // DEFAULT_FLOAT_PARSER/NUMERIC_UTILS_FLOAT_PARSER) so - // cache key includes - // DEFAULT_FLOAT_PARSER/NUMERIC_UTILS_FLOAT_PARSER: - try { - return wrapper.getFloats(reader, key.field, DEFAULT_FLOAT_PARSER, setDocsWithField); - } catch (NumberFormatException ne) { - return wrapper.getFloats(reader, key.field, NUMERIC_UTILS_FLOAT_PARSER, setDocsWithField); - } + // setting parser = NUMERIC_UTILS_FLOAT_PARSER) so + // cache key includes NUMERIC_UTILS_FLOAT_PARSER: + return wrapper.getFloats(reader, key.field, NUMERIC_UTILS_FLOAT_PARSER, setDocsWithField); } final HoldsOneThing valuesRef = new HoldsOneThing(); @@ -879,15 +704,17 @@ class FieldCacheImpl implements FieldCache { } static class LongsFromArray extends Longs { - private final long[] values; + private final PackedInts.Reader values; + private final long minValue; - public LongsFromArray(long[] values) { + public LongsFromArray(PackedInts.Reader values, long minValue) { this.values = values; + this.minValue = minValue; } @Override public long get(int docID) { - return values[docID]; + return minValue + values.get(docID); } } @@ -903,22 +730,17 @@ class FieldCacheImpl implements FieldCache { final LongParser parser = (LongParser) key.custom; if (parser == null) { // Confusing: must delegate to wrapper (vs simply - // setting parser = - // DEFAULT_LONG_PARSER/NUMERIC_UTILS_LONG_PARSER) so - // cache key includes - // DEFAULT_LONG_PARSER/NUMERIC_UTILS_LONG_PARSER: - try { - return wrapper.getLongs(reader, key.field, DEFAULT_LONG_PARSER, setDocsWithField); - } catch (NumberFormatException ne) { - return wrapper.getLongs(reader, key.field, NUMERIC_UTILS_LONG_PARSER, setDocsWithField); - } + // setting parser = NUMERIC_UTILS_LONG_PARSER) so + // cache key includes NUMERIC_UTILS_LONG_PARSER: + return wrapper.getLongs(reader, key.field, NUMERIC_UTILS_LONG_PARSER, setDocsWithField); } - final HoldsOneThing valuesRef = new HoldsOneThing(); + final HoldsOneThing valuesRef = new HoldsOneThing(); Uninvert u = new Uninvert() { + private long minValue; private long currentValue; - private long[] values; + private GrowableWriter values; @Override public void visitTerm(BytesRef term) { @@ -928,14 +750,26 @@ class FieldCacheImpl implements FieldCache { // (which will hit a NumberFormatException // when we first try the DEFAULT_INT_PARSER), // we don't double-alloc: - values = new long[reader.maxDoc()]; - valuesRef.set(values); + int startBitsPerValue; + // Make sure than missing values (0) can be stored without resizing + if (currentValue < 0) { + minValue = currentValue; + startBitsPerValue = minValue == Long.MIN_VALUE ? 64 : PackedInts.bitsRequired(-minValue); + } else { + minValue = 0; + startBitsPerValue = PackedInts.bitsRequired(currentValue); + } + values = new GrowableWriter(startBitsPerValue, reader.maxDoc(), PackedInts.FAST); + if (minValue != 0) { + values.fill(0, values.size(), -minValue); // default value must be 0 + } + valuesRef.set(new GrowableWriterAndMinValue(values, minValue)); } } @Override public void visitDoc(int docID) { - values[docID] = currentValue; + values.set(docID, currentValue - minValue); } @Override @@ -949,11 +783,11 @@ class FieldCacheImpl implements FieldCache { if (setDocsWithField) { wrapper.setDocsWithField(reader, key.field, u.docsWithField); } - long[] values = valuesRef.get(); + GrowableWriterAndMinValue values = valuesRef.get(); if (values == null) { - values = new long[reader.maxDoc()]; + return new LongsFromArray(new PackedInts.NullReader(reader.maxDoc()), 0L); } - return new LongsFromArray(values); + return new LongsFromArray(values.writer.getMutable(), values.minValue); } } @@ -1014,15 +848,9 @@ class FieldCacheImpl implements FieldCache { final DoubleParser parser = (DoubleParser) key.custom; if (parser == null) { // Confusing: must delegate to wrapper (vs simply - // setting parser = - // DEFAULT_DOUBLE_PARSER/NUMERIC_UTILS_DOUBLE_PARSER) so - // cache key includes - // DEFAULT_DOUBLE_PARSER/NUMERIC_UTILS_DOUBLE_PARSER: - try { - return wrapper.getDoubles(reader, key.field, DEFAULT_DOUBLE_PARSER, setDocsWithField); - } catch (NumberFormatException ne) { - return wrapper.getDoubles(reader, key.field, NUMERIC_UTILS_DOUBLE_PARSER, setDocsWithField); - } + // setting parser = NUMERIC_UTILS_DOUBLE_PARSER) so + // cache key includes NUMERIC_UTILS_DOUBLE_PARSER: + return wrapper.getDoubles(reader, key.field, NUMERIC_UTILS_DOUBLE_PARSER, setDocsWithField); } final HoldsOneThing valuesRef = new HoldsOneThing(); @@ -1146,7 +974,6 @@ class FieldCacheImpl implements FieldCache { final PagedBytes bytes = new PagedBytes(15); int startTermsBPV; - int startNumUniqueTerms; final int termCountHardLimit; if (maxDoc == Integer.MAX_VALUE) { @@ -1170,15 +997,11 @@ class FieldCacheImpl implements FieldCache { } startTermsBPV = PackedInts.bitsRequired(numUniqueTerms); - - startNumUniqueTerms = (int) numUniqueTerms; } else { startTermsBPV = 1; - startNumUniqueTerms = 1; } } else { startTermsBPV = 1; - startNumUniqueTerms = 1; } MonotonicAppendingLongBuffer termOrdToBytesOffset = new MonotonicAppendingLongBuffer(); diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java b/lucene/core/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java index 475a649779c..c262496500a 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java +++ b/lucene/core/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java @@ -193,109 +193,7 @@ public abstract class FieldCacheRangeFilter extends Filter { } }; } - - /** - * Creates a numeric range filter using {@link FieldCache#getBytes(AtomicReader,String,boolean)}. This works with all - * byte fields containing exactly one numeric term in the field. The range can be half-open by setting one - * of the values to null. - */ - public static FieldCacheRangeFilter newByteRange(String field, Byte lowerVal, Byte upperVal, boolean includeLower, boolean includeUpper) { - return newByteRange(field, null, lowerVal, upperVal, includeLower, includeUpper); - } - - /** - * Creates a numeric range filter using {@link FieldCache#getBytes(AtomicReader,String,FieldCache.ByteParser,boolean)}. This works with all - * byte fields containing exactly one numeric term in the field. The range can be half-open by setting one - * of the values to null. - */ - public static FieldCacheRangeFilter newByteRange(String field, FieldCache.ByteParser parser, Byte lowerVal, Byte upperVal, boolean includeLower, boolean includeUpper) { - return new FieldCacheRangeFilter(field, parser, lowerVal, upperVal, includeLower, includeUpper) { - @Override - public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException { - final byte inclusiveLowerPoint, inclusiveUpperPoint; - if (lowerVal != null) { - final byte i = lowerVal.byteValue(); - if (!includeLower && i == Byte.MAX_VALUE) - return null; - inclusiveLowerPoint = (byte) (includeLower ? i : (i + 1)); - } else { - inclusiveLowerPoint = Byte.MIN_VALUE; - } - if (upperVal != null) { - final byte i = upperVal.byteValue(); - if (!includeUpper && i == Byte.MIN_VALUE) - return null; - inclusiveUpperPoint = (byte) (includeUpper ? i : (i - 1)); - } else { - inclusiveUpperPoint = Byte.MAX_VALUE; - } - - if (inclusiveLowerPoint > inclusiveUpperPoint) - return null; - - final FieldCache.Bytes values = FieldCache.DEFAULT.getBytes(context.reader(), field, (FieldCache.ByteParser) parser, false); - return new FieldCacheDocIdSet(context.reader().maxDoc(), acceptDocs) { - @Override - protected boolean matchDoc(int doc) { - final byte value = values.get(doc); - return value >= inclusiveLowerPoint && value <= inclusiveUpperPoint; - } - }; - } - }; - } - - /** - * Creates a numeric range filter using {@link FieldCache#getShorts(AtomicReader,String,boolean)}. This works with all - * short fields containing exactly one numeric term in the field. The range can be half-open by setting one - * of the values to null. - */ - public static FieldCacheRangeFilter newShortRange(String field, Short lowerVal, Short upperVal, boolean includeLower, boolean includeUpper) { - return newShortRange(field, null, lowerVal, upperVal, includeLower, includeUpper); - } - - /** - * Creates a numeric range filter using {@link FieldCache#getShorts(AtomicReader,String,FieldCache.ShortParser,boolean)}. This works with all - * short fields containing exactly one numeric term in the field. The range can be half-open by setting one - * of the values to null. - */ - public static FieldCacheRangeFilter newShortRange(String field, FieldCache.ShortParser parser, Short lowerVal, Short upperVal, boolean includeLower, boolean includeUpper) { - return new FieldCacheRangeFilter(field, parser, lowerVal, upperVal, includeLower, includeUpper) { - @Override - public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException { - final short inclusiveLowerPoint, inclusiveUpperPoint; - if (lowerVal != null) { - short i = lowerVal.shortValue(); - if (!includeLower && i == Short.MAX_VALUE) - return null; - inclusiveLowerPoint = (short) (includeLower ? i : (i + 1)); - } else { - inclusiveLowerPoint = Short.MIN_VALUE; - } - if (upperVal != null) { - short i = upperVal.shortValue(); - if (!includeUpper && i == Short.MIN_VALUE) - return null; - inclusiveUpperPoint = (short) (includeUpper ? i : (i - 1)); - } else { - inclusiveUpperPoint = Short.MAX_VALUE; - } - - if (inclusiveLowerPoint > inclusiveUpperPoint) - return null; - - final FieldCache.Shorts values = FieldCache.DEFAULT.getShorts(context.reader(), field, (FieldCache.ShortParser) parser, false); - return new FieldCacheDocIdSet(context.reader().maxDoc(), acceptDocs) { - @Override - protected boolean matchDoc(int doc) { - final short value = values.get(doc); - return value >= inclusiveLowerPoint && value <= inclusiveUpperPoint; - } - }; - } - }; - } - + /** * Creates a numeric range filter using {@link FieldCache#getInts(AtomicReader,String,boolean)}. This works with all * int fields containing exactly one numeric term in the field. The range can be half-open by setting one @@ -521,7 +419,7 @@ public abstract class FieldCacheRangeFilter extends Filter { } @Override - @SuppressWarnings({"unchecked","rawtypes"}) + @SuppressWarnings({"rawtypes"}) public final boolean equals(Object o) { if (this == o) return true; if (!(o instanceof FieldCacheRangeFilter)) return false; diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java b/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java index ee9668e674e..28e6144bd42 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java +++ b/lucene/core/src/java/org/apache/lucene/search/FieldComparator.java @@ -22,12 +22,10 @@ import java.io.IOException; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.search.FieldCache.ByteParser; import org.apache.lucene.search.FieldCache.DoubleParser; import org.apache.lucene.search.FieldCache.FloatParser; import org.apache.lucene.search.FieldCache.IntParser; import org.apache.lucene.search.FieldCache.LongParser; -import org.apache.lucene.search.FieldCache.ShortParser; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; @@ -218,78 +216,6 @@ public abstract class FieldComparator { } } - /** Parses field's values as byte (using {@link - * FieldCache#getBytes} and sorts by ascending value */ - public static final class ByteComparator extends NumericComparator { - private final byte[] values; - private final ByteParser parser; - private FieldCache.Bytes currentReaderValues; - private byte bottom; - - ByteComparator(int numHits, String field, FieldCache.Parser parser, Byte missingValue) { - super(field, missingValue); - values = new byte[numHits]; - this.parser = (ByteParser) parser; - } - - @Override - public int compare(int slot1, int slot2) { - return Byte.compare(values[slot1], values[slot2]); - } - - @Override - public int compareBottom(int doc) { - byte v2 = currentReaderValues.get(doc); - // Test for v2 == 0 to save Bits.get method call for - // the common case (doc has value and value is non-zero): - if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) { - v2 = missingValue; - } - - return Byte.compare(bottom, v2); - } - - @Override - public void copy(int slot, int doc) { - byte v2 = currentReaderValues.get(doc); - // Test for v2 == 0 to save Bits.get method call for - // the common case (doc has value and value is non-zero): - if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) { - v2 = missingValue; - } - values[slot] = v2; - } - - @Override - public FieldComparator setNextReader(AtomicReaderContext context) throws IOException { - // NOTE: must do this before calling super otherwise - // we compute the docsWithField Bits twice! - currentReaderValues = FieldCache.DEFAULT.getBytes(context.reader(), field, parser, missingValue != null); - return super.setNextReader(context); - } - - @Override - public void setBottom(final int bottom) { - this.bottom = values[bottom]; - } - - @Override - public Byte value(int slot) { - return Byte.valueOf(values[slot]); - } - - @Override - public int compareDocToValue(int doc, Byte value) { - byte docValue = currentReaderValues.get(doc); - // Test for docValue == 0 to save Bits.get method call for - // the common case (doc has value and value is non-zero): - if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) { - docValue = missingValue; - } - return Byte.compare(docValue, value.byteValue()); - } - } - /** Parses field's values as double (using {@link * FieldCache#getDoubles} and sorts by ascending value */ public static final class DoubleComparator extends NumericComparator { @@ -439,80 +365,6 @@ public abstract class FieldComparator { } } - /** Parses field's values as short (using {@link - * FieldCache#getShorts} and sorts by ascending value */ - public static final class ShortComparator extends NumericComparator { - private final short[] values; - private final ShortParser parser; - private FieldCache.Shorts currentReaderValues; - private short bottom; - - ShortComparator(int numHits, String field, FieldCache.Parser parser, Short missingValue) { - super(field, missingValue); - values = new short[numHits]; - this.parser = (ShortParser) parser; - } - - @Override - public int compare(int slot1, int slot2) { - return Short.compare(values[slot1], values[slot2]); - } - - @Override - public int compareBottom(int doc) { - short v2 = currentReaderValues.get(doc); - // Test for v2 == 0 to save Bits.get method call for - // the common case (doc has value and value is non-zero): - if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) { - v2 = missingValue; - } - - return Short.compare(bottom, v2); - } - - @Override - public void copy(int slot, int doc) { - short v2 = currentReaderValues.get(doc); - // Test for v2 == 0 to save Bits.get method call for - // the common case (doc has value and value is non-zero): - if (docsWithField != null && v2 == 0 && !docsWithField.get(doc)) { - v2 = missingValue; - } - - values[slot] = v2; - } - - @Override - public FieldComparator setNextReader(AtomicReaderContext context) throws IOException { - // NOTE: must do this before calling super otherwise - // we compute the docsWithField Bits twice! - currentReaderValues = FieldCache.DEFAULT.getShorts(context.reader(), field, parser, missingValue != null); - return super.setNextReader(context); - } - - @Override - public void setBottom(final int bottom) { - this.bottom = values[bottom]; - } - - @Override - public Short value(int slot) { - return Short.valueOf(values[slot]); - } - - @Override - public int compareDocToValue(int doc, Short valueObj) { - final short value = valueObj.shortValue(); - short docValue = currentReaderValues.get(doc); - // Test for docValue == 0 to save Bits.get method call for - // the common case (doc has value and value is non-zero): - if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) { - docValue = missingValue; - } - return Short.compare(docValue, value); - } - } - /** Parses field's values as int (using {@link * FieldCache#getInts} and sorts by ascending value */ public static final class IntComparator extends NumericComparator { diff --git a/lucene/core/src/java/org/apache/lucene/search/SortField.java b/lucene/core/src/java/org/apache/lucene/search/SortField.java index d2251d19510..f7001bf9741 100644 --- a/lucene/core/src/java/org/apache/lucene/search/SortField.java +++ b/lucene/core/src/java/org/apache/lucene/search/SortField.java @@ -71,18 +71,10 @@ public class SortField { * lower values are at the front. */ DOUBLE, - /** Sort using term values as encoded Shorts. Sort values are Short and - * lower values are at the front. */ - SHORT, - /** Sort using a custom Comparator. Sort values are any Comparable and * sorting is done according to natural order. */ CUSTOM, - /** Sort using term values as encoded Bytes. Sort values are Byte and - * lower values are at the front. */ - BYTE, - /** Sort using term values as Strings, but comparing by * value (using String.compareTo) for all comparisons. * This is typically slower than {@link #STRING}, which @@ -164,8 +156,6 @@ public class SortField { public SortField(String field, FieldCache.Parser parser, boolean reverse) { if (parser instanceof FieldCache.IntParser) initFieldType(field, Type.INT); else if (parser instanceof FieldCache.FloatParser) initFieldType(field, Type.FLOAT); - else if (parser instanceof FieldCache.ShortParser) initFieldType(field, Type.SHORT); - else if (parser instanceof FieldCache.ByteParser) initFieldType(field, Type.BYTE); else if (parser instanceof FieldCache.LongParser) initFieldType(field, Type.LONG); else if (parser instanceof FieldCache.DoubleParser) initFieldType(field, Type.DOUBLE); else { @@ -177,7 +167,7 @@ public class SortField { } public SortField setMissingValue(Object missingValue) { - if (type != Type.BYTE && type != Type.SHORT && type != Type.INT && type != Type.FLOAT && type != Type.LONG && type != Type.DOUBLE) { + if (type != Type.INT && type != Type.FLOAT && type != Type.LONG && type != Type.DOUBLE) { throw new IllegalArgumentException( "Missing value only works for numeric types" ); } this.missingValue = missingValue; @@ -274,14 +264,6 @@ public class SortField { buffer.append(""); break; - case BYTE: - buffer.append(""); - break; - - case SHORT: - buffer.append(""); - break; - case INT: buffer.append(""); break; @@ -389,12 +371,6 @@ public class SortField { case DOUBLE: return new FieldComparator.DoubleComparator(numHits, field, parser, (Double) missingValue); - case BYTE: - return new FieldComparator.ByteComparator(numHits, field, parser, (Byte) missingValue); - - case SHORT: - return new FieldComparator.ShortComparator(numHits, field, parser, (Short) missingValue); - case CUSTOM: assert comparatorSource != null; return comparatorSource.newComparator(field, numHits, sortPos, reverse); diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java b/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java index ca7757fa491..45b333820a5 100644 --- a/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java +++ b/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java @@ -25,20 +25,30 @@ import org.apache.lucene.util.RamUsageEstimator; /** * Implements {@link PackedInts.Mutable}, but grows the * bit count of the underlying packed ints on-demand. + *

Beware that this class will accept to set negative values but in order + * to do this, it will grow the number of bits per value to 64. * *

@lucene.internal

*/ - public class GrowableWriter implements PackedInts.Mutable { - private long currentMaxValue; + private long currentMask; private PackedInts.Mutable current; private final float acceptableOverheadRatio; + /** + * @param startBitsPerValue the initial number of bits per value, may grow depending on the data + * @param valueCount the number of values + * @param acceptableOverheadRatio an acceptable overhead ratio + */ public GrowableWriter(int startBitsPerValue, int valueCount, float acceptableOverheadRatio) { this.acceptableOverheadRatio = acceptableOverheadRatio; current = PackedInts.getMutable(valueCount, startBitsPerValue, this.acceptableOverheadRatio); - currentMaxValue = PackedInts.maxValue(current.getBitsPerValue()); + currentMask = mask(current.getBitsPerValue()); + } + + private static long mask(int bitsPerValue) { + return bitsPerValue == 64 ? ~0L : PackedInts.maxValue(bitsPerValue); } @Override @@ -71,16 +81,16 @@ public class GrowableWriter implements PackedInts.Mutable { } private void ensureCapacity(long value) { - assert value >= 0; - if (value <= currentMaxValue) { + if ((value & currentMask) == value) { return; } - final int bitsRequired = PackedInts.bitsRequired(value); + final int bitsRequired = value < 0 ? 64 : PackedInts.bitsRequired(value); + assert bitsRequired > current.getBitsPerValue(); final int valueCount = size(); PackedInts.Mutable next = PackedInts.getMutable(valueCount, bitsRequired, acceptableOverheadRatio); PackedInts.copy(current, 0, next, 0, valueCount, PackedInts.DEFAULT_BUFFER_SIZE); current = next; - currentMaxValue = PackedInts.maxValue(current.getBitsPerValue()); + currentMask = mask(current.getBitsPerValue()); } @Override @@ -110,6 +120,10 @@ public class GrowableWriter implements PackedInts.Mutable { public int set(int index, long[] arr, int off, int len) { long max = 0; for (int i = off, end = off + len; i < end; ++i) { + // bitwise or is nice because either all values are positive and the + // or-ed result will require as many bits per value as the max of the + // values, or one of them is negative and the result will be negative, + // forcing GrowableWriter to use 64 bits per value max |= arr[i]; } ensureCapacity(max); diff --git a/lucene/core/src/test/org/apache/lucene/TestSearch.java b/lucene/core/src/test/org/apache/lucene/TestSearch.java index e4be172b73f..f2bedd4b40d 100644 --- a/lucene/core/src/test/org/apache/lucene/TestSearch.java +++ b/lucene/core/src/test/org/apache/lucene/TestSearch.java @@ -127,7 +127,7 @@ public class TestSearch extends LuceneTestCase { for (int j = 0; j < docs.length; j++) { Document d = new Document(); d.add(newTextField("contents", docs[j], Field.Store.YES)); - d.add(newStringField("id", ""+j, Field.Store.NO)); + d.add(new IntField("id", j, Field.Store.NO)); writer.addDocument(d); } writer.close(); diff --git a/lucene/core/src/test/org/apache/lucene/TestSearchForDuplicates.java b/lucene/core/src/test/org/apache/lucene/TestSearchForDuplicates.java index fd20985de5f..3d942dc3ba4 100644 --- a/lucene/core/src/test/org/apache/lucene/TestSearchForDuplicates.java +++ b/lucene/core/src/test/org/apache/lucene/TestSearchForDuplicates.java @@ -81,7 +81,7 @@ public class TestSearchForDuplicates extends LuceneTestCase { for (int j = 0; j < MAX_DOCS; j++) { Document d = new Document(); d.add(newTextField(PRIORITY_FIELD, HIGH_PRIORITY, Field.Store.YES)); - d.add(newTextField(ID_FIELD, Integer.toString(j), Field.Store.YES)); + d.add(new IntField(ID_FIELD, j, Field.Store.YES)); writer.addDocument(d); } writer.close(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java index 0ed240e57a8..102b7c9d7aa 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java @@ -33,6 +33,7 @@ import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.IntField; import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; @@ -764,7 +765,7 @@ public void testFilesOpenClose() throws IOException { setMergePolicy(newLogMergePolicy(10)) ); Document doc = new Document(); - doc.add(newStringField("number", "17", Field.Store.NO)); + doc.add(new IntField("number", 17, Field.Store.NO)); writer.addDocument(doc); writer.commit(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java b/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java index ff34c225836..17ce72b7c95 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDocValuesWithThreads.java @@ -87,23 +87,17 @@ public class TestDocValuesWithThreads extends LuceneTestCase { BytesRef scratch2 = new BytesRef(); for(int iter=0;iter>> 63); + assertEquals(0, Double.doubleToLongBits(v1) >>> 63); ir.close(); dir.close(); @@ -998,13 +767,13 @@ public class TestSort extends LuceneTestCase { Document doc = new Document(); writer.addDocument(doc); doc = new Document(); - doc.add(newStringField("value", "-1.3", Field.Store.YES)); + doc.add(new DoubleField("value", -1.3, Field.Store.YES)); writer.addDocument(doc); doc = new Document(); - doc.add(newStringField("value", "4.2333333333333", Field.Store.YES)); + doc.add(new DoubleField("value", 4.2333333333333, Field.Store.YES)); writer.addDocument(doc); doc = new Document(); - doc.add(newStringField("value", "4.2333333333332", Field.Store.YES)); + doc.add(new DoubleField("value", 4.2333333333332, Field.Store.YES)); writer.addDocument(doc); IndexReader ir = writer.getReader(); writer.close(); @@ -1031,13 +800,13 @@ public class TestSort extends LuceneTestCase { Document doc = new Document(); writer.addDocument(doc); doc = new Document(); - doc.add(newStringField("value", "-1.3", Field.Store.YES)); + doc.add(new DoubleField("value", -1.3, Field.Store.YES)); writer.addDocument(doc); doc = new Document(); - doc.add(newStringField("value", "4.2333333333333", Field.Store.YES)); + doc.add(new DoubleField("value", 4.2333333333333, Field.Store.YES)); writer.addDocument(doc); doc = new Document(); - doc.add(newStringField("value", "4.2333333333332", Field.Store.YES)); + doc.add(new DoubleField("value", 4.2333333333332, Field.Store.YES)); writer.addDocument(doc); IndexReader ir = writer.getReader(); writer.close(); @@ -1064,16 +833,16 @@ public class TestSort extends LuceneTestCase { Directory dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir); Document doc = new Document(); - doc.add(newStringField("value", "30.1", Field.Store.YES)); + doc.add(new DoubleField("value", 30.1, Field.Store.YES)); writer.addDocument(doc); doc = new Document(); - doc.add(newStringField("value", "-1.3", Field.Store.YES)); + doc.add(new DoubleField("value", -1.3, Field.Store.YES)); writer.addDocument(doc); doc = new Document(); - doc.add(newStringField("value", "4.2333333333333", Field.Store.YES)); + doc.add(new DoubleField("value", 4.2333333333333, Field.Store.YES)); writer.addDocument(doc); doc = new Document(); - doc.add(newStringField("value", "4.2333333333332", Field.Store.YES)); + doc.add(new DoubleField("value", 4.2333333333332, Field.Store.YES)); writer.addDocument(doc); IndexReader ir = writer.getReader(); writer.close(); @@ -1150,7 +919,7 @@ public class TestSort extends LuceneTestCase { for(int seg=0;seg<2;seg++) { for(int docIDX=0;docIDX<10;docIDX++) { Document doc = new Document(); - doc.add(newStringField("id", ""+docIDX, Field.Store.YES)); + doc.add(new IntField("id", docIDX, Field.Store.YES)); StringBuilder sb = new StringBuilder(); for(int i=0;i letters = Arrays.asList(new String[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J" }); - Collections.shuffle(letters, random()); - - Directory dir = newDirectory(); - RandomIndexWriter iw = new RandomIndexWriter(random(), dir); - for (String letter : letters) { - Document doc = new Document(); - doc.add(newStringField("parser", letter, Field.Store.YES)); - iw.addDocument(doc); - } - - IndexReader ir = iw.getReader(); - iw.close(); - - IndexSearcher searcher = newSearcher(ir); - Sort sort = new Sort(new SortField("parser", new FieldCache.ByteParser() { - @Override - public byte parseByte(BytesRef term) { - return (byte) (term.bytes[term.offset]-'A'); - } - - @Override - public TermsEnum termsEnum(Terms terms) throws IOException { - return terms.iterator(null); - } - }), SortField.FIELD_DOC ); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - - // results should be in alphabetical order - assertEquals(10, td.totalHits); - Collections.sort(letters); - for (int i = 0; i < letters.size(); i++) { - assertEquals(letters.get(i), searcher.doc(td.scoreDocs[i].doc).get("parser")); - } - - ir.close(); - dir.close(); - } - - /** - * test sorts for a custom short parser that uses a simple char encoding - */ - public void testCustomShortParser() throws Exception { - List letters = Arrays.asList(new String[] { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J" }); - Collections.shuffle(letters, random()); - - Directory dir = newDirectory(); - RandomIndexWriter iw = new RandomIndexWriter(random(), dir); - for (String letter : letters) { - Document doc = new Document(); - doc.add(newStringField("parser", letter, Field.Store.YES)); - iw.addDocument(doc); - } - - IndexReader ir = iw.getReader(); - iw.close(); - - IndexSearcher searcher = newSearcher(ir); - Sort sort = new Sort(new SortField("parser", new FieldCache.ShortParser() { - @Override - public short parseShort(BytesRef term) { - return (short) (term.bytes[term.offset]-'A'); - } - - @Override - public TermsEnum termsEnum(Terms terms) throws IOException { - return terms.iterator(null); - } - }), SortField.FIELD_DOC ); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - - // results should be in alphabetical order - assertEquals(10, td.totalHits); - Collections.sort(letters); - for (int i = 0; i < letters.size(); i++) { - assertEquals(letters.get(i), searcher.doc(td.scoreDocs[i].doc).get("parser")); - } - - ir.close(); - dir.close(); - } - /** * test sorts for a custom long parser that uses a simple char encoding */ diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java b/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java index 595b857919a..49f954d319f 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSortDocValues.java @@ -223,142 +223,6 @@ public class TestSortDocValues extends LuceneTestCase { dir.close(); } - /** Tests sorting on type byte */ - public void testByte() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(new NumericDocValuesField("value", 23)); - doc.add(newStringField("value", "23", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(new NumericDocValuesField("value", -1)); - doc.add(newStringField("value", "-1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(new NumericDocValuesField("value", 4)); - doc.add(newStringField("value", "4", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = newSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.BYTE)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // numeric order - assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("23", searcher.doc(td.scoreDocs[2].doc).get("value")); - assertNoFieldCaches(); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type byte in reverse */ - public void testByteReverse() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(new NumericDocValuesField("value", 23)); - doc.add(newStringField("value", "23", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(new NumericDocValuesField("value", -1)); - doc.add(newStringField("value", "-1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(new NumericDocValuesField("value", 4)); - doc.add(newStringField("value", "4", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = newSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.BYTE, true)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // reverse numeric order - assertEquals("23", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("-1", searcher.doc(td.scoreDocs[2].doc).get("value")); - assertNoFieldCaches(); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type short */ - public void testShort() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(new NumericDocValuesField("value", 300)); - doc.add(newStringField("value", "300", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(new NumericDocValuesField("value", -1)); - doc.add(newStringField("value", "-1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(new NumericDocValuesField("value", 4)); - doc.add(newStringField("value", "4", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = newSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.SHORT)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // numeric order - assertEquals("-1", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("300", searcher.doc(td.scoreDocs[2].doc).get("value")); - assertNoFieldCaches(); - - ir.close(); - dir.close(); - } - - /** Tests sorting on type short in reverse */ - public void testShortReverse() throws IOException { - Directory dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random(), dir); - Document doc = new Document(); - doc.add(new NumericDocValuesField("value", 300)); - doc.add(newStringField("value", "300", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(new NumericDocValuesField("value", -1)); - doc.add(newStringField("value", "-1", Field.Store.YES)); - writer.addDocument(doc); - doc = new Document(); - doc.add(new NumericDocValuesField("value", 4)); - doc.add(newStringField("value", "4", Field.Store.YES)); - writer.addDocument(doc); - IndexReader ir = writer.getReader(); - writer.close(); - - IndexSearcher searcher = newSearcher(ir); - Sort sort = new Sort(new SortField("value", SortField.Type.SHORT, true)); - - TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort); - assertEquals(3, td.totalHits); - // reverse numeric order - assertEquals("300", searcher.doc(td.scoreDocs[0].doc).get("value")); - assertEquals("4", searcher.doc(td.scoreDocs[1].doc).get("value")); - assertEquals("-1", searcher.doc(td.scoreDocs[2].doc).get("value")); - assertNoFieldCaches(); - - ir.close(); - dir.close(); - } - /** Tests sorting on type int */ public void testInt() throws IOException { Directory dir = newDirectory(); diff --git a/lucene/core/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java b/lucene/core/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java index fbfbded4316..895eaa603db 100644 --- a/lucene/core/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java +++ b/lucene/core/src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java @@ -16,21 +16,25 @@ package org.apache.lucene.util; * limitations under the License. */ +import java.io.IOException; + import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoubleField; import org.apache.lucene.document.Field; -import org.apache.lucene.search.FieldCache; +import org.apache.lucene.document.FloatField; +import org.apache.lucene.document.IntField; +import org.apache.lucene.document.LongField; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.SlowCompositeReaderWrapper; +import org.apache.lucene.search.FieldCache; import org.apache.lucene.store.Directory; import org.apache.lucene.util.FieldCacheSanityChecker.Insanity; import org.apache.lucene.util.FieldCacheSanityChecker.InsanityType; -import java.io.IOException; - public class TestFieldCacheSanityChecker extends LuceneTestCase { protected AtomicReader readerA; @@ -51,18 +55,14 @@ public class TestFieldCacheSanityChecker extends LuceneTestCase { long theLong = Long.MAX_VALUE; double theDouble = Double.MAX_VALUE; - byte theByte = Byte.MAX_VALUE; - short theShort = Short.MAX_VALUE; int theInt = Integer.MAX_VALUE; float theFloat = Float.MAX_VALUE; for (int i = 0; i < NUM_DOCS; i++){ Document doc = new Document(); - doc.add(newStringField("theLong", String.valueOf(theLong--), Field.Store.NO)); - doc.add(newStringField("theDouble", String.valueOf(theDouble--), Field.Store.NO)); - doc.add(newStringField("theByte", String.valueOf(theByte--), Field.Store.NO)); - doc.add(newStringField("theShort", String.valueOf(theShort--), Field.Store.NO)); - doc.add(newStringField("theInt", String.valueOf(theInt--), Field.Store.NO)); - doc.add(newStringField("theFloat", String.valueOf(theFloat--), Field.Store.NO)); + doc.add(new LongField("theLong", theLong--, Field.Store.NO)); + doc.add(new DoubleField("theDouble", theDouble--, Field.Store.NO)); + doc.add(new IntField("theInt", theInt--, Field.Store.NO)); + doc.add(new FloatField("theFloat", theFloat--, Field.Store.NO)); if (0 == i % 3) { wA.addDocument(doc); } else { @@ -95,12 +95,12 @@ public class TestFieldCacheSanityChecker extends LuceneTestCase { cache.purgeAllCaches(); cache.getDoubles(readerA, "theDouble", false); - cache.getDoubles(readerA, "theDouble", FieldCache.DEFAULT_DOUBLE_PARSER, false); - cache.getDoubles(readerAclone, "theDouble", FieldCache.DEFAULT_DOUBLE_PARSER, false); - cache.getDoubles(readerB, "theDouble", FieldCache.DEFAULT_DOUBLE_PARSER, false); + cache.getDoubles(readerA, "theDouble", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, false); + cache.getDoubles(readerAclone, "theDouble", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, false); + cache.getDoubles(readerB, "theDouble", FieldCache.NUMERIC_UTILS_DOUBLE_PARSER, false); cache.getInts(readerX, "theInt", false); - cache.getInts(readerX, "theInt", FieldCache.DEFAULT_INT_PARSER, false); + cache.getInts(readerX, "theInt", FieldCache.NUMERIC_UTILS_INT_PARSER, false); // // // @@ -119,9 +119,8 @@ public class TestFieldCacheSanityChecker extends LuceneTestCase { FieldCache cache = FieldCache.DEFAULT; cache.purgeAllCaches(); - cache.getInts(readerX, "theInt", FieldCache.DEFAULT_INT_PARSER, false); + cache.getInts(readerX, "theInt", FieldCache.NUMERIC_UTILS_INT_PARSER, false); cache.getTerms(readerX, "theInt"); - cache.getBytes(readerX, "theByte", false); // // // @@ -147,8 +146,6 @@ public class TestFieldCacheSanityChecker extends LuceneTestCase { cache.getTerms(readerB, "theInt"); cache.getTerms(readerX, "theInt"); - cache.getBytes(readerX, "theByte", false); - // // // diff --git a/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java b/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java index a527868acca..c60ce1638c3 100644 --- a/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java +++ b/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java @@ -650,8 +650,11 @@ public class TestPackedInts extends LuceneTestCase { wrt.set(99, (1 << 23) - 1); assertEquals(1 << 10, wrt.get(valueCount - 1)); wrt.set(1, Long.MAX_VALUE); + wrt.set(2, -3); + assertEquals(64, wrt.getBitsPerValue()); assertEquals(1 << 10, wrt.get(valueCount - 1)); assertEquals(Long.MAX_VALUE, wrt.get(1)); + assertEquals(-3L, wrt.get(2)); assertEquals(2, wrt.get(4)); assertEquals((1 << 23) - 1, wrt.get(99)); assertEquals(10, wrt.get(7)); diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java index 1c5efcfe1a5..795a984167e 100644 --- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java +++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java @@ -78,7 +78,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase { Document doc = new Document(); addGroupField(doc, groupField, "author1", valueType); doc.add(newTextField("content", "random text", Field.Store.NO)); - doc.add(newStringField("id_1", "1", Field.Store.NO)); + doc.add(new IntField("id_1", 1, Field.Store.NO)); doc.add(newStringField("id_2", "1", Field.Store.NO)); w.addDocument(doc); @@ -86,7 +86,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase { doc = new Document(); addGroupField(doc, groupField, "author1", valueType); doc.add(newTextField("content", "some more random text blob", Field.Store.NO)); - doc.add(newStringField("id_1", "2", Field.Store.NO)); + doc.add(new IntField("id_1", 2, Field.Store.NO)); doc.add(newStringField("id_2", "2", Field.Store.NO)); w.addDocument(doc); @@ -94,7 +94,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase { doc = new Document(); addGroupField(doc, groupField, "author1", valueType); doc.add(newTextField("content", "some more random textual data", Field.Store.NO)); - doc.add(newStringField("id_1", "3", Field.Store.NO)); + doc.add(new IntField("id_1", 3, Field.Store.NO)); doc.add(newStringField("id_2", "3", Field.Store.NO)); w.addDocument(doc); w.commit(); // To ensure a second segment @@ -103,7 +103,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase { doc = new Document(); addGroupField(doc, groupField, "author2", valueType); doc.add(newTextField("content", "some random text", Field.Store.NO)); - doc.add(newStringField("id_1", "4", Field.Store.NO)); + doc.add(new IntField("id_1", 4, Field.Store.NO)); doc.add(newStringField("id_2", "4", Field.Store.NO)); w.addDocument(doc); @@ -111,7 +111,7 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase { doc = new Document(); addGroupField(doc, groupField, "author3", valueType); doc.add(newTextField("content", "some more random text", Field.Store.NO)); - doc.add(newStringField("id_1", "5", Field.Store.NO)); + doc.add(new IntField("id_1", 5, Field.Store.NO)); doc.add(newStringField("id_2", "5", Field.Store.NO)); w.addDocument(doc); @@ -119,21 +119,21 @@ public class AllGroupHeadsCollectorTest extends LuceneTestCase { doc = new Document(); addGroupField(doc, groupField, "author3", valueType); doc.add(newTextField("content", "random blob", Field.Store.NO)); - doc.add(newStringField("id_1", "6", Field.Store.NO)); + doc.add(new IntField("id_1", 6, Field.Store.NO)); doc.add(newStringField("id_2", "6", Field.Store.NO)); w.addDocument(doc); // 6 -- no author field doc = new Document(); doc.add(newTextField("content", "random word stuck in alot of other text", Field.Store.NO)); - doc.add(newStringField("id_1", "6", Field.Store.NO)); + doc.add(new IntField("id_1", 6, Field.Store.NO)); doc.add(newStringField("id_2", "6", Field.Store.NO)); w.addDocument(doc); // 7 -- no author field doc = new Document(); doc.add(newTextField("content", "random word stuck in alot of other text", Field.Store.NO)); - doc.add(newStringField("id_1", "7", Field.Store.NO)); + doc.add(new IntField("id_1", 7, Field.Store.NO)); doc.add(newStringField("id_2", "7", Field.Store.NO)); w.addDocument(doc); diff --git a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java index 5dfd746ff0c..147b4a5deb9 100644 --- a/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java +++ b/lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java @@ -459,7 +459,7 @@ public class TestBlockJoin extends LuceneTestCase { for(int parentDocID=0;parentDocID joinDocs = new ArrayList<>(); @@ -497,7 +497,7 @@ public class TestBlockJoin extends LuceneTestCase { Document joinChildDoc = new Document(); joinDocs.add(joinChildDoc); - Field childID = newStringField("childID", ""+childDocID, Field.Store.YES); + Field childID = new IntField("childID", childDocID, Field.Store.YES); childDoc.add(childID); joinChildDoc.add(childID); @@ -522,7 +522,7 @@ public class TestBlockJoin extends LuceneTestCase { } if (doDeletes) { - joinChildDoc.add(newStringField("blockID", ""+parentDocID, Field.Store.NO)); + joinChildDoc.add(new IntField("blockID", parentDocID, Field.Store.NO)); } w.addDocument(childDoc); @@ -541,8 +541,10 @@ public class TestBlockJoin extends LuceneTestCase { if (VERBOSE) { System.out.println("DELETE parentID=" + deleteID); } - w.deleteDocuments(new Term("blockID", ""+deleteID)); - joinW.deleteDocuments(new Term("blockID", ""+deleteID)); + BytesRef term = new BytesRef(); + NumericUtils.intToPrefixCodedBytes(deleteID, 0, term); + w.deleteDocuments(new Term("blockID", term)); + joinW.deleteDocuments(new Term("blockID", term)); } final IndexReader r = w.getReader(); diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ByteFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ByteFieldSource.java deleted file mode 100644 index da66797d7f9..00000000000 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ByteFieldSource.java +++ /dev/null @@ -1,121 +0,0 @@ -package org.apache.lucene.queries.function.valuesource; -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Map; - -import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.queries.function.FunctionValues; -import org.apache.lucene.search.FieldCache; - -/** - * Obtains int field values from the {@link org.apache.lucene.search.FieldCache} - * using getInts() - * and makes those values available as other numeric types, casting as needed. * - * - * - */ - -public class ByteFieldSource extends FieldCacheSource { - - private final FieldCache.ByteParser parser; - - public ByteFieldSource(String field) { - this(field, null); - } - - public ByteFieldSource(String field, FieldCache.ByteParser parser) { - super(field); - this.parser = parser; - } - - @Override - public String description() { - return "byte(" + field + ')'; - } - - @Override - public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException { - final FieldCache.Bytes arr = cache.getBytes(readerContext.reader(), field, parser, false); - - return new FunctionValues() { - @Override - public byte byteVal(int doc) { - return arr.get(doc); - } - - @Override - public short shortVal(int doc) { - return (short) arr.get(doc); - } - - @Override - public float floatVal(int doc) { - return (float) arr.get(doc); - } - - @Override - public int intVal(int doc) { - return (int) arr.get(doc); - } - - @Override - public long longVal(int doc) { - return (long) arr.get(doc); - } - - @Override - public double doubleVal(int doc) { - return (double) arr.get(doc); - } - - @Override - public String strVal(int doc) { - return Byte.toString(arr.get(doc)); - } - - @Override - public String toString(int doc) { - return description() + '=' + byteVal(doc); - } - - @Override - public Object objectVal(int doc) { - return arr.get(doc); // TODO: valid? - } - - }; - } - - @Override - public boolean equals(Object o) { - if (o.getClass() != ByteFieldSource.class) return false; - ByteFieldSource - other = (ByteFieldSource) o; - return super.equals(other) - && (this.parser == null ? other.parser == null : - this.parser.getClass() == other.parser.getClass()); - } - - @Override - public int hashCode() { - int h = parser == null ? Byte.class.hashCode() : parser.getClass().hashCode(); - h += super.hashCode(); - return h; - } -} diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ShortFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ShortFieldSource.java deleted file mode 100644 index 2909ebbee69..00000000000 --- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ShortFieldSource.java +++ /dev/null @@ -1,114 +0,0 @@ -package org.apache.lucene.queries.function.valuesource; -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Map; - -import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.queries.function.FunctionValues; -import org.apache.lucene.search.FieldCache; - - -/** - * Obtains short field values from the {@link org.apache.lucene.search.FieldCache} - * using getShorts() - * and makes those values available as other numeric types, casting as needed. - **/ -public class ShortFieldSource extends FieldCacheSource { - - final FieldCache.ShortParser parser; - - public ShortFieldSource(String field) { - this(field, null); - } - - public ShortFieldSource(String field, FieldCache.ShortParser parser) { - super(field); - this.parser = parser; - } - - @Override - public String description() { - return "short(" + field + ')'; - } - - @Override - public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException { - final FieldCache.Shorts arr = cache.getShorts(readerContext.reader(), field, parser, false); - - return new FunctionValues() { - @Override - public byte byteVal(int doc) { - return (byte) arr.get(doc); - } - - @Override - public short shortVal(int doc) { - return arr.get(doc); - } - - @Override - public float floatVal(int doc) { - return (float) arr.get(doc); - } - - @Override - public int intVal(int doc) { - return (int) arr.get(doc); - } - - @Override - public long longVal(int doc) { - return (long) arr.get(doc); - } - - @Override - public double doubleVal(int doc) { - return (double) arr.get(doc); - } - - @Override - public String strVal(int doc) { - return Short.toString(arr.get(doc)); - } - - @Override - public String toString(int doc) { - return description() + '=' + shortVal(doc); - } - - }; - } - - @Override - public boolean equals(Object o) { - if (o.getClass() != ShortFieldSource.class) return false; - ShortFieldSource - other = (ShortFieldSource) o; - return super.equals(other) - && (this.parser == null ? other.parser == null : - this.parser.getClass() == other.parser.getClass()); - } - - @Override - public int hashCode() { - int h = parser == null ? Short.class.hashCode() : parser.getClass().hashCode(); - h += super.hashCode(); - return h; - } -} diff --git a/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java b/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java index 41a903e5811..d976d805475 100755 --- a/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java @@ -52,26 +52,6 @@ public class TestCustomScoreQuery extends FunctionTestSetup { createIndex(true); } - /** - * Test that CustomScoreQuery of Type.BYTE returns the expected scores. - */ - @Test - public void testCustomScoreByte() throws Exception { - // INT field values are small enough to be parsed as byte - doTestCustomScore(BYTE_VALUESOURCE, 1.0); - doTestCustomScore(BYTE_VALUESOURCE, 2.0); - } - - /** - * Test that CustomScoreQuery of Type.SHORT returns the expected scores. - */ - @Test - public void testCustomScoreShort() throws Exception { - // INT field values are small enough to be parsed as short - doTestCustomScore(SHORT_VALUESOURCE, 1.0); - doTestCustomScore(SHORT_VALUESOURCE, 3.0); - } - /** * Test that CustomScoreQuery of Type.INT returns the expected scores. */ diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java b/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java index 416967feb25..4f3c0126458 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java @@ -1,18 +1,25 @@ package org.apache.lucene.queries.function; +import java.io.IOException; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.FloatField; +import org.apache.lucene.document.IntField; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.document.Field.Store; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.queries.function.valuesource.ByteFieldSource; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.queries.function.valuesource.FloatFieldSource; import org.apache.lucene.queries.function.valuesource.IntFieldSource; -import org.apache.lucene.queries.function.valuesource.ShortFieldSource; +import org.apache.lucene.search.FieldCache; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; import org.junit.AfterClass; @@ -53,10 +60,21 @@ public abstract class FunctionTestSetup extends LuceneTestCase { protected static final String INT_FIELD = "iii"; protected static final String FLOAT_FIELD = "fff"; - protected ValueSource BYTE_VALUESOURCE = new ByteFieldSource(INT_FIELD); - protected ValueSource SHORT_VALUESOURCE = new ShortFieldSource(INT_FIELD); + private static final FieldCache.FloatParser CUSTOM_FLOAT_PARSER = new FieldCache.FloatParser() { + + @Override + public TermsEnum termsEnum(Terms terms) throws IOException { + return FieldCache.NUMERIC_UTILS_INT_PARSER.termsEnum(terms); + } + + @Override + public float parseFloat(BytesRef term) { + return (float) FieldCache.NUMERIC_UTILS_INT_PARSER.parseInt(term); + } + }; + protected ValueSource INT_VALUESOURCE = new IntFieldSource(INT_FIELD); - protected ValueSource INT_AS_FLOAT_VALUESOURCE = new FloatFieldSource(INT_FIELD); + protected ValueSource INT_AS_FLOAT_VALUESOURCE = new FloatFieldSource(INT_FIELD, CUSTOM_FLOAT_PARSER); protected ValueSource FLOAT_VALUESOURCE = new FloatFieldSource(FLOAT_FIELD); private static final String DOC_TEXT_LINES[] = { @@ -140,10 +158,10 @@ public abstract class FunctionTestSetup extends LuceneTestCase { f = newField(TEXT_FIELD, "text of doc" + scoreAndID + textLine(i), customType2); // for regular search d.add(f); - f = newField(INT_FIELD, "" + scoreAndID, customType); // for function scoring + f = new IntField(INT_FIELD, scoreAndID, Store.YES); // for function scoring d.add(f); - f = newField(FLOAT_FIELD, scoreAndID + ".000", customType); // for function scoring + f = new FloatField(FLOAT_FIELD, scoreAndID, Store.YES); // for function scoring d.add(f); iw.addDocument(d); diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java index 41d3c50cbf9..7aab17edaa7 100755 --- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java @@ -19,12 +19,6 @@ package org.apache.lucene.queries.function; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.queries.function.FunctionQuery; -import org.apache.lucene.queries.function.ValueSource; -import org.apache.lucene.queries.function.valuesource.ByteFieldSource; -import org.apache.lucene.queries.function.valuesource.FloatFieldSource; -import org.apache.lucene.queries.function.valuesource.IntFieldSource; -import org.apache.lucene.queries.function.valuesource.ShortFieldSource; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.QueryUtils; import org.apache.lucene.search.ScoreDoc; @@ -50,20 +44,6 @@ public class TestFieldScoreQuery extends FunctionTestSetup { createIndex(true); } - /** Test that FieldScoreQuery of Type.BYTE returns docs in expected order. */ - @Test - public void testRankByte () throws Exception { - // INT field values are small enough to be parsed as byte - doTestRank(BYTE_VALUESOURCE); - } - - /** Test that FieldScoreQuery of Type.SHORT returns docs in expected order. */ - @Test - public void testRankShort () throws Exception { - // INT field values are small enough to be parsed as short - doTestRank(SHORT_VALUESOURCE); - } - /** Test that FieldScoreQuery of Type.INT returns docs in expected order. */ @Test public void testRankInt () throws Exception { @@ -99,20 +79,6 @@ public class TestFieldScoreQuery extends FunctionTestSetup { r.close(); } - /** Test that FieldScoreQuery of Type.BYTE returns the expected scores. */ - @Test - public void testExactScoreByte () throws Exception { - // INT field values are small enough to be parsed as byte - doTestExactScore(BYTE_VALUESOURCE); - } - - /** Test that FieldScoreQuery of Type.SHORT returns the expected scores. */ - @Test - public void testExactScoreShort () throws Exception { - // INT field values are small enough to be parsed as short - doTestExactScore(SHORT_VALUESOURCE); - } - /** Test that FieldScoreQuery of Type.INT returns the expected scores. */ @Test public void testExactScoreInt () throws Exception { diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java index 572188f1920..760aab340d0 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java @@ -18,9 +18,10 @@ package org.apache.lucene.queries.function; */ import java.io.IOException; + import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; +import org.apache.lucene.document.IntField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.RandomIndexWriter; @@ -46,13 +47,13 @@ public class TestFunctionQuerySort extends LuceneTestCase { RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc); Document doc = new Document(); - Field field = new StringField("value", "", Field.Store.YES); + Field field = new IntField("value", 0, Field.Store.YES); doc.add(field); // Save docs unsorted (decreasing value n, n-1, ...) final int NUM_VALS = 5; for (int val = NUM_VALS; val > 0; val--) { - field.setStringValue(Integer.toString(val)); + field.setIntValue(val); writer.addDocument(doc); } diff --git a/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java b/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java index cd856d3639d..e781b22b419 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java @@ -22,13 +22,16 @@ import java.util.List; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoubleField; import org.apache.lucene.document.Field; +import org.apache.lucene.document.FloatField; +import org.apache.lucene.document.IntField; +import org.apache.lucene.document.LongField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.queries.function.valuesource.ByteFieldSource; import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource; import org.apache.lucene.queries.function.valuesource.ConstValueSource; import org.apache.lucene.queries.function.valuesource.DivFloatFunction; @@ -54,7 +57,6 @@ import org.apache.lucene.queries.function.valuesource.QueryValueSource; import org.apache.lucene.queries.function.valuesource.RangeMapFloatFunction; import org.apache.lucene.queries.function.valuesource.ReciprocalFloatFunction; import org.apache.lucene.queries.function.valuesource.ScaleFloatFunction; -import org.apache.lucene.queries.function.valuesource.ShortFieldSource; import org.apache.lucene.queries.function.valuesource.SumFloatFunction; import org.apache.lucene.queries.function.valuesource.SumTotalTermFreqValueSource; import org.apache.lucene.queries.function.valuesource.TFValueSource; @@ -85,9 +87,9 @@ public class TestValueSources extends LuceneTestCase { static IndexSearcher searcher; static final List documents = Arrays.asList(new String[][] { - /* id, byte, double, float, int, long, short, string, text */ - new String[] { "0", "5", "3.63", "5.2", "35", "4343", "945", "test", "this is a test test test" }, - new String[] { "1", "12", "5.65", "9.3", "54", "1954", "123", "bar", "second test" }, + /* id, double, float, int, long, string, text */ + new String[] { "0", "3.63", "5.2", "35", "4343", "test", "this is a test test test" }, + new String[] { "1", "5.65", "9.3", "54", "1954", "bar", "second test" }, }); @BeforeClass @@ -99,18 +101,14 @@ public class TestValueSources extends LuceneTestCase { Document document = new Document(); Field idField = new StringField("id", "", Field.Store.NO); document.add(idField); - Field byteField = new StringField("byte", "", Field.Store.NO); - document.add(byteField); - Field doubleField = new StringField("double", "", Field.Store.NO); + Field doubleField = new DoubleField("double", 0d, Field.Store.NO); document.add(doubleField); - Field floatField = new StringField("float", "", Field.Store.NO); + Field floatField = new FloatField("float", 0f, Field.Store.NO); document.add(floatField); - Field intField = new StringField("int", "", Field.Store.NO); + Field intField = new IntField("int", 0, Field.Store.NO); document.add(intField); - Field longField = new StringField("long", "", Field.Store.NO); + Field longField = new LongField("long", 0L, Field.Store.NO); document.add(longField); - Field shortField = new StringField("short", "", Field.Store.NO); - document.add(shortField); Field stringField = new StringField("string", "", Field.Store.NO); document.add(stringField); Field textField = new TextField("text", "", Field.Store.NO); @@ -118,14 +116,12 @@ public class TestValueSources extends LuceneTestCase { for (String [] doc : documents) { idField.setStringValue(doc[0]); - byteField.setStringValue(doc[1]); - doubleField.setStringValue(doc[2]); - floatField.setStringValue(doc[3]); - intField.setStringValue(doc[4]); - longField.setStringValue(doc[5]); - shortField.setStringValue(doc[6]); - stringField.setStringValue(doc[7]); - textField.setStringValue(doc[8]); + doubleField.setDoubleValue(Double.valueOf(doc[1])); + floatField.setFloatValue(Float.valueOf(doc[2])); + intField.setIntValue(Integer.valueOf(doc[3])); + longField.setLongValue(Long.valueOf(doc[4])); + stringField.setStringValue(doc[5]); + textField.setStringValue(doc[6]); iw.addDocument(document); } @@ -143,11 +139,6 @@ public class TestValueSources extends LuceneTestCase { dir = null; } - public void testByte() throws Exception { - assertHits(new FunctionQuery(new ByteFieldSource("byte")), - new float[] { 5f, 12f }); - } - public void testConst() throws Exception { assertHits(new FunctionQuery(new ConstValueSource(0.3f)), new float[] { 0.3f, 0.3f }); @@ -298,11 +289,6 @@ public class TestValueSources extends LuceneTestCase { new float[] { 0.0f, 1.0f }); } - public void testShort() throws Exception { - assertHits(new FunctionQuery(new ShortFieldSource("short")), - new float[] { 945f, 123f }); - } - public void testSumFloat() throws Exception { assertHits(new FunctionQuery(new SumFloatFunction(new ValueSource[] { new ConstValueSource(1f), new ConstValueSource(2f)})), diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java b/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java index 93f1f24007f..063dec381a9 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/LineFileDocs.java @@ -36,6 +36,7 @@ import java.util.zip.GZIPInputStream; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.IntField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; @@ -158,6 +159,7 @@ public class LineFileDocs implements Closeable { final Field titleDV; final Field body; final Field id; + final Field idNum; final Field date; public DocState(boolean useDocValues) { @@ -180,6 +182,9 @@ public class LineFileDocs implements Closeable { id = new StringField("docid", "", Field.Store.YES); doc.add(id); + idNum = new IntField("docid_int", 0, Field.Store.NO); + doc.add(idNum); + date = new StringField("date", "", Field.Store.YES); doc.add(date); @@ -233,7 +238,9 @@ public class LineFileDocs implements Closeable { } docState.titleTokenized.setStringValue(title); docState.date.setStringValue(line.substring(1+spot, spot2)); - docState.id.setStringValue(Integer.toString(id.getAndIncrement())); + final int i = id.getAndIncrement(); + docState.id.setStringValue(Integer.toString(i)); + docState.idNum.setIntValue(i); return docState.doc; } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java index a5fa9a5ce52..89a62e5dc84 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java @@ -49,7 +49,12 @@ import org.apache.lucene.codecs.lucene42.Lucene42Codec; import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoubleField; import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType.NumericType; +import org.apache.lucene.document.FloatField; +import org.apache.lucene.document.IntField; +import org.apache.lucene.document.LongField; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.index.AtomicReader; @@ -78,14 +83,15 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.FilteredQuery.FilterStrategy; import org.apache.lucene.search.FilteredQuery; +import org.apache.lucene.search.FilteredQuery.FilterStrategy; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.junit.Assert; + import com.carrotsearch.randomizedtesting.RandomizedContext; import com.carrotsearch.randomizedtesting.generators.RandomInts; import com.carrotsearch.randomizedtesting.generators.RandomPicks; @@ -872,6 +878,7 @@ public class _TestUtil { final Field field1 = (Field) f; final Field field2; final DocValuesType dvType = field1.fieldType().docValueType(); + final NumericType numType = field1.fieldType().numericType(); if (dvType != null) { switch(dvType) { case NUMERIC: @@ -886,6 +893,23 @@ public class _TestUtil { default: throw new IllegalStateException("unknown Type: " + dvType); } + } else if (numType != null) { + switch (numType) { + case INT: + field2 = new IntField(field1.name(), field1.numericValue().intValue(), field1.fieldType()); + break; + case FLOAT: + field2 = new FloatField(field1.name(), field1.numericValue().intValue(), field1.fieldType()); + break; + case LONG: + field2 = new LongField(field1.name(), field1.numericValue().intValue(), field1.fieldType()); + break; + case DOUBLE: + field2 = new DoubleField(field1.name(), field1.numericValue().intValue(), field1.fieldType()); + break; + default: + throw new IllegalStateException("unknown Type: " + numType); + } } else { field2 = new Field(field1.name(), field1.stringValue(), field1.fieldType()); } diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index eeec11460d4..330fdafa9d4 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -79,6 +79,10 @@ Upgrading from Solr 4.3.0 * SOLR-4778: The signature of LogWatcher.registerListener has changed, from (ListenerConfig, CoreContainer) to (ListenerConfig). Users implementing their own LogWatcher classes will need to change their code accordingly. + +* LUCENE-5063: ByteField and ShortField have been deprecated and will be removed + in 5.0. If you are still using these field types, you should migrate your + fields to TrieIntField. Detailed Change List ---------------------- diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java index c350dd5d936..d334ffc6d59 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java @@ -49,9 +49,7 @@ public class StatsValuesFactory { if (DoubleField.class.isInstance(fieldType) || IntField.class.isInstance(fieldType) || LongField.class.isInstance(fieldType) || - ShortField.class.isInstance(fieldType) || FloatField.class.isInstance(fieldType) || - ByteField.class.isInstance(fieldType) || TrieField.class.isInstance(fieldType) || SortableDoubleField.class.isInstance(fieldType) || SortableIntField.class.isInstance(fieldType) || diff --git a/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java b/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java index 0e2b49427a3..b89445aad75 100755 --- a/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java @@ -260,11 +260,9 @@ public class BinaryResponseWriter implements BinaryQueryResponseWriter { KNOWN_TYPES.add(BCDIntField.class); KNOWN_TYPES.add(BCDLongField.class); KNOWN_TYPES.add(BCDStrField.class); - KNOWN_TYPES.add(ByteField.class); KNOWN_TYPES.add(DateField.class); KNOWN_TYPES.add(DoubleField.class); KNOWN_TYPES.add(FloatField.class); - KNOWN_TYPES.add(ShortField.class); KNOWN_TYPES.add(IntField.class); KNOWN_TYPES.add(LongField.class); KNOWN_TYPES.add(SortableLongField.class); diff --git a/solr/core/src/java/org/apache/solr/schema/ByteField.java b/solr/core/src/java/org/apache/solr/schema/ByteField.java deleted file mode 100644 index 873ce7d09a4..00000000000 --- a/solr/core/src/java/org/apache/solr/schema/ByteField.java +++ /dev/null @@ -1,98 +0,0 @@ -package org.apache.solr.schema; -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.queries.function.ValueSource; -import org.apache.lucene.queries.function.valuesource.ByteFieldSource; -import org.apache.lucene.index.GeneralField; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.StorableField; -import org.apache.lucene.search.SortField; - -import org.apache.solr.response.TextResponseWriter; -import org.apache.solr.search.QParser; - -import java.io.IOException; -import java.util.Map; - -/** - * A numeric field that can contain 8-bit signed two's complement integer - * values, encoded as simple Strings. - * - *

- * Field values will sort numerically, but Range Queries (and other features - * that rely on numeric ranges) will not work as expected: values will be - * evaluated in unicode String order, not numeric order. - *

- * - *
    - *
  • Min Value Allowed: -128
    • - *
  • Max Value Allowed: 127
    • - *
- * - * @see Byte - */ -public class ByteField extends PrimitiveFieldType { - @Override - protected void init(IndexSchema schema, Map args) { - super.init(schema, args); - restrictProps(SORT_MISSING_FIRST | SORT_MISSING_LAST); - } - - ///////////////////////////////////////////////////////////// - @Override - public SortField getSortField(SchemaField field, boolean reverse) { - field.checkSortability(); - return new SortField(field.name, SortField.Type.BYTE, reverse); - } - - @Override - public ValueSource getValueSource(SchemaField field, QParser qparser) { - field.checkFieldCacheSource(qparser); - return new ByteFieldSource(field.name); - } - - @Override - public void write(TextResponseWriter writer, String name, StorableField f) throws IOException { - String s = f.stringValue(); - - // these values may be from a legacy lucene index, which may - // not be properly formatted in some output formats, or may - // incorrectly have a zero length. - - if (s.length()==0) { - // zero length value means someone mistakenly indexed the value - // instead of simply leaving it out. Write a null value instead of a numeric. - writer.writeNull(name); - return; - } - - try { - byte val = Byte.parseByte(s); - writer.writeInt(name, val); - } catch (NumberFormatException e){ - // can't parse - write out the contents as a string so nothing is lost and - // clients don't get a parse error. - writer.writeStr(name, s, true); - } - } - - @Override - public Byte toObject(StorableField f) { - return Byte.valueOf(toExternal(f)); - } -} diff --git a/solr/core/src/java/org/apache/solr/schema/DoubleField.java b/solr/core/src/java/org/apache/solr/schema/DoubleField.java index 5d2968fa139..c76b99c2592 100644 --- a/solr/core/src/java/org/apache/solr/schema/DoubleField.java +++ b/solr/core/src/java/org/apache/solr/schema/DoubleField.java @@ -17,18 +17,20 @@ package org.apache.solr.schema; -import org.apache.lucene.queries.function.ValueSource; -import org.apache.lucene.queries.function.valuesource.DoubleFieldSource; -import org.apache.lucene.index.GeneralField; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.StorableField; -import org.apache.lucene.search.SortField; -import org.apache.solr.response.TextResponseWriter; -import org.apache.solr.search.QParser; - import java.io.IOException; import java.util.Map; +import org.apache.lucene.index.StorableField; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.valuesource.DoubleFieldSource; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.search.SortField; +import org.apache.lucene.util.BytesRef; +import org.apache.solr.response.TextResponseWriter; +import org.apache.solr.search.QParser; + /** * A legacy numeric field type that encodes "Double" values as simple Strings. * This class should not be used except by people with existing indexes that @@ -44,6 +46,20 @@ import java.util.Map; * @see TrieDoubleField */ public class DoubleField extends PrimitiveFieldType { + + private static final FieldCache.DoubleParser PARSER = new FieldCache.DoubleParser() { + + @Override + public TermsEnum termsEnum(Terms terms) throws IOException { + return terms.iterator(null); + } + + @Override + public double parseDouble(BytesRef term) { + return Double.parseDouble(term.utf8ToString()); + } + }; + @Override protected void init(IndexSchema schema, Map args) { super.init(schema, args); @@ -54,13 +70,13 @@ public class DoubleField extends PrimitiveFieldType { @Override public SortField getSortField(SchemaField field, boolean reverse) { field.checkSortability(); - return new SortField(field.name, SortField.Type.DOUBLE, reverse); + return new SortField(field.name, PARSER, reverse); } @Override public ValueSource getValueSource(SchemaField field, QParser qparser) { field.checkFieldCacheSource(qparser); - return new DoubleFieldSource(field.name); + return new DoubleFieldSource(field.name, PARSER); } @Override diff --git a/solr/core/src/java/org/apache/solr/schema/FloatField.java b/solr/core/src/java/org/apache/solr/schema/FloatField.java index 75cf0873dd0..2c1ed4dbb44 100644 --- a/solr/core/src/java/org/apache/solr/schema/FloatField.java +++ b/solr/core/src/java/org/apache/solr/schema/FloatField.java @@ -19,11 +19,15 @@ package org.apache.solr.schema; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.FloatFieldSource; +import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.SortField; +import org.apache.lucene.util.BytesRef; import org.apache.solr.search.QParser; import org.apache.lucene.index.GeneralField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.StorableField; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.solr.response.TextResponseWriter; import java.util.Map; @@ -43,6 +47,20 @@ import java.io.IOException; * @see TrieFloatField */ public class FloatField extends PrimitiveFieldType { + + private static final FieldCache.FloatParser PARSER = new FieldCache.FloatParser() { + + @Override + public TermsEnum termsEnum(Terms terms) throws IOException { + return terms.iterator(null); + } + + @Override + public float parseFloat(BytesRef term) { + return Float.parseFloat(term.utf8ToString()); + } + }; + @Override protected void init(IndexSchema schema, Map args) { super.init(schema, args); @@ -58,7 +76,7 @@ public class FloatField extends PrimitiveFieldType { @Override public ValueSource getValueSource(SchemaField field, QParser qparser) { field.checkFieldCacheSource(qparser); - return new FloatFieldSource(field.name); + return new FloatFieldSource(field.name, PARSER); } @Override diff --git a/solr/core/src/java/org/apache/solr/schema/IntField.java b/solr/core/src/java/org/apache/solr/schema/IntField.java index 3c9320c40f0..85e3c3b3b7d 100644 --- a/solr/core/src/java/org/apache/solr/schema/IntField.java +++ b/solr/core/src/java/org/apache/solr/schema/IntField.java @@ -19,11 +19,15 @@ package org.apache.solr.schema; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.IntFieldSource; +import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.SortField; +import org.apache.lucene.util.BytesRef; import org.apache.solr.search.QParser; import org.apache.lucene.index.GeneralField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.StorableField; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.solr.response.TextResponseWriter; import java.util.Map; @@ -43,6 +47,20 @@ import java.io.IOException; * @see TrieIntField */ public class IntField extends PrimitiveFieldType { + + private static final FieldCache.IntParser PARSER = new FieldCache.IntParser() { + + @Override + public TermsEnum termsEnum(Terms terms) throws IOException { + return terms.iterator(null); + } + + @Override + public int parseInt(BytesRef term) { + return Integer.parseInt(term.utf8ToString()); + } + }; + @Override protected void init(IndexSchema schema, Map args) { super.init(schema, args); @@ -52,13 +70,13 @@ public class IntField extends PrimitiveFieldType { @Override public SortField getSortField(SchemaField field,boolean reverse) { field.checkSortability(); - return new SortField(field.name,SortField.Type.INT, reverse); + return new SortField(field.name, PARSER, reverse); } @Override public ValueSource getValueSource(SchemaField field, QParser qparser) { field.checkFieldCacheSource(qparser); - return new IntFieldSource(field.name); + return new IntFieldSource(field.name, PARSER); } @Override diff --git a/solr/core/src/java/org/apache/solr/schema/LongField.java b/solr/core/src/java/org/apache/solr/schema/LongField.java index 9f795a7a083..93f3389c2b1 100644 --- a/solr/core/src/java/org/apache/solr/schema/LongField.java +++ b/solr/core/src/java/org/apache/solr/schema/LongField.java @@ -22,7 +22,11 @@ import org.apache.lucene.queries.function.valuesource.LongFieldSource; import org.apache.lucene.index.GeneralField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.StorableField; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.SortField; +import org.apache.lucene.util.BytesRef; import org.apache.solr.response.TextResponseWriter; import org.apache.solr.search.QParser; @@ -43,6 +47,20 @@ import java.util.Map; * @see TrieLongField */ public class LongField extends PrimitiveFieldType { + + private static final FieldCache.LongParser PARSER = new FieldCache.LongParser() { + + @Override + public TermsEnum termsEnum(Terms terms) throws IOException { + return terms.iterator(null); + } + + @Override + public long parseLong(BytesRef term) { + return Long.parseLong(term.utf8ToString()); + } + }; + @Override protected void init(IndexSchema schema, Map args) { super.init(schema, args); @@ -54,13 +72,13 @@ public class LongField extends PrimitiveFieldType { @Override public SortField getSortField(SchemaField field,boolean reverse) { field.checkSortability(); - return new SortField(field.name,SortField.Type.LONG, reverse); + return new SortField(field.name, PARSER, reverse); } @Override public ValueSource getValueSource(SchemaField field, QParser qparser) { field.checkFieldCacheSource(qparser); - return new LongFieldSource(field.name); + return new LongFieldSource(field.name, PARSER); } @Override diff --git a/solr/core/src/java/org/apache/solr/schema/ShortField.java b/solr/core/src/java/org/apache/solr/schema/ShortField.java deleted file mode 100644 index 78456097898..00000000000 --- a/solr/core/src/java/org/apache/solr/schema/ShortField.java +++ /dev/null @@ -1,101 +0,0 @@ -package org.apache.solr.schema; -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.queries.function.ValueSource; -import org.apache.lucene.queries.function.valuesource.ShortFieldSource; -import org.apache.lucene.index.GeneralField; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.StorableField; -import org.apache.lucene.search.SortField; - -import org.apache.solr.response.TextResponseWriter; -import org.apache.solr.search.QParser; - -import java.io.IOException; -import java.util.Map; - - -/** - * A numeric field that can contain 16-bit signed two's complement integer - * values, encoded as simple Strings. - * - *

- * Field values will sort numerically, but Range Queries (and other features - * that rely on numeric ranges) will not work as expected: values will be - * evaluated in unicode String order, not numeric order. - *

- * - *
    - *
  • Min Value Allowed: -32768
    • - *
  • Max Value Allowed: 32767
    • - *
- * - * @see Short - **/ -public class ShortField extends PrimitiveFieldType { - @Override - protected void init(IndexSchema schema, Map args) { - super.init(schema, args); - restrictProps(SORT_MISSING_FIRST | SORT_MISSING_LAST); - } - - ///////////////////////////////////////////////////////////// - - @Override - public SortField getSortField(SchemaField field, boolean reverse) { - field.checkSortability(); - return new SortField(field.name, SortField.Type.SHORT, reverse); - } - - @Override - public ValueSource getValueSource(SchemaField field, QParser qparser) { - field.checkFieldCacheSource(qparser); - return new ShortFieldSource(field.name); - } - - @Override - public void write(TextResponseWriter writer, String name, StorableField f) throws IOException { - String s = f.stringValue(); - - // these values may be from a legacy lucene index, which may - // not be properly formatted in some output formats, or may - // incorrectly have a zero length. - - if (s.length()==0) { - // zero length value means someone mistakenly indexed the value - // instead of simply leaving it out. Write a null value instead of a numeric. - writer.writeNull(name); - return; - } - - try { - short val = Short.parseShort(s); - writer.writeInt(name, val); - } catch (NumberFormatException e){ - // can't parse - write out the contents as a string so nothing is lost and - // clients don't get a parse error. - writer.writeStr(name, s, true); - } - } - - @Override - public Short toObject(StorableField f) { - return Short.valueOf(toExternal(f)); - } - -} diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-folding.xml b/solr/core/src/test-files/solr/collection1/conf/schema-folding.xml index 88a6d66fecd..c2a0e60f3ed 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-folding.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-folding.xml @@ -221,8 +221,6 @@ - - @@ -233,8 +231,6 @@ - - diff --git a/solr/core/src/test-files/solr/collection1/conf/schema.xml b/solr/core/src/test-files/solr/collection1/conf/schema.xml index 74525cab85b..6ab77435f00 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema.xml @@ -48,8 +48,6 @@ - - @@ -609,10 +607,6 @@ - - - - diff --git a/solr/core/src/test/org/apache/solr/core/QueryResultKeyTest.java b/solr/core/src/test/org/apache/solr/core/QueryResultKeyTest.java index 3702d254f54..8ad39d526ae 100644 --- a/solr/core/src/test/org/apache/solr/core/QueryResultKeyTest.java +++ b/solr/core/src/test/org/apache/solr/core/QueryResultKeyTest.java @@ -38,7 +38,7 @@ public class QueryResultKeyTest extends SolrTestCaseJ4 { // the hashcode should be the same even when the list // of filters is in a different order - Sort sort = new Sort(new SortField("test", SortField.Type.BYTE)); + Sort sort = new Sort(new SortField("test", SortField.Type.INT)); List filters = new ArrayList(); filters.add(new TermQuery(new Term("test", "field"))); filters.add(new TermQuery(new Term("test2", "field2"))); diff --git a/solr/core/src/test/org/apache/solr/schema/PrimitiveFieldTypeTest.java b/solr/core/src/test/org/apache/solr/schema/PrimitiveFieldTypeTest.java index 34d2e45ec37..be283608319 100644 --- a/solr/core/src/test/org/apache/solr/schema/PrimitiveFieldTypeTest.java +++ b/solr/core/src/test/org/apache/solr/schema/PrimitiveFieldTypeTest.java @@ -59,11 +59,9 @@ public class PrimitiveFieldTypeTest extends SolrTestCaseJ4 { TrieIntField ti; SortableIntField si; LongField l; - ShortField sf; FloatField f; DoubleField d; BoolField b; - ByteField bf; // *********************** @@ -95,10 +93,6 @@ public class PrimitiveFieldTypeTest extends SolrTestCaseJ4 { l.init(schema, initMap); assertFalse(l.hasProperty(FieldType.OMIT_NORMS)); - sf = new ShortField(); - sf.init(schema, initMap); - assertFalse(sf.hasProperty(FieldType.OMIT_NORMS)); - f = new FloatField(); f.init(schema, initMap); assertFalse(f.hasProperty(FieldType.OMIT_NORMS)); @@ -115,10 +109,6 @@ public class PrimitiveFieldTypeTest extends SolrTestCaseJ4 { b.init(schema, initMap); assertFalse(b.hasProperty(FieldType.OMIT_NORMS)); - bf = new ByteField(); - bf.init(schema, initMap); - assertFalse(bf.hasProperty(FieldType.OMIT_NORMS)); - // Non-primitive fields t = new TextField(); t.init(schema, initMap); @@ -157,10 +147,6 @@ public class PrimitiveFieldTypeTest extends SolrTestCaseJ4 { l.init(schema, initMap); assertTrue(l.hasProperty(FieldType.OMIT_NORMS)); - sf = new ShortField(); - sf.init(schema, initMap); - assertTrue(sf.hasProperty(FieldType.OMIT_NORMS)); - f = new FloatField(); f.init(schema, initMap); assertTrue(f.hasProperty(FieldType.OMIT_NORMS)); @@ -177,10 +163,6 @@ public class PrimitiveFieldTypeTest extends SolrTestCaseJ4 { b.init(schema, initMap); assertTrue(b.hasProperty(FieldType.OMIT_NORMS)); - bf = new ByteField(); - bf.init(schema, initMap); - assertTrue(bf.hasProperty(FieldType.OMIT_NORMS)); - // Non-primitive fields t = new TextField(); t.init(schema, initMap); diff --git a/solr/core/src/test/org/apache/solr/search/TestFoldingMultitermQuery.java b/solr/core/src/test/org/apache/solr/search/TestFoldingMultitermQuery.java index db800fc7023..08fcfa8d19b 100644 --- a/solr/core/src/test/org/apache/solr/search/TestFoldingMultitermQuery.java +++ b/solr/core/src/test/org/apache/solr/search/TestFoldingMultitermQuery.java @@ -46,8 +46,6 @@ public class TestFoldingMultitermQuery extends SolrTestCaseJ4 { "float_f", num, "long_f", num, "double_f", num, - "byte_f", num, - "short_f", num, "bool_f", boolVal, "date_f", "200" + Integer.toString(i % 10) + "-01-01T00:00:00Z", "content", docs[i], @@ -269,7 +267,7 @@ public class TestFoldingMultitermQuery extends SolrTestCaseJ4 { @Test public void testNonTextTypes() { - String[] intTypes = {"int_f", "float_f", "long_f", "double_f", "byte_f", "short_f"}; + String[] intTypes = {"int_f", "float_f", "long_f", "double_f"}; for (String str : intTypes) { assertQ(req("q", str + ":" + "0"), diff --git a/solr/core/src/test/org/apache/solr/search/TestValueSourceCache.java b/solr/core/src/test/org/apache/solr/search/TestValueSourceCache.java index b6da59185c1..57c40645817 100644 --- a/solr/core/src/test/org/apache/solr/search/TestValueSourceCache.java +++ b/solr/core/src/test/org/apache/solr/search/TestValueSourceCache.java @@ -78,8 +78,6 @@ public class TestValueSourceCache extends SolrTestCaseJ4 { "val1_i1", "val1_l1", "val1_b1", - "val1_by1", - "val1_sh1" }; for (String template : templates) { for (String nums : numbers) {