Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-1291 SOLR-1322 SOLR-1325 SOLR-1328 : more trie integration 

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@800565 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley 2009-08-03 21:09:05 +00:00
parent 0d3d010633
commit d8027f2de0
16 changed files with 518 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
CHANGES.txt
View File

@ -180,10 +180,10 @@ New Features
and lucene query parser (the latter existed as an undocumented and lucene query parser (the latter existed as an undocumented
feature in 1.3) (yonik) feature in 1.3) (yonik)
40. SOLR-940: Add support for Lucene's Trie Range Queries by providing new FieldTypes in schema for int, float, long, 40. SOLR-940: Add support for Lucene's Trie Range Queries by providing new FieldTypes in
double and date. Range searches and term queries on such fields will automatically use the corresponding trie schema for int, float, long, double and date. Single-valued Trie based
range filter in Lucene contrib-queries and can be dramatically faster than normal range queries. fields with a precisionStep will index multiple precisions and enable
(Uwe Schindler, shalin) faster range queries. (Uwe Schindler, yonik, shalin)
41. SOLR-1038: Enhance CommonsHttpSolrServer to add docs in batch using an iterator API (Noble Paul via shalin) 41. SOLR-1038: Enhance CommonsHttpSolrServer to add docs in batch using an iterator API (Noble Paul via shalin)

124
example/solr/conf/schema.xml
View File

@ -63,6 +63,7 @@
<!-- The optional sortMissingLast and sortMissingFirst attributes are <!-- The optional sortMissingLast and sortMissingFirst attributes are
currently supported on types that are sorted internally as strings. currently supported on types that are sorted internally as strings.
This includes "string","boolean","sint","slong","sfloat","sdouble","pdate"
- If sortMissingLast="true", then a sort on this field will cause documents - If sortMissingLast="true", then a sort on this field will cause documents
without the field to come after documents with the field, without the field to come after documents with the field,
regardless of the requested sort order (asc or desc). regardless of the requested sort order (asc or desc).
@ -74,25 +75,31 @@
field first in an ascending sort and last in a descending sort. field first in an ascending sort and last in a descending sort.
--> -->
<!-- Default numeric field types. For faster range queries, use the tint/tfloat/tlong/tdouble types.
Note: the statistics component does not yet work with these field types.
-->
<fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
<!-- numeric field types that store and index the text <!--
value verbatim (and hence don't support range queries, since the Numeric field types for single-valued fields that index extra tokens with
lexicographic ordering isn't equal to the numeric ordering) --> lower precision to accelerate range queries when the number of values between
<fieldType name="integer" class="solr.IntField" omitNorms="true"/> the range endpoints is large. See the javadoc for NumericRangeQuery for
<fieldType name="long" class="solr.LongField" omitNorms="true"/> internal implementation details.
<fieldType name="float" class="solr.FloatField" omitNorms="true"/>
<fieldType name="double" class="solr.DoubleField" omitNorms="true"/>
For single-valued fields, smaller precisionStep values (specified in bits)
will lead to more tokens indexed per value, slightly higher index size, and
faster range queries.
<!-- Numeric field types that manipulate the value into Note: precisionStep is disabled for multiValued fields.
a string value that isn't human-readable in its internal form, Note: faceting does not currently work for these fields.
but with a lexicographic ordering the same as the numeric ordering, -->
so that range queries work correctly. --> <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="sint" class="solr.SortableIntField" sortMissingLast="true" omitNorms="true"/> <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="slong" class="solr.SortableLongField" sortMissingLast="true" omitNorms="true"/> <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="sfloat" class="solr.SortableFloatField" sortMissingLast="true" omitNorms="true"/> <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="sdouble" class="solr.SortableDoubleField" sortMissingLast="true" omitNorms="true"/>
<!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and <!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
is a more restricted form of the canonical representation of dateTime is a more restricted form of the canonical representation of dateTime
@ -114,35 +121,33 @@
Consult the DateField javadocs for more information. Consult the DateField javadocs for more information.
--> -->
<fieldType name="date" class="solr.DateField" sortMissingLast="true" omitNorms="true"/> <fieldType name="date" class="solr.TrieDateField" omitNorms="true" precisionStep="0" positionIncrementGap="0"/>
<!-- A Trie based single-valued date field for faster date range queries and date faceting -->
<fieldType name="tdate" class="solr.TrieDateField" omitNorms="true" precisionStep="8" positionIncrementGap="0"/>
<!-- plain numeric field types that store and index the text
value verbatim (and hence don't support range queries, since the
lexicographic ordering isn't equal to the numeric ordering)
These should only be used for compatibility with existing indexes.
Use Trie based fields instead.
-->
<fieldType name="pint" class="solr.IntField" omitNorms="true"/>
<fieldType name="plong" class="solr.LongField" omitNorms="true"/>
<fieldType name="pfloat" class="solr.FloatField" omitNorms="true"/>
<fieldType name="pdouble" class="solr.DoubleField" omitNorms="true"/>
<fieldType name="pdate" class="solr.DateField" sortMissingLast="true" omitNorms="true"/>
<!-- <!--
Numeric field types that manipulate the value into trie encoded strings which are not These types should only be used for back compatibility with existing
human readable in the internal form. Range searches on such fields use the fast Trie Range Queries indexes, or if "sortMissingLast" funcitonallity is needed. Use Trie based fields instead.
which are much faster than range searches on the SortableNumberField types. -->
<fieldType name="sint" class="solr.SortableIntField" sortMissingLast="true" omitNorms="true"/>
For the fast range search to work, trie fields must be indexed. <fieldType name="slong" class="solr.SortableLongField" sortMissingLast="true" omitNorms="true"/>
<fieldType name="sfloat" class="solr.SortableFloatField" sortMissingLast="true" omitNorms="true"/>
For each number being added to this field, multiple terms are generated as per the algorithm described in <fieldType name="sdouble" class="solr.SortableDoubleField" sortMissingLast="true" omitNorms="true"/>
org.apache.lucene.search.trie package description. The possible number of terms depend on the precisionStep
attribute and increase dramatically with higher precision steps (factor 2**precisionStep). The default
value of precisionStep is 8.
Note that if you use a precisionStep of 32 for int/float and 64 for long/double, then multiple terms
will not be generated, and range search will be no faster than any other number field.
-->
<fieldType name="tint" class="solr.TrieField" type="integer" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<fieldType name="tfloat" class="solr.TrieField" type="float" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<fieldType name="tlong" class="solr.TrieField" type="long" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<fieldType name="tdouble" class="solr.TrieField" type="double" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<fieldType name="tdouble4" class="solr.TrieField" type="double" precisionStep="4" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<!--
This date field manipulates the value into a trie encoded strings for fast range searches. They follow the
same format and semantics as the normal DateField and support the date math syntax.
-->
<fieldType name="tdate" class="solr.TrieField" type="date" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<!-- The "RandomSortField" is not used to store or search any <!-- The "RandomSortField" is not used to store or search any
@ -356,17 +361,17 @@
<field name="features" type="text" indexed="true" stored="true" multiValued="true" termVectors="true" termPositions="true" termOffsets="true"/> <field name="features" type="text" indexed="true" stored="true" multiValued="true" termVectors="true" termPositions="true" termOffsets="true"/>
<field name="includes" type="text" indexed="true" stored="true"/> <field name="includes" type="text" indexed="true" stored="true"/>
<field name="weight" type="sfloat" indexed="true" stored="true"/> <field name="weight" type="float" indexed="true" stored="true"/>
<field name="price" type="sfloat" indexed="true" stored="true"/> <field name="price" type="float" indexed="true" stored="true"/>
<!-- "default" values can be specified for fields, indicating which <!-- "default" values can be specified for fields, indicating which
value should be used if no value is specified when adding a document. value should be used if no value is specified when adding a document.
--> -->
<field name="popularity" type="sint" indexed="true" stored="true"/> <field name="popularity" type="int" indexed="true" stored="true"/>
<field name="inStock" type="boolean" indexed="true" stored="true"/> <field name="inStock" type="boolean" indexed="true" stored="true"/>
<!-- Some sample docs exists solely to demonstrate the spellchecker <!-- Some sample docs exists solely to demonstrate the spellchecker
functionality, this is the only field they contain. functionality, this is the only field they contain.
Typically you might build the spellchecker of "catchall" type field Typically you might build the spellchecker off "catchall" type field
containing all of the text in each document. containing all of the text in each document.
--> -->
<field name="word" type="string" indexed="true" stored="true"/> <field name="word" type="string" indexed="true" stored="true"/>
@ -389,17 +394,6 @@
<field name="spell" type="textSpell" indexed="true" stored="true" multiValued="true"/> <field name="spell" type="textSpell" indexed="true" stored="true" multiValued="true"/>
<!-- Some examples of trie fields -->
<field name="tint" type="tint" indexed="true" stored="false" />
<field name="tfloat" type="tfloat" indexed="true" stored="false" />
<field name="tlong" type="tlong" indexed="true" stored="false" />
<field name="tdouble" type="tdouble" indexed="true" stored="false" />
<!-- A double with a custom precisionStep -->
<field name="tdouble4" type="tdouble4" indexed="true" stored="false" />
<!-- An example for the trie date field -->
<field name="tdate" type="tdate" indexed="true" stored="true" />
<!-- Dynamic field definitions. If a field name is not found, dynamicFields <!-- Dynamic field definitions. If a field name is not found, dynamicFields
will be used if the name matches any of the patterns. will be used if the name matches any of the patterns.
@ -408,19 +402,21 @@
EXAMPLE: name="*_i" will match any field ending in _i (like myid_i, z_i) EXAMPLE: name="*_i" will match any field ending in _i (like myid_i, z_i)
Longer patterns will be matched first. if equal size patterns Longer patterns will be matched first. if equal size patterns
both match, the first appearing in the schema will be used. --> both match, the first appearing in the schema will be used. -->
<dynamicField name="*_i" type="sint" indexed="true" stored="true"/> <dynamicField name="*_i" type="int" indexed="true" stored="true"/>
<dynamicField name="*_s" type="string" indexed="true" stored="true"/> <dynamicField name="*_s" type="string" indexed="true" stored="true"/>
<dynamicField name="*_l" type="slong" indexed="true" stored="true"/> <dynamicField name="*_l" type="long" indexed="true" stored="true"/>
<dynamicField name="*_t" type="text" indexed="true" stored="true"/> <dynamicField name="*_t" type="text" indexed="true" stored="true"/>
<dynamicField name="*_b" type="boolean" indexed="true" stored="true"/> <dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
<dynamicField name="*_f" type="sfloat" indexed="true" stored="true"/> <dynamicField name="*_f" type="float" indexed="true" stored="true"/>
<dynamicField name="*_d" type="sdouble" indexed="true" stored="true"/> <dynamicField name="*_d" type="double" indexed="true" stored="true"/>
<dynamicField name="*_dt" type="date" indexed="true" stored="true"/> <dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
<dynamicField name="ignored_*" type="ignored"/> <dynamicField name="*_pi" type="pint" indexed="true" stored="true"/>
<dynamicField name="ignored_*" type="ignored" multiValued="true"/>
<dynamicField name="attr_*" type="text" indexed="true" stored="true" multiValued="true"/> <dynamicField name="attr_*" type="text" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="random*" type="random" /> <dynamicField name="random_*" type="random" />
<!-- uncomment the following to ignore any fields that don't already match an existing <!-- uncomment the following to ignore any fields that don't already match an existing
field name or dynamic field, rather than reporting them as an error. field name or dynamic field, rather than reporting them as an error.

5
src/java/org/apache/solr/request/BinaryResponseWriter.java
View File

@ -211,6 +211,11 @@ public class BinaryResponseWriter implements BinaryQueryResponseWriter {
KNOWN_TYPES.add(StrField.class); KNOWN_TYPES.add(StrField.class);
KNOWN_TYPES.add(TextField.class); KNOWN_TYPES.add(TextField.class);
KNOWN_TYPES.add(TrieField.class); KNOWN_TYPES.add(TrieField.class);
KNOWN_TYPES.add(TrieIntField.class);
KNOWN_TYPES.add(TrieLongField.class);
KNOWN_TYPES.add(TrieFloatField.class);
KNOWN_TYPES.add(TrieDoubleField.class);
KNOWN_TYPES.add(TrieDateField.class);
KNOWN_TYPES.add(BinaryField.class); KNOWN_TYPES.add(BinaryField.class);
// We do not add UUIDField because UUID object is not a supported type in JavaBinCodec // We do not add UUIDField because UUID object is not a supported type in JavaBinCodec
// and if we write UUIDField.toObject, we wouldn't know how to handle it in the client side // and if we write UUIDField.toObject, we wouldn't know how to handle it in the client side

39
src/java/org/apache/solr/request/SimpleFacets.java
View File

@ -33,11 +33,7 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrCore;
import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.*;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.BoolField;
import org.apache.solr.schema.DateField;
import org.apache.solr.search.*; import org.apache.solr.search.*;
import org.apache.solr.util.BoundedTreeSet; import org.apache.solr.util.BoundedTreeSet;
import org.apache.solr.util.DateMathParser; import org.apache.solr.util.DateMathParser;
@ -556,13 +552,13 @@ public class SimpleFacets {
final NamedList resInner = new SimpleOrderedMap(); final NamedList resInner = new SimpleOrderedMap();
resOuter.add(key, resInner); resOuter.add(key, resInner);
final FieldType trash = schema.getFieldType(f); final SchemaField sf = schema.getField(f);
if (! (trash instanceof DateField)) { if (! (sf.getType() instanceof DateField)) {
throw new SolrException throw new SolrException
(SolrException.ErrorCode.BAD_REQUEST, (SolrException.ErrorCode.BAD_REQUEST,
"Can not date facet on a field which is not a DateField: " + f); "Can not date facet on a field which is not a DateField: " + f);
} }
final DateField ft = (DateField) trash; final DateField ft = (DateField) sf.getType();
final String startS final String startS
= required.getFieldParam(f,FacetParams.FACET_DATE_START); = required.getFieldParam(f,FacetParams.FACET_DATE_START);
final Date start; final Date start;
@ -600,7 +596,9 @@ public class SimpleFacets {
while (low.before(end)) { while (low.before(end)) {
dmp.setNow(low); dmp.setNow(low);
final String lowI = ft.toInternal(low); final String lowI = ft.toInternal(low);
final String label = ft.indexedToReadable(lowI); // final String label = ft.indexedToReadable(lowI);
String label = ft.toExternal(low);
Date high = dmp.parseMath(gap); Date high = dmp.parseMath(gap);
if (end.before(high)) { if (end.before(high)) {
if (params.getFieldBool(f,FacetParams.FACET_DATE_HARD_END,false)) { if (params.getFieldBool(f,FacetParams.FACET_DATE_HARD_END,false)) {
@ -615,7 +613,8 @@ public class SimpleFacets {
"date facet infinite loop (is gap negative?)"); "date facet infinite loop (is gap negative?)");
} }
final String highI = ft.toInternal(high); final String highI = ft.toInternal(high);
resInner.add(label, rangeCount(f,lowI,highI,true,true)); // resInner.add(label, rangeCount(sf,lowI,highI,true,true));
resInner.add(label, rangeCount(sf,low,high,true,true));
low = high; low = high;
} }
} catch (java.text.ParseException e) { } catch (java.text.ParseException e) {
@ -647,15 +646,15 @@ public class SimpleFacets {
if (all || others.contains(FacetDateOther.BEFORE)) { if (all || others.contains(FacetDateOther.BEFORE)) {
resInner.add(FacetDateOther.BEFORE.toString(), resInner.add(FacetDateOther.BEFORE.toString(),
rangeCount(f,null,startI,false,false)); rangeCount(sf,null,start,false,false));
} }
if (all || others.contains(FacetDateOther.AFTER)) { if (all || others.contains(FacetDateOther.AFTER)) {
resInner.add(FacetDateOther.AFTER.toString(), resInner.add(FacetDateOther.AFTER.toString(),
rangeCount(f,endI,null,false,false)); rangeCount(sf,end,null,false,false));
} }
if (all || others.contains(FacetDateOther.BETWEEN)) { if (all || others.contains(FacetDateOther.BETWEEN)) {
resInner.add(FacetDateOther.BETWEEN.toString(), resInner.add(FacetDateOther.BETWEEN.toString(),
rangeCount(f,startI,endI,true,true)); rangeCount(sf,start,end,true,true));
} }
} }
} }
@ -665,14 +664,20 @@ public class SimpleFacets {
} }
/** /**
* Macro for getting the numDocs of a TermRangeQuery over docs * Macro for getting the numDocs of range over docs
* @see SolrIndexSearcher#numDocs * @see SolrIndexSearcher#numDocs
* @see TermRangeQuery * @see TermRangeQuery
*/ */
protected int rangeCount(String field, String low, String high, protected int rangeCount(SchemaField sf, String low, String high,
boolean iLow, boolean iHigh) throws IOException { boolean iLow, boolean iHigh) throws IOException {
return searcher.numDocs(new TermRangeQuery(field,low,high,iLow,iHigh), Query rangeQ = sf.getType().getRangeQuery(null, sf,low,high,iLow,iHigh);
base); return searcher.numDocs(rangeQ ,base);
}
protected int rangeCount(SchemaField sf, Date low, Date high,
boolean iLow, boolean iHigh) throws IOException {
Query rangeQ = ((DateField)(sf.getType())).getRangeQuery(null, sf,low,high,iLow,iHigh);
return searcher.numDocs(rangeQ ,base);
} }
/** /**

23
src/java/org/apache/solr/schema/DateField.java
View File

@ -22,6 +22,8 @@ import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter; import org.apache.solr.request.TextResponseWriter;
import org.apache.lucene.document.Fieldable; import org.apache.lucene.document.Fieldable;
import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortField;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader;
import org.apache.solr.search.function.*; import org.apache.solr.search.function.*;
import org.apache.solr.search.QParser; import org.apache.solr.search.QParser;
@ -158,7 +160,8 @@ public class DateField extends FieldType {
if (0 < zz) { if (0 < zz) {
math = val.substring(zz+1); math = val.substring(zz+1);
try { try {
p.setNow(toObject(val.substring(0,zz))); // p.setNow(toObject(val.substring(0,zz)));
p.setNow(parseDate(val.substring(0,zz+1)));
} catch (ParseException e) { } catch (ParseException e) {
throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
"Invalid Date in Date Math String:'" "Invalid Date in Date Math String:'"
@ -193,6 +196,7 @@ public class DateField extends FieldType {
public String toExternal(Fieldable f) { public String toExternal(Fieldable f) {
return indexedToReadable(f.stringValue()); return indexedToReadable(f.stringValue());
} }
public Date toObject(String indexedForm) throws java.text.ParseException { public Date toObject(String indexedForm) throws java.text.ParseException {
return parseDate(indexedToReadable(indexedForm)); return parseDate(indexedToReadable(indexedForm));
} }
@ -243,6 +247,13 @@ public class DateField extends FieldType {
return fmtThreadLocal.get().format(d); return fmtThreadLocal.get().format(d);
} }
/**
* Return the standard human readable form of the date
*/
public String toExternal(Date d) {
return fmtThreadLocal.get().format(d) + 'Z';
}
/** /**
* Thread safe method that can be used by subclasses to parse a Date * Thread safe method that can be used by subclasses to parse a Date
* that is already in the internal representation * that is already in the internal representation
@ -336,6 +347,16 @@ public class DateField extends FieldType {
public ValueSource getValueSource(SchemaField field, QParser parser) { public ValueSource getValueSource(SchemaField field, QParser parser) {
return new DateFieldSource(field.getName(), field.getType()); return new DateFieldSource(field.getName(), field.getType());
} }
/** DateField specific range query */
public Query getRangeQuery(QParser parser, SchemaField sf, Date part1, Date part2, boolean minInclusive, boolean maxInclusive) {
return new TermRangeQuery(
sf.getName(),
part1 == null ? null : toInternal(part1),
part2 == null ? null : toInternal(part2),
minInclusive, maxInclusive);
}
} }

14
src/java/org/apache/solr/schema/FieldType.java
View File

@ -233,7 +233,7 @@ public abstract class FieldType extends FieldProperties {
/** /**
* Convert an external value (from XML update command or from query string) * Convert an external value (from XML update command or from query string)
* into the internal format. * into the internal format for both storing and indexing (which can be modified by any analyzers).
* @see #toExternal * @see #toExternal
*/ */
public String toInternal(String val) { public String toInternal(String val) {
@ -280,7 +280,11 @@ public abstract class FieldType extends FieldProperties {
// that the indexed form is the same as the stored field form. // that the indexed form is the same as the stored field form.
return f.stringValue(); return f.stringValue();
} }
/** Given the readable value, return the term value that will match it. */
public String readableToIndexed(String val) {
return toInternal(val);
}
/********* /*********
// default analyzer for non-text fields. // default analyzer for non-text fields.
@ -437,7 +441,7 @@ public abstract class FieldType extends FieldProperties {
* handle nulls in part1 and/or part2 as well as unequal minInclusive and maxInclusive parameters gracefully. * handle nulls in part1 and/or part2 as well as unequal minInclusive and maxInclusive parameters gracefully.
* *
* @param parser * @param parser
* @param field the name of the field * @param field the schema field
* @param part1 the lower boundary of the range, nulls are allowed. * @param part1 the lower boundary of the range, nulls are allowed.
* @param part2 the upper boundary of the range, nulls are allowed * @param part2 the upper boundary of the range, nulls are allowed
* @param minInclusive whether the minimum of the range is inclusive or not * @param minInclusive whether the minimum of the range is inclusive or not
@ -446,10 +450,10 @@ public abstract class FieldType extends FieldProperties {
* *
* @see org.apache.solr.search.SolrQueryParser#getRangeQuery(String, String, String, boolean) * @see org.apache.solr.search.SolrQueryParser#getRangeQuery(String, String, String, boolean)
*/ */
public Query getRangeQuery(QParser parser, String field, String part1, String part2, boolean minInclusive, boolean maxInclusive) { public Query getRangeQuery(QParser parser, SchemaField field, String part1, String part2, boolean minInclusive, boolean maxInclusive) {
// constant score mode is now enabled per default // constant score mode is now enabled per default
return new TermRangeQuery( return new TermRangeQuery(
field, field.getName(),
part1 == null ? null : toInternal(part1), part1 == null ? null : toInternal(part1),
part2 == null ? null : toInternal(part2), part2 == null ? null : toInternal(part2),
minInclusive, maxInclusive); minInclusive, maxInclusive);

308
src/java/org/apache/solr/schema/TrieField.java
View File

@ -17,10 +17,14 @@
package org.apache.solr.schema; package org.apache.solr.schema;
import org.apache.lucene.document.Fieldable; import org.apache.lucene.document.Fieldable;
import org.apache.lucene.document.Field;
import org.apache.lucene.search.Query; import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortField;
import org.apache.lucene.search.NumericRangeQuery; import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.NumericTokenStream;
import org.apache.solr.analysis.*; import org.apache.solr.analysis.*;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.apache.solr.request.TextResponseWriter; import org.apache.solr.request.TextResponseWriter;
@ -30,6 +34,7 @@ import org.apache.solr.search.function.*;
import java.io.IOException; import java.io.IOException;
import java.util.Map; import java.util.Map;
import java.util.Date;
/** /**
* Provides field types to support for Lucene's Trie Range Queries. * Provides field types to support for Lucene's Trie Range Queries.
@ -43,7 +48,7 @@ import java.util.Map;
* Trie fields are sortable in numerical order and can be used in function queries. * Trie fields are sortable in numerical order and can be used in function queries.
* <p/> * <p/>
* Note that if you use a precisionStep of 32 for int/float and 64 for long/double, then multiple terms will not be * Note that if you use a precisionStep of 32 for int/float and 64 for long/double, then multiple terms will not be
* generated, range search will be no faster than any other number field, but sorting will be possible. * generated, range search will be no faster than any other number field, but sorting will still be possible.
* *
* @version $Id$ * @version $Id$
* @see org.apache.lucene.search.NumericRangeQuery * @see org.apache.lucene.search.NumericRangeQuery
@ -52,25 +57,27 @@ import java.util.Map;
public class TrieField extends FieldType { public class TrieField extends FieldType {
public static final int DEFAULT_PRECISION_STEP = 8; public static final int DEFAULT_PRECISION_STEP = 8;
protected int precisionStep = TrieField.DEFAULT_PRECISION_STEP; protected int precisionStepArg = TrieField.DEFAULT_PRECISION_STEP; // the one passed in or defaulted
protected int precisionStep; // normalized
protected TrieTypes type; protected TrieTypes type;
/** /**
* Used for handling date types following the same semantics as DateField * Used for handling date types following the same semantics as DateField
*/ */
private static final DateField dateField = new DateField(); static final DateField dateField = new DateField();
@Override @Override
protected void init(IndexSchema schema, Map<String, String> args) { protected void init(IndexSchema schema, Map<String, String> args) {
String p = args.remove("precisionStep"); String p = args.remove("precisionStep");
if (p != null) { if (p != null) {
precisionStep = Integer.parseInt(p); precisionStepArg = Integer.parseInt(p);
} }
// normalize the precisionStep
precisionStep = precisionStepArg;
if (precisionStep<=0 || precisionStep>=64) precisionStep=Integer.MAX_VALUE;
String t = args.remove("type"); String t = args.remove("type");
if (t == null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, if (t != null) {
"Invalid type specified in schema.xml for field: " + args.get("name"));
} else {
try { try {
type = TrieTypes.valueOf(t.toUpperCase()); type = TrieTypes.valueOf(t.toUpperCase());
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
@ -78,6 +85,7 @@ public class TrieField extends FieldType {
"Invalid type specified in schema.xml for field: " + args.get("name"), e); "Invalid type specified in schema.xml for field: " + args.get("name"), e);
} }
} }
CharFilterFactory[] filterFactories = new CharFilterFactory[0]; CharFilterFactory[] filterFactories = new CharFilterFactory[0];
TokenFilterFactory[] tokenFilterFactories = new TokenFilterFactory[0]; TokenFilterFactory[] tokenFilterFactories = new TokenFilterFactory[0];
@ -88,18 +96,19 @@ public class TrieField extends FieldType {
@Override @Override
public Object toObject(Fieldable f) { public Object toObject(Fieldable f) {
String s = f.stringValue(); byte[] arr = f.binaryValue();
if (arr==null) return badFieldString(f);
switch (type) { switch (type) {
case INTEGER: case INTEGER:
return Integer.parseInt(s); return toInt(arr);
case FLOAT: case FLOAT:
return Float.parseFloat(s); return toFloat(arr);
case LONG: case LONG:
return Long.parseLong(s); return toLong(arr);
case DOUBLE: case DOUBLE:
return Double.parseDouble(s); return toDouble(arr);
case DATE: case DATE:
return dateField.toObject(f); return new Date(toLong(arr));
default: default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name()); throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
} }
@ -111,7 +120,7 @@ public class TrieField extends FieldType {
return new SortField(field.getName(), FieldCache.NUMERIC_UTILS_INT_PARSER, top); return new SortField(field.getName(), FieldCache.NUMERIC_UTILS_INT_PARSER, top);
case FLOAT: case FLOAT:
return new SortField(field.getName(), FieldCache.NUMERIC_UTILS_FLOAT_PARSER, top); return new SortField(field.getName(), FieldCache.NUMERIC_UTILS_FLOAT_PARSER, top);
case DATE: case DATE: // fallthrough
case LONG: case LONG:
return new SortField(field.getName(), FieldCache.NUMERIC_UTILS_LONG_PARSER, top); return new SortField(field.getName(), FieldCache.NUMERIC_UTILS_LONG_PARSER, top);
case DOUBLE: case DOUBLE:
@ -128,6 +137,7 @@ public class TrieField extends FieldType {
case FLOAT: case FLOAT:
return new FloatFieldSource(field.getName(), FieldCache.NUMERIC_UTILS_FLOAT_PARSER); return new FloatFieldSource(field.getName(), FieldCache.NUMERIC_UTILS_FLOAT_PARSER);
case DATE: case DATE:
return new TrieDateFieldSource(field.getName(), FieldCache.NUMERIC_UTILS_LONG_PARSER);
case LONG: case LONG:
return new LongFieldSource(field.getName(), FieldCache.NUMERIC_UTILS_LONG_PARSER); return new LongFieldSource(field.getName(), FieldCache.NUMERIC_UTILS_LONG_PARSER);
case DOUBLE: case DOUBLE:
@ -138,11 +148,58 @@ public class TrieField extends FieldType {
} }
public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException { public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeVal(name, toObject(f)); byte[] arr = f.binaryValue();
if (arr==null) {
xmlWriter.writeStr(name, badFieldString(f));
return;
}
switch (type) {
case INTEGER:
xmlWriter.writeInt(name,toInt(arr));
break;
case FLOAT:
xmlWriter.writeFloat(name,toFloat(arr));
break;
case LONG:
xmlWriter.writeLong(name,toLong(arr));
break;
case DOUBLE:
xmlWriter.writeDouble(name,toDouble(arr));
break;
case DATE:
xmlWriter.writeDate(name,new Date(toLong(arr)));
break;
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
}
} }
public void write(TextResponseWriter writer, String name, Fieldable f) throws IOException { public void write(TextResponseWriter writer, String name, Fieldable f) throws IOException {
writer.writeVal(name, toObject(f)); byte[] arr = f.binaryValue();
if (arr==null) {
writer.writeStr(name, badFieldString(f),true);
return;
}
switch (type) {
case INTEGER:
writer.writeInt(name,toInt(arr));
break;
case FLOAT:
writer.writeFloat(name,toFloat(arr));
break;
case LONG:
writer.writeLong(name,toLong(arr));
break;
case DOUBLE:
writer.writeDouble(name,toDouble(arr));
break;
case DATE:
writer.writeDate(name,new Date(toLong(arr)));
break;
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
}
} }
@Override @Override
@ -154,7 +211,7 @@ public class TrieField extends FieldType {
* @return the precisionStep used to index values into the field * @return the precisionStep used to index values into the field
*/ */
public int getPrecisionStep() { public int getPrecisionStep() {
return precisionStep; return precisionStepArg;
} }
/** /**
@ -165,35 +222,38 @@ public class TrieField extends FieldType {
} }
@Override @Override
public Query getRangeQuery(QParser parser, String field, String min, String max, boolean minInclusive, boolean maxInclusive) { public Query getRangeQuery(QParser parser, SchemaField field, String min, String max, boolean minInclusive, boolean maxInclusive) {
// don't use a precisionStep if the field is multiValued
int ps = field.multiValued() ? Integer.MAX_VALUE : precisionStep;
Query query = null; Query query = null;
switch (type) { switch (type) {
case INTEGER: case INTEGER:
query = NumericRangeQuery.newIntRange(field, precisionStep, query = NumericRangeQuery.newIntRange(field.getName(), ps,
min == null ? null : Integer.parseInt(min), min == null ? null : Integer.parseInt(min),
max == null ? null : Integer.parseInt(max), max == null ? null : Integer.parseInt(max),
minInclusive, maxInclusive); minInclusive, maxInclusive);
break; break;
case FLOAT: case FLOAT:
query = NumericRangeQuery.newFloatRange(field, precisionStep, query = NumericRangeQuery.newFloatRange(field.getName(), ps,
min == null ? null : Float.parseFloat(min), min == null ? null : Float.parseFloat(min),
max == null ? null : Float.parseFloat(max), max == null ? null : Float.parseFloat(max),
minInclusive, maxInclusive); minInclusive, maxInclusive);
break; break;
case LONG: case LONG:
query = NumericRangeQuery.newLongRange(field, precisionStep, query = NumericRangeQuery.newLongRange(field.getName(), ps,
min == null ? null : Long.parseLong(min), min == null ? null : Long.parseLong(min),
max == null ? null : Long.parseLong(max), max == null ? null : Long.parseLong(max),
minInclusive, maxInclusive); minInclusive, maxInclusive);
break; break;
case DOUBLE: case DOUBLE:
query = NumericRangeQuery.newDoubleRange(field, precisionStep, query = NumericRangeQuery.newDoubleRange(field.getName(), ps,
min == null ? null : Double.parseDouble(min), min == null ? null : Double.parseDouble(min),
max == null ? null : Double.parseDouble(max), max == null ? null : Double.parseDouble(max),
minInclusive, maxInclusive); minInclusive, maxInclusive);
break; break;
case DATE: case DATE:
query = NumericRangeQuery.newLongRange(field, precisionStep, query = NumericRangeQuery.newLongRange(field.getName(), ps,
min == null ? null : dateField.parseMath(null, min).getTime(), min == null ? null : dateField.parseMath(null, min).getTime(),
max == null ? null : dateField.parseMath(null, max).getTime(), max == null ? null : dateField.parseMath(null, max).getTime(),
minInclusive, maxInclusive); minInclusive, maxInclusive);
@ -205,24 +265,196 @@ public class TrieField extends FieldType {
return query; return query;
} }
static int toInt(byte[] arr) {
return (arr[0]<<24) | ((arr[1]&0xff)<<16) | ((arr[2]&0xff)<<8) | (arr[3]&0xff);
}
static long toLong(byte[] arr) {
int high = (arr[0]<<24) | ((arr[1]&0xff)<<16) | ((arr[2]&0xff)<<8) | (arr[3]&0xff);
int low = (arr[4]<<24) | ((arr[5]&0xff)<<16) | ((arr[6]&0xff)<<8) | (arr[7]&0xff);
return (((long)high)<<32) | (low&0x0ffffffffL);
}
static float toFloat(byte[] arr) {
return Float.intBitsToFloat(toInt(arr));
}
static double toDouble(byte[] arr) {
return Double.longBitsToDouble(toLong(arr));
}
static byte[] toArr(int val) {
byte[] arr = new byte[4];
arr[0] = (byte)(val>>>24);
arr[1] = (byte)(val>>>16);
arr[2] = (byte)(val>>>8);
arr[3] = (byte)(val);
return arr;
}
static byte[] toArr(long val) {
byte[] arr = new byte[8];
arr[0] = (byte)(val>>>56);
arr[1] = (byte)(val>>>48);
arr[2] = (byte)(val>>>40);
arr[3] = (byte)(val>>>32);
arr[4] = (byte)(val>>>24);
arr[5] = (byte)(val>>>16);
arr[6] = (byte)(val>>>8);
arr[7] = (byte)(val);
return arr;
}
static byte[] toArr(float val) {
return toArr(Float.floatToRawIntBits(val));
}
static byte[] toArr(double val) {
return toArr(Double.doubleToRawLongBits(val));
}
@Override
public String storedToReadable(Fieldable f) {
return toExternal(f);
}
@Override
public String readableToIndexed(String val) {
switch (type) {
case INTEGER:
return NumericUtils.intToPrefixCoded(Integer.parseInt(val));
case FLOAT:
return NumericUtils.intToPrefixCoded(NumericUtils.floatToSortableInt(Float.parseFloat(val)));
case LONG:
return NumericUtils.longToPrefixCoded(Long.parseLong(val));
case DOUBLE:
return NumericUtils.longToPrefixCoded(NumericUtils.doubleToSortableLong(Double.parseDouble(val)));
case DATE:
return NumericUtils.longToPrefixCoded(dateField.parseMath(null, val).getTime());
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type);
}
}
@Override @Override
public String toInternal(String val) { public String toInternal(String val) {
return super.toInternal(val); return readableToIndexed(val);
}
static String badFieldString(Fieldable f) {
String s = f.stringValue();
return "ERROR:SCHEMA-INDEX-MISMATCH,stringValue="+s;
} }
@Override @Override
public String toExternal(Fieldable f) { public String toExternal(Fieldable f) {
return super.toExternal(f); byte[] arr = f.binaryValue();
if (arr==null) return badFieldString(f);
switch (type) {
case INTEGER:
return Integer.toString(toInt(arr));
case FLOAT:
return Float.toString(toFloat(arr));
case LONG:
return Long.toString(toLong(arr));
case DOUBLE:
return Double.toString(toDouble(arr));
case DATE:
return dateField.formatDate(new Date(toLong(arr)));
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
}
} }
@Override @Override
public String indexedToReadable(String indexedForm) { public String indexedToReadable(String indexedForm) {
return super.indexedToReadable(indexedForm); switch (type) {
case INTEGER:
return Integer.toString( NumericUtils.prefixCodedToInt(indexedForm) );
case FLOAT:
return Float.toString( NumericUtils.sortableIntToFloat(NumericUtils.prefixCodedToInt(indexedForm)) );
case LONG:
return Long.toString( NumericUtils.prefixCodedToLong(indexedForm) );
case DOUBLE:
return Double.toString( NumericUtils.sortableLongToDouble(NumericUtils.prefixCodedToLong(indexedForm)) );
case DATE:
return dateField.formatDate( new Date(NumericUtils.prefixCodedToLong(indexedForm)) );
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type);
}
} }
@Override @Override
public String storedToIndexed(Fieldable f) { public String storedToIndexed(Fieldable f) {
return super.storedToIndexed(f); // TODO: optimize to remove redundant string conversion
return readableToIndexed(storedToReadable(f));
}
@Override
public Field createField(SchemaField field, String externalVal, float boost) {
boolean indexed = field.indexed();
boolean stored = field.stored();
if (!indexed && !stored) {
if (log.isTraceEnabled())
log.trace("Ignoring unindexed/unstored field: " + field);
return null;
}
int ps = field.multiValued() ? Integer.MAX_VALUE : precisionStep;
byte[] arr=null;
TokenStream ts=null;
// String indexedVal = indexed && precisionStep==0 ? readableToIndexed(externalVal) : null;
switch (type) {
case INTEGER:
int i = Integer.parseInt(externalVal);
if (stored) arr = toArr(i);
if (indexed) ts = new NumericTokenStream(ps).setIntValue(i);
break;
case FLOAT:
float f = Float.parseFloat(externalVal);
if (stored) arr = toArr(f);
if (indexed) ts = new NumericTokenStream(ps).setFloatValue(f);
break;
case LONG:
long l = Long.parseLong(externalVal);
if (stored) arr = toArr(l);
if (indexed) ts = new NumericTokenStream(ps).setLongValue(l);
break;
case DOUBLE:
double d = Double.parseDouble(externalVal);
if (stored) arr = toArr(d);
if (indexed) ts = new NumericTokenStream(ps).setDoubleValue(d);
break;
case DATE:
long time = dateField.parseMath(null, externalVal).getTime();
if (stored) arr = toArr(time);
if (indexed) ts = new NumericTokenStream(ps).setLongValue(time);
break;
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type);
}
Field f;
if (stored) {
f = new Field(field.getName(), arr, Field.Store.YES);
if (indexed) f.setTokenStream(ts);
} else {
f = new Field(field.getName(), ts);
}
// term vectors aren't supported
f.setOmitNorms(field.omitNorms());
f.setOmitTermFreqAndPositions(field.omitTf());
f.setBoost(boost);
return f;
} }
public enum TrieTypes { public enum TrieTypes {
@ -233,3 +465,23 @@ public class TrieField extends FieldType {
DATE DATE
} }
} }
class TrieDateFieldSource extends LongFieldSource {
public TrieDateFieldSource(String field, FieldCache.LongParser parser) {
super(field, parser);
}
public TrieDateFieldSource(String field) {
super(field);
}
public String description() {
return "date(" + field + ')';
}
@Override
public long externalToLong(String extVal) {
return TrieField.dateField.parseMath(null, extVal).getTime();
}
}

12
src/java/org/apache/solr/search/QueryParsing.java
View File

@ -303,7 +303,13 @@ public class QueryParsing {
static void writeFieldVal(String val, FieldType ft, Appendable out, int flags) throws IOException { static void writeFieldVal(String val, FieldType ft, Appendable out, int flags) throws IOException {
if (ft!=null) { if (ft!=null) {
out.append(ft.toExternal(new Field("",val, Field.Store.YES, Field.Index.UN_TOKENIZED))); try {
out.append(ft.indexedToReadable(val));
} catch (Exception e) {
out.append("EXCEPTION(val=");
out.append(val);
out.append(")");
}
} else { } else {
out.append(val); out.append(val);
} }
@ -349,7 +355,7 @@ public class QueryParsing {
if (lt==null) { if (lt==null) {
out.append('*'); out.append('*');
} else { } else {
writeFieldVal(lt.toString(), ft, out, flags); out.append(lt.toString());
} }
out.append(" TO "); out.append(" TO ");
@ -357,7 +363,7 @@ public class QueryParsing {
if (ut==null) { if (ut==null) {
out.append('*'); out.append('*');
} else { } else {
writeFieldVal(ut.toString(), ft, out, flags); out.append(ut.toString());
} }
out.append( q.includesMax() ? ']' : '}' ); out.append( q.includesMax() ? ']' : '}' );

5
src/java/org/apache/solr/search/SolrQueryParser.java
View File

@ -26,6 +26,7 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.schema.FieldType; import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.TrieField; import org.apache.solr.schema.TrieField;
import org.apache.solr.schema.SchemaField;
// TODO: implement the analysis of simple fields with // TODO: implement the analysis of simple fields with
// FieldType.toInternal() instead of going through the // FieldType.toInternal() instead of going through the
@ -118,8 +119,8 @@ public class SolrQueryParser extends QueryParser {
protected Query getRangeQuery(String field, String part1, String part2, boolean inclusive) throws ParseException { protected Query getRangeQuery(String field, String part1, String part2, boolean inclusive) throws ParseException {
checkNullField(field); checkNullField(field);
FieldType ft = schema.getFieldType(field); SchemaField sf = schema.getField(field);
return ft.getRangeQuery(parser, field, return sf.getType().getRangeQuery(parser, sf,
"*".equals(part1) ? null : part1, "*".equals(part1) ? null : part1,
"*".equals(part2) ? null : part2, "*".equals(part2) ? null : part2,
inclusive, inclusive); inclusive, inclusive);

14
src/java/org/apache/solr/search/function/LongFieldSource.java
View File

@ -47,6 +47,11 @@ public class LongFieldSource extends FieldCacheSource {
return "long(" + field + ')'; return "long(" + field + ')';
} }
public long externalToLong(String extVal) {
return Long.parseLong(extVal);
}
public DocValues getValues(IndexReader reader) throws IOException { public DocValues getValues(IndexReader reader) throws IOException {
final long[] arr = (parser == null) ? final long[] arr = (parser == null) ?
((FieldCache) cache).getLongs(reader, field) : ((FieldCache) cache).getLongs(reader, field) :
@ -76,7 +81,6 @@ public class LongFieldSource extends FieldCacheSource {
return description() + '=' + longVal(doc); return description() + '=' + longVal(doc);
} }
@Override @Override
public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) { public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
long lower,upper; long lower,upper;
@ -86,14 +90,14 @@ public class LongFieldSource extends FieldCacheSource {
if (lowerVal==null) { if (lowerVal==null) {
lower = Long.MIN_VALUE; lower = Long.MIN_VALUE;
} else { } else {
lower = Long.parseLong(lowerVal); lower = externalToLong(lowerVal);
if (!includeLower && lower < Long.MAX_VALUE) lower++; if (!includeLower && lower < Long.MAX_VALUE) lower++;
} }
if (upperVal==null) { if (upperVal==null) {
upper = Long.MAX_VALUE; upper = Long.MAX_VALUE;
} else { } else {
upper = Long.parseLong(upperVal); upper = externalToLong(upperVal);
if (!includeUpper && upper > Long.MIN_VALUE) upper--; if (!includeUpper && upper > Long.MIN_VALUE) upper--;
} }
@ -116,7 +120,7 @@ public class LongFieldSource extends FieldCacheSource {
} }
public boolean equals(Object o) { public boolean equals(Object o) {
if (o.getClass() != LongFieldSource.class) return false; if (o.getClass() != this.getClass()) return false;
LongFieldSource other = (LongFieldSource) o; LongFieldSource other = (LongFieldSource) o;
return super.equals(other) return super.equals(other)
&& this.parser == null ? other.parser == null : && this.parser == null ? other.parser == null :
@ -124,7 +128,7 @@ public class LongFieldSource extends FieldCacheSource {
} }
public int hashCode() { public int hashCode() {
int h = parser == null ? Long.class.hashCode() : parser.getClass().hashCode(); int h = parser == null ? this.getClass().hashCode() : parser.getClass().hashCode();
h += super.hashCode(); h += super.hashCode();
return h; return h;
} }

3
src/java/org/apache/solr/util/AbstractSolrTestCase.java
View File

@ -180,7 +180,8 @@ public abstract class AbstractSolrTestCase extends TestCase {
String results = h.validateXPath(response, tests); String results = h.validateXPath(response, tests);
if (null != results) { if (null != results) {
fail(m + "query failed XPath: " + results + fail(m + "query failed XPath: " + results +
" xml response was: " + response); "\n xml response was: " + response +
"\n request was: " + req.getParamString());
} }
} catch (XPathExpressionException e1) { } catch (XPathExpressionException e1) {
throw new RuntimeException("XPath is invalid", e1); throw new RuntimeException("XPath is invalid", e1);

20
src/test/org/apache/solr/client/solrj/SolrExampleTests.java
View File

@ -48,7 +48,7 @@ import org.apache.solr.common.params.FacetParams;
* @version $Id$ * @version $Id$
* @since solr 1.3 * @since solr 1.3
*/ */
abstract public class SolrExampleTests extends SolrExampleTestBase abstract public class SolrExampleTests extends SolrExampleTestBase
{ {
/** /**
* query the example * query the example
@ -336,6 +336,8 @@ abstract public class SolrExampleTests extends SolrExampleTestBase
server.deleteByQuery( "*:*" );// delete everything! server.deleteByQuery( "*:*" );// delete everything!
server.commit(); server.commit();
assertNumFound( "*:*", 0 ); // make sure it got in assertNumFound( "*:*", 0 ); // make sure it got in
String f = "val_pi";
int i=0; // 0 1 2 3 4 5 6 7 8 9 int i=0; // 0 1 2 3 4 5 6 7 8 9
int[] nums = new int[] { 23, 26, 38, 46, 55, 63, 77, 84, 92, 94 }; int[] nums = new int[] { 23, 26, 38, 46, 55, 63, 77, 84, 92, 94 };
@ -343,7 +345,7 @@ abstract public class SolrExampleTests extends SolrExampleTestBase
SolrInputDocument doc = new SolrInputDocument(); SolrInputDocument doc = new SolrInputDocument();
doc.setField( "id", "doc"+i++ ); doc.setField( "id", "doc"+i++ );
doc.setField( "name", "doc: "+num ); doc.setField( "name", "doc: "+num );
doc.setField( "popularity", num ); doc.setField( f, num );
server.add( doc ); server.add( doc );
} }
server.commit(); server.commit();
@ -351,10 +353,10 @@ abstract public class SolrExampleTests extends SolrExampleTestBase
SolrQuery query = new SolrQuery( "*:*" ); SolrQuery query = new SolrQuery( "*:*" );
query.setRows( 0 ); query.setRows( 0 );
query.setGetFieldStatistics( "popularity" ); query.setGetFieldStatistics( f );
QueryResponse rsp = server.query( query ); QueryResponse rsp = server.query( query );
FieldStatsInfo stats = rsp.getFieldStatsInfo().get( "popularity" ); FieldStatsInfo stats = rsp.getFieldStatsInfo().get( f );
assertNotNull( stats ); assertNotNull( stats );
assertEquals( 23.0, stats.getMin() ); assertEquals( 23.0, stats.getMin() );
@ -373,14 +375,14 @@ abstract public class SolrExampleTests extends SolrExampleTestBase
SolrInputDocument doc = new SolrInputDocument(); SolrInputDocument doc = new SolrInputDocument();
doc.setField( "id", "doc"+i++ ); doc.setField( "id", "doc"+i++ );
doc.setField( "name", "doc: "+num ); doc.setField( "name", "doc: "+num );
doc.setField( "popularity", num ); doc.setField( f, num );
server.add( doc ); server.add( doc );
} }
server.commit(); server.commit();
assertNumFound( "*:*", nums.length ); // make sure they all got in assertNumFound( "*:*", nums.length ); // make sure they all got in
rsp = server.query( query ); rsp = server.query( query );
stats = rsp.getFieldStatsInfo().get( "popularity" ); stats = rsp.getFieldStatsInfo().get( f );
assertNotNull( stats ); assertNotNull( stats );
assertEquals( 5.0, stats.getMin() ); assertEquals( 5.0, stats.getMin() );
@ -399,7 +401,7 @@ abstract public class SolrExampleTests extends SolrExampleTestBase
SolrInputDocument doc = new SolrInputDocument(); SolrInputDocument doc = new SolrInputDocument();
doc.setField( "id", "doc"+i ); doc.setField( "id", "doc"+i );
doc.setField( "name", "doc: "+num ); doc.setField( "name", "doc: "+num );
doc.setField( "popularity", num ); doc.setField( f, num );
doc.setField( "inStock", i < 5 ); doc.setField( "inStock", i < 5 );
server.add( doc ); server.add( doc );
} }
@ -408,9 +410,9 @@ abstract public class SolrExampleTests extends SolrExampleTestBase
assertNumFound( "inStock:false", 5 ); // make sure they all got in assertNumFound( "inStock:false", 5 ); // make sure they all got in
// facet on 'inStock' // facet on 'inStock'
query.addStatsFieldFacets( "popularity", "inStock" ); query.addStatsFieldFacets( f, "inStock" );
rsp = server.query( query ); rsp = server.query( query );
stats = rsp.getFieldStatsInfo().get( "popularity" ); stats = rsp.getFieldStatsInfo().get( f );
assertNotNull( stats ); assertNotNull( stats );
List<FieldStatsInfo> facets = stats.getFacets().get( "inStock" ); List<FieldStatsInfo> facets = stats.getFacets().get( "inStock" );

59
src/test/org/apache/solr/search/TestQueryTypes.java
View File

@ -46,8 +46,65 @@ public class TestQueryTypes extends AbstractSolrTestCase {
assertU(adoc("id","6", "v_f","8983")); assertU(adoc("id","6", "v_f","8983"));
assertU(adoc("id","7", "v_f","1.5")); assertU(adoc("id","7", "v_f","1.5"));
assertU(adoc("id","8", "v_ti","5")); assertU(adoc("id","8", "v_ti","5"));
Object[] arr = new Object[] {
"id",999.0
,"v_s","wow dude"
,"v_t","wow"
,"v_ti",-1
,"v_tis",-1
,"v_tl",-1234567891234567890L
,"v_tls",-1234567891234567890L
,"v_tf",-2.0f
,"v_tfs",-2.0f
,"v_td",-2.0
,"v_tds",-2.0
,"v_tdt","2000-05-10T01:01:01Z"
,"v_tdts","2002-08-26T01:01:01Z"
};
String[] sarr = new String[arr.length];
for (int i=0; i<arr.length; i++) {
sarr[i] = arr[i].toString();
}
assertU(adoc(sarr));
assertU(optimize()); assertU(optimize());
// test field queries
for (int i=0; i<arr.length; i+=2) {
String f = arr[i].toString();
String v = arr[i+1].toString();
// normal lucene fielded query
assertQ(req( "q",f+":\""+v+'"')
,"//result[@numFound='1']"
,"//*[@name='id'][.='999.0']"
,"//*[@name='" + f + "'][.='" + v + "']"
);
// System.out.println("#########################################" + f + "=" + v);
// field qparser
assertQ(req( "q", "{!field f="+f+"}"+v)
,"//result[@numFound='1']"
);
// lucene range
assertQ(req( "q", f + ":[\"" + v + "\" TO \"" + v + "\"]" )
,"//result[@numFound='1']"
);
// frange qparser
assertQ(req( "q", "{!frange v="+f+" l='"+v+"' u='"+v+"'}" )
,"//result[@numFound='1']"
);
// function query... just make sure it doesn't throw an exception
assertQ(req( "q", "+id:999 _val_:\"" + f + "\"")
,"//result[@numFound='1']"
);
}
// Some basic tests to ensure that parsing local params is working // Some basic tests to ensure that parsing local params is working
assertQ("test prefix query", assertQ("test prefix query",
@ -85,12 +142,10 @@ public class TestQueryTypes extends AbstractSolrTestCase {
,"//result[@numFound='1']" ,"//result[@numFound='1']"
); );
/** future test
assertQ( assertQ(
req("q","{!field f=v_ti}5") req("q","{!field f=v_ti}5")
,"//result[@numFound='1']" ,"//result[@numFound='1']"
); );
**/
assertQ("test multi term field query on text type", assertQ("test multi term field query on text type",
req("q","{!field f=v_t}Hello DUDE") req("q","{!field f=v_t}Hello DUDE")

2
src/test/org/apache/solr/search/TestRangeQuery.java
View File

@ -94,6 +94,7 @@ public class TestRangeQuery extends AbstractSolrTestCase {
norm_fields.put("foo_ti", ints); norm_fields.put("foo_ti", ints);
norm_fields.put("foo_tl", longs); norm_fields.put("foo_tl", longs);
norm_fields.put("foo_td", doubles); norm_fields.put("foo_td", doubles);
norm_fields.put("foo_tdt", dates);
norm_fields.put("foo_s", strings); norm_fields.put("foo_s", strings);
norm_fields.put("foo_dt", dates); norm_fields.put("foo_dt", dates);
@ -108,6 +109,7 @@ public class TestRangeQuery extends AbstractSolrTestCase {
frange_fields.put("foo_ti", ints); frange_fields.put("foo_ti", ints);
frange_fields.put("foo_tl", longs); frange_fields.put("foo_tl", longs);
frange_fields.put("foo_td", doubles); frange_fields.put("foo_td", doubles);
frange_fields.put("foo_tdt", dates);
frange_fields.put("foo_pi", ints); frange_fields.put("foo_pi", ints);
frange_fields.put("foo_pl", longs); frange_fields.put("foo_pl", longs);

12
src/test/test-files/solr/conf/schema-trie.xml
View File

@ -88,14 +88,14 @@
<fieldType name="sfloat" class="solr.SortableFloatField" sortMissingLast="true" omitNorms="true"/> <fieldType name="sfloat" class="solr.SortableFloatField" sortMissingLast="true" omitNorms="true"/>
<fieldType name="sdouble" class="solr.SortableDoubleField" sortMissingLast="true" omitNorms="true"/> <fieldType name="sdouble" class="solr.SortableDoubleField" sortMissingLast="true" omitNorms="true"/>
<fieldType name="tint" class="solr.TrieField" type="integer" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" /> <fieldType name="tint" class="solr.TrieIntField" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<fieldType name="tfloat" class="solr.TrieField" type="float" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" /> <fieldType name="tfloat" class="solr.TrieFloatField" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<fieldType name="tlong" class="solr.TrieField" type="long" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" /> <fieldType name="tlong" class="solr.TrieLongField" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<fieldType name="tdouble" class="solr.TrieField" type="double" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" /> <fieldType name="tdouble" class="solr.TrieDoubleField" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<fieldType name="tdouble4" class="solr.TrieField" type="double" precisionStep="4" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" /> <fieldType name="tdouble4" class="solr.TrieDoubleField" type="double" precisionStep="4" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<fieldType name="tdate" class="solr.TrieField" type="date" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" /> <fieldType name="tdate" class="solr.TrieDateField" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and <!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and

24
src/test/test-files/solr/conf/schema11.xml
View File

@ -237,15 +237,19 @@
<fieldType name="file" keyField="id" defVal="1" stored="false" indexed="false" class="solr.ExternalFileField" valType="float"/> <fieldType name="file" keyField="id" defVal="1" stored="false" indexed="false" class="solr.ExternalFileField" valType="float"/>
<fieldType name="tint" class="solr.TrieField" type="integer" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" /> <fieldType name="tint" class="solr.TrieIntField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tfloat" class="solr.TrieField" type="float" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" /> <fieldType name="tfloat" class="solr.TrieFloatField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tlong" class="solr.TrieField" type="long" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" /> <fieldType name="tlong" class="solr.TrieLongField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tdouble" class="solr.TrieField" type="double" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" /> <fieldType name="tdouble" class="solr.TrieDoubleField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tdouble4" class="solr.TrieDoubleField" precisionStep="4" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tdate" class="solr.TrieDateField" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tdouble4" class="solr.TrieField" type="double" precisionStep="4" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<fieldType name="tdate" class="solr.TrieField" type="date" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<fieldType name="tints" class="solr.TrieIntField" omitNorms="true" positionIncrementGap="0" precisionStep="0" multiValued="true" />
<fieldType name="tfloats" class="solr.TrieFloatField" omitNorms="true" positionIncrementGap="0" precisionStep="0" multiValued="true"/>
<fieldType name="tlongs" class="solr.TrieLongField" omitNorms="true" positionIncrementGap="0" precisionStep="0" multiValued="true"/>
<fieldType name="tdoubles" class="solr.TrieDoubleField" omitNorms="true" positionIncrementGap="0" precisionStep="0" multiValued="true" />
<fieldType name="tdates" class="solr.TrieDateField" omitNorms="true" positionIncrementGap="0" precisionStep="0" multiValued="true" />
</types> </types>
@ -298,6 +302,12 @@
<dynamicField name="*_td" type="tdouble" indexed="true" stored="true"/> <dynamicField name="*_td" type="tdouble" indexed="true" stored="true"/>
<dynamicField name="*_tdt" type="tdate" indexed="true" stored="true"/> <dynamicField name="*_tdt" type="tdate" indexed="true" stored="true"/>
<dynamicField name="*_tis" type="tints" indexed="true" stored="true"/>
<dynamicField name="*_tls" type="tlongs" indexed="true" stored="true"/>
<dynamicField name="*_tfs" type="tfloats" indexed="true" stored="true"/>
<dynamicField name="*_tds" type="tdoubles" indexed="true" stored="true"/>
<dynamicField name="*_tdts" type="tdates" indexed="true" stored="true"/>
<dynamicField name="*_t" type="text" indexed="true" stored="true"/> <dynamicField name="*_t" type="text" indexed="true" stored="true"/>
<dynamicField name="*_b" type="boolean" indexed="true" stored="true"/> <dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
<dynamicField name="*_dt" type="date" indexed="true" stored="true"/> <dynamicField name="*_dt" type="date" indexed="true" stored="true"/>