();
diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java
index 54a0206490c..4fca55aff99 100644
--- a/solr/core/src/java/org/apache/solr/schema/FieldType.java
+++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java
@@ -428,21 +428,6 @@ public abstract class FieldType extends FieldProperties {
*/
protected Analyzer queryAnalyzer=analyzer;
- /**
- * Analyzer set by schema for text types to use when searching fields
- * of this type, subclasses can set analyzer themselves or override
- * getAnalyzer()
- * This analyzer is used to process wildcard, prefix, regex and other multiterm queries. It
- * assembles a list of tokenizer +filters that "make sense" for this, primarily accent folding and
- * lowercasing filters, and charfilters.
- *
- * If users require old-style behavior, they can specify 'legacyMultiterm="true" ' in the schema file
- * @see #getMultiTermAnalyzer
- * @see #setMultiTermAnalyzer
- */
- protected Analyzer multiTermAnalyzer=null;
-
-
/**
* Returns the Analyzer to be used when indexing fields of this type.
*
@@ -465,20 +450,6 @@ public abstract class FieldType extends FieldProperties {
return queryAnalyzer;
}
- /**
- * Returns the Analyzer to be used when searching fields of this type when mult-term queries are specified.
- *
- * This method may be called many times, at any time.
- *
- * @see #getAnalyzer
- */
- public Analyzer getMultiTermAnalyzer() {
- return multiTermAnalyzer;
- }
-
- private final String analyzerError =
- "FieldType: " + this.getClass().getSimpleName() +
- " (" + typeName + ") does not support specifying an analyzer";
/**
* Sets the Analyzer to be used when indexing fields of this type.
@@ -524,28 +495,6 @@ public abstract class FieldType extends FieldProperties {
throw e;
}
- /**
- * Sets the Analyzer to be used when querying fields of this type.
- *
- *
- *
- * Subclasses that override this method need to ensure the behavior
- * of the analyzer is consistent with the implementation of toInternal.
- *
- *
- * @see #toInternal
- * @see #setAnalyzer
- * @see #getQueryAnalyzer
- */
- public void setMultiTermAnalyzer(Analyzer analyzer) {
- SolrException e = new SolrException
- (ErrorCode.SERVER_ERROR,
- "FieldType: " + this.getClass().getSimpleName() +
- " (" + typeName + ") does not support specifying an analyzer");
- SolrException.logOnce(log,null,e);
- throw e;
- }
-
/** @lucene.internal */
protected Similarity similarity;
diff --git a/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java b/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java
index 090c7b0ada2..37277e87cd3 100644
--- a/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java
+++ b/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java
@@ -102,15 +102,13 @@ public final class FieldTypePluginLoader
if (queryAnalyzer==null) queryAnalyzer=analyzer;
if (analyzer==null) analyzer=queryAnalyzer;
if (multiAnalyzer == null) {
- Boolean legacyMatch = ! schema.getDefaultLuceneMatchVersion().onOrAfter(Version.LUCENE_36);
- legacyMatch = (DOMUtil.getAttr(node, "legacyMultiTerm", null) == null) ? legacyMatch :
- Boolean.parseBoolean(DOMUtil.getAttr(node, "legacyMultiTerm", null));
- multiAnalyzer = constructMultiTermAnalyzer(queryAnalyzer, legacyMatch);
+ multiAnalyzer = constructMultiTermAnalyzer(queryAnalyzer);
}
if (analyzer!=null) {
ft.setAnalyzer(analyzer);
ft.setQueryAnalyzer(queryAnalyzer);
- ft.setMultiTermAnalyzer(multiAnalyzer);
+ if (ft instanceof TextField)
+ ((TextField)ft).setMultiTermAnalyzer(multiAnalyzer);
}
if (similarity!=null) {
ft.setSimilarity(similarity);
@@ -143,36 +141,75 @@ public final class FieldTypePluginLoader
// 2> If letacyMultiTerm == true just construct the analyzer from a KeywordTokenizer. That should mimic current behavior.
// Do the same if they've specified that the old behavior is required (legacyMultiTerm="true")
- private Analyzer constructMultiTermAnalyzer(Analyzer queryAnalyzer, Boolean legacyMultiTerm) {
+ private Analyzer constructMultiTermAnalyzer(Analyzer queryAnalyzer) {
if (queryAnalyzer == null) return null;
- if (legacyMultiTerm || (!(queryAnalyzer instanceof TokenizerChain))) {
+ if (!(queryAnalyzer instanceof TokenizerChain)) {
return new KeywordAnalyzer();
}
TokenizerChain tc = (TokenizerChain) queryAnalyzer;
+ MultiTermChainBuilder builder = new MultiTermChainBuilder();
- // we know it'll never be longer than this unless the code below is explicitly changed
- TokenFilterFactory[] filters = new TokenFilterFactory[2];
- int idx = 0;
- for (TokenFilterFactory factory : tc.getTokenFilterFactories()) {
- if (factory instanceof LowerCaseFilterFactory) {
- filters[idx] = new LowerCaseFilterFactory();
- filters[idx++].init(factory.getArgs());
- }
- if (factory instanceof ASCIIFoldingFilterFactory) {
- filters[idx] = new ASCIIFoldingFilterFactory();
- filters[idx++].init(factory.getArgs());
+ CharFilterFactory[] charFactories = tc.getCharFilterFactories();
+ if (charFactories != null) {
+ for (CharFilterFactory fact : charFactories) {
+ builder.add(fact);
}
}
- WhitespaceTokenizerFactory white = new WhitespaceTokenizerFactory();
- white.init(tc.getTokenizerFactory().getArgs());
- return new TokenizerChain(tc.getCharFilterFactories(),
- white,
- Arrays.copyOfRange(filters, 0, idx));
+ builder.add(tc.getTokenizerFactory());
+
+ for (TokenFilterFactory fact : tc.getTokenFilterFactories()) {
+ builder.add(fact);
+ }
+
+ return builder.build();
}
+ private static class MultiTermChainBuilder {
+ static final KeywordTokenizerFactory keyFactory;
+
+ static {
+ keyFactory = new KeywordTokenizerFactory();
+ keyFactory.init(new HashMap());
+ }
+
+ ArrayList charFilters = null;
+ ArrayList filters = new ArrayList(2);
+ TokenizerFactory tokenizer = keyFactory;
+
+ public void add(Object current) {
+ if (!(current instanceof MultiTermAwareComponent)) return;
+ Object newComponent = ((MultiTermAwareComponent)current).getMultiTermComponent();
+ if (newComponent instanceof TokenFilterFactory) {
+ if (filters == null) {
+ filters = new ArrayList(2);
+ }
+ filters.add((TokenFilterFactory)newComponent);
+ } else if (newComponent instanceof TokenizerFactory) {
+ tokenizer = (TokenizerFactory)newComponent;
+ } else if (newComponent instanceof CharFilterFactory) {
+ if (charFilters == null) {
+ charFilters = new ArrayList(1);
+ }
+ charFilters.add( (CharFilterFactory)newComponent);
+
+ } else {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown analysis component from MultiTermAwareComponent: " + newComponent);
+ }
+ }
+
+ public TokenizerChain build() {
+ CharFilterFactory[] charFilterArr = charFilters == null ? null : charFilters.toArray(new CharFilterFactory[charFilters.size()]);
+ TokenFilterFactory[] filterArr = filters == null ? new TokenFilterFactory[0] : filters.toArray(new TokenFilterFactory[filters.size()]);
+ return new TokenizerChain(charFilterArr, tokenizer, filterArr);
+ }
+
+
+ }
+
+
//
//
//
diff --git a/solr/core/src/java/org/apache/solr/schema/SchemaField.java b/solr/core/src/java/org/apache/solr/schema/SchemaField.java
index 34990ff9655..aaf5c06f30e 100644
--- a/solr/core/src/java/org/apache/solr/schema/SchemaField.java
+++ b/solr/core/src/java/org/apache/solr/schema/SchemaField.java
@@ -97,10 +97,6 @@ public final class SchemaField extends FieldProperties {
boolean isTokenized() { return (properties & TOKENIZED)!=0; }
boolean isBinary() { return (properties & BINARY)!=0; }
- boolean legacyMultiTerm() {
- return (properties & LEGACY_MULTITERM) != 0;
- }
-
public IndexableField createField(Object val, float boost) {
return type.createField(this,val,boost);
}
diff --git a/solr/core/src/java/org/apache/solr/schema/TextField.java b/solr/core/src/java/org/apache/solr/schema/TextField.java
index c3f76324d0e..8af29f06af8 100644
--- a/solr/core/src/java/org/apache/solr/schema/TextField.java
+++ b/solr/core/src/java/org/apache/solr/schema/TextField.java
@@ -17,13 +17,8 @@
package org.apache.solr.schema;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.PhraseQuery;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.MultiPhraseQuery;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
+import org.apache.lucene.search.*;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
@@ -32,6 +27,7 @@ import org.apache.lucene.analysis.CachingTokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.util.BytesRef;
+import org.apache.solr.common.SolrException;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
@@ -48,6 +44,19 @@ import java.io.StringReader;
public class TextField extends FieldType {
protected boolean autoGeneratePhraseQueries;
+ /**
+ * Analyzer set by schema for text types to use when searching fields
+ * of this type, subclasses can set analyzer themselves or override
+ * getAnalyzer()
+ * This analyzer is used to process wildcard, prefix, regex and other multiterm queries. It
+ * assembles a list of tokenizer +filters that "make sense" for this, primarily accent folding and
+ * lowercasing filters, and charfilters.
+ *
+ * @see #getMultiTermAnalyzer
+ * @see #setMultiTermAnalyzer
+ */
+ protected Analyzer multiTermAnalyzer=null;
+
@Override
protected void init(IndexSchema schema, Map args) {
properties |= TOKENIZED;
@@ -63,6 +72,21 @@ public class TextField extends FieldType {
super.init(schema, args);
}
+ /**
+ * Returns the Analyzer to be used when searching fields of this type when mult-term queries are specified.
+ *
+ * This method may be called many times, at any time.
+ *
+ * @see #getAnalyzer
+ */
+ public Analyzer getMultiTermAnalyzer() {
+ return multiTermAnalyzer;
+ }
+
+ public void setMultiTermAnalyzer(Analyzer analyzer) {
+ this.multiTermAnalyzer = analyzer;
+ }
+
public boolean getAutoGeneratePhraseQueries() {
return autoGeneratePhraseQueries;
}
@@ -98,11 +122,50 @@ public class TextField extends FieldType {
this.queryAnalyzer = analyzer;
}
+
@Override
- public void setMultiTermAnalyzer(Analyzer analyzer) {
- this.multiTermAnalyzer = analyzer;
+ public Query getRangeQuery(QParser parser, SchemaField field, String part1, String part2, boolean minInclusive, boolean maxInclusive) {
+ Analyzer multiAnalyzer = getMultiTermAnalyzer();
+ BytesRef lower = analyzeMultiTerm(field.getName(), part1, multiAnalyzer);
+ BytesRef upper = analyzeMultiTerm(field.getName(), part2, multiAnalyzer);
+ return new TermRangeQuery(field.getName(), lower, upper, minInclusive, maxInclusive);
}
+ public static BytesRef analyzeMultiTerm(String field, String part, Analyzer analyzerIn) {
+ if (part == null) return null;
+
+ TokenStream source;
+ try {
+ source = analyzerIn.tokenStream(field, new StringReader(part));
+ source.reset();
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unable to initialize TokenStream to analyze multiTerm term: " + part, e);
+ }
+
+ TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class);
+ BytesRef bytes = termAtt.getBytesRef();
+
+ try {
+ if (!source.incrementToken())
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,"analyzer returned no terms for multiTerm term: " + part);
+ termAtt.fillBytesRef();
+ if (source.incrementToken())
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,"analyzer returned too many terms for multiTerm term: " + part);
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,"error analyzing range part: " + part, e);
+ }
+
+ try {
+ source.end();
+ source.close();
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to end & close TokenStream after analyzing multiTerm term: " + part, e);
+ }
+
+ return BytesRef.deepCopyOf(bytes);
+ }
+
+
static Query parseFieldQuery(QParser parser, Analyzer analyzer, String field, String queryText) {
int phraseSlop = 0;
boolean enablePositionIncrements = true;
diff --git a/solr/core/src/java/org/apache/solr/search/SolrQueryParser.java b/solr/core/src/java/org/apache/solr/search/SolrQueryParser.java
index 0c5584d9c3a..441a26a9137 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrQueryParser.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrQueryParser.java
@@ -58,8 +58,9 @@ public class SolrQueryParser extends QueryParser {
protected final IndexSchema schema;
protected final QParser parser;
protected final String defaultField;
- protected final Map leadingWildcards =
- new HashMap();
+
+ // implementation detail - caching ReversedWildcardFilterFactory based on type
+ private Map leadingWildcards;
public SolrQueryParser(QParser parser, String defaultField) {
this(parser, defaultField, parser.getReq().getSchema().getQueryAnalyzer());
@@ -71,30 +72,34 @@ public class SolrQueryParser extends QueryParser {
this.parser = parser;
this.defaultField = defaultField;
setEnablePositionIncrements(true);
- checkAllowLeadingWildcards();
+ setLowercaseExpandedTerms(false);
+ setAllowLeadingWildcard(true);
}
- protected void checkAllowLeadingWildcards() {
- boolean allow = false;
- for (Entry e : schema.getFieldTypes().entrySet()) {
- Analyzer a = e.getValue().getAnalyzer();
- if (a instanceof TokenizerChain) {
- // examine the indexing analysis chain if it supports leading wildcards
- TokenizerChain tc = (TokenizerChain)a;
- TokenFilterFactory[] factories = tc.getTokenFilterFactories();
- for (TokenFilterFactory factory : factories) {
- if (factory instanceof ReversedWildcardFilterFactory) {
- allow = true;
- leadingWildcards.put(e.getKey(), (ReversedWildcardFilterFactory)factory);
- }
+ protected ReversedWildcardFilterFactory getReversedWildcardFilterFactory(FieldType fieldType) {
+ if (leadingWildcards == null) leadingWildcards = new HashMap();
+ ReversedWildcardFilterFactory fac = leadingWildcards.get(fieldType);
+ if (fac == null && leadingWildcards.containsKey(fac)) {
+ return fac;
+ }
+
+ Analyzer a = fieldType.getAnalyzer();
+ if (a instanceof TokenizerChain) {
+ // examine the indexing analysis chain if it supports leading wildcards
+ TokenizerChain tc = (TokenizerChain)a;
+ TokenFilterFactory[] factories = tc.getTokenFilterFactories();
+ for (TokenFilterFactory factory : factories) {
+ if (factory instanceof ReversedWildcardFilterFactory) {
+ fac = (ReversedWildcardFilterFactory)factory;
+ break;
}
}
}
- // XXX should be enabled on a per-field basis
- if (allow) {
- setAllowLeadingWildcard(true);
- }
+
+ leadingWildcards.put(fieldType, fac);
+ return fac;
}
+
private void checkNullField(String field) throws SolrException {
if (field == null && defaultField == null) {
@@ -104,12 +109,14 @@ public class SolrQueryParser extends QueryParser {
}
}
- protected String analyzeIfMultitermTermText(String field, String part, Analyzer analyzer) {
+ protected String analyzeIfMultitermTermText(String field, String part, FieldType fieldType) {
if (part == null) return part;
SchemaField sf = schema.getFieldOrNull((field));
- if (sf == null || ! (sf.getType() instanceof TextField)) return part;
- return analyzeMultitermTerm(field, part, analyzer).utf8ToString();
+ if (sf == null || ! (fieldType instanceof TextField)) return part;
+ String out = TextField.analyzeMultiTerm(field, part, ((TextField)fieldType).getMultiTermAnalyzer()).utf8ToString();
+ // System.out.println("INPUT="+part + " OUTPUT="+out);
+ return out;
}
@Override
@@ -143,8 +150,6 @@ public class SolrQueryParser extends QueryParser {
@Override
protected Query getRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) throws ParseException {
checkNullField(field);
- part1 = analyzeIfMultitermTermText(field, part1, schema.getFieldType(field).getMultiTermAnalyzer());
- part2 = analyzeIfMultitermTermText(field, part2, schema.getFieldType(field).getMultiTermAnalyzer());
SchemaField sf = schema.getField(field);
return sf.getType().getRangeQuery(parser, sf, part1, part2, startInclusive, endInclusive);
}
@@ -153,21 +158,10 @@ public class SolrQueryParser extends QueryParser {
protected Query getPrefixQuery(String field, String termStr) throws ParseException {
checkNullField(field);
- termStr = analyzeIfMultitermTermText(field, termStr, schema.getFieldType(field).getMultiTermAnalyzer());
+ termStr = analyzeIfMultitermTermText(field, termStr, schema.getFieldType(field));
- // TODO: toInternal() won't necessarily work on partial
- // values, so it looks like we need a getPrefix() function
- // on fieldtype? Or at the minimum, a method on fieldType
- // that can tell me if I should lowercase or not...
- // Schema could tell if lowercase filter is in the chain,
- // but a more sure way would be to run something through
- // the first time and check if it got lowercased.
-
- // TODO: throw exception if field type doesn't support prefixes?
- // (sortable numeric types don't do prefixes, but can do range queries)
- Term t = new Term(field, termStr);
- PrefixQuery prefixQuery = new PrefixQuery(t);
- return prefixQuery;
+ // Solr has always used constant scoring for prefix queries. This should return constant scoring by default.
+ return newPrefixQuery(new Term(field, termStr));
}
@Override
protected Query getWildcardQuery(String field, String termStr) throws ParseException {
@@ -175,10 +169,10 @@ public class SolrQueryParser extends QueryParser {
if ("*".equals(field) && "*".equals(termStr)) {
return newMatchAllDocsQuery();
}
- termStr = analyzeIfMultitermTermText(field, termStr, schema.getFieldType(field).getMultiTermAnalyzer());
+ FieldType fieldType = schema.getFieldType(field);
+ termStr = analyzeIfMultitermTermText(field, termStr, fieldType);
// can we use reversed wildcards in this field?
- String type = schema.getFieldType(field).getTypeName();
- ReversedWildcardFilterFactory factory = leadingWildcards.get(type);
+ ReversedWildcardFilterFactory factory = getReversedWildcardFilterFactory(fieldType);
if (factory != null) {
Term term = new Term(field, termStr);
// fsa representing the query
@@ -211,19 +205,15 @@ public class SolrQueryParser extends QueryParser {
}
};
}
- Query q = super.getWildcardQuery(field, termStr);
- if (q instanceof WildcardQuery) {
- // use a constant score query to avoid overflowing clauses
- WildcardQuery wildcardQuery = new WildcardQuery(((WildcardQuery)q).getTerm());
- return wildcardQuery;
- }
- return q;
+
+ // Solr has always used constant scoring for wildcard queries. This should return constant scoring by default.
+ return newWildcardQuery(new Term(field, termStr));
}
-
+ @Override
protected Query getRegexpQuery(String field, String termStr) throws ParseException
{
- termStr = analyzeIfMultitermTermText(field, termStr, schema.getFieldType(field).getMultiTermAnalyzer());
- return super.getRegexpQuery(field, termStr);
+ termStr = analyzeIfMultitermTermText(field, termStr, schema.getFieldType(field));
+ return newRegexpQuery(new Term(field, termStr));
}
}
diff --git a/solr/core/src/test-files/solr/conf/schema-folding.xml b/solr/core/src/test-files/solr/conf/schema-folding.xml
index 798ca301217..0e77b8b59a6 100644
--- a/solr/core/src/test-files/solr/conf/schema-folding.xml
+++ b/solr/core/src/test-files/solr/conf/schema-folding.xml
@@ -64,7 +64,7 @@